Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.7.3.1'

Browse Source
This commit is contained in:
gnosygnu
2016-07-17 21:10:59 -04:00
parent b333db45f8
commit 7a851a41a5
290 changed files with 3048 additions and 2124 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
400_xowa/src/gplx/xowa/addons/bldrs/files/Xoax_builds_files_addon.java
View File

@@ -18,8 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.files.cmds.*;
import gplx.xowa.addons.bldrs.mass_parses.inits.*;
import gplx.xowa.addons.bldrs.mass_parses.makes.*;
import gplx.xowa.addons.bldrs.mass_parses.inits.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.makes.*;
public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
public Xob_cmd[] Bldr_cmds() {
return new Xob_cmd[]
@@ -43,6 +42,7 @@ public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__
, Xomp_init_cmd.Prototype
, Xomp_parse_cmd.Prototype
, Xomp_make_cmd.Prototype
};
}

4
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__lnki_temp__create.java
View File

@@ -111,11 +111,11 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
parser.Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), wiki.Ns_mgr().Ns_template(), ttl_bry, page_src);
else {
parser.Parse_page_all_clear(root, ctx, ctx.Tkn_mkr(), page_src);
if ( gen_html
if ( gen_html
&& page.Redirect().Itms__len() == 0) // don't generate html for redirected pages
wiki.Html_mgr().Page_wtr_mgr().Gen(ctx.Page().Root_(root), Xopg_page_.Tid_read);
if (gen_hdump)
hdump_bldr.Insert(page.Root_(root));
hdump_bldr.Insert(ctx, page.Root_(root));
root.Clear();
}
}

59
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_db_core.java Normal file
View File

@@ -0,0 +1,59 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
public class Xomp_db_core {
private final Object thread_lock = new Object();
private final Io_url root_dir;
Xomp_db_core(Io_url root_dir) {
this.root_dir = root_dir;
Io_url mgr_url = root_dir.GenSubFil("xomp.sqlite3");
this.mgr_db = new Xomp_mgr_db(mgr_url);
}
public Xomp_mgr_db Mgr_db() {return mgr_db;} private Xomp_mgr_db mgr_db;
public Xomp_wkr_db Wkr_db(boolean delete, int idx) {
Io_url wkr_url = root_dir.GenSubFil_nest("xomp_" + Int_.To_str_fmt(idx, "000"), "xomp_wkr.sqlite3");
if (delete) Io_mgr.Instance.DeleteFil(wkr_url);
return new Xomp_wkr_db(idx, wkr_url);
}
public int Wkr_count() {
Io_url[] wkr_dirs = Io_mgr.Instance.QueryDir_args(root_dir).DirOnly_().ExecAsUrlAry();
return wkr_dirs.length;
}
public void Update_wkr_id(int idx, Db_conn wkr_conn) {
synchronized (thread_lock) {
Db_attach_mgr attach_mgr = new Db_attach_mgr(mgr_db.Conn(), new Db_attach_itm("wkr_db", wkr_conn));
attach_mgr.Exec_sql_w_msg("updating page_regy: wkr_id=" + idx, String_.Concat_lines_nl_skip_last // ANSI.Y
( "UPDATE xomp_page"
, "SET xomp_wkr_id = " + Int_.To_str(idx)
, ", html_len = (SELECT length(body) FROM <wkr_db>html h WHERE h.page_id = xomp_page.page_id)"
, "WHERE page_id IN (SELECT page_id FROM <wkr_db>html h)"
));
}
}
public static Xomp_db_core New__make(Xowe_wiki wiki) {
Io_url root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
Io_mgr.Instance.DeleteDirDeep(root_dir);
return new Xomp_db_core(root_dir);
}
public static Xomp_db_core New__load(Xowe_wiki wiki) {
Io_url root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
return new Xomp_db_core(root_dir);
}
}

29
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_mgr_db.java Normal file
View File

@@ -0,0 +1,29 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
public class Xomp_mgr_db {
public Xomp_mgr_db(Io_url url) {
this.url = url;
this.conn = Db_conn_bldr.Instance.Get_or_autocreate(true, url);
this.page_tbl = new Xomp_page_tbl(conn);
}
public Io_url Url() {return url;} private Io_url url;
public Db_conn Conn() {return conn;} private Db_conn conn;
public Xomp_page_tbl Page_tbl() {return page_tbl;} private Xomp_page_tbl page_tbl;
}

39
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_page_tbl.java Normal file
View File

@@ -0,0 +1,39 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
public class Xomp_page_tbl implements Db_tbl {
// private final String fld_page_id, fld_page_status, fld_page_mgr_id;
private final Db_conn conn;
public Xomp_page_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "xomp_page";
flds.Add_int_pkey("page_id");
flds.Add_int("page_ns");
flds.Add_byte("page_status"); // 0=wait; 1=done; 2=fail
flds.Add_int_dflt("html_len", -1);
flds.Add_int_dflt("xomp_wkr_id", -1);
conn.Rls_reg(this);
}
public String Tbl_name() {return tbl_name;} private final String tbl_name;
public Dbmeta_fld_list Flds() {return flds;} private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
public void Create_tbl() {
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
}
public void Rls() {}
}

33
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_wkr_db.java Normal file
View File

@@ -0,0 +1,33 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.htmls.core.dbs.*;
public class Xomp_wkr_db {
public Xomp_wkr_db(int idx, Io_url url) {
this.idx = idx;
this.url = url;
this.conn = Db_conn_bldr.Instance.Get_or_autocreate(true, url);
this.html_tbl = new Xowd_html_tbl(conn);
conn.Meta_tbl_assert(html_tbl);
}
public int Idx() {return idx;} private final int idx;
public Io_url Url() {return url;} private Io_url url;
public Db_conn Conn() {return conn;} private Db_conn conn;
public Xowd_html_tbl Html_tbl() {return html_tbl;} private final Xowd_html_tbl html_tbl;
}

34
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_mgr.java
View File

@@ -18,35 +18,33 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.mass_parses.inits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
class Xomp_init_mgr {
private final Xow_wiki wiki;
public Xomp_init_mgr(Xow_wiki wiki) {this.wiki = wiki;}
private final Xowe_wiki wiki;
public Xomp_init_mgr(Xowe_wiki wiki) {this.wiki = wiki;}
public void Exec() {
Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
Db_conn conn = make_db.Conn();
// init
Xomp_db_core db_core = Xomp_db_core.New__make(wiki);
Xomp_page_tbl page_tbl = db_core.Mgr_db().Page_tbl();
// make table
conn.Meta_tbl_remake(Dbmeta_tbl_itm.New("mp_page", new Dbmeta_fld_itm[]
{ Dbmeta_fld_itm.new_int("page_id").Primary_y_()
, Dbmeta_fld_itm.new_bool("page_done")
}
, Dbmeta_idx_itm.new_normal_by_tbl("mp_page", "page_id__page_done", "page_id", "page_done")
));
// rebuild table
Db_conn mgr_conn = db_core.Mgr_db().Conn();
mgr_conn.Meta_tbl_remake(page_tbl);
// fill table
Db_attach_mgr attach_mgr = new Db_attach_mgr(conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
Db_attach_mgr attach_mgr = new Db_attach_mgr(mgr_conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
int[] ns_ary = new int[] {0, 4, 14};
int len = ns_ary.length;
String sql = String_.Concat_lines_nl_skip_last
( "INSERT INTO mp_page (page_id, page_done)"
, "SELECT page_id, 0"
, "FROM <page_db>page"
, "WHERE page_namespace = {0}"
, "AND page_is_redirect = 0"
( "INSERT INTO xomp_page (page_id, page_ns, page_status, html_len, xomp_wkr_id)"
, "SELECT p.page_id, p.page_namespace, 0, 0, 0"
, "FROM <page_db>page p"
, "WHERE p.page_namespace = {0}"
, "AND p.page_is_redirect = 0"
);
for (int i = 0; i < len; ++i) {
int ns_id = ns_ary[i];
attach_mgr.Exec_sql_w_msg("adding rows for mp_page: ns=" + ns_id, sql, ns_id);// ANSI.Y
attach_mgr.Exec_sql_w_msg("adding rows for xomp_page: ns=" + ns_id, sql, ns_id);// ANSI.Y
}
}
}

41
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_rdr.java Normal file
View File

@@ -0,0 +1,41 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*;
import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
class Xomp_html_db_rdr {
private final Xowd_html_tbl[] src_tbls;
private final Xomp_db_core db;
public Xomp_html_db_rdr(Xowe_wiki wiki) {
this.db = Xomp_db_core.New__load(wiki);
this.src_tbls = new Xowd_html_tbl[db.Wkr_count()];
}
public void Rows__get(Xowd_html_row rv, int wkr_id, int page_id) {
Xowd_html_tbl src_tbl = src_tbls[wkr_id];
if (src_tbl == null) {
Db_conn wkr_conn = db.Wkr_db(Bool_.N, wkr_id).Conn();
src_tbl = new Xowd_html_tbl(wkr_conn);
src_tbls[wkr_id] = src_tbl;
}
src_tbl.Select_as_row(rv, page_id);
}
public void Rls() {
for (Xowd_html_tbl src_tbl : src_tbls)
src_tbl.Conn().Rls_conn();
}
}

66
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_wtr.java Normal file
View File

@@ -0,0 +1,66 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.wikis.data.*;
class Xomp_html_db_wtr {
private final long len_max;
private final Xowe_wiki wiki; private final Xow_db_mgr db_mgr;
private long len_cur;
private Xow_db_file html_db; private Xowd_html_tbl html_tbl;
public Xomp_html_db_wtr(Xowe_wiki wiki) {
this.wiki = wiki; this.db_mgr = wiki.Data__core_mgr();
this.len_max = wiki.Appe().Api_root().Bldr().Wiki().Import().Html_db_max();
// delete all existing tbls
if (!db_mgr.Props().Layout_html().Tid_is_all())
wiki.Data__core_mgr().Dbs__delete_by_tid(Xow_db_file_.Tid__html_data);
}
public int Cur_db_id() {return html_db.Id();}
public Xowd_html_tbl Tbls__get_or_new(int ns_id, long html_len) {
long len_new = len_cur + html_len;
if (html_tbl == null || len_new > len_max) {
Commit();
this.html_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__html_data);
if (html_db == null) {
html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
html_db.Conn().Txn_bgn("xomp.html_db_wtr");
this.html_tbl = new Xowd_html_tbl(html_db.Conn());
html_tbl.Create_tbl();
}
}
len_cur = len_new;
return html_tbl;
}
public void Rls() {
this.Commit();
}
private void Commit() {
if (html_tbl == null) return;
html_tbl.Conn().Txn_end();
html_tbl.Conn().Rls_conn();
// update page_ids
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
( "UPDATE page"
, "SET page_html_db_id = {0}"
, "WHERE page_id IN (SELECT page_id FROM <html_db>html h)"
), html_db.Id());
Db_attach_mgr attach_mgr = new Db_attach_mgr(db_mgr.Db__core().Conn(), new Db_attach_itm("html_db", html_db.Conn()));
attach_mgr.Exec_sql_w_msg("updating page_ids: " + Int_.To_str(html_db.Id()), sql);
}
}

31
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java Normal file
View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xomp_make_cmd extends Xob_cmd__base {
public Xomp_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
Xomp_make_wkr wkr = new Xomp_make_wkr(wiki);
wkr.Exec();
}
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "wiki.mass_parse.make";
public static final Xob_cmd Prototype = new Xomp_make_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xomp_make_cmd(bldr, wiki);}
}

71
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_wkr.java Normal file
View File

@@ -0,0 +1,71 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.brys.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
class Xomp_make_wkr {
private final Db_conn mgr_conn;
private final Xomp_html_db_wtr html_db_wtr;
private final Xomp_html_db_rdr html_db_rdr;
private final Int_flag_bldr src_body_flag_bldr = Xowd_html_tbl.Make_body_flag_bldr();
public Xomp_make_wkr(Xowe_wiki wiki) {
this.db = Xomp_db_core.New__load(wiki);
this.mgr_conn = db.Mgr_db().Conn();
this.html_db_rdr = new Xomp_html_db_rdr(wiki);
this.html_db_wtr = new Xomp_html_db_wtr(wiki);
}
public Xomp_db_core Db() {return db;} private Xomp_db_core db;
public void Exec() {
Xowd_html_row src_row = new Xowd_html_row();
int[] ns_ary = new int[] {0, 4, 14};
int ns_ary_len = ns_ary.length;
for (int i = 0; i < ns_ary_len; ++i) {
int ns_id = ns_ary[i];
String sql = String_.Format("SELECT * FROM xomp_page WHERE page_ns = {0} AND html_len != 0 ORDER BY page_id;", ns_id); // NOTE: html_len == 0 when page failed
int count = 0;
Db_rdr rdr = mgr_conn.Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
try {
while (rdr.Move_next()) {
Make_page(rdr, src_row, ns_id);
if (++count % 10000 == 0)
Gfo_usr_dlg_.Instance.Prog_many("", "", "xomp.html.insert: ns=~{0} db=~{1} count=~{2}", Int_.To_str_pad_bgn_space(ns_id, 3), Int_.To_str_pad_bgn_space(html_db_wtr.Cur_db_id(), 3), Int_.To_str_pad_bgn_space(count, 8));
}
} finally {rdr.Rls();}
}
this.Rls();
}
private void Make_page(Db_rdr rdr, Xowd_html_row src_row, int ns_id) {
// get src_row
int page_id = rdr.Read_int("page_id");
int html_len = rdr.Read_int("html_len");
int wkr_id = rdr.Read_int("xomp_wkr_id");
html_db_rdr.Rows__get(src_row, wkr_id, page_id);
src_body_flag_bldr.Decode(src_row.Body_flag());
// get trg_tbl and write
Xowd_html_tbl trg_tbl = html_db_wtr.Tbls__get_or_new(ns_id, html_len);
trg_tbl.Insert(src_row.Page_id(), src_row.Head_flag(), src_body_flag_bldr.Get_as_int(0), src_body_flag_bldr.Get_as_int(1), src_row.Display_ttl(), src_row.Content_sub(), src_row.Sidebar_div(), src_row.Body());
}
private void Rls() {
mgr_conn.Rls_conn();
html_db_rdr.Rls();
html_db_wtr.Rls();
}
}

84
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_page_pool.java
View File

@@ -1,84 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xomp_page_pool {
private final Object thread_lock = new Object();
private Xomp_page_pool_loader loader;
private List_adp pool = List_adp_.New(); private int pool_idx = 0, pool_len = 0;
private Db_conn make_conn;
private final Bry_bfr prog_bfr = Bry_bfr_.New();
private int pages_done, pages_total;
private long time_bgn, time_prv, time_done;
public void Init(Xow_wiki wiki, int num_pages_per_load) {
this.make_conn = gplx.xowa.bldrs.Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
this.loader = new Xomp_page_pool_loader(wiki, make_conn, num_pages_per_load);
this.pages_done = 0;
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
this.pages_total = loader.Get_pending_count();
}
public boolean Empty() {return empty;} private boolean empty = false;
public void Get_next(List_adp wkr_list, int num_pages_per_wkr) {
synchronized (thread_lock) {
// pool already marked exhausted by another wkr; return;
if (empty) return;
int wkr_end = pool_idx + num_pages_per_wkr;
// need pages to fulfill request
if (wkr_end > pool_len) {
this.pool = loader.Load(pool, pool_idx, pool_len);
this.pool_idx = 0;
this.pool_len = pool.Len();
if (pool_len == 0) { // no more pages; return;
empty = true;
return;
}
wkr_end = num_pages_per_wkr; // recalc wkr_end
}
// reset wkr_end; needed for very last set
if (wkr_end >= pool_len)
wkr_end = pool_len;
// add pages to wkr_list
for (int i = pool_idx; i < wkr_end; ++i) {
Xomp_page_itm page = (Xomp_page_itm)pool.Get_at(i);
wkr_list.Add(page);
}
pool_idx = wkr_end;
}
}
public void Mark_done(int id) {
synchronized (thread_lock) {
pages_done += 1;
if (pages_done % 1000 == 0) {
long time_cur = gplx.core.envs.Env_.TickCount();
int pages_left = pages_total - pages_done;
time_done += (time_cur - time_prv);
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{0} left=~{1} rate=~{2} time_past=~{3} time_left=~{4}", pages_done, pages_left, (int)rate_cur, time_past, time_left);
time_prv = time_cur;
}
}
}
public void Rls() {
make_conn.Rls_conn();
}
}

9
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xob_hdump_tbl_retriever__xomp.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xob_hdump_tbl_retriever__xomp.java
View File

@@ -15,16 +15,15 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.htmls.core.dbs.*;
class Xob_hdump_tbl_retriever__xomp implements Xob_hdump_tbl_retriever {
private final Db_conn conn;
private final Xowd_html_tbl tbl;
public Xob_hdump_tbl_retriever__xomp(Db_conn conn) {
this.conn = conn;
this.tbl = new Xowd_html_tbl(conn);
conn.Meta_tbl_assert(tbl);
public Xob_hdump_tbl_retriever__xomp(Xowd_html_tbl tbl) {
this.tbl = tbl;
this.conn = tbl.Conn();
}
public Xowd_html_tbl Get_html_tbl(Xow_ns ns, int prv_row_len) {
return tbl;

126
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_load_wkr.java Normal file
View File

@@ -0,0 +1,126 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*; import gplx.core.threads.utils.*;
class Xomp_load_wkr implements Gfo_invk {
private final Object thread_lock = new Object();
private final Xow_wiki wiki;
private final Db_conn mgr_conn;
private final Db_attach_mgr attach_mgr;
private final Gfo_blocking_queue queue;
private final int num_wkrs;
private final Bry_bfr prog_bfr = Bry_bfr_.New();
private int pages_done, pages_total;
private long time_bgn, time_prv, time_done;
public Xomp_load_wkr(Xow_wiki wiki, Db_conn mgr_conn, int num_pages_in_pool, int num_wkrs) {
this.wiki = wiki;
this.mgr_conn = mgr_conn;
this.attach_mgr = new Db_attach_mgr(mgr_conn);
this.queue = new Gfo_blocking_queue(num_pages_in_pool);
this.num_wkrs = num_wkrs;
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
this.pages_total = this.Get_pending_count();
}
public int Get_pending_count() {
Db_rdr rdr = mgr_conn.Stmt_sql("SELECT Count(*) AS Count_of FROM xomp_page mp WHERE mp.page_status = 0").Exec_select__rls_auto();
try {return rdr.Move_next() ? rdr.Read_int("Count_of") : 0;}
finally {rdr.Rls();}
}
public Xomp_page_itm Take() {return (Xomp_page_itm)queue.Take();}
private void Exec() {
int prv_page_id = 0;
while (prv_page_id != -1) {
prv_page_id = Load_pages(prv_page_id);
}
for (int i = 0; i < num_wkrs; ++i)
queue.Put(Xomp_page_itm.Null);
}
private int Load_pages(int prv_page_id) {
// page_tbl.prep_sql
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
( "SELECT mp.page_id"
, ", pp.page_namespace"
, ", pp.page_title"
, ", pp.page_text_db_id"
, "FROM xomp_page mp"
, " JOIN <page_db>page pp ON mp.page_id = pp.page_id"
, "WHERE mp.page_id > {0}"
, "AND mp.page_status = 0"
, "LIMIT {1}"
), prv_page_id, queue.Capacity());
this.attach_mgr.Conn_others_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
sql = attach_mgr.Resolve_sql(sql);
// page_tbl.load_sql
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
attach_mgr.Attach();
Db_rdr rdr = mgr_conn.Stmt_sql(sql).Exec_select__rls_auto();
List_adp list = List_adp_.New();
int count = 0;
try {
while (rdr.Move_next()) {
prv_page_id = rdr.Read_int("page_id");
int text_db_id = rdr.Read_int("page_text_db_id");
Xomp_page_itm ppg = new Xomp_page_itm(prv_page_id);
ppg.Init_by_page
( rdr.Read_int("page_namespace")
, rdr.Read_bry_by_str("page_title")
, text_db_id
);
list.Add(ppg);
text_db_loader.Add(text_db_id, ppg);
++count;
}
} finally {rdr.Rls();}
attach_mgr.Detach();
text_db_loader.Load();
int len = list.Len();
for (int i = 0; i < len; ++i) {
queue.Put((Xomp_page_itm)list.Get_at(i));
}
return count == 0 ? -1 : prv_page_id;
}
public void Mark_done(int id) {
synchronized (thread_lock) {
pages_done += 1;
if (pages_done % 1000 == 0) {
long time_cur = gplx.core.envs.Env_.TickCount();
int pages_left = pages_total - pages_done;
time_done += (time_cur - time_prv);
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{0} left=~{1} rate=~{2} time_past=~{3} time_left=~{4}", pages_done, pages_left, (int)rate_cur, time_past, time_left);
time_prv = time_cur;
}
}
}
public void Rls() {
mgr_conn.Rls_conn();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__exec)) this.Exec();
else return Gfo_invk_.Rv_unhandled;
return this;
}
public static final String Invk__exec = "exec";
}

11
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_page_itm.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_page_itm.java
View File

@@ -15,8 +15,9 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_page_itm {
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.xowa.wikis.data.tbls.*;
class Xomp_page_itm implements Xowd_text_bry_owner {
public Xomp_page_itm(int id) {this.id = id;}
public int Id() {return id;} private final int id;
public int Ns_id() {return ns_id;} private int ns_id;
@@ -29,9 +30,9 @@ class Xomp_page_itm {
this.ttl_bry = ttl_bry;
this.text_db_id = text_db_id;
}
public void Init_by_text(byte[] text) {
this.text = text;
}
public int Page_id() {return id;}
public void Set_text_bry_by_db(byte[] v) {this.text = v;}
public static final Xomp_page_itm Null = new Xomp_page_itm(-1);
}

62
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_page_pool.java Normal file
View File

@@ -0,0 +1,62 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xomp_page_pool {
private final Object thread_lock = new Object();
private final Xomp_page_pool_loader loader;
private final int num_pages_per_wkr;
private List_adp pool = List_adp_.New(); private int pool_idx = 0, pool_len = 0;
public Xomp_page_pool(Xomp_page_pool_loader loader, int num_pages_per_wkr) {
this.loader = loader; this.num_pages_per_wkr = num_pages_per_wkr;
}
public boolean Empty() {synchronized (thread_lock) {return empty;}} private boolean empty = false;
public void Get_next(List_adp wkr_list) {
synchronized (thread_lock) {
// pool already marked exhausted by another wkr; return;
if (empty) return;
int wkr_end = pool_idx + num_pages_per_wkr;
// need pages to fulfill request
if (wkr_end > pool_len) {
this.pool = loader.Load(pool, pool_idx, pool_len);
this.pool_idx = 0;
this.pool_len = pool.Len();
if (pool_len == 0) { // no more pages; return;
empty = true;
return;
}
wkr_end = num_pages_per_wkr; // recalc wkr_end
}
// reset wkr_end; needed for very last set
if (wkr_end >= pool_len)
wkr_end = pool_len;
// add pages to wkr_list
for (int i = pool_idx; i < wkr_end; ++i) {
Xomp_page_itm page = (Xomp_page_itm)pool.Get_at(i);
wkr_list.Add(page);
}
pool_idx = wkr_end;
}
}
public void Rls() {
loader.Conn().Rls_conn();
}
}

12
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_page_pool_loader.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_page_pool_loader.java
View File

@@ -15,22 +15,22 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xomp_page_pool_loader {
private final Xow_wiki wiki;
private int prv_page_id = -1;
private final Db_conn make_conn;
private final int num_pages_per_load;
private final Db_attach_mgr attach_mgr;
private int prv_page_id = -1;
public Xomp_page_pool_loader(Xow_wiki wiki, Db_conn make_conn, int num_pages_per_load) {
this.wiki = wiki;
this.make_conn = make_conn;
this.attach_mgr = new Db_attach_mgr(make_conn);
this.num_pages_per_load = num_pages_per_load;
}
public Db_conn Conn() {return make_conn;} private final Db_conn make_conn;
public int Get_pending_count() {
Db_rdr rdr = make_conn.Stmt_sql("SELECT Count(*) AS Count_of FROM mp_page mp WHERE mp.page_done = 0").Exec_select__rls_auto();
Db_rdr rdr = make_conn.Stmt_sql("SELECT Count(*) AS Count_of FROM xomp_page mp WHERE mp.page_status = 0").Exec_select__rls_auto();
try {
return rdr.Move_next() ? rdr.Read_int("Count_of") : 0;
} finally {rdr.Rls();}
@@ -53,10 +53,10 @@ class Xomp_page_pool_loader {
, ", pp.page_namespace"
, ", pp.page_title"
, ", pp.page_text_db_id"
, "FROM mp_page mp"
, "FROM xomp_page mp"
, " JOIN <page_db>page pp ON mp.page_id = pp.page_id"
, "WHERE mp.page_id > {0}"
, "AND mp.page_done = 0"
, "AND mp.page_status = 0"
, "LIMIT {1}"
), prv_page_id, num_pages_per_load);
this.attach_mgr.Conn_others_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_parse_cmd.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_cmd.java
View File

@@ -15,7 +15,7 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xomp_parse_cmd extends Xob_cmd__base {
private final Xomp_parse_mgr mgr = new Xomp_parse_mgr();

43
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_parse_mgr.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_mgr.java
View File

@@ -15,24 +15,38 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.threads.*;
import gplx.xowa.langs.*;
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.threads.*; import gplx.core.threads.utils.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.cases.*;
import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.wikis.caches.*;
class Xomp_parse_mgr {
private final Xomp_page_pool page_pool = new Xomp_page_pool();
private Gfo_countdown_latch latch;
public Xomp_parse_mgr_cfg Cfg() {return cfg;} private final Xomp_parse_mgr_cfg cfg = new Xomp_parse_mgr_cfg();
private int wkrs_done;
public void Wkrs_done_add_1() {synchronized (page_pool) {++wkrs_done;}}
public Xomp_db_core Db_core() {return db_core;} private Xomp_db_core db_core;
public Xomp_prog_mgr Prog_mgr() {return prog_mgr;} private final Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
public void Wkrs_done_add_1() {latch.Countdown();}
public void Run(Xowe_wiki wiki) {
// init pool
// init db, pool_loader, pool, prog_mgr
cfg.Init(wiki);
page_pool.Init(wiki, cfg.Num_pages_in_pool());
this.db_core = Xomp_db_core.New__load(wiki);
Xomp_page_pool_loader pool_loader = new Xomp_page_pool_loader(wiki, db_core.Mgr_db().Conn(), cfg.Num_pages_in_pool());
Xomp_page_pool page_pool = new Xomp_page_pool(pool_loader, cfg.Num_pages_per_wkr());
prog_mgr.Init(pool_loader.Get_pending_count());
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, true);
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of cache
// init threads
// load_wkr: init and start
// Xomp_load_wkr load_wkr = new Xomp_load_wkr(wiki, db_core.Mgr_db().Conn(), cfg.Num_pages_in_pool(), cfg.Num_wkrs());
// Thread_adp_.Start_by_key("xomp.load", Cancelable_.Never, load_wkr, Xomp_load_wkr.Invk__exec);
// init parse_wkrs
int wkr_len = cfg.Num_wkrs();
latch = new Gfo_countdown_latch(wkr_len);
Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len];
for (int i = 0; i < wkr_len; ++i) {
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, Clone_wiki(wiki), page_pool, i, cfg.Num_pages_per_wkr());
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, Clone_wiki(wiki), page_pool, i);
wkr.Wiki().Cache_mgr().Page_cache_(page_cache);
wkrs[i] = wkr;
}
@@ -43,12 +57,7 @@ class Xomp_parse_mgr {
}
// wait until wkrs are wkrs_done
while (true) {
synchronized (page_pool) {
if (wkrs_done == wkr_len) break;
}
Thread_adp_.Sleep(1000);
}
latch.Await();
page_pool.Rls();
// print stats
@@ -61,7 +70,7 @@ class Xomp_parse_mgr {
private static Xowe_wiki Clone_wiki(Xowe_wiki wiki) {
Xol_lang_itm lang = new Xol_lang_itm(wiki.App().Lang_mgr(), wiki.Lang().Key_bry());
Xol_lang_itm_.Lang_init(lang);
Xowe_wiki rv = new Xowe_wiki(wiki.Appe(), lang, wiki.Ns_mgr(), wiki.Domain_itm(), wiki.Fsys_mgr().Root_dir());
Xowe_wiki rv = new Xowe_wiki(wiki.Appe(), lang, gplx.xowa.wikis.nss.Xow_ns_mgr_.default_(lang.Case_mgr()), wiki.Domain_itm(), wiki.Fsys_mgr().Root_dir());
rv.Init_by_wiki();
return rv;
}

10
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_parse_mgr_cfg.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_mgr_cfg.java
View File

@@ -15,14 +15,14 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_parse_mgr_cfg implements Gfo_invk {
public Io_url Root_dir() {return root_dir;} private Io_url root_dir;
// public Io_url Root_dir() {return root_dir;} private Io_url root_dir;
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = 1000;
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
public void Init(Xowe_wiki wiki) {
if (root_dir == null) root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
// if (root_dir == null) root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Env_.System_cpu_count();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
@@ -30,11 +30,11 @@ class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__root_dir_)) root_dir = m.ReadIoUrl("v");
// else if (ctx.Match(k, Invk__root_dir_)) root_dir = m.ReadIoUrl("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
, Invk__root_dir_ = "root_dir_"
// , Invk__root_dir_ = "root_dir_"
;
}

47
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_parse_wkr.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_wkr.java
View File

@@ -15,33 +15,33 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.htmls.core.bldrs.*;
import gplx.xowa.parsers.*;
import gplx.xowa.parsers.*;
class Xomp_parse_wkr implements Gfo_invk {
private final Xomp_parse_mgr mgr;
private final Xowe_wiki wiki;
private final Xomp_parse_mgr mgr;
private final Xomp_page_pool page_pool;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private final int num_pages_per_wkr;
private final int idx;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
// private boolean gen_html = true;// gen_hdump = true;
private int done_count; private long done_time;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx, int num_pages_per_wkr) {
private Xomp_wkr_db wkr_db;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx) {
this.mgr = mgr; this.wiki = wiki;
this.page_pool = page_pool; this.num_pages_per_wkr = num_pages_per_wkr;
this.page_pool = page_pool;
this.idx = idx;
this.wkr_db = mgr.Db_core().Wkr_db(Bool_.Y, idx); // NOTE: must go in ctor, or else thread issues
}
public Xowe_wiki Wiki() {return wiki;} private final Xowe_wiki wiki;
public void Exec() {
// init
Db_conn wkr_conn = Db_conn_bldr.Instance.Get_or_autocreate(true, mgr.Cfg().Root_dir().GenSubFil_nest("xomp_" + Int_.To_str_fmt(idx, "000"), "xomp_wkr.sqlite3"));
Xow_parser_mgr parser = new Xow_parser_mgr(wiki);
Xow_parser_mgr parser_mgr = new Xow_parser_mgr(wiki);
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(false); // disable categories else progress messages written (also for PERF)
if (wiki.File__bin_mgr() != null)
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // remove wmf wkr, else will try to download images during parsing
hdump_bldr.Init(wiki, wkr_conn, new Xob_hdump_tbl_retriever__xomp(wkr_conn));
hdump_bldr.Enabled_(true).Hzip_enabled_(true).Hzip_diff_(true).Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
wkr_db.Conn().Txn_bgn("xomp");
while (true) {
Xomp_page_itm ppg = Get_next(); if (ppg == Xomp_page_itm.Null) break; // no more pages
@@ -56,18 +56,15 @@ class Xomp_parse_wkr implements Gfo_invk {
wpg.Db().Page().Id_(ppg.Id());
// parse page
parser.Ctx().Clear_all();
parser.Parse(wpg, true);
Xop_ctx pctx = parser_mgr.Ctx();
pctx.Clear_all();
parser_mgr.Parse(wpg, true);
// gen_html
// if ( gen_html
// && wpg.Redirect().Itms__len() == 0) // don't generate html for redirected pages
// wiki.Html_mgr().Page_wtr_mgr().Gen(wpg, gplx.xowa.wikis.pages.Xopg_page_.Tid_read);
// if (gen_hdump)
// hdump_bldr.Insert(wpg);
hdump_bldr.Insert(pctx, wpg);
// mark done for sake of progress
page_pool.Mark_done(ppg.Id());
mgr.Prog_mgr().Mark_done(ppg.Id());
// update stats
long time_cur = gplx.core.envs.Env_.TickCount();
@@ -80,15 +77,17 @@ class Xomp_parse_wkr implements Gfo_invk {
if (wiki.Cache_mgr().Tmpl_result_cache().Count() > 50000)
wiki.Cache_mgr().Tmpl_result_cache().Clear();
if (done_count % 50 == 0) {
wiki.Cache_mgr().Free_mem_all();
wiki.Cache_mgr().Free_mem_all(Bool_.N);
wiki.Parser_mgr().Scrib().Core_term();
}
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "mass_parse.fail:ns=~{0} ttl=~{1} err=~{2}", ppg.Ns_id(), ppg.Ttl_bry(), Err_.Message_gplx_log(e));
}
}
wkr_db.Conn().Txn_end(); // NOTE: must end txn before running update wkr_id
mgr.Db_core().Update_wkr_id(idx, wkr_db.Conn());
mgr.Wkrs_done_add_1();
wkr_conn.Rls_conn();
wkr_db.Conn().Rls_conn();
}
public void Bld_stats(Bry_bfr bfr) {
int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1;
@@ -101,7 +100,7 @@ class Xomp_parse_wkr implements Gfo_invk {
private Xomp_page_itm Get_next() {
if (list_idx == list_len) {
list.Clear();
page_pool.Get_next(list, num_pages_per_wkr);
page_pool.Get_next(list);
list_len = list.Len();
if (list_len == 0) return Xomp_page_itm.Null;
list_idx = 0;

43
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_prog_mgr.java Normal file
View File

@@ -0,0 +1,43 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_prog_mgr {
private final Object thread_lock = new Object();
private int pages_done, pages_total;
private long time_bgn, time_prv, time_done;
private final Bry_bfr prog_bfr = Bry_bfr_.New();
public void Init(int pages_total) {
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
this.pages_total = pages_total;
}
public void Mark_done(int id) {
synchronized (thread_lock) {
pages_done += 1;
if (pages_done % 1000 == 0) {
long time_cur = gplx.core.envs.Env_.TickCount();
int pages_left = pages_total - pages_done;
time_done += (time_cur - time_prv);
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left);
time_prv = time_cur;
}
}
}
}

13
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_text_db_loader.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_text_db_loader.java
View File

@@ -15,9 +15,10 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.core.ios.*;
import gplx.xowa.wikis.data.tbls.*;
class Xomp_text_db_loader {
private final Xow_wiki wiki;
private final Ordered_hash text_db_hash = Ordered_hash_.New();
@@ -25,7 +26,7 @@ class Xomp_text_db_loader {
public Xomp_text_db_loader(Xow_wiki wiki) {
this.wiki = wiki;
}
public void Add(int text_db_id, Xomp_page_itm ppg) {
public void Add(int text_db_id, Xowd_text_bry_owner ppg) {
Xomp_text_db_itm itm = (Xomp_text_db_itm)text_db_hash.Get_by(text_db_id);
if (itm == null) {
itm = new Xomp_text_db_itm(text_db_id);
@@ -53,8 +54,8 @@ class Xomp_text_db_loader {
}
// build WHERE IN for page_ids; EX: "1, 2, 3, 4"
Xomp_page_itm ppg = (Xomp_page_itm)list.Get_at(i);
int page_id = ppg.Id();
Xowd_text_bry_owner ppg = (Xowd_text_bry_owner)list.Get_at(i);
int page_id = ppg.Page_id();
if (batch_idx != 0) bry.Add_byte_comma();
bry.Add_int_variable(page_id);
page_hash.Add(page_id, ppg);
@@ -77,8 +78,8 @@ class Xomp_text_db_loader {
int page_id = rdr.Read_int("page_id");
byte[] text_data = rdr.Read_bry("text_data");
text_data = zip_mgr.Unzip(zip_tid, text_data);
Xomp_page_itm ppg = (Xomp_page_itm)page_hash.Get_by(page_id);
ppg.Init_by_text(text_data);
Xowd_text_bry_owner ppg = (Xowd_text_bry_owner)page_hash.Get_by(page_id);
ppg.Set_text_bry_by_db(text_data);
}
}
finally {

81
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_tmpl_cache_bldr.java Normal file
View File

@@ -0,0 +1,81 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.wikis.caches.*;
class Xomp_tmpl_cache_bldr {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all) {
Xow_page_cache rv = new Xow_page_cache(wiki);
if (fill_all) Fill_all(rv, wiki);
return rv;
}
private static void Fill_all(Xow_page_cache cache, Xowe_wiki wiki) {
String sql = String_.Concat_lines_nl_skip_last // ANSI.Y
( "SELECT pp.page_id"
, ", pp.page_namespace"
, ", pp.page_title"
, ", pp.page_text_db_id"
, ", pp.page_redirect_id"
, "FROM page pp"
, "WHERE pp.page_namespace IN (10, 828)"
);
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
// load pages
int count = 0;
List_adp redirect_list = List_adp_.New();
Ordered_hash page_regy = Ordered_hash_.New();
Db_rdr rdr = wiki.Data__core_mgr().Db__core().Tbl__page().Conn().Stmt_sql(sql).Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
// get ttl
Xoa_ttl page_ttl = wiki.Ttl_parse(rdr.Read_int("page_namespace"), rdr.Read_bry_by_str("page_title"));
// add to text_db_loader
int page_id = rdr.Read_int("page_id");
int page_redirect_id = rdr.Read_int("page_redirect_id");
Xow_page_cache_itm itm = new Xow_page_cache_itm(page_ttl, Bry_.Empty, Bry_.Empty);
itm.Set_page_ids(page_id, page_redirect_id);
text_db_loader.Add(rdr.Read_int("page_text_db_id"), itm);
cache.Add(page_ttl.Full_db(), itm);
page_regy.Add(page_id, itm);
if (page_redirect_id != -1)
redirect_list.Add(itm);
if ((++count % 10000) == 0)
Gfo_usr_dlg_.Instance.Prog_many("", "", "loading tmpls: ~{0}", count);
}
} finally {rdr.Rls();}
// load wikitext
text_db_loader.Load();
// handle redirects
int redirect_len = redirect_list.Len();
for (int i = 0; i < redirect_len; ++i) {
Xow_page_cache_itm src_itm = (Xow_page_cache_itm)redirect_list.Get_at(i);
Xow_page_cache_itm trg_itm = (Xow_page_cache_itm)page_regy.Get_by(src_itm.Redirect_id());
if (trg_itm == null) {
Gfo_usr_dlg_.Instance.Prog_many("", "", "missing redirect for tmpl: ~{0}", src_itm.Ttl().Full_db());
continue;
}
src_itm.Set_redirect_bry(trg_itm.Wtxt__direct());
}
}
}

92
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_htmlr.java Normal file
View File

@@ -0,0 +1,92 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import gplx.langs.htmls.*; import gplx.xowa.htmls.core.htmls.*;
class Xoh_toc_htmlr implements gplx.core.brys.Bfr_arg {
private final Bry_bfr numbering_bfr = Bry_bfr_.New();
private byte[] toc_label;
private int prv_lvl;
private Ordered_hash toc_itms;
public void Clear() {
prv_lvl = 0;
}
public void Init(byte[] toc_label) {
this.toc_label = toc_label;
}
public void To_html(Bry_bfr rv, Xoh_wtr_ctx hctx, Ordered_hash toc_itms, boolean toc_mode_is_pgbnr) {
this.toc_itms = toc_itms;
fmtr_div.Bld_many(rv, toc_mode_is_pgbnr ? Bry_.Empty : Bry_toc_cls, toc_label, this);
}
public void Test__to_html(Bry_bfr rv, Ordered_hash toc_itms) {
this.toc_itms = toc_itms;
Bfr_arg__add(rv);
}
public void Bfr_arg__add(Bry_bfr bfr) {
int len = toc_itms.Len();
prv_lvl = 0;
for (int i = 0; i < len; ++i) {
Xoh_toc_itm itm = (Xoh_toc_itm)toc_itms.Get_at(i);
Write(bfr, itm);
}
// close all open levels
for (int i = prv_lvl; i > 0; --i) {
int indent = i * 2;
bfr.Add_byte_repeat(Byte_ascii.Space, indent + 2).Add(Gfh_tag_.Li_rhs).Add_byte_nl(); // EX: " </li>\n"
bfr.Add_byte_repeat(Byte_ascii.Space, indent ).Add(Gfh_tag_.Ul_rhs).Add_byte_nl(); // EX: " </ul>\n"
}
}
private void Write(Bry_bfr bfr, Xoh_toc_itm itm) {
int cur_lvl = itm.Lvl();
int indent = cur_lvl * 2;
switch (CompareAble_.Compare(cur_lvl, prv_lvl)) {
case CompareAble_.More: // start new "<ul>"
bfr.Add_byte_repeat(Byte_ascii.Space, indent).Add(Gfh_tag_.Ul_lhs).Add_byte_nl(); // EX: " <ul>\n"
break;
case CompareAble_.Same: // close old "</li>"; NOTE: Comparable_.Same will never be 1st item (so won't ever get </li><li>)
bfr.Add_byte_repeat(Byte_ascii.Space, indent + 2).Add(Gfh_tag_.Li_rhs).Add_byte_nl(); // EX: " </li>\n"
break;
case CompareAble_.Less: // close old "</ul>" and "</li>"
for (int j = prv_lvl; j > cur_lvl; --j) {
bfr.Add_byte_repeat(Byte_ascii.Space, (j * 2) + 2).Add(Gfh_tag_.Li_rhs).Add_byte_nl(); // EX: " </li>\n"
bfr.Add_byte_repeat(Byte_ascii.Space, (j * 2) ).Add(Gfh_tag_.Ul_rhs).Add_byte_nl(); // EX: " </ul>\n"
}
bfr.Add_byte_repeat(Byte_ascii.Space, indent + 2).Add(Gfh_tag_.Li_rhs).Add_byte_nl(); // EX: " </li>\n"
break;
default: throw Err_.new_unhandled_default(CompareAble_.Compare(cur_lvl, prv_lvl));
}
// write "<li ..."
bfr.Add_byte_repeat(Byte_ascii.Space, indent); // indent
fmtr_itm.Bld_many(bfr, itm.Lvl(), itm.Uid(), itm.Anch(), itm.Path_to_bry(numbering_bfr), itm.Text());
prv_lvl = cur_lvl;
}
private static final byte[] Bry_toc_cls = Bry_.new_a7(" id=\"toc\" class=\"toc\"");
private final Bry_fmt
fmtr_div = Bry_fmt.Auto(String_.Concat_lines_nl_skip_last
( "<div~{toc}>"
, " <div id=\"toctitle\">"
, " <h2>~{contents_title}</h2>"
, " </div>"
, "~{itms}</div>"
, ""
))
, fmtr_itm = Bry_fmt.Auto
( " <li class=\"toclevel-~{level} tocsection-~{toc_idx}\"><a href=\"#~{anchor}\"><span class=\"tocnumber\">~{heading}</span> <span class=\"toctext\">~{text}</span></a>\n"
);
}

177
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_htmlr__basic__tst.java Normal file
View File

@@ -0,0 +1,177 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import org.junit.*; import gplx.core.tests.*;
public class Xoh_toc_htmlr__basic__tst {
@Before public void init() {fxt.Clear();} private final Xoh_toc_htmlr__basic__fxt fxt = new Xoh_toc_htmlr__basic__fxt();
@Test public void D1_S0_S0() {
fxt.Init__add(2, "a");
fxt.Init__add(2, "b");
fxt.Init__add(2, "c");
fxt.Test__html_itms
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-2\"><a href=\"#b\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-3\"><a href=\"#c\"><span class=\"tocnumber\">3</span> <span class=\"toctext\">c</span></a>"
, " </li>"
, " </ul>"
);
}
@Test public void D1_D1_D1() {
fxt.Init__add(2, "a");
fxt.Init__add(3, "a_a");
fxt.Init__add(4, "a_a_a");
fxt.Test__html_itms
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#a_a\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">a_a</span></a>"
, " <ul>"
, " <li class=\"toclevel-3 tocsection-3\"><a href=\"#a_a_a\"><span class=\"tocnumber\">1.1.1</span> <span class=\"toctext\">a_a_a</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
);
}
@Test public void D1_D1_S0_U1() {
fxt.Init__add(2, "a");
fxt.Init__add(3, "a_a");
fxt.Init__add(3, "a_b");
fxt.Init__add(2, "b");
fxt.Test__html_itms
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#a_a\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">a_a</span></a>"
, " </li>"
, " <li class=\"toclevel-2 tocsection-3\"><a href=\"#a_b\"><span class=\"tocnumber\">1.2</span> <span class=\"toctext\">a_b</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-4\"><a href=\"#b\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " </ul>"
);
}
@Test public void D1_D1_U1_D1() {
fxt.Init__add(2, "a");
fxt.Init__add(3, "a_a");
fxt.Init__add(2, "b");
fxt.Init__add(3, "b_a");
fxt.Test__html_itms
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#a_a\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">a_a</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-3\"><a href=\"#b\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">b</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-4\"><a href=\"#b_a\"><span class=\"tocnumber\">2.1</span> <span class=\"toctext\">b_a</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
);
}
@Test public void D1_D1_D1_U2() {
fxt.Init__add(2, "a");
fxt.Init__add(3, "a_a");
fxt.Init__add(4, "a_a_a");
fxt.Init__add(2, "b");
fxt.Test__html_itms
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#a_a\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">a_a</span></a>"
, " <ul>"
, " <li class=\"toclevel-3 tocsection-3\"><a href=\"#a_a_a\"><span class=\"tocnumber\">1.1.1</span> <span class=\"toctext\">a_a_a</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-4\"><a href=\"#b\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " </ul>"
);
}
@Test public void D1_D2_U1_D1() {
fxt.Init__add(2, "a");
fxt.Init__add(4, "a_a_a_a");
fxt.Init__add(3, "a_a_a");
fxt.Init__add(4, "a_a_a_b");
fxt.Test__html_itms
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#a_a_a_a\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">a_a_a_a</span></a>"
, " </li>"
, " <li class=\"toclevel-2 tocsection-3\"><a href=\"#a_a_a\"><span class=\"tocnumber\">1.2</span> <span class=\"toctext\">a_a_a</span></a>"
, " <ul>"
, " <li class=\"toclevel-3 tocsection-4\"><a href=\"#a_a_a_b\"><span class=\"tocnumber\">1.2.1</span> <span class=\"toctext\">a_a_a_b</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
);
}
@Test public void Div() {
fxt.Init__init_page("Table of contents", false);
fxt.Init__add(2, "a");
fxt.Init__add(2, "b");
fxt.Init__add(2, "c");
fxt.Test__html_div
( "<div id=\"toc\" class=\"toc\">"
, " <div id=\"toctitle\">"
, " <h2>Table of contents</h2>"
, " </div>"
, " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-2\"><a href=\"#b\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-3\"><a href=\"#c\"><span class=\"tocnumber\">3</span> <span class=\"toctext\">c</span></a>"
, " </li>"
, " </ul>"
, "</div>"
);
}
}
class Xoh_toc_htmlr__basic__fxt {
private final Xoh_toc_mgr wtr = new Xoh_toc_mgr();
private final Bry_bfr bfr = Bry_bfr_.New();
public void Clear() {wtr.Clear();}
public void Init__add(int hdr_num, String hdr_txt) {wtr.Add(hdr_num, Bry_.new_u8(hdr_txt));}
public void Init__init_page(String toc_title, boolean page_banner) {wtr.Init(Bry_.new_u8(toc_title), Bry_.Empty);}
public void Test__html_itms(String... expd_ary) {
Gftest.Eq__ary(expd_ary, String_.Ary(Bry_split_.Split_lines(wtr.Test__to_html())));
}
public void Test__html_div(String... expd_ary) {
wtr.To_html(bfr, gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx.Basic, false);
Gftest.Eq__ary(expd_ary, String_.Ary(Bry_split_.Split_lines(bfr.To_bry_and_clear())));
}
}

35
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_itm.java Normal file
View File

@@ -0,0 +1,35 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
public class Xoh_toc_itm {// EX: <li class="toclevel-3 tocsection-3"><a href="#aaa"><span class="tocnumber">1.1.1</span> <span class="toctext">aaa</span></a></li>
public int Uid() {return uid;} private int uid; // uid of itm; HTML: "tocsection-3"
public int Lvl() {return lvl;} private int lvl; // indent level; HTML: "toclevel-3"
public int[] Path() {return path;} private int[] path; // path of itm; HTML: "1.1.1"
public byte[] Anch() {return anch;} private byte[] anch; // HTML: "#aaa"
public byte[] Text() {return text;} private byte[] text; // HTML: "aaa"
public byte[] Path_to_bry(Bry_bfr bfr) {
int len = path.length;
for (int i = 0; i < len; ++i) {
if (i != 0) bfr.Add_byte_dot();
bfr.Add_int_variable(path[i]);
}
return bfr.To_bry_and_clear();
}
public void Set__lvl(int uid, int lvl, int[] path) {this.uid = uid; this.lvl = lvl; this.path = path;}
public Xoh_toc_itm Set__txt(byte[] anch, byte[] text) {this.anch = anch; this.text = text; return this;}
}

56
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_mgr.java Normal file
View File

@@ -0,0 +1,56 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.htmls.core.wkrs.tocs.*;
public class Xoh_toc_mgr {
private final Ordered_hash itms = Ordered_hash_.New_bry();
private final Xoh_toc_wkr__lvl lvl_wkr = new Xoh_toc_wkr__lvl();
private final Xoh_toc_wkr__txt txt_wkr = new Xoh_toc_wkr__txt();
private final Xoh_toc_htmlr htmlr = new Xoh_toc_htmlr();
public boolean Exists() {return exists && Enabled;} private boolean exists;
public void Exists_y_() {exists = true;}
public int Toc_bgn() {return toc_bgn;} private int toc_bgn;
public void Toc_bgn_(int v) {this.toc_bgn = v;}
public void Clear() {
this.exists = false;
itms.Clear();
lvl_wkr.Clear();
txt_wkr.Clear();
htmlr.Clear();
toc_bgn = -1;
}
public void Init(byte[] toc_title, byte[] page_name) {
this.Clear();
htmlr.Init(toc_title);
txt_wkr.Init(page_name);
}
public Xoh_toc_itm Add(int hdr_num, byte[] hdr_txt) {
Xoh_toc_itm itm = new Xoh_toc_itm();
lvl_wkr.Calc_level(itm, hdr_num);
txt_wkr.Calc_anch_text(itm, hdr_txt);
itms.Add(itm.Anch(), itm);
return itm;
}
public void To_html(Bry_bfr rv, Xoh_wtr_ctx hctx, boolean toc_mode_is_pgbnr) {htmlr.To_html(rv, hctx, itms, toc_mode_is_pgbnr);}
public byte[] Test__to_html() {
Bry_bfr bfr = Bry_bfr_.New();
htmlr.Test__to_html(bfr, itms);
return bfr.To_bry_and_clear();
}
public static boolean Enabled = true; // TEST
}

75
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__lvl.java Normal file
View File

@@ -0,0 +1,75 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
class Xoh_toc_wkr__lvl {
private static final int Toc_lvls_max = 7;
private final int[] sub_lvl_count = new int[Toc_lvls_max], lvl_count = new int[Toc_lvls_max];
private int prv_lvl, toc_lvl, prv_toc_lvl;
private int uid = 0;
public void Clear() {
uid = prv_lvl = toc_lvl = prv_toc_lvl = 0;
}
public void Calc_level(Xoh_toc_itm rv, int lvl) { // REF.MW:Parser.php!formatHeadings
if (lvl > prv_lvl) { // Increase TOC lvl
toc_lvl++;
sub_lvl_count[toc_lvl - List_adp_.Base1] = 0;
if (toc_lvl < Toc_lvls_max) {
prv_toc_lvl = toc_lvl;
// $toc .= Linker::tocIndent();
}
}
else if (lvl < prv_lvl && toc_lvl > 1) {// Decrease TOC lvl, find lvl to jump to
int i = toc_lvl;
for (; i > 0; i--) {
int cur_lvl_count = lvl_count[i];
if (cur_lvl_count == lvl) { // Found last matching lvl
toc_lvl = i;
break;
}
else if (cur_lvl_count < lvl) { // Found first matching lvl below current lvl
toc_lvl = i + 1;
break;
}
}
if (i == 0)
toc_lvl = 1;
if (toc_lvl < Toc_lvls_max) {
if (prv_toc_lvl < Toc_lvls_max) {
// Unindent only if the previous toc lvl was shown :p
// $toc .= Linker::tocUnindent( $prv_toc_lvl - $toc_lvl );
prv_toc_lvl = toc_lvl;
} else {
// $toc .= Linker::tocLineEnd();
}
}
}
else { // No change in lvl, end TOC line
if (toc_lvl < Toc_lvls_max) {
// $toc .= Linker::tocLineEnd();
}
}
lvl_count[toc_lvl] = lvl;
sub_lvl_count[toc_lvl - List_adp_.Base1] = sub_lvl_count[toc_lvl - List_adp_.Base1] + 1;
prv_lvl = lvl; // NOTE: same as "if ( $toclevel ) $prevlevel = $level;" but at end of block
// Tfds.Write(lvl, prv_lvl, lvl, toc_lvl, Int_.Ary_concat(",", lvl_count), Int_.Ary_concat(",", sub_lvl_count));
int[] copy = new int[toc_lvl];
Int_.Ary_copy_to(sub_lvl_count, toc_lvl, copy);
rv.Set__lvl(++uid, toc_lvl, copy);
}
}

66
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__lvl__basic__tst.java Normal file
View File

@@ -0,0 +1,66 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import org.junit.*; import gplx.core.tests.*;
public class Xoh_toc_wkr__lvl__basic__tst {
@Before public void init() {fxt.Clear();} private final Xoh_toc_wkr__lvl__fxt fxt = new Xoh_toc_wkr__lvl__fxt();
@Test public void D1_S0_S0() {
fxt.Test__calc(2, fxt.Make(1, 1, Int_.Ary(1)));
fxt.Test__calc(2, fxt.Make(2, 1, Int_.Ary(2)));
fxt.Test__calc(2, fxt.Make(3, 1, Int_.Ary(3)));
}
@Test public void D1_D1_D1() {
fxt.Test__calc(2, fxt.Make(1, 1, Int_.Ary(1)));
fxt.Test__calc(3, fxt.Make(2, 2, Int_.Ary(1, 1)));
fxt.Test__calc(4, fxt.Make(3, 3, Int_.Ary(1, 1, 1)));
}
@Test public void D1_D1_S0_U1() {
fxt.Test__calc(2, fxt.Make(1, 1, Int_.Ary(1)));
fxt.Test__calc(3, fxt.Make(2, 2, Int_.Ary(1, 1)));
fxt.Test__calc(3, fxt.Make(3, 2, Int_.Ary(1, 2)));
fxt.Test__calc(2, fxt.Make(4, 1, Int_.Ary(2)));
}
@Test public void D1_D1_U1_D1() {
fxt.Test__calc(2, fxt.Make(1, 1, Int_.Ary(1)));
fxt.Test__calc(3, fxt.Make(2, 2, Int_.Ary(1, 1)));
fxt.Test__calc(2, fxt.Make(3, 1, Int_.Ary(2)));
fxt.Test__calc(3, fxt.Make(4, 2, Int_.Ary(2, 1)));
}
@Test public void D1_D1_D1_U2() {
fxt.Test__calc(2, fxt.Make(1, 1, Int_.Ary(1)));
fxt.Test__calc(3, fxt.Make(2, 2, Int_.Ary(1, 1)));
fxt.Test__calc(4, fxt.Make(3, 3, Int_.Ary(1, 1, 1)));
fxt.Test__calc(2, fxt.Make(4, 1, Int_.Ary(2)));
}
}
class Xoh_toc_wkr__lvl__fxt {
private final Xoh_toc_wkr__lvl wkr = new Xoh_toc_wkr__lvl();
private final Xoh_toc_itm actl = new Xoh_toc_itm();
public void Clear() {wkr.Clear();}
public Xoh_toc_itm Make(int uid, int lvl, int[] path) {
Xoh_toc_itm rv = new Xoh_toc_itm();
rv.Set__lvl(uid, lvl, path);
return rv;
}
public void Test__calc(int lvl, Xoh_toc_itm expd) {
wkr.Calc_level(actl, lvl);
Gftest.Eq__int(expd.Uid(), actl.Uid(), "uid");
Gftest.Eq__int(expd.Lvl(), actl.Lvl(), "lvl");
Gftest.Eq__ary(expd.Path(), actl.Path(), "path");
}
}

181
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt.java Normal file
View File

@@ -0,0 +1,181 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import gplx.langs.htmls.*; import gplx.langs.htmls.docs.*; import gplx.langs.htmls.encoders.*;
import gplx.xowa.parsers.amps.*; import gplx.core.primitives.*;
class Xoh_toc_wkr__txt {
private final Gfh_tag_rdr tag_rdr = Gfh_tag_rdr.New__html();
private final Bry_bfr anch_bfr = Bry_bfr_.New(), text_bfr = Bry_bfr_.New();
private final Gfo_url_encoder anch_encoder = Gfo_url_encoder_.New__id();
private final Xop_amp_mgr amp_mgr = Xop_amp_mgr.Instance;
private final Hash_adp anch_hash = Hash_adp_bry.ci_u8(gplx.xowa.langs.cases.Xol_case_mgr_.U8());
private byte[] page_name;
public void Clear() {
anch_bfr.Clear();
text_bfr.Clear();
anch_hash.Clear();
}
public void Init(byte[] page_name) {this.page_name = page_name;}
public void Calc_anch_text(Xoh_toc_itm rv, byte[] src) { // text within hdr; EX: <h2>Abc</h2> -> Abc
int end = src.length;
src = Remove_comment(text_bfr, src, 0, end);
end = src.length;
tag_rdr.Init(page_name, src, 0, end);
try {
Calc_anch_text_recurse(src, 0, end);
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "toc:failed while generating anch_text; page=~{0} src=~{1} err=~{2}", page_name, Err_.Message_gplx_log(e));
text_bfr.Clear().Add(src);
anch_encoder.Encode(anch_bfr, src);
}
byte[] anch_bry = anch_bfr.To_bry_and_clear_and_trim(Bool_.Y, Bool_.Y, id_trim_ary);
if (anch_hash.Has(anch_bry)) {
int anch_idx = 2;
while (true) { // NOTE: this is not big-O performant, but it mirrors MW; DATE:2016-07-09
byte[] anch_tmp = Bry_.Add(anch_bry, Byte_ascii.Underline_bry, Int_.To_bry(anch_idx++));
if (!anch_hash.Has(anch_tmp)) {
anch_bry = anch_tmp;
break;
}
}
}
anch_hash.Add_as_key_and_val(anch_bry);
rv.Set__txt
( anch_bry
, text_bfr.To_bry_and_clear_and_trim()); // NOTE: both id and text trim ends
}
private void Calc_anch_text_recurse(byte[] src, int pos, int end) {
tag_rdr.Src_rng_(pos, end);
while (pos < end) {
Gfh_tag lhs = tag_rdr.Tag__move_fwd_head();
int tag_id = lhs.Name_id();
byte[] span_dir = null;
// add any text before lhs;
int txt_end = lhs.Src_bgn();
switch (tag_id) {
case Gfh_tag_.Id__eos: txt_end = end; break; // eos; print everything until end
}
// add any text before tag
if (pos < txt_end) {
byte[] anch_bry = amp_mgr.Decode_as_bry(Bry_.Mid(src, pos, txt_end));
anch_encoder.Encode(anch_bfr, anch_bry);
text_bfr.Add_mid(src, pos, txt_end);
}
// set print_tag tag; REF.MW:Parser.php!formatHeadings
boolean print_tag = false;
switch (tag_id) {
case Gfh_tag_.Id__eos: // eos; return;
return;
case Gfh_tag_.Id__sup: // always print tag; REF.MW:Parser.php!formatHeadings!"Allowed tags are"
case Gfh_tag_.Id__sub:
case Gfh_tag_.Id__i:
case Gfh_tag_.Id__b:
case Gfh_tag_.Id__bdi:
print_tag = true;
break;
case Gfh_tag_.Id__span: // print span only if it has a dir attribute
span_dir = lhs.Atrs__get_as_bry(Gfh_atr_.Bry__dir);
print_tag = Bry_.Len_gt_0(span_dir);
break;
case Gfh_tag_.Id__comment: // never print tag
default:
print_tag = false;
break;
case Gfh_tag_.Id__any: // unknown tags print
print_tag = true;
break;
}
// get lhs / rhs vars
byte[] lhs_bry = lhs.Name_bry();
int lhs_end = lhs.Src_end();
// ignore tags which are not closed by tidy as default; EX: <br> not <br>a</br> or <br/>
boolean lhs_is_dangling = false;
switch (tag_id) {
case Gfh_tag_.Id__img:
case Gfh_tag_.Id__br:
case Gfh_tag_.Id__hr:
lhs_is_dangling = true;
break;
}
boolean lhs_is_pair = !lhs.Tag_is_inline() && !lhs_is_dangling;
int rhs_bgn = -1, rhs_end = -1, new_pos = lhs_end;
if (lhs_is_pair) { // get rhs unless inline
if (tag_id == Gfh_tag_.Id__any) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "unknown tag: page=~{0} tag=~{1}", page_name, lhs_bry);
Gfh_tag rhs = tag_rdr.Tag__move_fwd_tail(lhs_bry);
rhs_bgn = rhs.Src_bgn(); rhs_end = rhs.Src_end();
new_pos = rhs_end;
}
else {
Gfh_tag rhs = tag_rdr.Tag__move_fwd_tail(tag_id);
rhs_bgn = rhs.Src_bgn(); rhs_end = rhs.Src_end();
new_pos = rhs_end;
}
}
// print "<tag></tag>"; also, recurse
if (print_tag) {
text_bfr.Add_byte(Byte_ascii.Angle_bgn).Add(lhs_bry);
if (span_dir != null) // if span has dir, add it; EX: <span id='1' dir='rtl'> -> <span dir='rtl'>
Gfh_atr_.Add(text_bfr, Gfh_atr_.Bry__dir, span_dir);
text_bfr.Add_byte(Byte_ascii.Angle_end); // only add name; do not add atrs; EX: <i id='1'> -> <i>
}
Calc_anch_text_recurse(src, lhs_end, rhs_bgn);
if (print_tag && lhs_is_pair)
text_bfr.Add_mid(src, rhs_bgn, rhs_end);
// set new_pos
pos = new_pos;
tag_rdr.Src_rng_(new_pos, end); // NOTE: must reinit pos and especially end
}
}
public static byte[] Remove_comment(Bry_bfr tmp, byte[] src, int bgn, int end) {
boolean dirty = false, append_to_eos = true;
int pos = bgn;
while (true) {
int comm_bgn = Bry_find_.Find_fwd(src, Gfh_tag_.Comm_bgn, pos, end);
if (comm_bgn != -1) { // comment found
int tmp_pos = comm_bgn + Gfh_tag_.Comm_bgn_len;
int comm_end = Bry_find_.Find_fwd(src, Gfh_tag_.Comm_end, tmp_pos, end);
if (comm_end == -1) { // dangling
tmp.Add_mid(src, pos, comm_bgn);
append_to_eos = false;
}
else {
dirty = true;
tmp.Add_mid(src, pos, comm_bgn);
pos = comm_end + Gfh_tag_.Comm_end_len;
continue;
}
}
break;
}
if (dirty && append_to_eos) {
tmp.Add_mid(src, pos, end);
}
return dirty ? tmp.To_bry_and_clear() : src;
}
private static final byte[] id_trim_ary = Bry_.mask_(256, Byte_ascii.Underline_bry);
}

67
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt__basic__tst.java Normal file
View File

@@ -0,0 +1,67 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import org.junit.*; import gplx.core.tests.*;
public class Xoh_toc_wkr__txt__basic__tst {
@Before public void init() {fxt.Clear();} private final Xoh_toc_wkr__txt__fxt fxt = new Xoh_toc_wkr__txt__fxt();
@Test public void Basic() {
fxt.Test__both("a b c", "a_b_c", "a b c");
}
@Test public void Ws() {
fxt.Test__both(" a b ", "a_b", "a b");
}
@Test public void Empty() { // PAGE:s.w:Colac,_Victoria DATE:2016-07-17
fxt.Test__both("", "", "");
}
@Test public void Amp__ncr() {
fxt.Test__both("&#91;a&#93;", ".5Ba.5D", "&#91;a&#93;");
}
@Test public void Encode() {
fxt.Test__both("a+b", "a.2Bb", "a+b");
}
@Test public void Comment() {
fxt.Test__text("a<!--b-->c", "ac");
}
@Test public void Remove_comment__one() {
fxt.Test__remove_comment("a<!--b-->c", "ac");
}
@Test public void Remove_comment__many() {
fxt.Test__remove_comment("1<!--2-->3<!--4-->5", "135");
}
@Test public void Remove_comment__dangling() {
fxt.Test__remove_comment("1<!--2-->3<!--4->5", "13");
}
}
class Xoh_toc_wkr__txt__fxt {
private final Xoh_toc_wkr__txt wkr = new Xoh_toc_wkr__txt();
private final Xoh_toc_itm itm = new Xoh_toc_itm();
private final Bry_bfr tmp = Bry_bfr_.New();
public void Clear() {wkr.Clear();}
public void Test__anch(String html, String expd_anch) {Test__both(html, expd_anch, null);}
public void Test__text(String html, String expd_text) {Test__both(html, null, expd_text);}
public void Test__both(String html, String expd) {Test__both(html, expd, expd);}
public void Test__both(String html, String expd_anch, String expd_text) {
wkr.Calc_anch_text(itm, Bry_.new_u8(html));
if (expd_anch != null) Gftest.Eq__str(expd_anch, itm.Anch(), "anch");
if (expd_text != null) Gftest.Eq__str(expd_text, itm.Text(), "text");
}
public void Test__remove_comment(String html, String expd) {
byte[] html_bry = Bry_.new_u8(html);
Gftest.Eq__str(expd, Xoh_toc_wkr__txt.Remove_comment(tmp, html_bry, 0, html_bry.length));
}
}

42
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt__dupe__tst.java Normal file
View File

@@ -0,0 +1,42 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import org.junit.*; import gplx.core.tests.*;
public class Xoh_toc_wkr__txt__dupe__tst {
@Before public void init() {fxt.Clear();} private final Xoh_toc_wkr__txt__fxt fxt = new Xoh_toc_wkr__txt__fxt();
@Test public void Basic() {
fxt.Test__anch("a" , "a");
fxt.Test__anch("a" , "a_2");
}
@Test public void Case_insensitive() {
fxt.Test__anch("a" , "a");
fxt.Test__anch("A" , "A_2");
}
@Test public void Autonumber_exists() { // PAGE:fr.w:Itanium; EX: Itanium,Itanium_2,Itanium
fxt.Test__anch("a" , "a");
fxt.Test__anch("a_2" , "a_2");
fxt.Test__anch("a" , "a_3");
}
@Test public void Autonumber_exists_2() {
fxt.Test__anch("a_2" , "a_2");
fxt.Test__anch("a" , "a");
fxt.Test__anch("a" , "a_3");
fxt.Test__anch("a" , "a_4");
fxt.Test__anch("a_2" , "a_2_2");
}
}

42
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt__xnde__tst.java Normal file
View File

@@ -0,0 +1,42 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import org.junit.*; import gplx.core.tests.*;
public class Xoh_toc_wkr__txt__xnde__tst {
@Before public void init() {fxt.Clear();} private final Xoh_toc_wkr__txt__fxt fxt = new Xoh_toc_wkr__txt__fxt();
@Test public void I() {fxt.Test__both("<i>a</i>" , "a", "<i>a</i>");}
@Test public void I__id() {fxt.Test__both("<i id='1'>a</i>" , "a", "<i>a</i>");}
@Test public void B() {fxt.Test__both("<b>a</b>" , "a", "<b>a</b>");}
@Test public void Sup() {fxt.Test__both("<sup>a</sup>" , "a", "<sup>a</sup>");}
@Test public void Sub() {fxt.Test__both("<sub>a</sub>" , "a", "<sub>a</sub>");}
@Test public void Bdi() {fxt.Test__both("<bdi>a</bdi>" , "a", "<bdi>a</bdi>");}
@Test public void Span() {fxt.Test__both("<span>a</span>" , "a", "a");}
@Test public void Span__id() {fxt.Test__both("<span id='1'>a</span>" , "a", "a");}
@Test public void Span__dir() {fxt.Test__both("<span dir=\"ltr\">a</span>" , "a", "<span dir=\"ltr\">a</span>");}
@Test public void Span__dir_id() {fxt.Test__both("<span id='1' dir=\"ltr\">a</span>" , "a", "<span dir=\"ltr\">a</span>");}
@Test public void Small() {fxt.Test__text("<small>a</small>" , "a");}
@Test public void A() {fxt.Test__both("<a href=\"/wiki/A\">b</a>" , "b");}
@Test public void A__nest() {fxt.Test__both("<a href=\"/wiki/A\">b<i>c</i>d</a>" , "bcd", "b<i>c</i>d");}
@Test public void Br() {fxt.Test__both("a<br/>b" , "ab");}
@Test public void Br__dangling() {fxt.Test__both("a<br>b" , "ab");}
@Test public void H2() {fxt.Test__both("a<h2>b</h2>c" , "abc");} // NOTE: not a valid test; MW actually generates "ab" b/c of tidy; see corresponding edit test; DATE:2016-06-28
@Test public void Li() {fxt.Test__text("a<ul><li>b</li></ul>c" , "abc");}
@Test public void Table() {fxt.Test__text("a<table><tr><td>b</td></tr></table>c" , "abc");}
@Test public void Unknown__i() {fxt.Test__both("a<unknown>b<i>c</i>d</unknown>e" , "abcde", "a<unknown>b<i>c</i>d</unknown>e");} // NOTE: technically, anch should be href_encoded a<unknown>b<i>c</i>d</unknown>e b/c <unknown> is not a valid tag; compare with known tags like <li> / <table> which are just stripped
@Test public void Unknown__a() {fxt.Test__both("a<unknown>b<a>c</a>d</unknown>e" , "abcde", "a<unknown>bcd</unknown>e");}
}

549
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xowe_hdr_bldr__tst.java Normal file
View File

@@ -0,0 +1,549 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import org.junit.*; import gplx.xowa.parsers.*; import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.htmls.*;
import gplx.xowa.parsers.hdrs.*;
public class Xowe_hdr_bldr__tst {
@Before public void init() {fxt.Clear();} private final Xowe_hdr_bldr_fxt fxt = new Xowe_hdr_bldr_fxt();
@Test public void Basic() {
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "==a=="
, "==b=="
, "==c=="
, "==d=="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-2\"><a href=\"#b\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-3\"><a href=\"#c\"><span class=\"tocnumber\">3</span> <span class=\"toctext\">c</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-4\"><a href=\"#d\"><span class=\"tocnumber\">4</span> <span class=\"toctext\">d</span></a>"
, " </li>"
, " </ul>"
)
);
}
@Test public void Hier_down() {
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "==a=="
, "===b==="
, "====c===="
, "=====d====="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#b\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">b</span></a>"
, " <ul>"
, " <li class=\"toclevel-3 tocsection-3\"><a href=\"#c\"><span class=\"tocnumber\">1.1.1</span> <span class=\"toctext\">c</span></a>"
, " <ul>"
, " <li class=\"toclevel-4 tocsection-4\"><a href=\"#d\"><span class=\"tocnumber\">1.1.1.1</span> <span class=\"toctext\">d</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
));
}
@Test public void Hier_up() {
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "==a=="
, "===b==="
, "===c==="
, "==d=="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#b\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " <li class=\"toclevel-2 tocsection-3\"><a href=\"#c\"><span class=\"tocnumber\">1.2</span> <span class=\"toctext\">c</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-4\"><a href=\"#d\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">d</span></a>"
, " </li>"
, " </ul>"
));
}
@Test public void Down_up() {
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "==a=="
, "===b==="
, "==c=="
, "===d==="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#b\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-3\"><a href=\"#c\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">c</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-4\"><a href=\"#d\"><span class=\"tocnumber\">2.1</span> <span class=\"toctext\">d</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
));
}
@Test public void D1_D1_D1_U2() {
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "==a=="
, "===b==="
, "====c===="
, "==d=="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#b\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">b</span></a>"
, " <ul>"
, " <li class=\"toclevel-3 tocsection-3\"><a href=\"#c\"><span class=\"tocnumber\">1.1.1</span> <span class=\"toctext\">c</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-4\"><a href=\"#d\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">d</span></a>"
, " </li>"
, " </ul>"
));
}
@Test public void Err() { // PURPOSE: models strange case wherein jumping down does not work
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "==a=="
, "====b===="
, "===c==="
, "====d===="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-2\"><a href=\"#b\"><span class=\"tocnumber\">1.1</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " <li class=\"toclevel-2 tocsection-3\"><a href=\"#c\"><span class=\"tocnumber\">1.2</span> <span class=\"toctext\">c</span></a>"
, " <ul>"
, " <li class=\"toclevel-3 tocsection-4\"><a href=\"#d\"><span class=\"tocnumber\">1.2.1</span> <span class=\"toctext\">d</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
, " </li>"
, " </ul>"
));
}
@Test public void Repeat_name() {
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "==a=="
, "==a=="
, "==a=="
, "==a=="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-2\"><a href=\"#a_2\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-3\"><a href=\"#a_3\"><span class=\"tocnumber\">3</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-4\"><a href=\"#a_4\"><span class=\"tocnumber\">4</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " </ul>"
));
}
@Test public void Id__encode() {
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==a+b=="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a.2Bb\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a+b</span></a>"
, " </li>"
, " </ul>"
));
}
@Test public void Ws() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "== a b =="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a_b\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a b</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='a_b'> a b </span></h2>"
));
}
@Test public void Apos_italic() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==''a''=="
)
, String_.Concat_lines_nl_skip_last
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\"><i>a</i></span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='a'><i>a</i></span></h2>"
, ""
));
}
@Test public void Xnde__italic() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==<i>a</i>=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\"><i>a</i></span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='a'><i>a</i></span></h2>"
));
}
@Test public void Xnde__small() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==<small>a</small>=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='a'><small>a</small></span></h2>"
));
}
@Test public void Xnde__li() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==a<ul><li>b</li></ul>c=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#abc\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">abc</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='abc'>a<ul><li>b</li></ul>c</span></h2>"
));
}
@Test public void Xnde__table() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==a<table><tr><td>b</td></tr></table>c=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#abc\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">abc</span></a>" // NOTE: toc id should be "abc"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='abc'>a<table><tr><td>b</td></tr></table>c</span></h2>"
));
}
// TOMBSTONE: on MW, shows up as 'href="#ab"; <span class="toctext">ab</span>; '; TIDY doing strange things; ignore for now; DATE:2016-06-28
//@Test public void Xnde__h2() {
// fxt.Test_html_all(String_.Concat_lines_nl_skip_last
// ( "__FORCETOC__"
// , "==a<h2>b</h2>c=="
// )
// , String_.Concat_lines_nl
// ( fxt.toc_tbl_nl_n
// ( " <ul>"
// , " <li class=\"toclevel-1 tocsection-1\"><a href=\"#abc\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">ac</span></a>"
// , " </li>"
// , " </ul>"
// )
// , "<h2><span class='mw-headline' id='abc'>a<h2>b</h2>c</span></h2>"
// ));
//}
@Test public void Xnde__dangling() { // PURPOSE: do not render dangling xndes; EX: Casualties_of_the_Iraq_War; ===<small>Iraqi Health Ministry<small>===
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==<small>a<small>=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='a'><small>a<small></small></small></span></h2>"
));
}
@Test public void Nest__xnde__small() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==a <sup>b<small>c</small>d</sup> e=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a_bcd_e\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a <sup>bcd</sup> e</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='a_bcd_e'>a <sup>b<small>c</small>d</sup> e</span></h2>"
));
}
@Test public void Nest__lnki() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==<small>[[a|b]]</small>=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#b\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='b'><small><a href=\"/wiki/A\">b</a></small></span></h2>"
));
}
@Test public void Nest__br() { // PURPOSE: do not render inline xndes; EX: Magnetic_resonance_imaging
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==a<span id=\"b\">b<br/></span>=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#ab\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">ab</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='ab'>a<span id='b'>b<br/></span></span></h2>"
));
}
@Test public void Lnki_link() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==[[a]]=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='a'><a href=\"/wiki/A\">a</a></span></h2>"
));
}
@Test public void Lnki_caption() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==[[a|b]]=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#b\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='b'><a href=\"/wiki/A\">b</a></span></h2>"
));
}
@Test public void Lnki_caption_nest() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==[[a|b<i>c</i>d]]=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#bcd\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">b<i>c</i>d</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='bcd'><a href=\"/wiki/A\">b<i>c</i>d</a></span></h2>"
));
}
@Test public void Html_ncr() {
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==&#91;a&#93;=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#.5Ba.5D\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">&#91;a&#93;</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='.5Ba.5D'>&#91;a&#93;</span></h2>"
));
}
@Test public void Fix_large_before_small() { // PURPOSE.fix: "===a===\n===b===\n" followed by "==c==" causes improper formatting; DATE:2013-05-16
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "===a==="
, "===b==="
, "==c=="
, "==d=="
), fxt.toc_tbl_nl_y // NOTE: should all be level 2
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-2\"><a href=\"#b\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-3\"><a href=\"#c\"><span class=\"tocnumber\">3</span> <span class=\"toctext\">c</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-4\"><a href=\"#d\"><span class=\"tocnumber\">4</span> <span class=\"toctext\">d</span></a>"
, " </li>"
, " </ul>"
));
}
@Test public void Fix_large_before_small_2() { // PURPOSE.fix: similar to above, but has h3 after h2; PAGE:https://en.wikipedia.org/wiki/Wikipedia:Articles_for_creation/2006-08-27 DATE:2014-06-09
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "===a_0==="
, "==b_0=="
, "===b_1==="
, "==c_0=="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a_0\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a_0</span></a>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-2\"><a href=\"#b_0\"><span class=\"tocnumber\">2</span> <span class=\"toctext\">b_0</span></a>"
, " <ul>"
, " <li class=\"toclevel-2 tocsection-3\"><a href=\"#b_1\"><span class=\"tocnumber\">2.1</span> <span class=\"toctext\">b_1</span></a>"
, " </li>"
, " </ul>"
, " </li>"
, " <li class=\"toclevel-1 tocsection-4\"><a href=\"#c_0\"><span class=\"tocnumber\">3</span> <span class=\"toctext\">c_0</span></a>"
, " </li>"
, " </ul>"
));
}
@Test public void Translate_and_comment() { // PURPOSE: <translate> is an xtn and parses its innerText separately; meanwhile, toc_mgr defaults to using the innerText to build toc; EX:Wikidata:Introduction; DATE:2013-07-16
fxt.Test_html_toc(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==<translate><!--b-->ac</translate>=="
), fxt.toc_tbl_nl_y
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#ac\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">ac</span></a>"
, " </li>"
, " </ul>"
));
}
@Test public void Ref() { // PURPOSE: ref contents should not print in TOC; DATE:2013-07-23
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==a<ref>b</ref>=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#a.5B1.5D\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">a<sup>[1]</sup></span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='a.5B1.5D'>a<sup id=\"cite_ref-0\" class=\"reference\"><a href=\"#cite_note-0\">[1]</a></sup></span></h2>"
));
}
@Test public void Category() { // PURPOSE: Category should not show in in TOC; DATE:2013-12-09
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==A[[Category:B]]=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#A\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">A</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='A'>A</span></h2>"
));
}
@Test public void Category_literal() { // PURPOSE: literal Category should show in in TOC; EX: de.w:1234; DATE:2014-01-21
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==A[[:Category:B]]=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#ACategory:B\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">ACategory:B</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='ACategory:B'>A<a href=\"/wiki/Category:B\">Category:B</a></span></h2>"
));
}
@Test public void File() { // PURPOSE: file should show in in TOC; EX: tr.w:D<>nya_Miraslari; DATE:2014-06-06
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==[[File:A.png]] b=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#b\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">b</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='b'><a href=\"/wiki/File:A.png\" class=\"image\" xowa_title=\"A.png\"><img id=\"xoimg_0\" alt=\"\" src=\"file:///mem/wiki/repo/trg/orig/7/0/A.png\" width=\"0\" height=\"0\" /></a> b</span></h2>"
));
}
@Test public void Lnki_invalid() { // PURPOSE: invalid lnki was causing null ref; DATE:2014-02-07
fxt.Test_html_all(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==[[]]=="
)
, String_.Concat_lines_nl
( fxt.toc_tbl_nl_n
( " <ul>"
, " <li class=\"toclevel-1 tocsection-1\"><a href=\"#.5B.5B.5D.5D\"><span class=\"tocnumber\">1</span> <span class=\"toctext\">[[]]</span></a>"
, " </li>"
, " </ul>"
)
, "<h2><span class='mw-headline' id='.5B.5B.5D.5D'>[[]]</span></h2>"
));
}
@Test public void File_in_tbl() { // PURPOSE: two issues (a) don't show file if in tbl; (b) if v2, file inside tbl fails; PAGE:en.w:Holmes County,_Mississippi; DATE:2014-06-22
fxt.Test_html_frag(String_.Concat_lines_nl_skip_last
( "__FORCETOC__"
, "==a <table><tr><td>[[File:A.png]]b</td></tr></table> c=="
)
, "<span class=\"toctext\">a b c</span>" // note that "b" inside tbl shows
);
}
}

69
400_xowa/src/gplx/xowa/addons/htmls/tocs/Xowe_hdr_bldr_fxt.java Normal file
View File

@@ -0,0 +1,69 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.htmls.*;
public class Xowe_hdr_bldr_fxt {
private final Bry_bfr tmp = Bry_bfr_.New();
public Xop_fxt Fxt() {return fxt;} private final Xop_fxt fxt = new Xop_fxt();
public void Clear() {
fxt.Reset();
tmp.Clear();
}
public void Test_html_toc(String raw, String expd) {
fxt.Wtr_cfg().Toc__show_(Bool_.Y);
String actl = Bld_page_with_toc(tmp, fxt, raw);
// HACK: proc only tests TOC; remove <h#><span class="mw-> section; ugly hack, but this is only test code
int span_pos = String_.FindFwd(actl, "<span class=\"mw-");
actl = String_.Mid(actl, 0, span_pos - 5);
Tfds.Eq_str_lines(expd, actl);
fxt.Wtr_cfg().Toc__show_(Bool_.N);
}
public void Test_html_all(String raw, String expd) {
expd = Xoh_consts.Escape_apos(expd);
fxt.Wtr_cfg().Toc__show_(Bool_.Y);
String actl = Bld_page_with_toc(tmp, fxt, raw);
Tfds.Eq(expd, actl);
fxt.Wtr_cfg().Toc__show_(Bool_.N);
}
public void Test_html_frag(String raw, String frag) {
fxt.Wtr_cfg().Toc__show_(Bool_.Y);
String actl = Bld_page_with_toc(tmp, fxt, raw);
fxt.Test_str_part_y(actl, frag);
fxt.Wtr_cfg().Toc__show_(Bool_.N);
}
public String toc_tbl_nl_y(String... ary) {return toc_tbl(Bool_.Y, ary);}
public String toc_tbl_nl_n(String... ary) {return toc_tbl(Bool_.N, ary);}
public String toc_tbl(boolean nl, String... ary) {
return String_.Concat_lines_nl_skip_last
( "<div id=\"toc\" class=\"toc\">"
, " <div id=\"toctitle\">"
, " <h2>Contents</h2>"
, " </div>"
, String_.Concat_lines_nl_skip_last(ary)
, "</div>" + (nl ? "\n" : "")
);
}
public static String Bld_page_with_toc(Bry_bfr bfr, Xop_fxt fxt, String raw) {
String rv = fxt.Exec_parse_page_all_as_str(raw);
bfr.Add_str_u8(rv);
gplx.xowa.htmls.core.wkrs.tocs.Xoh_toc_wtr.Write_toc(bfr, fxt.Page(), Xoh_wtr_ctx.Basic);
return bfr.To_str_and_clear();
}
}

9
400_xowa/src/gplx/xowa/addons/wikis/searchs/parsers/Srch_text_parser.java
View File

@@ -19,6 +19,7 @@ package gplx.xowa.addons.wikis.searchs.parsers; import gplx.*; import gplx.xowa.
import gplx.core.btries.*; import gplx.xowa.langs.cases.*;
public class Srch_text_parser {
private Btrie_slim_mgr parser_trie = Btrie_slim_mgr.cs(); public Btrie_slim_mgr word_end_trie = Btrie_slim_mgr.cs(); private Btrie_slim_mgr word_bgn_trie = Btrie_slim_mgr.cs();
private final Btrie_rv trv = new Btrie_rv();
private Xol_case_mgr case_mgr;
public final Bry_bfr Tmp_bfr = Bry_bfr_.New_w_size(32);
private byte[] src; private int end;
@@ -87,14 +88,14 @@ public class Srch_text_parser {
break;
}
byte b = src[pos];
Object o = parser_trie.Match_bgn_w_byte(b, src, pos, end);
Object o = parser_trie.Match_at_w_b0(trv, b, src, pos, end);
if (o == null) { // unknown sequence; word-char
if (cur_bgn == -1) cur_bgn = pos; // set 1st char for word
++pos;
}
else {
Srch_sym_parser parser = (Srch_sym_parser)o;
pos = parser.Parse(this, src, end, pos, parser_trie.Match_pos());
pos = parser.Parse(this, src, end, pos, trv.Pos());
}
}
}
@@ -133,9 +134,9 @@ public class Srch_text_parser {
int pos = 0; int len = bry.length;
while (pos < len) {
byte b = bry[pos];
if (word_bgn_trie.Match_bgn_w_byte(b, bry, pos, len) != null) { // b is symbol;
if (word_bgn_trie.Match_at_w_b0(trv, b, bry, pos, len) != null) { // b is symbol;
dirty = true;
pos = word_bgn_trie.Match_pos();
pos = trv.Pos();
}
else {
break;

8
400_xowa/src/gplx/xowa/addons/wikis/searchs/searchers/crts/Srch_crt_scanner.java
View File

@@ -20,8 +20,8 @@ import gplx.core.btries.*; import gplx.xowa.langs.cases.*;
import gplx.xowa.addons.wikis.searchs.parsers.*;
class Srch_crt_scanner {
private final List_adp tkns = List_adp_.New(); private byte[] src; private int src_len, pos, txt_bgn;
private final Srch_crt_scanner_syms trie_bldr; private final Btrie_slim_mgr trie;
private final Bry_bfr word_bfr = Bry_bfr_.New(); private boolean word_is_dirty;
private final Srch_crt_scanner_syms trie_bldr; private final Btrie_slim_mgr trie; private final Btrie_rv trv = new Btrie_rv();
private final Bry_bfr word_bfr = Bry_bfr_.New(); private boolean word_is_dirty;
public Srch_crt_scanner(Srch_crt_scanner_syms trie_bldr) {
this.trie_bldr = trie_bldr;
this.trie = trie_bldr.Trie();
@@ -31,14 +31,14 @@ class Srch_crt_scanner {
tkns.Clear(); pos = 0; txt_bgn = -1;
while (pos < src_len) {
byte cur_b = src[pos];
byte cur_tid = trie.Match_byte_or(cur_b, src, pos, src_len, Byte_.Max_value_127);
byte cur_tid = trie.Match_byte_or(trv, cur_b, src, pos, src_len, Byte_.Max_value_127);
if (cur_tid == Byte_.Max_value_127) { // text character
if (txt_bgn == -1) txt_bgn = pos; // 1st character not set; set it
if (word_is_dirty) word_bfr.Add_byte(cur_b);
++pos;
}
else { // \ \s " - & | ( )
int pos_end = trie.Match_pos();
int pos_end = trv.Pos();
if ( cur_tid == Srch_crt_tkn.Tid__not // if "-"
&& txt_bgn != -1) { // && "word has started"
++pos;

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/specials/Srch_special_cmd.java
View File

@@ -35,7 +35,7 @@ public class Srch_special_cmd implements Gfo_invk, Srch_rslt_cbk, Xog_tab_close_
}
public void Search() {
if (async) { // NOTE: async useful with multiple wikis; allows parallel searches;
Srch_html_row_bldr html_row_bldr = new Srch_html_row_bldr(new gplx.xowa.htmls.core.htmls.utls.Xoh_lnki_bldr(wiki.App(), wiki.App().Html__href_wtr()));
Srch_html_row_bldr html_row_bldr = new Srch_html_row_bldr(new gplx.xowa.htmls.core.htmls.utls.Xoh_lnki_bldr(wiki.App(), wiki.Html__href_wtr()));
html_row_wkr = new Srch_html_row_wkr(html_row_bldr, js_wkr, qry.Slab_end - qry.Slab_bgn, wiki.Domain_bry());
Thread_adp_.Start_by_key(gplx.xowa.apps.Xoa_thread_.Key_special_search_db, this, Invk_search_db);
}