mirror of https://github.com/gnosygnu/xowa
gnosygnu
8 years ago
parent
b333db45f8
commit
7a851a41a5
@ -0,0 +1,35 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.core.threads.utils; import gplx.*; import gplx.core.*; import gplx.core.threads.*;
|
||||
import java.util.concurrent.ArrayBlockingQueue;
|
||||
public class Gfo_blocking_queue {
|
||||
private final ArrayBlockingQueue queue;
|
||||
public Gfo_blocking_queue(int capacity) {
|
||||
this.capacity = capacity;
|
||||
this.queue = new ArrayBlockingQueue(capacity);
|
||||
}
|
||||
public int Capacity() {return capacity;} private final int capacity;
|
||||
public void Put(Object o) {
|
||||
try {queue.put(o);}
|
||||
catch (InterruptedException e) {throw Err_.new_exc(e, "threads", "put interrupted");}
|
||||
}
|
||||
public Object Take() {
|
||||
try {return queue.take();}
|
||||
catch (InterruptedException e) {throw Err_.new_exc(e, "threads", "take interrupted");}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,32 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.core.threads.utils; import gplx.*; import gplx.core.*; import gplx.core.threads.*;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
public class Gfo_countdown_latch {
|
||||
private final CountDownLatch latch;
|
||||
public Gfo_countdown_latch(int count) {
|
||||
latch = new CountDownLatch(count);
|
||||
}
|
||||
public void Countdown() {
|
||||
latch.countDown();
|
||||
}
|
||||
public void Await() {
|
||||
try {latch.await();}
|
||||
catch (InterruptedException e) {throw Err_.new_exc(e, "threads", "await interrupted");}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,59 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
public class Xomp_db_core {
|
||||
private final Object thread_lock = new Object();
|
||||
private final Io_url root_dir;
|
||||
Xomp_db_core(Io_url root_dir) {
|
||||
this.root_dir = root_dir;
|
||||
Io_url mgr_url = root_dir.GenSubFil("xomp.sqlite3");
|
||||
this.mgr_db = new Xomp_mgr_db(mgr_url);
|
||||
}
|
||||
public Xomp_mgr_db Mgr_db() {return mgr_db;} private Xomp_mgr_db mgr_db;
|
||||
public Xomp_wkr_db Wkr_db(boolean delete, int idx) {
|
||||
Io_url wkr_url = root_dir.GenSubFil_nest("xomp_" + Int_.To_str_fmt(idx, "000"), "xomp_wkr.sqlite3");
|
||||
if (delete) Io_mgr.Instance.DeleteFil(wkr_url);
|
||||
return new Xomp_wkr_db(idx, wkr_url);
|
||||
}
|
||||
public int Wkr_count() {
|
||||
Io_url[] wkr_dirs = Io_mgr.Instance.QueryDir_args(root_dir).DirOnly_().ExecAsUrlAry();
|
||||
return wkr_dirs.length;
|
||||
}
|
||||
public void Update_wkr_id(int idx, Db_conn wkr_conn) {
|
||||
synchronized (thread_lock) {
|
||||
Db_attach_mgr attach_mgr = new Db_attach_mgr(mgr_db.Conn(), new Db_attach_itm("wkr_db", wkr_conn));
|
||||
attach_mgr.Exec_sql_w_msg("updating page_regy: wkr_id=" + idx, String_.Concat_lines_nl_skip_last // ANSI.Y
|
||||
( "UPDATE xomp_page"
|
||||
, "SET xomp_wkr_id = " + Int_.To_str(idx)
|
||||
, ", html_len = (SELECT length(body) FROM <wkr_db>html h WHERE h.page_id = xomp_page.page_id)"
|
||||
, "WHERE page_id IN (SELECT page_id FROM <wkr_db>html h)"
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
public static Xomp_db_core New__make(Xowe_wiki wiki) {
|
||||
Io_url root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
|
||||
Io_mgr.Instance.DeleteDirDeep(root_dir);
|
||||
return new Xomp_db_core(root_dir);
|
||||
}
|
||||
public static Xomp_db_core New__load(Xowe_wiki wiki) {
|
||||
Io_url root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
|
||||
return new Xomp_db_core(root_dir);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,29 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
public class Xomp_mgr_db {
|
||||
public Xomp_mgr_db(Io_url url) {
|
||||
this.url = url;
|
||||
this.conn = Db_conn_bldr.Instance.Get_or_autocreate(true, url);
|
||||
this.page_tbl = new Xomp_page_tbl(conn);
|
||||
}
|
||||
public Io_url Url() {return url;} private Io_url url;
|
||||
public Db_conn Conn() {return conn;} private Db_conn conn;
|
||||
public Xomp_page_tbl Page_tbl() {return page_tbl;} private Xomp_page_tbl page_tbl;
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
public class Xomp_page_tbl implements Db_tbl {
|
||||
// private final String fld_page_id, fld_page_status, fld_page_mgr_id;
|
||||
private final Db_conn conn;
|
||||
public Xomp_page_tbl(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
this.tbl_name = "xomp_page";
|
||||
flds.Add_int_pkey("page_id");
|
||||
flds.Add_int("page_ns");
|
||||
flds.Add_byte("page_status"); // 0=wait; 1=done; 2=fail
|
||||
flds.Add_int_dflt("html_len", -1);
|
||||
flds.Add_int_dflt("xomp_wkr_id", -1);
|
||||
conn.Rls_reg(this);
|
||||
}
|
||||
public String Tbl_name() {return tbl_name;} private final String tbl_name;
|
||||
public Dbmeta_fld_list Flds() {return flds;} private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
|
||||
public void Create_tbl() {
|
||||
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
|
||||
}
|
||||
public void Rls() {}
|
||||
}
|
||||
@ -0,0 +1,33 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.htmls.core.dbs.*;
|
||||
public class Xomp_wkr_db {
|
||||
public Xomp_wkr_db(int idx, Io_url url) {
|
||||
this.idx = idx;
|
||||
this.url = url;
|
||||
this.conn = Db_conn_bldr.Instance.Get_or_autocreate(true, url);
|
||||
this.html_tbl = new Xowd_html_tbl(conn);
|
||||
conn.Meta_tbl_assert(html_tbl);
|
||||
}
|
||||
public int Idx() {return idx;} private final int idx;
|
||||
public Io_url Url() {return url;} private Io_url url;
|
||||
public Db_conn Conn() {return conn;} private Db_conn conn;
|
||||
public Xowd_html_tbl Html_tbl() {return html_tbl;} private final Xowd_html_tbl html_tbl;
|
||||
}
|
||||
@ -0,0 +1,41 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*;
|
||||
import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
|
||||
class Xomp_html_db_rdr {
|
||||
private final Xowd_html_tbl[] src_tbls;
|
||||
private final Xomp_db_core db;
|
||||
public Xomp_html_db_rdr(Xowe_wiki wiki) {
|
||||
this.db = Xomp_db_core.New__load(wiki);
|
||||
this.src_tbls = new Xowd_html_tbl[db.Wkr_count()];
|
||||
}
|
||||
public void Rows__get(Xowd_html_row rv, int wkr_id, int page_id) {
|
||||
Xowd_html_tbl src_tbl = src_tbls[wkr_id];
|
||||
if (src_tbl == null) {
|
||||
Db_conn wkr_conn = db.Wkr_db(Bool_.N, wkr_id).Conn();
|
||||
src_tbl = new Xowd_html_tbl(wkr_conn);
|
||||
src_tbls[wkr_id] = src_tbl;
|
||||
}
|
||||
src_tbl.Select_as_row(rv, page_id);
|
||||
}
|
||||
public void Rls() {
|
||||
for (Xowd_html_tbl src_tbl : src_tbls)
|
||||
src_tbl.Conn().Rls_conn();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,66 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.wikis.data.*;
|
||||
class Xomp_html_db_wtr {
|
||||
private final long len_max;
|
||||
private final Xowe_wiki wiki; private final Xow_db_mgr db_mgr;
|
||||
private long len_cur;
|
||||
private Xow_db_file html_db; private Xowd_html_tbl html_tbl;
|
||||
public Xomp_html_db_wtr(Xowe_wiki wiki) {
|
||||
this.wiki = wiki; this.db_mgr = wiki.Data__core_mgr();
|
||||
this.len_max = wiki.Appe().Api_root().Bldr().Wiki().Import().Html_db_max();
|
||||
|
||||
// delete all existing tbls
|
||||
if (!db_mgr.Props().Layout_html().Tid_is_all())
|
||||
wiki.Data__core_mgr().Dbs__delete_by_tid(Xow_db_file_.Tid__html_data);
|
||||
}
|
||||
public int Cur_db_id() {return html_db.Id();}
|
||||
public Xowd_html_tbl Tbls__get_or_new(int ns_id, long html_len) {
|
||||
long len_new = len_cur + html_len;
|
||||
if (html_tbl == null || len_new > len_max) {
|
||||
Commit();
|
||||
this.html_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__html_data);
|
||||
if (html_db == null) {
|
||||
html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
|
||||
html_db.Conn().Txn_bgn("xomp.html_db_wtr");
|
||||
this.html_tbl = new Xowd_html_tbl(html_db.Conn());
|
||||
html_tbl.Create_tbl();
|
||||
}
|
||||
}
|
||||
len_cur = len_new;
|
||||
return html_tbl;
|
||||
}
|
||||
public void Rls() {
|
||||
this.Commit();
|
||||
}
|
||||
private void Commit() {
|
||||
if (html_tbl == null) return;
|
||||
html_tbl.Conn().Txn_end();
|
||||
html_tbl.Conn().Rls_conn();
|
||||
|
||||
// update page_ids
|
||||
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
|
||||
( "UPDATE page"
|
||||
, "SET page_html_db_id = {0}"
|
||||
, "WHERE page_id IN (SELECT page_id FROM <html_db>html h)"
|
||||
), html_db.Id());
|
||||
Db_attach_mgr attach_mgr = new Db_attach_mgr(db_mgr.Db__core().Conn(), new Db_attach_itm("html_db", html_db.Conn()));
|
||||
attach_mgr.Exec_sql_w_msg("updating page_ids: " + Int_.To_str(html_db.Id()), sql);
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,31 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xomp_make_cmd extends Xob_cmd__base {
|
||||
public Xomp_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
|
||||
@Override public void Cmd_run() {
|
||||
wiki.Init_assert();
|
||||
Xomp_make_wkr wkr = new Xomp_make_wkr(wiki);
|
||||
wkr.Exec();
|
||||
}
|
||||
|
||||
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "wiki.mass_parse.make";
|
||||
public static final Xob_cmd Prototype = new Xomp_make_cmd(null, null);
|
||||
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xomp_make_cmd(bldr, wiki);}
|
||||
}
|
||||
@ -0,0 +1,71 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.core.brys.*;
|
||||
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
|
||||
class Xomp_make_wkr {
|
||||
private final Db_conn mgr_conn;
|
||||
private final Xomp_html_db_wtr html_db_wtr;
|
||||
private final Xomp_html_db_rdr html_db_rdr;
|
||||
private final Int_flag_bldr src_body_flag_bldr = Xowd_html_tbl.Make_body_flag_bldr();
|
||||
public Xomp_make_wkr(Xowe_wiki wiki) {
|
||||
this.db = Xomp_db_core.New__load(wiki);
|
||||
this.mgr_conn = db.Mgr_db().Conn();
|
||||
this.html_db_rdr = new Xomp_html_db_rdr(wiki);
|
||||
this.html_db_wtr = new Xomp_html_db_wtr(wiki);
|
||||
}
|
||||
public Xomp_db_core Db() {return db;} private Xomp_db_core db;
|
||||
public void Exec() {
|
||||
Xowd_html_row src_row = new Xowd_html_row();
|
||||
|
||||
int[] ns_ary = new int[] {0, 4, 14};
|
||||
int ns_ary_len = ns_ary.length;
|
||||
for (int i = 0; i < ns_ary_len; ++i) {
|
||||
int ns_id = ns_ary[i];
|
||||
String sql = String_.Format("SELECT * FROM xomp_page WHERE page_ns = {0} AND html_len != 0 ORDER BY page_id;", ns_id); // NOTE: html_len == 0 when page failed
|
||||
int count = 0;
|
||||
Db_rdr rdr = mgr_conn.Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
Make_page(rdr, src_row, ns_id);
|
||||
if (++count % 10000 == 0)
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "xomp.html.insert: ns=~{0} db=~{1} count=~{2}", Int_.To_str_pad_bgn_space(ns_id, 3), Int_.To_str_pad_bgn_space(html_db_wtr.Cur_db_id(), 3), Int_.To_str_pad_bgn_space(count, 8));
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
}
|
||||
|
||||
this.Rls();
|
||||
}
|
||||
private void Make_page(Db_rdr rdr, Xowd_html_row src_row, int ns_id) {
|
||||
// get src_row
|
||||
int page_id = rdr.Read_int("page_id");
|
||||
int html_len = rdr.Read_int("html_len");
|
||||
int wkr_id = rdr.Read_int("xomp_wkr_id");
|
||||
html_db_rdr.Rows__get(src_row, wkr_id, page_id);
|
||||
src_body_flag_bldr.Decode(src_row.Body_flag());
|
||||
|
||||
// get trg_tbl and write
|
||||
Xowd_html_tbl trg_tbl = html_db_wtr.Tbls__get_or_new(ns_id, html_len);
|
||||
trg_tbl.Insert(src_row.Page_id(), src_row.Head_flag(), src_body_flag_bldr.Get_as_int(0), src_body_flag_bldr.Get_as_int(1), src_row.Display_ttl(), src_row.Content_sub(), src_row.Sidebar_div(), src_row.Body());
|
||||
}
|
||||
private void Rls() {
|
||||
mgr_conn.Rls_conn();
|
||||
html_db_rdr.Rls();
|
||||
html_db_wtr.Rls();
|
||||
}
|
||||
}
|
||||
@ -1,84 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
class Xomp_page_pool {
|
||||
private final Object thread_lock = new Object();
|
||||
private Xomp_page_pool_loader loader;
|
||||
private List_adp pool = List_adp_.New(); private int pool_idx = 0, pool_len = 0;
|
||||
private Db_conn make_conn;
|
||||
private final Bry_bfr prog_bfr = Bry_bfr_.New();
|
||||
private int pages_done, pages_total;
|
||||
private long time_bgn, time_prv, time_done;
|
||||
public void Init(Xow_wiki wiki, int num_pages_per_load) {
|
||||
this.make_conn = gplx.xowa.bldrs.Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
|
||||
this.loader = new Xomp_page_pool_loader(wiki, make_conn, num_pages_per_load);
|
||||
this.pages_done = 0;
|
||||
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
|
||||
this.pages_total = loader.Get_pending_count();
|
||||
}
|
||||
public boolean Empty() {return empty;} private boolean empty = false;
|
||||
public void Get_next(List_adp wkr_list, int num_pages_per_wkr) {
|
||||
synchronized (thread_lock) {
|
||||
// pool already marked exhausted by another wkr; return;
|
||||
if (empty) return;
|
||||
int wkr_end = pool_idx + num_pages_per_wkr;
|
||||
|
||||
// need pages to fulfill request
|
||||
if (wkr_end > pool_len) {
|
||||
this.pool = loader.Load(pool, pool_idx, pool_len);
|
||||
this.pool_idx = 0;
|
||||
this.pool_len = pool.Len();
|
||||
if (pool_len == 0) { // no more pages; return;
|
||||
empty = true;
|
||||
return;
|
||||
}
|
||||
wkr_end = num_pages_per_wkr; // recalc wkr_end
|
||||
}
|
||||
|
||||
// reset wkr_end; needed for very last set
|
||||
if (wkr_end >= pool_len)
|
||||
wkr_end = pool_len;
|
||||
|
||||
// add pages to wkr_list
|
||||
for (int i = pool_idx; i < wkr_end; ++i) {
|
||||
Xomp_page_itm page = (Xomp_page_itm)pool.Get_at(i);
|
||||
wkr_list.Add(page);
|
||||
}
|
||||
pool_idx = wkr_end;
|
||||
}
|
||||
}
|
||||
public void Mark_done(int id) {
|
||||
synchronized (thread_lock) {
|
||||
pages_done += 1;
|
||||
if (pages_done % 1000 == 0) {
|
||||
long time_cur = gplx.core.envs.Env_.TickCount();
|
||||
int pages_left = pages_total - pages_done;
|
||||
time_done += (time_cur - time_prv);
|
||||
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
|
||||
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
|
||||
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{0} left=~{1} rate=~{2} time_past=~{3} time_left=~{4}", pages_done, pages_left, (int)rate_cur, time_past, time_left);
|
||||
time_prv = time_cur;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void Rls() {
|
||||
make_conn.Rls_conn();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,126 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*; import gplx.core.threads.utils.*;
|
||||
class Xomp_load_wkr implements Gfo_invk {
|
||||
private final Object thread_lock = new Object();
|
||||
private final Xow_wiki wiki;
|
||||
private final Db_conn mgr_conn;
|
||||
private final Db_attach_mgr attach_mgr;
|
||||
private final Gfo_blocking_queue queue;
|
||||
private final int num_wkrs;
|
||||
|
||||
private final Bry_bfr prog_bfr = Bry_bfr_.New();
|
||||
private int pages_done, pages_total;
|
||||
private long time_bgn, time_prv, time_done;
|
||||
public Xomp_load_wkr(Xow_wiki wiki, Db_conn mgr_conn, int num_pages_in_pool, int num_wkrs) {
|
||||
this.wiki = wiki;
|
||||
this.mgr_conn = mgr_conn;
|
||||
this.attach_mgr = new Db_attach_mgr(mgr_conn);
|
||||
this.queue = new Gfo_blocking_queue(num_pages_in_pool);
|
||||
this.num_wkrs = num_wkrs;
|
||||
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
|
||||
this.pages_total = this.Get_pending_count();
|
||||
}
|
||||
public int Get_pending_count() {
|
||||
Db_rdr rdr = mgr_conn.Stmt_sql("SELECT Count(*) AS Count_of FROM xomp_page mp WHERE mp.page_status = 0").Exec_select__rls_auto();
|
||||
try {return rdr.Move_next() ? rdr.Read_int("Count_of") : 0;}
|
||||
finally {rdr.Rls();}
|
||||
}
|
||||
public Xomp_page_itm Take() {return (Xomp_page_itm)queue.Take();}
|
||||
private void Exec() {
|
||||
int prv_page_id = 0;
|
||||
while (prv_page_id != -1) {
|
||||
prv_page_id = Load_pages(prv_page_id);
|
||||
}
|
||||
for (int i = 0; i < num_wkrs; ++i)
|
||||
queue.Put(Xomp_page_itm.Null);
|
||||
}
|
||||
private int Load_pages(int prv_page_id) {
|
||||
// page_tbl.prep_sql
|
||||
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
|
||||
( "SELECT mp.page_id"
|
||||
, ", pp.page_namespace"
|
||||
, ", pp.page_title"
|
||||
, ", pp.page_text_db_id"
|
||||
, "FROM xomp_page mp"
|
||||
, " JOIN <page_db>page pp ON mp.page_id = pp.page_id"
|
||||
, "WHERE mp.page_id > {0}"
|
||||
, "AND mp.page_status = 0"
|
||||
, "LIMIT {1}"
|
||||
), prv_page_id, queue.Capacity());
|
||||
this.attach_mgr.Conn_others_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
|
||||
sql = attach_mgr.Resolve_sql(sql);
|
||||
|
||||
// page_tbl.load_sql
|
||||
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
|
||||
attach_mgr.Attach();
|
||||
Db_rdr rdr = mgr_conn.Stmt_sql(sql).Exec_select__rls_auto();
|
||||
List_adp list = List_adp_.New();
|
||||
int count = 0;
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
prv_page_id = rdr.Read_int("page_id");
|
||||
int text_db_id = rdr.Read_int("page_text_db_id");
|
||||
Xomp_page_itm ppg = new Xomp_page_itm(prv_page_id);
|
||||
ppg.Init_by_page
|
||||
( rdr.Read_int("page_namespace")
|
||||
, rdr.Read_bry_by_str("page_title")
|
||||
, text_db_id
|
||||
);
|
||||
list.Add(ppg);
|
||||
text_db_loader.Add(text_db_id, ppg);
|
||||
++count;
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
attach_mgr.Detach();
|
||||
|
||||
text_db_loader.Load();
|
||||
|
||||
int len = list.Len();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
queue.Put((Xomp_page_itm)list.Get_at(i));
|
||||
}
|
||||
|
||||
return count == 0 ? -1 : prv_page_id;
|
||||
}
|
||||
public void Mark_done(int id) {
|
||||
synchronized (thread_lock) {
|
||||
pages_done += 1;
|
||||
if (pages_done % 1000 == 0) {
|
||||
long time_cur = gplx.core.envs.Env_.TickCount();
|
||||
int pages_left = pages_total - pages_done;
|
||||
time_done += (time_cur - time_prv);
|
||||
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
|
||||
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
|
||||
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{0} left=~{1} rate=~{2} time_past=~{3} time_left=~{4}", pages_done, pages_left, (int)rate_cur, time_past, time_left);
|
||||
time_prv = time_cur;
|
||||
}
|
||||
}
|
||||
}
|
||||
public void Rls() {
|
||||
mgr_conn.Rls_conn();
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk__exec)) this.Exec();
|
||||
else return Gfo_invk_.Rv_unhandled;
|
||||
return this;
|
||||
}
|
||||
public static final String Invk__exec = "exec";
|
||||
}
|
||||
@ -0,0 +1,62 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
class Xomp_page_pool {
|
||||
private final Object thread_lock = new Object();
|
||||
private final Xomp_page_pool_loader loader;
|
||||
private final int num_pages_per_wkr;
|
||||
private List_adp pool = List_adp_.New(); private int pool_idx = 0, pool_len = 0;
|
||||
public Xomp_page_pool(Xomp_page_pool_loader loader, int num_pages_per_wkr) {
|
||||
this.loader = loader; this.num_pages_per_wkr = num_pages_per_wkr;
|
||||
}
|
||||
public boolean Empty() {synchronized (thread_lock) {return empty;}} private boolean empty = false;
|
||||
public void Get_next(List_adp wkr_list) {
|
||||
synchronized (thread_lock) {
|
||||
// pool already marked exhausted by another wkr; return;
|
||||
if (empty) return;
|
||||
int wkr_end = pool_idx + num_pages_per_wkr;
|
||||
|
||||
// need pages to fulfill request
|
||||
if (wkr_end > pool_len) {
|
||||
this.pool = loader.Load(pool, pool_idx, pool_len);
|
||||
this.pool_idx = 0;
|
||||
this.pool_len = pool.Len();
|
||||
if (pool_len == 0) { // no more pages; return;
|
||||
empty = true;
|
||||
return;
|
||||
}
|
||||
wkr_end = num_pages_per_wkr; // recalc wkr_end
|
||||
}
|
||||
|
||||
// reset wkr_end; needed for very last set
|
||||
if (wkr_end >= pool_len)
|
||||
wkr_end = pool_len;
|
||||
|
||||
// add pages to wkr_list
|
||||
for (int i = pool_idx; i < wkr_end; ++i) {
|
||||
Xomp_page_itm page = (Xomp_page_itm)pool.Get_at(i);
|
||||
wkr_list.Add(page);
|
||||
}
|
||||
pool_idx = wkr_end;
|
||||
}
|
||||
}
|
||||
public void Rls() {
|
||||
loader.Conn().Rls_conn();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,43 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
class Xomp_prog_mgr {
|
||||
private final Object thread_lock = new Object();
|
||||
private int pages_done, pages_total;
|
||||
private long time_bgn, time_prv, time_done;
|
||||
private final Bry_bfr prog_bfr = Bry_bfr_.New();
|
||||
public void Init(int pages_total) {
|
||||
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
|
||||
this.pages_total = pages_total;
|
||||
}
|
||||
public void Mark_done(int id) {
|
||||
synchronized (thread_lock) {
|
||||
pages_done += 1;
|
||||
if (pages_done % 1000 == 0) {
|
||||
long time_cur = gplx.core.envs.Env_.TickCount();
|
||||
int pages_left = pages_total - pages_done;
|
||||
time_done += (time_cur - time_prv);
|
||||
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);
|
||||
String time_past = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)((time_cur - time_bgn) / 1000), true, 0);
|
||||
String time_left = gplx.xowa.addons.bldrs.centrals.utils.Time_dhms_.To_str(prog_bfr, (int)(pages_left / rate_cur), true, 0);
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "done=~{1} left=~{2} rate=~{3} time_past=~{4} time_left=~{5}", id, pages_done, pages_left, (int)rate_cur, time_past, time_left);
|
||||
time_prv = time_cur;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,81 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.wikis.caches.*;
|
||||
class Xomp_tmpl_cache_bldr {
|
||||
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all) {
|
||||
Xow_page_cache rv = new Xow_page_cache(wiki);
|
||||
if (fill_all) Fill_all(rv, wiki);
|
||||
return rv;
|
||||
}
|
||||
private static void Fill_all(Xow_page_cache cache, Xowe_wiki wiki) {
|
||||
String sql = String_.Concat_lines_nl_skip_last // ANSI.Y
|
||||
( "SELECT pp.page_id"
|
||||
, ", pp.page_namespace"
|
||||
, ", pp.page_title"
|
||||
, ", pp.page_text_db_id"
|
||||
, ", pp.page_redirect_id"
|
||||
, "FROM page pp"
|
||||
, "WHERE pp.page_namespace IN (10, 828)"
|
||||
);
|
||||
|
||||
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
|
||||
|
||||
// load pages
|
||||
int count = 0;
|
||||
List_adp redirect_list = List_adp_.New();
|
||||
Ordered_hash page_regy = Ordered_hash_.New();
|
||||
Db_rdr rdr = wiki.Data__core_mgr().Db__core().Tbl__page().Conn().Stmt_sql(sql).Exec_select__rls_auto();
|
||||
try {
|
||||
while (rdr.Move_next()) {
|
||||
// get ttl
|
||||
Xoa_ttl page_ttl = wiki.Ttl_parse(rdr.Read_int("page_namespace"), rdr.Read_bry_by_str("page_title"));
|
||||
|
||||
// add to text_db_loader
|
||||
int page_id = rdr.Read_int("page_id");
|
||||
int page_redirect_id = rdr.Read_int("page_redirect_id");
|
||||
Xow_page_cache_itm itm = new Xow_page_cache_itm(page_ttl, Bry_.Empty, Bry_.Empty);
|
||||
itm.Set_page_ids(page_id, page_redirect_id);
|
||||
text_db_loader.Add(rdr.Read_int("page_text_db_id"), itm);
|
||||
cache.Add(page_ttl.Full_db(), itm);
|
||||
page_regy.Add(page_id, itm);
|
||||
|
||||
if (page_redirect_id != -1)
|
||||
redirect_list.Add(itm);
|
||||
if ((++count % 10000) == 0)
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "loading tmpls: ~{0}", count);
|
||||
}
|
||||
} finally {rdr.Rls();}
|
||||
|
||||
// load wikitext
|
||||
text_db_loader.Load();
|
||||
|
||||
// handle redirects
|
||||
int redirect_len = redirect_list.Len();
|
||||
for (int i = 0; i < redirect_len; ++i) {
|
||||
Xow_page_cache_itm src_itm = (Xow_page_cache_itm)redirect_list.Get_at(i);
|
||||
Xow_page_cache_itm trg_itm = (Xow_page_cache_itm)page_regy.Get_by(src_itm.Redirect_id());
|
||||
if (trg_itm == null) {
|
||||
Gfo_usr_dlg_.Instance.Prog_many("", "", "missing redirect for tmpl: ~{0}", src_itm.Ttl().Full_db());
|
||||
continue;
|
||||
}
|
||||
src_itm.Set_redirect_bry(trg_itm.Wtxt__direct());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,69 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.addons.htmls.tocs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.htmls.*;
|
||||
import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.htmls.*;
|
||||
public class Xowe_hdr_bldr_fxt {
|
||||
private final Bry_bfr tmp = Bry_bfr_.New();
|
||||
public Xop_fxt Fxt() {return fxt;} private final Xop_fxt fxt = new Xop_fxt();
|
||||
public void Clear() {
|
||||
fxt.Reset();
|
||||
tmp.Clear();
|
||||
}
|
||||
public void Test_html_toc(String raw, String expd) {
|
||||
fxt.Wtr_cfg().Toc__show_(Bool_.Y);
|
||||
String actl = Bld_page_with_toc(tmp, fxt, raw);
|
||||
|
||||
// HACK: proc only tests TOC; remove <h#><span class="mw-> section; ugly hack, but this is only test code
|
||||
int span_pos = String_.FindFwd(actl, "<span class=\"mw-");
|
||||
actl = String_.Mid(actl, 0, span_pos - 5);
|
||||
|
||||
Tfds.Eq_str_lines(expd, actl);
|
||||
fxt.Wtr_cfg().Toc__show_(Bool_.N);
|
||||
}
|
||||
public void Test_html_all(String raw, String expd) {
|
||||
expd = Xoh_consts.Escape_apos(expd);
|
||||
fxt.Wtr_cfg().Toc__show_(Bool_.Y);
|
||||
String actl = Bld_page_with_toc(tmp, fxt, raw);
|
||||
Tfds.Eq(expd, actl);
|
||||
fxt.Wtr_cfg().Toc__show_(Bool_.N);
|
||||
}
|
||||
public void Test_html_frag(String raw, String frag) {
|
||||
fxt.Wtr_cfg().Toc__show_(Bool_.Y);
|
||||
String actl = Bld_page_with_toc(tmp, fxt, raw);
|
||||
fxt.Test_str_part_y(actl, frag);
|
||||
fxt.Wtr_cfg().Toc__show_(Bool_.N);
|
||||
}
|
||||
public String toc_tbl_nl_y(String... ary) {return toc_tbl(Bool_.Y, ary);}
|
||||
public String toc_tbl_nl_n(String... ary) {return toc_tbl(Bool_.N, ary);}
|
||||
public String toc_tbl(boolean nl, String... ary) {
|
||||
return String_.Concat_lines_nl_skip_last
|
||||
( "<div id=\"toc\" class=\"toc\">"
|
||||
, " <div id=\"toctitle\">"
|
||||
, " <h2>Contents</h2>"
|
||||
, " </div>"
|
||||
, String_.Concat_lines_nl_skip_last(ary)
|
||||
, "</div>" + (nl ? "\n" : "")
|
||||
);
|
||||
}
|
||||
public static String Bld_page_with_toc(Bry_bfr bfr, Xop_fxt fxt, String raw) {
|
||||
String rv = fxt.Exec_parse_page_all_as_str(raw);
|
||||
bfr.Add_str_u8(rv);
|
||||
gplx.xowa.htmls.core.wkrs.tocs.Xoh_toc_wtr.Write_toc(bfr, fxt.Page(), Xoh_wtr_ctx.Basic);
|
||||
return bfr.To_str_and_clear();
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.htmls; import gplx.*; import gplx.xowa.*;
|
||||
import gplx.xowa.htmls.core.wkrs.tocs.*;
|
||||
public class Xoh_page_bfr {
|
||||
private byte toc_mode;
|
||||
private Bry_bfr head_bfr; private final Bry_bfr body_bfr = Bry_bfr_.New();
|
||||
public void Init(Bry_bfr head_bfr) {
|
||||
this.toc_mode = Xoh_toc_data.Toc_mode__none;
|
||||
this.head_bfr = head_bfr;
|
||||
body_bfr.Clear();
|
||||
}
|
||||
public Bry_bfr Split_by_toc(byte toc_mode) {
|
||||
this.toc_mode = toc_mode;
|
||||
return body_bfr;
|
||||
}
|
||||
public void Commit(Xoa_page pg) {
|
||||
boolean toc_mode_enabled = true, toc_mode_is_pgbnr = false; // default to Xoh_toc_data_.Toc_mode__basic
|
||||
switch (toc_mode) {
|
||||
case Xoh_toc_data.Toc_mode__none : toc_mode_enabled = false; break;
|
||||
case Xoh_toc_data.Toc_mode__pgbnr : toc_mode_is_pgbnr = true; break;
|
||||
}
|
||||
|
||||
// set flags
|
||||
pg.Html_data().Toc_mgr().Exists_y_();
|
||||
pg.Html_data().Head_mgr().Itm__pgbnr().Enabled_(toc_mode_is_pgbnr);
|
||||
|
||||
// build bfr by add bfr_0, toc, body_bfr
|
||||
if (toc_mode_enabled) {
|
||||
pg.Html_data().Toc_mgr().To_html(head_bfr, gplx.xowa.htmls.core.htmls.Xoh_wtr_ctx.Basic, toc_mode_is_pgbnr);
|
||||
head_bfr.Add_bfr_and_clear(body_bfr);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,47 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa.htmls.core.htmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.*;
|
||||
import gplx.core.btries.*;
|
||||
import gplx.xowa.parsers.htmls.*;
|
||||
class Xoh_display_ttl_wtr {
|
||||
public static boolean Is_style_restricted(Bry_bfr bfr, Xoh_wtr_ctx hctx, byte[] src, Mwh_atr_itm atr, byte[] atr_key) {
|
||||
if (atr_key != null
|
||||
&& Bry_.Eq(atr_key, Atr_key_style)
|
||||
) {
|
||||
byte[] atr_val = atr.Val_as_bry(); if (atr_val == null) return false; // bounds_chk
|
||||
int atr_val_len = atr_val.length;
|
||||
int atr_pos = 0;
|
||||
while (atr_pos < atr_val_len) {
|
||||
byte b = atr_val[atr_pos];
|
||||
Object o = style_trie.Match_bgn_w_byte(b, atr_val, atr_pos, atr_val_len);
|
||||
if (o != null) {
|
||||
bfr.Add(Msg_style_restricted);
|
||||
return true;
|
||||
}
|
||||
++atr_pos;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
private static final byte[]
|
||||
Atr_key_style = Bry_.new_a7("style")
|
||||
, Msg_style_restricted = Bry_.new_a7(" style='/* attempt to bypass $wgRestrictDisplayTitle */'")
|
||||
;
|
||||
private static final Btrie_slim_mgr style_trie = Btrie_slim_mgr.ci_a7()
|
||||
.Add_str_byte__many(Byte_.By_int(0), "display", "user-select", "visibility"); // if ( preg_match( '/(display|user-select|visibility)\s*:/i', $decoded['style'] ) ) {
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Loading…
Reference in new issue