Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.9.2.1'

Browse Source
This commit is contained in:
gnosygnu
2016-09-11 21:49:20 -04:00
parent 232838c732
commit 35d78f6106
310 changed files with 4358 additions and 5116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/Sync_addon.java → 400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/Xosync_addon.java
View File

@@ -19,17 +19,17 @@ package gplx.xowa.addons.wikis.pages.syncs; import gplx.*; import gplx.xowa.*; i
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.specials.*;
import gplx.xowa.addons.wikis.pages.syncs.specials.*;
public class Sync_addon implements Xoax_addon_itm, Xoax_addon_itm__special {
public class Xosync_addon implements Xoax_addon_itm, Xoax_addon_itm__special {
public Xow_special_page[] Special_pages() {
return new Xow_special_page[]
{ Sync_html_special.Prototype
};
}
public static Sync_addon Get(Xow_wiki wiki) {
Sync_addon rv = (Sync_addon)wiki.Addon_mgr().Itms__get_or_null(ADDON_KEY);
public static Xosync_addon Get(Xow_wiki wiki) {
Xosync_addon rv = (Xosync_addon)wiki.Addon_mgr().Itms__get_or_null(ADDON_KEY);
if (rv == null) {
rv = new Sync_addon();
rv = new Xosync_addon();
wiki.Addon_mgr().Itms__add(rv);
}
return rv;

55
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/Xosync_read_mgr.java Normal file
View File

@@ -0,0 +1,55 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*;
import gplx.dbs.*;
import gplx.xowa.htmls.*;
import gplx.xowa.addons.wikis.pages.syncs.dbs.*;
import gplx.xowa.apps.apis.xowa.addons.bldrs.*;
import gplx.xowa.wikis.data.tbls.*;
public class Xosync_read_mgr {
private final Xowd_page_itm tmp_dbpg = new Xowd_page_itm();
private Db_conn sync_conn; private Xosync_sync_tbl sync_tbl;
private final Xosync_update_mgr update_mgr = new Xosync_update_mgr();
public void Auto_update(Xow_wiki wiki, Xoa_page page, Xoa_ttl page_ttl) {
if (wiki.Domain_itm().Domain_type_id() == gplx.xowa.wikis.domains.Xow_domain_tid_.Tid__home) return;
if (page_ttl.Ns().Id_is_special()) return;
Xoapi_sync_api sync_api = wiki.App().Api_root().Addon().Bldr().Sync();
if (!sync_api.Auto_enabled()) return;
if (!sync_api.Auto_page_matcher().Match(page_ttl.Full_db())) return;
wiki.Data__core_mgr().Db__core().Tbl__page().Select_by_ttl(tmp_dbpg, page_ttl.Ns(), page_ttl.Page_db());
if (sync_conn == null) {
Io_url sync_db_url = wiki.Fsys_mgr().Root_dir().GenSubFil(wiki.Domain_str() + "-sync.xowa");
sync_conn = Db_conn_bldr.Instance.Get_or_autocreate(true, sync_db_url);
sync_tbl = new Xosync_sync_tbl(sync_conn);
sync_conn.Meta_tbl_assert(sync_tbl);
}
DateAdp sync_date = sync_tbl.Select_sync_date_or_min(tmp_dbpg.Id());
if (Datetime_now.Get().Diff(sync_date).Total_mins().To_int() < sync_api.Auto_interval()) return;
Xoa_app app = wiki.App();
Xoh_page hpg = new Xoh_page();
update_mgr.Init_by_app(app);
update_mgr.Init_by_page(wiki, hpg);
update_mgr.Update(app.Wmf_mgr().Download_wkr(), wiki, page_ttl);
sync_tbl.Upsert(tmp_dbpg.Id(), Datetime_now.Get());
}
}

81
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/Xosync_update_mgr.java Normal file
View File

@@ -0,0 +1,81 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*;
import gplx.xowa.files.downloads.*;
import gplx.xowa.wikis.data.*;
import gplx.xowa.htmls.*; import gplx.langs.htmls.docs.*;
import gplx.xowa.htmls.core.wkrs.*; import gplx.xowa.htmls.core.wkrs.txts.*; import gplx.xowa.htmls.core.hzips.*;
import gplx.xowa.htmls.core.dbs.*;
import gplx.xowa.addons.wikis.pages.syncs.wmapis.*;
import gplx.xowa.addons.wikis.pages.syncs.core.parsers.*;
public class Xosync_update_mgr {
private final Xoh_hzip_bfr bfr = new Xoh_hzip_bfr(Io_mgr.Len_mb, Bool_.N, Byte_.Max_value_127);
private final Gfh_doc_parser hdoc_parser_mgr;
private final Xoh_hdoc_ctx hctx = new Xoh_hdoc_ctx();
private final Xosync_hdoc_wtr hdoc_bldr = new Xosync_hdoc_wtr();
private final Xosync_hdoc_parser hdoc_parser_wkr;
private final Xowd_html_tbl_mgr html_tbl_mgr = new Xowd_html_tbl_mgr();
public Xosync_update_mgr() {
hdoc_parser_wkr = new Xosync_hdoc_parser(hdoc_bldr);
hdoc_parser_mgr = new Gfh_doc_parser(new Xoh_txt_parser(hdoc_bldr), hdoc_parser_wkr);
}
public void Init_by_app(Xoa_app app) {
hctx.Init_by_app(app);
}
public void Init_by_page(Xow_wiki wiki, Xoa_page page) {
hctx.Init_by_page(wiki, page);
page.Hdump_mgr().Clear();
}
public void Update(Xof_download_wkr download_wkr, Xow_wiki wiki, Xoa_ttl page_ttl) {
Xoh_page hpg = (Xoh_page)hctx.Page();
// call wmf api
Xowm_parse_wmf parse_wkr = new Xowm_parse_wmf();
Xowm_parse_data data = parse_wkr.Get_parse_or_null(download_wkr, wiki, page_ttl);
if (data == null) return;
// parse
Parse(hpg, wiki, hctx.Page__url(), data.Revn_html());
// get existing html_tbl
Xow_db_file html_db = html_tbl_mgr.Get_html_db(wiki);
html_tbl_mgr.Save_html(wiki, html_db, data.Page_id(), data.Revn_id(), hpg.Db().Html().Html_bry());
// download files
hpg.Ctor_by_hview(wiki, wiki.Utl__url_parser().Parse(page_ttl.Full_db()), page_ttl, data.Page_id());
gplx.xowa.files.Xof_file_wkr file_thread = new gplx.xowa.files.Xof_file_wkr
( wiki.File__orig_mgr(), wiki.File__bin_mgr(), wiki.File__mnt_mgr()
, wiki.App().User().User_db_mgr().Cache_mgr(), wiki.File__repo_mgr(), gplx.xowa.guis.cbks.js.Xog_js_wkr_.Noop, hpg, hpg.Hdump_mgr().Imgs()
);
gplx.core.threads.Gfo_thread_pool thread_pool = new gplx.core.threads.Gfo_thread_pool();
thread_pool.Add_at_end(file_thread);
thread_pool.Run();
}
public void Parse(Xoh_page hpg, Xow_wiki wiki, byte[] page_url, byte[] src) {
int src_len = src.length;
// init_by_page for bldr, parser, hdoc
hctx.Init_by_page(wiki, hpg);
hdoc_bldr.Init_by_page(bfr, hpg, hctx, src, 0, src_len);
hdoc_parser_wkr.Init_by_page(hctx, src, 0, src_len);
// parse
hdoc_parser_mgr.Parse(page_url, src, 0, src_len);
hpg.Db().Html().Html_bry_(bfr.To_bry_and_clear());
}
}

79
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader.java Normal file
View File

@@ -0,0 +1,79 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.loaders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import gplx.core.brys.*;
import gplx.langs.htmls.*; import gplx.xowa.htmls.*; import gplx.langs.htmls.docs.*; import gplx.xowa.htmls.core.wkrs.*; import gplx.xowa.htmls.core.wkrs.imgs.atrs.*;
import gplx.xowa.files.*; import gplx.xowa.files.repos.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.addons.wikis.pages.syncs.core.parsers.*;
public class Xosync_page_loader {
private final Xoh_hdoc_ctx hctx = new Xoh_hdoc_ctx();
private final Gfh_tag_rdr tag_rdr = Gfh_tag_rdr.New__html();
private final Bry_err_wkr err_wkr = new Bry_err_wkr();
private final Xoh_img_src_data img_src_parser = new Xoh_img_src_data();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
public byte[] Parse(Xow_wiki wiki, Xoh_page hpg, byte[] src) {
// init hctx, tag_rdr, err_wkr
int src_len = src.length;
hctx.Init_by_page(wiki, hpg);
tag_rdr.Init(hpg.Url_bry_safe(), src, 0, src_len);
err_wkr.Init_by_page(String_.new_u8(hpg.Url_bry_safe()), src);
// loop for all <img>
int pos = 0;
while (true) {
// get next "<img>"
Gfh_tag img_tag = tag_rdr.Tag__find_fwd_head(pos, src_len, Gfh_tag_.Id__img);
// none found; add and exit
if (img_tag.Name_id() == Gfh_tag_.Id__eos) {
tmp_bfr.Add_mid(src, pos, src_len); // add bytes between img_end and prv_pos
break;
}
// add bytes between prv_pos and img_bgn
tmp_bfr.Add_mid(src, pos, img_tag.Src_bgn());
// do simple replace for @src
Gfh_atr img_src_atr = img_tag.Atrs__get_by_or_fail(Gfh_atr_.Bry__src);
byte[] img_src_val = img_src_atr.Val();
img_src_val = Bry_.Replace(img_src_val, Xosync_img_src_parser.Bry__xowa_src_bgn, wiki.App().Fsys_mgr().File_dir().To_http_file_bry());
// parse other atrs for fsdb
img_src_parser.Parse(err_wkr, hctx, wiki.Domain_bry(), img_src_atr.Val_bgn(), img_src_atr.Val_end());
if (img_src_parser.File_ttl_bry() != null) { // NOTE: need to skip images that don't follow MW image format ("commons.wikimedia.org/thumb/7/70/A.png"); for example, math images
Xof_fsdb_itm img = hpg.Img_mgr().Make_img(false);
byte[] file_ttl_bry = gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url.Decode(img_src_parser.File_ttl_bry());
img.Init_by_wm_parse(hctx.Wiki__domain_itm().Abrv_xo(), img_src_parser.Repo_is_commons(), img_src_parser.File_is_orig(), file_ttl_bry, img_src_parser.File_w(), img_src_parser.File_time(), img_src_parser.File_page());
hctx.File__url_bldr().Init_by_root(img_src_parser.Repo_is_commons() ? hctx.Fsys__file__comm() : hctx.Fsys__file__wiki(), Bool_.N, Byte_ascii.Slash, Bool_.N, Bool_.N, 4);
hctx.File__url_bldr().Init_by_itm(img_src_parser.File_is_orig() ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb, file_ttl_bry, Xof_file_wkr_.Md5(file_ttl_bry), Xof_ext_.new_by_ttl_(file_ttl_bry), img_src_parser.File_w(), img_src_parser.File_time(), img_src_parser.File_page());
img.Orig_repo_name_(img_src_parser.Repo_is_commons() ? Xow_domain_itm_.Bry__commons : wiki.Domain_bry());
Io_url html_view_url = hctx.File__url_bldr().Xto_url_by_http();
img.Init_at_gallery_end(img_tag.Atrs__get_as_int(Gfh_atr_.Bry__width), img_tag.Atrs__get_as_int(Gfh_atr_.Bry__height), html_view_url, html_view_url);
Xosync_hdoc_parser.Write_img_tag(tmp_bfr, img_tag, img_src_val, img.Html_uid());
}
pos = img_tag.Src_end();
}
// overwrite html
src = tmp_bfr.To_bry_and_clear();
hpg.Db().Html().Html_bry_(src);
return src;
}
}

51
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__fxt.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.loaders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import gplx.core.tests.*;
import gplx.langs.htmls.*; import gplx.xowa.htmls.*;
import gplx.xowa.files.*; import gplx.xowa.files.repos.*;
import gplx.xowa.addons.wikis.pages.syncs.core.parsers.*;
public class Xosync_page_loader__fxt {
private final Xosync_page_loader mgr = new Xosync_page_loader();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private final Xoh_page hpg = new Xoh_page();
private Xowe_wiki wiki;
public void Clear() {
Xoae_app app = Xoa_app_fxt.Make__app__edit();
this.wiki = Xoa_app_fxt.Make__wiki__edit(app);
hpg.Clear();
}
public Xosync_page_loader__fxt Exec__parse(String raw) {
mgr.Parse(wiki, hpg, Bry_.new_u8(raw));
return this;
}
public Xosync_page_loader__fxt Test__html(String expd) {
Gftest.Eq__ary__lines(expd, hpg.Db().Html().Html_bry(), "converted html");
return this;
}
public Xof_fsdb_itm Make__fsdb(boolean repo_is_commons, boolean file_is_thumb, String file_ttl, int file_w, double file_time, int file_page) {
Xof_fsdb_itm itm = new Xof_fsdb_itm();
itm.Init_by_wm_parse(wiki.Domain_itm().Abrv_xo(), repo_is_commons, file_is_thumb, Bry_.new_u8(file_ttl), file_w, file_time, file_page);
return itm;
}
public Xosync_page_loader__fxt Test__fsdb(Xof_fsdb_itm expd) {
Xof_fsdb_itm actl = (Xof_fsdb_itm)hpg.Img_mgr().Get_at(0);
Gftest.Eq__str(Xosync_hdoc_parser__fxt.To_str(tmp_bfr, expd), Xosync_hdoc_parser__fxt.To_str(tmp_bfr, actl));
return this;
}
}

29
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__tst.java Normal file
View File

@@ -0,0 +1,29 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.loaders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import org.junit.*;
import gplx.langs.htmls.*;
public class Xosync_page_loader__tst {
@Before public void init() {fxt.Clear();} private final Xosync_page_loader__fxt fxt = new Xosync_page_loader__fxt();
@Test public void Basic() {
fxt.Exec__parse(Gfh_utl.Replace_apos("a<img src='xowa:/file/commons.wikimedia.org/thumb/4/a/6/9/Commons-logo.svg/12px.png' width='12' height='20'>b"))
.Test__html(Gfh_utl.Replace_apos("a<img id='xoimg_0' src='file:///mem/xowa/file/commons.wikimedia.org/thumb/4/a/6/9/Commons-logo.svg/12px.png' width='12' height='20'>b"))
.Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "Commons-logo.svg", 12, -1, -1))
;
}
}

106
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser.java Normal file
View File

@@ -0,0 +1,106 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import gplx.langs.htmls.*; import gplx.langs.htmls.docs.*; import gplx.xowa.htmls.core.wkrs.*;
public class Xosync_hdoc_parser implements Gfh_doc_wkr {
private final Xosync_hdoc_wtr hdoc_wtr;
private final Gfh_tag_rdr tag_rdr = Gfh_tag_rdr.New__html();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private final Xosync_img_src_parser img_src_parser = new Xosync_img_src_parser();
public Xosync_hdoc_parser(Xosync_hdoc_wtr hdoc_wtr) {this.hdoc_wtr = hdoc_wtr;}
public byte[] Hook() {return Byte_ascii.Angle_bgn_bry;}
public void Init_by_page(Xoh_hdoc_ctx hctx, byte[] src, int src_bgn, int src_end) {
tag_rdr.Init(hctx.Page__url(), src, src_bgn, src_end);
img_src_parser.Init_by_page(hctx);
}
public int Parse(byte[] src, int src_bgn, int src_end, int pos) {
// note that entry point is at "<"
tag_rdr.Pos_(pos);
int nxt_pos = tag_rdr.Pos() + 1; if (nxt_pos == src_end) return src_end;
// check if head or tail; EX: "<a>" vs "</a>"
byte nxt_byte = src[nxt_pos];
// skip comment; needed else comment may gobble up rest of text; see test; DATE:2016-09-10
if (nxt_byte == Byte_ascii.Bang) { // assume comment; EX:"<!--"
int end_comm = Bry_find_.Move_fwd(src, Gfh_tag_.Comm_end, nxt_pos);
if (end_comm == Bry_find_.Not_found) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "end comment not found; src=~{0}", String_.new_u8(src));
end_comm = src_end;
}
return end_comm;
}
Gfh_tag cur = nxt_byte == Byte_ascii.Slash ? tag_rdr.Tag__move_fwd_tail(Gfh_tag_.Id__any) : tag_rdr.Tag__move_fwd_head();
if (cur.Tag_is_tail()) {}
else {
int cur_name_id = cur.Name_id();
switch (cur_name_id) {
case Gfh_tag_.Id__span:
if (cur.Atrs__cls_has(Bry__span__edit_section)) { // remove edit-section
tag_rdr.Tag__move_fwd_tail(cur_name_id);
return tag_rdr.Pos();
}
break;
case Gfh_tag_.Id__img: // rewrite src for XOWA; especially necessary for relative protocol; EX: "//upload.wikimedia.org"; note do not use <super> tag b/c of issues with anchors like "href=#section"
return Parse_img_src(cur);
default:
break;
}
}
hdoc_wtr.On_txt(cur.Src_bgn(), cur.Src_end());
return cur.Src_end();
}
private int Parse_img_src(Gfh_tag img_tag) {
// get @src and parse it
Gfh_atr src_atr = img_tag.Atrs__get_by_or_empty(Gfh_atr_.Bry__src);
img_src_parser.Parse(src_atr.Val());
// if error, write comment; EX: <!--error--><img ...>
String err_msg = img_src_parser.Err_msg();
if (err_msg != null) {
hdoc_wtr.Add_bry(Gfh_tag_.Comm_bgn);
hdoc_wtr.Add_str(img_src_parser.Err_msg());
hdoc_wtr.Add_bry(Gfh_tag_.Comm_end);
}
// get img_src; use img_src_parser if no error, else use original value
byte[] img_src_val = err_msg == null ? img_src_parser.To_bry() : src_atr.Val();
// write html
Write_img_tag(tmp_bfr, img_tag, img_src_val, -1);
hdoc_wtr.Add_bfr(tmp_bfr);
return img_tag.Src_end();
}
public static void Write_img_tag(Bry_bfr bfr, Gfh_tag img_tag, byte[] img_src_val, int uid) {
// rewrite <img> tag with custom img_src_val
int atrs_len = img_tag.Atrs__len();
bfr.Add(Byte_ascii.Angle_bgn_bry);
bfr.Add(Gfh_tag_.Bry__img);
if (uid != -1) {
Gfh_atr_.Add(bfr, Gfh_atr_.Bry__id, Bry_.new_a7("xoimg_" + Int_.To_str(uid)));
}
for (int i = 0; i < atrs_len; ++i) {
Gfh_atr atr = img_tag.Atrs__get_at(i);
// if atr is src use img_src_val; EX: ' src="//upload.wikimedia.org/..."' -> ' src="xowa:/file/..."
Gfh_atr_.Add(bfr, atr.Key(), Bry_.Eq(atr.Key(), Gfh_atr_.Bry__src) ? img_src_val : atr.Val());
}
bfr.Add(Byte_ascii.Angle_end_bry);
}
private static final byte[] Bry__span__edit_section = Bry_.new_a7("mw-editsection");
}

47
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__err__tst.java Normal file
View File

@@ -0,0 +1,47 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import org.junit.*;
import gplx.langs.htmls.*;
public class Xosync_hdoc_parser__err__tst {
@Before public void init() {fxt.Clear();} private final Xosync_hdoc_parser__fxt fxt = new Xosync_hdoc_parser__fxt();
@Test public void Url_does_not_start_with_upload_wikimedia_org() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//fail/wikipedia/commons/thumb/7/70/A.png/220px-A.png'>"))
.Test__html(Gfh_utl.Replace_apos("<!--wm.parse:url does not start with //upload.wikimedia.org--><img src='//fail/wikipedia/commons/thumb/7/70/A.png/220px-A.png'>"));
}
@Test public void Unknown_repo() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//upload.wikimedia.org/wiktionary/fr/thumb/7/70/A.png/220px-A.png'>"))
.Test__html(Gfh_utl.Replace_apos("<!--wm.parse:unknown repo--><img src='//upload.wikimedia.org/wiktionary/fr/thumb/7/70/A.png/220px-A.png'>"));
}
@Test public void Bad_md5() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//upload.wikimedia.org/wikipedia/commons/thumb/fail/A.png/220px-A.png'>"))
.Test__html(Gfh_utl.Replace_apos("<!--wm.parse:invalid md5--><img src='//upload.wikimedia.org/wikipedia/commons/thumb/fail/A.png/220px-A.png'>"));
}
@Test public void Missing_px() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//upload.wikimedia.org/wikipedia/commons/thumb/7/70/A.png/220-A.png'>"))
.Test__html(Gfh_utl.Replace_apos("<!--wm.parse:missing px--><img src='//upload.wikimedia.org/wikipedia/commons/thumb/7/70/A.png/220-A.png'>"));
}
@Test public void Bad_file_w() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//upload.wikimedia.org/wikipedia/commons/thumb/7/70/A.png/220_fail_px-A.png'>"))
.Test__html(Gfh_utl.Replace_apos("<!--wm.parse:invalid file_w--><img src='//upload.wikimedia.org/wikipedia/commons/thumb/7/70/A.png/220_fail_px-A.png'>"));
}
@Test public void Comment() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<i>a<!-- - --></i><b>b</b><i>c</i>"))
.Test__html(Gfh_utl.Replace_apos("<i>a</i><b>b</b><i>c</i>"));
}
}

46
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__file__tst.java Normal file
View File

@@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import org.junit.*;
import gplx.langs.htmls.*;
public class Xosync_hdoc_parser__file__tst {
@Before public void init() {fxt.Clear();} private final Xosync_hdoc_parser__fxt fxt = new Xosync_hdoc_parser__fxt();
@Test public void Commons__thumb() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//upload.wikimedia.org/wikipedia/commons/thumb/7/70/A.png/220px-A.png'>"))
.Test__html(Gfh_utl.Replace_apos("<img src='xowa:/file/commons.wikimedia.org/thumb/7/0/1/c/A.png/220px.png'>"))
.Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A.png", 220, -1, -1));
}
@Test public void Url_encoded() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//upload.wikimedia.org/wikipedia/commons/thumb/9/91/A%2CB.png/220px-A%2CB.png'>"))
.Test__html(Gfh_utl.Replace_apos("<img src='xowa:/file/commons.wikimedia.org/thumb/9/1/0/8/A%2CB.png/220px.png'>"))
.Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A,B.png", 220, -1, -1));
}
@Test public void Local__orig() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//upload.wikimedia.org/wikipedia/en/7/70/A.png'>"))
.Test__html(Gfh_utl.Replace_apos("<img src='xowa:/file/en.wikipedia.org/orig/7/0/1/c/A.png'>"))
.Test__fsdb(fxt.Make__fsdb(Bool_.N, Bool_.Y, "A.png", -1, -1, -1));
}
@Test public void Svg() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//upload.wikimedia.org/wikipedia/commons/thumb/4/4a/Commons-logo.svg/12px-Commons-logo.svg.png'>"))
.Test__html(Gfh_utl.Replace_apos("<img src='xowa:/file/commons.wikimedia.org/thumb/4/a/6/9/Commons-logo.svg/12px.png'>"))
.Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "Commons-logo.svg", 12, -1, -1));
// https://upload.wikimedia.org/wikipedia/commons/thumb/f/fc/Papilio_dardanus_emerging.ogg/320px--Papilio_dardanus_emerging.ogg.jpg
// https://upload.wikimedia.org/wikipedia/commons/thumb/3/30/Clip_from_My_Man_Godfrey.ogg/240px-seek%3D67-Clip_from_My_Man_Godfrey.ogg.jpg
// https://upload.wikimedia.org/wikipedia/commons/thumb/7/7a/PL_Henryk_Sienkiewicz-Pisma_zapomniane_i_niewydane.djvu/page6-250px-PL_Henryk_Sienkiewicz-Pisma_zapomniane_i_niewydane.djvu.jpg
}
}

64
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__fxt.java Normal file
View File

@@ -0,0 +1,64 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import gplx.core.tests.*;
import gplx.langs.htmls.*; import gplx.xowa.htmls.*;
import gplx.xowa.files.*; import gplx.xowa.files.repos.*;
public class Xosync_hdoc_parser__fxt {
private final Xosync_update_mgr mgr = new Xosync_update_mgr();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private final Xoh_page hpg = new Xoh_page();
private Xowe_wiki wiki;
public void Clear() {
Xoae_app app = Xoa_app_fxt.Make__app__edit();
this.wiki = Xoa_app_fxt.Make__wiki__edit(app);
Xoa_app_fxt.repo2_(app, wiki);
mgr.Init_by_app(app);
mgr.Init_by_page(wiki, hpg);
}
public Xosync_hdoc_parser__fxt Exec__parse(String raw) {
mgr.Parse(hpg, wiki, Bry_.Empty, Bry_.new_u8(raw));
return this;
}
public Xosync_hdoc_parser__fxt Test__html(String expd) {
Gftest.Eq__ary__lines(expd, hpg.Db().Html().Html_bry(), "converted html");
return this;
}
public Xof_fsdb_itm Make__fsdb(boolean repo_is_commons, boolean file_is_thumb, String file_ttl, int file_w, double file_time, int file_page) {
Xof_fsdb_itm itm = new Xof_fsdb_itm();
itm.Init_by_wm_parse(wiki.Domain_itm().Abrv_xo(), repo_is_commons, file_is_thumb, Bry_.new_u8(file_ttl), file_w, file_time, file_page);
return itm;
}
public Xosync_hdoc_parser__fxt Test__fsdb(Xof_fsdb_itm expd) {
Xof_fsdb_itm actl = (Xof_fsdb_itm)hpg.Hdump_mgr().Imgs().Get_at(0);
Gftest.Eq__str(To_str(tmp_bfr, expd), To_str(tmp_bfr, actl));
return this;
}
public static String To_str(Bry_bfr tmp_bfr, Xof_fsdb_itm itm) {
To_bfr(tmp_bfr, itm);
return tmp_bfr.To_str_and_clear();
}
private static void To_bfr(Bry_bfr bfr, Xof_fsdb_itm itm) {
bfr.Add_str_a7(itm.Orig_repo_id() == Xof_repo_itm_.Repo_remote ? "remote" : "local").Add_byte_pipe();
bfr.Add_str_a7(itm.File_is_orig() ? "orig" : "thumb").Add_byte_pipe();
bfr.Add(itm.Orig_ttl()).Add_byte_pipe();
bfr.Add_int_variable(itm.File_w()).Add_byte_pipe();
bfr.Add_double(itm.Lnki_time()).Add_byte_pipe();
bfr.Add_int_variable(itm.Lnki_page()).Add_byte_pipe();
}
}

49
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__tst.java Normal file
View File

@@ -0,0 +1,49 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import org.junit.*;
import gplx.langs.htmls.*;
public class Xosync_hdoc_parser__tst {
@Before public void init() {fxt.Clear();} private final Xosync_hdoc_parser__fxt fxt = new Xosync_hdoc_parser__fxt();
@Test public void Remove_edit() {
fxt.Exec__parse(Gfh_utl.Replace_apos_concat_lines
( "<h2><span class='mw-headline' id='Section_1'>Section_1</span>"
, "<span class='mw-editsection'>"
, "<span class='mw-editsection-bracket'>[</span><a href='/w/index.php?title=Page_1&amp;action=edit&amp;section=1' title='Edit section: Section_1'>edit</a>"
, "<span class='mw-editsection-bracket'>]</span>"
, "</span>"
, "</h2>"
)).Test__html(Gfh_utl.Replace_apos_concat_lines
( "<h2><span class='mw-headline' id='Section_1'>Section_1</span>"
, ""
, "</h2>"
));
}
@Test public void File() {
fxt.Exec__parse(Gfh_utl.Replace_apos("<img src='//upload.wikimedia.org/wikipedia/commons/thumb/4/4a/Commons-logo.svg/12px-Commons-logo.svg.png'>"))
.Test__html(Gfh_utl.Replace_apos("<img src='xowa:/file/commons.wikimedia.org/thumb/4/a/6/9/Commons-logo.svg/12px.png'>"))
.Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "Commons-logo.svg", 12, -1, -1));
// https://upload.wikimedia.org/wikipedia/commons/thumb/f/fc/Papilio_dardanus_emerging.ogg/320px--Papilio_dardanus_emerging.ogg.jpg
// https://upload.wikimedia.org/wikipedia/commons/thumb/3/30/Clip_from_My_Man_Godfrey.ogg/240px-seek%3D67-Clip_from_My_Man_Godfrey.ogg.jpg
// https://upload.wikimedia.org/wikipedia/commons/thumb/7/7a/PL_Henryk_Sienkiewicz-Pisma_zapomniane_i_niewydane.djvu/page6-250px-PL_Henryk_Sienkiewicz-Pisma_zapomniane_i_niewydane.djvu.jpg
}
// @Test public void Smoke() {
// fxt.Exec__parse(Io_mgr.Instance.LoadFilStr("C:\\xowa\\dev\\wm.updater.src.html"));
// Io_mgr.Instance.SaveFilBry("C:\\xowa\\dev\\wm.updater.trg.html", fxt.Hdoc().Converted());
// }
}

43
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_wtr.java Normal file
View File

@@ -0,0 +1,43 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.wkrs.*; import gplx.xowa.htmls.core.hzips.*;
public class Xosync_hdoc_wtr implements Xoh_hdoc_wkr {
private Xoh_hzip_bfr bfr;
private byte[] src;
public void Init_by_page(Xoh_hzip_bfr bfr, Xoh_page hpg, Xoh_hdoc_ctx hctx, byte[] src, int src_bgn, int src_end) {
this.On_new_page(bfr, hpg, hctx, src, src_bgn, src_end);
}
public void On_new_page(Xoh_hzip_bfr bfr, Xoh_page hpg, Xoh_hdoc_ctx hctx, byte[] src, int src_bgn, int src_end) {
this.bfr = bfr;
this.src = src;
}
public void On_txt (int rng_bgn, int rng_end) {bfr.Add_mid(src, rng_bgn, rng_end);}
public void Add_bfr (Bry_bfr v) {bfr.Add_bfr_and_clear(v);}
public void Add_str (String v) {bfr.Add_str_u8(v);}
public void Add_bry (byte[] v) {bfr.Add(v);}
// not used
public void On_escape (gplx.xowa.htmls.core.wkrs.escapes.Xoh_escape_data data) {}
public void On_xnde (gplx.xowa.htmls.core.wkrs.xndes.Xoh_xnde_parser parser) {}
public void On_lnki (gplx.xowa.htmls.core.wkrs.lnkis.Xoh_lnki_data parser) {}
public void On_thm (gplx.xowa.htmls.core.wkrs.thms.Xoh_thm_data parser) {}
public void On_gly (gplx.xowa.htmls.core.wkrs.glys.Xoh_gly_grp_data parser) {}
public boolean Process_parse(Xoh_data_itm data) {return false;}
}

167
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_img_src_parser.java Normal file
View File

@@ -0,0 +1,167 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import gplx.core.brys.*;
import gplx.xowa.files.*; import gplx.xowa.files.repos.*;
import gplx.langs.htmls.*; import gplx.xowa.htmls.core.wkrs.*;
import gplx.xowa.wikis.domains.*;
public class Xosync_img_src_parser {
private final Bry_rdr rdr = new Bry_rdr().Dflt_dlm_(Byte_ascii.Slash);
private final Xof_url_bldr url_bldr = Xof_url_bldr.new_v2();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private Xoh_hdoc_ctx hctx;
private byte[] img_src_bgn_local, img_src_bgn_remote;
private byte[] page_url, repo_local;
private byte[] raw;
public boolean Repo_is_commons() {return repo_is_commons;} private boolean repo_is_commons;
public byte[] File_ttl_bry() {return file_ttl_bry;} private byte[] file_ttl_bry;
public boolean File_is_orig() {return file_is_orig;} private boolean file_is_orig;
public int File_w() {return file_w;} private int file_w;
public double File_time() {return file_time;} private double file_time;
public int File_page() {return file_page;} private int file_page;
public String Err_msg() {return err_msg;} private String err_msg;
public Xosync_img_src_parser() {
rdr.Err_wkr().Fail_throws_err_(false);
img_src_bgn_remote = tmp_bfr.Add(Bry__xowa_src_bgn).Add(Xow_domain_itm_.Bry__commons).Add_byte_slash().To_bry_and_clear();
}
public void Init_by_page(Xoh_hdoc_ctx hctx) {
this.hctx = hctx;
this.page_url = hctx.Page__url();
this.repo_local = To_wmf_repo_or_null(tmp_bfr, hctx.Wiki__domain_itm());
if (repo_local == null) Gfo_usr_dlg_.Instance.Warn_many("", "", "unsupported wmf repo; domain=~{0}", hctx.Wiki__domain_itm().Domain_bry());
img_src_bgn_local = tmp_bfr.Add(Bry__xowa_src_bgn).Add(hctx.Wiki__domain_bry()).Add_byte_slash().To_bry_and_clear(); // EX: "xowa:/file/en.wikipedia.org/"
}
public boolean Parse(byte[] raw) {
// init
this.Clear();
// set raw, raw_len; exit if empty
this.raw = raw;
int raw_len = raw.length;
if (raw_len == 0) return Fail("empty img_src");
rdr.Init_by_src(raw);
// check "//upload.wikimedia.org/" at bgn
if (!rdr.Is(Bry__upload_wikimedia_org)) return Fail("url does not start with //upload.wikimedia.org");
// get repo: either "wikipedia/commons/" or "wiki_type/wiki_lang/"; EX:"wiktionary/fr"
if (rdr.Is(Bry__repo_remote))
this.repo_is_commons = true;
else {
if (!rdr.Is(repo_local)) return Fail("unknown repo");
}
// get file_is_orig; note omitting "else" b/c default is file_is_orig == false
if (!rdr.Is(Bry__thumb)) file_is_orig = true; // no "/thumb";
// check md5
if (!Check_md5()) return Fail("invalid md5");
// get file_ttl
int file_ttl_bgn = rdr.Pos();
int file_ttl_end = rdr.Find_fwd_lr_or(Byte_ascii.Slash, raw_len);
file_ttl_bry = Bry_.Mid(raw, file_ttl_bgn, file_ttl_end);
file_ttl_bry = gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url.Decode(file_ttl_bry); // NOTE: @src is always url-encoded; file_ttl_bry is un-encoded (for MD5, database lookups, etc.)
// if thumb, get width, time, page
if (!file_is_orig) {
int file_w_bgn = rdr.Pos();
int file_w_end = rdr.Find_fwd_lr(Bry__px);
if (file_w_end == -1) return Fail("missing px");
file_w = Bry_.To_int_or(raw, file_w_bgn, file_w_end, -1);
if (file_w == -1) return Fail("invalid file_w");
}
// make image
Xof_fsdb_itm itm = new Xof_fsdb_itm();
itm.Init_by_wm_parse(hctx.Wiki__domain_itm().Abrv_xo(), repo_is_commons, file_is_orig, file_ttl_bry, file_w, file_time, file_page);
hctx.Page().Hdump_mgr().Imgs().Add(itm);
return true;
}
public byte[] To_bry() {
To_bfr(tmp_bfr);
return tmp_bfr.To_bry_and_clear();
}
public void To_bfr(Bry_bfr bfr) { // EX:'xowa:file/commons.wikimedia.org/thumb/7/0/1/c/A.png/220px.png'
// init repo; either "xowa:file/commons.wikimedia.org" or "xowa:file/en.wikipedia.org"
url_bldr.Init_by_root(repo_is_commons ? img_src_bgn_remote : img_src_bgn_local, Bool_.N, Byte_ascii.Slash, Bool_.N, Bool_.N, 4);
// set other props and generate url;
url_bldr.Init_by_itm(file_is_orig ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb, gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url.Encode(file_ttl_bry), Xof_file_wkr_.Md5(file_ttl_bry), Xof_ext_.new_by_ttl_(file_ttl_bry), file_w, file_time, file_page);
bfr.Add(url_bldr.Xto_bry());
}
private void Clear() {
this.file_ttl_bry = null;
this.repo_is_commons = false;
this.file_is_orig = false;
this.file_w = -1;
this.file_time = -1;
this.file_page = -1;
this.err_msg = null;
this.raw = null;
}
private boolean Fail(String fmt) {
this.err_msg = "wm.parse:" + fmt;
String msg = String_.Format("", err_msg + "; page={0} raw={1}", page_url, raw);
Gfo_usr_dlg_.Instance.Warn_many("", "", msg);
return false;
}
private boolean Check_md5() { // check if md5; also, skip past md5; EX: "a/a0/"
int pos = rdr.Pos();
if (!Byte_.Eq_many(Byte_ascii.Slash, raw[pos + 1], raw[pos + 4])) return false; // check slashes
byte b_0 = raw[pos + 0], b_2 = raw[pos + 2];
if (b_0 != b_2) return false; // WM repeats 1st MD5 char; EX: "a" in "a/a0"
if (!gplx.core.encoders.Hex_utl_.Is_hex_many(b_0, b_2, raw[pos + 3])) return false; // check rest is hex
rdr.Move_to(pos + 5);
return true;
}
private static final byte[]
Bry__upload_wikimedia_org = Bry_.new_a7("//upload.wikimedia.org/")
, Bry__repo_remote = Bry_.new_a7("wikipedia/commons/")
, Bry__thumb = Bry_.new_a7("thumb/")
, Bry__px = Bry_.new_a7("px")
;
public static final byte[] Bry__xowa_src_bgn = Bry_.new_a7("xowa:/file/");
private static byte[] To_wmf_repo_or_null(Bry_bfr bfr, Xow_domain_itm domain_itm) {
// add type; EX: "fr.wiktionary.org" -> "wiktionary/"
switch (domain_itm.Domain_type_id()) {
case Xow_domain_tid_.Tid__wikipedia:
case Xow_domain_tid_.Tid__wiktionary:
case Xow_domain_tid_.Tid__wikisource:
case Xow_domain_tid_.Tid__wikivoyage:
case Xow_domain_tid_.Tid__wikiquote:
case Xow_domain_tid_.Tid__wikibooks:
case Xow_domain_tid_.Tid__wikiversity:
case Xow_domain_tid_.Tid__wikinews:
bfr.Add(domain_itm.Domain_type().Key_bry()).Add_byte_slash();
break;
default:
return null;
}
// add lang; EX: "fr.wiktionary.org" -> "fr/"
bfr.Add(domain_itm.Lang_orig_key()).Add_byte_slash();
return bfr.To_bry_and_clear();
}
}

38
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xowd_html_tbl_mgr.java Normal file
View File

@@ -0,0 +1,38 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*;
import gplx.core.ios.streams.*;
import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.htmls.core.hzips.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.pages.*;
public class Xowd_html_tbl_mgr {
public Xow_db_file Get_html_db(Xow_wiki wiki) {
return wiki.Data__core_mgr().Dbs__assert_by_tid(Xow_db_file_.Tid__html_user);
}
public void Save_html(Xow_wiki wiki, Xow_db_file db, int page_id, int revn_id, byte[] src) {
Xowd_html_tbl tbl = new Xowd_html_tbl(db.Conn());
// set other html props to null; TODO_FUTURE:need to parse html to get these
byte[] display_ttl = null;
byte[] content_sub = null;
byte[] sidebar_div = null;
db.Conn().Meta_tbl_assert(tbl);
tbl.Upsert(page_id, Xopg_module_mgr.Tid_null, Io_stream_.Tid_raw, Xoh_hzip_dict_.Hzip__plain, display_ttl, content_sub, sidebar_div, src);
wiki.Data__core_mgr().Db__core().Tbl__page().Update__html_db_id(page_id, db.Id());
}
}

41
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/dbs/Xosync_sync_tbl.java Normal file
View File

@@ -0,0 +1,41 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*;
import gplx.dbs.*; import gplx.dbs.utls.*;
public class Xosync_sync_tbl implements Db_tbl {
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private final String fld_page_id, fld_sync_date;
private final Db_conn conn;
public Xosync_sync_tbl(Db_conn conn) {
this.conn = conn;
this.fld_page_id = flds.Add_int_pkey("page_id");
this.fld_sync_date = flds.Add_str("sync_date", 32);
conn.Rls_reg(this);
}
public String Tbl_name() {return tbl_name;} private final String tbl_name = "sync";
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public DateAdp Select_sync_date_or_min(int page_id) {
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_page_id).Crt_int(fld_page_id, page_id).Exec_select__rls_auto();
try {return rdr.Move_next() ? rdr.Read_date_by_str(fld_sync_date) : DateAdp_.MinValue;}
finally {rdr.Rls();}
}
public void Upsert(int page_id, DateAdp sync_date) {
Db_tbl__crud_.Upsert(conn, tbl_name, flds, String_.Ary(fld_page_id), page_id, sync_date.XtoStr_fmt_yyyyMMdd_HHmmss());
}
public void Rls() {}
}

3
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/specials/Sync_html_special.java
View File

@@ -19,6 +19,7 @@ package gplx.xowa.addons.wikis.pages.syncs.specials; import gplx.*; import gplx.
import gplx.core.net.qargs.*;
import gplx.xowa.specials.*; import gplx.xowa.wikis.nss.*;
import gplx.xowa.htmls.*;
import gplx.xowa.addons.wikis.pages.syncs.core.*;
public class Sync_html_special implements Xow_special_page {
public void Special__gen(Xow_wiki wiki, Xoa_page page, Xoa_url url, Xoa_ttl ttl) {
Gfo_qarg_mgr url_args = new Gfo_qarg_mgr().Init(url.Qargs_ary());
@@ -29,7 +30,7 @@ public class Sync_html_special implements Xow_special_page {
// Xoa_url redirect_url = wiki.Utl__url_parser().Parse(redirect_bry);
// update
gplx.xowa.apps.wms.apis.parses.Wm_page_updater updater = new gplx.xowa.apps.wms.apis.parses.Wm_page_updater();
Xosync_update_mgr updater = new Xosync_update_mgr();
updater.Init_by_app(wiki.App());
Xoh_page hpg = new Xoh_page();
updater.Init_by_page(wiki, hpg);

38
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/wmapis/Xowm_parse_data.java Normal file
View File

@@ -0,0 +1,38 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.wmapis; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*;
public class Xowm_parse_data {
public Xowm_parse_data(byte[] wiki_domain
, int page_id, byte[] page_ttl, byte[] redirect_src
, int revn_id, byte[] revn_html) {
this.wiki_domain = wiki_domain;
this.page_id = page_id;
this.page_ttl = page_ttl;
this.redirect_src = redirect_src;
this.revn_id = revn_id;
this.revn_html = revn_html;
}
public byte[] Wiki_domain() {return wiki_domain;} private final byte[] wiki_domain;
public int Page_id() {return page_id;} private final int page_id;
public byte[] Page_ttl() {return page_ttl;} private final byte[] page_ttl;
public byte[] Redirect_src() {return redirect_src;} private final byte[] redirect_src;
public int Revn_id() {return revn_id;} private final int revn_id;
public byte[] Revn_html() {return revn_html;} private final byte[] revn_html;
}

54
400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/wmapis/Xowm_parse_wmf.java Normal file
View File

@@ -0,0 +1,54 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.wikis.pages.syncs.wmapis; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*;
import gplx.langs.jsons.*;
import gplx.xowa.files.downloads.*;
import gplx.xowa.apps.wms.apis.*;
public class Xowm_parse_wmf {
private final Bry_bfr bfr = Bry_bfr_.New();
private final Json_parser json_parser = new Json_parser();
public Xowm_parse_data Get_parse_or_null(Xof_download_wkr download_wkr, Xow_wiki wiki, Xoa_ttl page_ttl) {
if (!gplx.core.ios.IoEngine_system.Web_access_enabled) return null;
byte[] wiki_domain = wiki.Domain_bry();
byte[] page_full_db = page_ttl.Full_db();
byte[] json = Download(bfr, download_wkr, wiki_domain, page_full_db);
return Parse(json_parser, wiki_domain, page_full_db, json);
}
private static byte[] Download(Bry_bfr bfr, Xof_download_wkr download_wkr, byte[] wiki_domain, byte[] page_full_db) {
// build url; EX: "https://en.wikipedia.org/w/api.php?action=parse&format=json&redirects=1&page=Wikipedia:Main%20Page"
Xowm_api_bldr.Bld_bgn(bfr, wiki_domain);
bfr.Add_str_a7("action=parse&format=json&redirects=1&page="); // NOTE:redirects=1 needed to resolve redirects
bfr.Add(page_full_db);
// download
return download_wkr.Download_xrg().Exec_as_bry(bfr.To_str_and_clear());
}
private static Xowm_parse_data Parse(Json_parser json_parser, byte[] wiki_domain, byte[] page_full_db, byte[] json) {
Json_doc jdoc = json_parser.Parse(json);
// get data
Json_nde parse_nde = jdoc.Root_nde().Get_as_nde("parse");
int page_id = parse_nde.Get_as_int("pageid");
int revn_id = parse_nde.Get_as_int("revid");
byte[] page_ttl = Xoa_ttl.Replace_spaces(parse_nde.Get_as_bry("title"));
byte[] redirect_src = Bry_.Eq(page_ttl, page_full_db) ? null : page_full_db;
byte[] revn_html = parse_nde.Get_as_nde("text").Get_as_bry("*");
return new Xowm_parse_data(wiki_domain, page_id, page_ttl, redirect_src, revn_id, revn_html);
}
}