1
0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00

'v3.8.2.1'

This commit is contained in:
gnosygnu
2016-08-07 21:36:50 -04:00
parent b0fdf78a41
commit e4a2af026b
165 changed files with 2534 additions and 1247 deletions

View File

@@ -32,6 +32,7 @@ public class Xob_db_file {
public static Xob_db_file New__temp_log(Io_url dir) {return New(dir, Name__temp_log);}
public static Xob_db_file New__redlink(Io_url dir) {return New(dir, Name__redlink);}
public static Xob_db_file New__page_link(Xow_wiki wiki) {return New(wiki.Fsys_mgr().Root_dir(), Name__page_link);}
public static Xob_db_file New__img_link(Xow_wiki wiki) {return New(wiki.Fsys_mgr().Root_dir(), "xowa.wiki.imglinks.sqlite3");}
public static Xob_db_file New__page_file_map(Xow_wiki wiki) {return New(wiki.Fsys_mgr().Root_dir(), wiki.Domain_str() + "-file-page_map.xowa");}
public static Xob_db_file New__deletion_db(Xow_wiki wiki) {
String name = String_.Format("{0}-file-core-deletion_db-{1}.xowa", wiki.Domain_str(), Datetime_now.Get().XtoStr_fmt("yyyy.MM"));

View File

@@ -46,7 +46,7 @@ public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xo
root = ctx.Tkn_mkr().Root(Bry_.Empty);
wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql
wiki.Db_mgr_as_sql().Core_data_mgr().Init_by_load(gplx.xowa.wikis.data.Xow_db_file__core_.Find_core_fil(wiki)); // NOTE: must reinit providers as previous steps may have rls'd (and left member variable conn which is closed)
wiki.File__orig_mgr().Wkrs_del(Xof_orig_wkr_.Tid_wmf_api);
wiki.File__orig_mgr().Wkrs__del(Xof_orig_wkr_.Tid_wmf_api);
db_fsys_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
db_ary = Xob_dump_mgr_base_.Init_text_files_ary(db_fsys_mgr);
poll_interval = poll_mgr.Poll_interval();

View File

@@ -19,7 +19,7 @@ package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx
import gplx.core.ios.*; import gplx.core.ios.streams.*;
import gplx.xowa.bldrs.wms.dumps.*;
public class Xoi_dump_mgr implements Gfo_invk {
public String[] Server_urls() {return server_urls;} private String[] server_urls = String_.Ary(Xowm_dump_file_.Server_wmf_https, Xowm_dump_file_.Server_your_org, Xowm_dump_file_.Server_c3sl, Xowm_dump_file_.Server_masaryk);
public String[] Server_urls() {return server_urls;} private String[] server_urls = String_.Ary(Xowm_dump_file_.Server_your_org, Xowm_dump_file_.Server_wmf_https, Xowm_dump_file_.Server_c3sl, Xowm_dump_file_.Server_masaryk); // promote your.org to primary url; DATE:2016-08-07
public String[] Custom_cmds() {return custom_cmds;} private String[] custom_cmds = String_.Ary(Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_import.KEY);
public byte Data_storage_format() {return data_storage_format;} public Xoi_dump_mgr Data_storage_format_(byte v) {data_storage_format = v; return this;} private byte data_storage_format = gplx.core.ios.streams.Io_stream_.Tid_gzip;
public long Db_text_max() {return db_text_max;} private long db_text_max = (long)3000 * Io_mgr.Len_mb;

View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.wms; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.xowa.bldrs.wms.apis.*; import gplx.xowa.wikis.*;
import gplx.xowa.apps.wms.apis.*; import gplx.xowa.wikis.*;
public class Xoa_wmf_mgr implements Gfo_invk {
private final Xoae_wiki_mgr wiki_mgr;
public Xoa_wmf_mgr(Gfo_usr_dlg usr_dlg, Xoae_wiki_mgr wiki_mgr) {

View File

@@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.wms; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.net.*;
import gplx.xowa.bldrs.wms.apis.*; import gplx.xowa.files.downloads.*;
import gplx.xowa.apps.wms.apis.*; import gplx.xowa.files.downloads.*;
public class Xowmf_mgr {
public Xowmf_mgr() {
download_wkr.Download_xrg().User_agent_(Xoa_app_.User_agent);
@@ -26,5 +26,5 @@ public class Xowmf_mgr {
download_wkr.Download_xrg().Prog_dlg_(Xoa_app_.Usr_dlg());
}
public Xowmf_api_mgr Api_mgr() {return api_mgr;} private Xowmf_api_mgr api_mgr = new Xowmf_api_mgr();
public Xof_download_wkr Download_wkr() {return download_wkr;} private final Xof_download_wkr download_wkr = new Xof_download_wkr_io();
public Xof_download_wkr Download_wkr() {return download_wkr;} private final Xof_download_wkr download_wkr = new Xof_download_wkr_io();
}

View File

@@ -1,34 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.wms.apis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wms.*;
import gplx.core.ios.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.downloads.*;
public abstract class Xoapi_orig_base {
public boolean Api_query_size(Xoapi_orig_rslts rv, Xof_download_wkr download_wkr, Xow_repo_mgr repo_mgr, byte[] ttl, int width, int height) {
if (!gplx.core.ios.IoEngine_system.Web_access_enabled) return false; // don't check api if download disabled else "download_failed" messages in log (particularly during pkg_make) DATE:2015-02-12
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Instance;
Xof_repo_pair[] repos = repo_mgr.Repos_ary();
int len = repos.length;
for (int i = 0; i < len; i++) {
Xof_repo_pair repo_pair = repos[i];
if (Api_query_size_exec(rv, download_wkr, ttl, width, height, usr_dlg, repo_pair.Wiki_domain())) return true;
}
usr_dlg.Warn_many(Xoapi_orig_wmf.GRP_KEY, "download_failed", "download_failed: ~{0}", String_.new_u8(ttl));
return false;
}
public abstract boolean Api_query_size_exec(Xoapi_orig_rslts rv, Xof_download_wkr download_wkr, byte[] ttl, int width, int height, Gfo_usr_dlg usr_dlg, byte[] repo_wiki_key);
}

View File

@@ -1,61 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.wms.apis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wms.*;
import org.junit.*;
public class Xoapi_orig_base_tst {
Xoapi_orig_base_fxt fxt = new Xoapi_orig_base_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Bld_api_url() {
fxt.Bld_api_url_tst("A.png" , 220, 110, "https://en.wikipedia.org/w/api.php?action=query&format=xml&prop=imageinfo&iiprop=size|url&redirects&titles=File:A.png&iiurlwidth=220&iiurlheight=110");
fxt.Bld_api_url_tst("A.png" , 220, 0, "https://en.wikipedia.org/w/api.php?action=query&format=xml&prop=imageinfo&iiprop=size|url&redirects&titles=File:A.png&iiurlwidth=220");
fxt.Bld_api_url_tst("A.png" , 0, 110, "https://en.wikipedia.org/w/api.php?action=query&format=xml&prop=imageinfo&iiprop=size|url&redirects&titles=File:A.png"); // assert that null width does not write height
fxt.Bld_api_url_tst("A b.png" , 220, 0, "https://en.wikipedia.org/w/api.php?action=query&format=xml&prop=imageinfo&iiprop=size|url&redirects&titles=File:A_b.png&iiurlwidth=220");
fxt.Bld_api_url_tst("A&b.png" , 220, 0, "https://en.wikipedia.org/w/api.php?action=query&format=xml&prop=imageinfo&iiprop=size|url&redirects&titles=File:A%26b.png&iiurlwidth=220");
}
@Test public void Parse_size() {
String raw = "<api><query><pages><page ns=\"6\" title=\"File:A.png\" missing=\"\" imagerepository=\"shared\"><imageinfo><ii size=\"1234\" width=\"220\" height=\"110\" /></imageinfo></page></pages></query></api>";
fxt.Parse_size_tst(raw, 220, 110);
}
@Test public void Parse_reg() {
String raw = "<api><query><pages><page ns=\"6\" title=\"File:A.png\" missing=\"\" imagerepository=\"shared\"><imageinfo><ii descriptionurl=\"http://commons.wikimedia.org/wiki/File:Berkheyde-Haarlem.png\" /></imageinfo></page></pages></query></api>";
fxt.Parse_reg_tst(raw, "commons.wikimedia.org", "Berkheyde-Haarlem.png");
}
}
class Xoapi_orig_base_fxt {
private Xoae_app app; private Xowe_wiki wiki; private Xoapi_orig_rslts rv = new Xoapi_orig_rslts();
public void Clear() {
this.app = Xoa_app_fxt.Make__app__edit();
this.wiki = Xoa_app_fxt.Make__wiki__edit(app);
}
public void Bld_api_url_tst(String ttl_str, int w, int h, String expd) {
String actl = Xoapi_orig_wmf.Bld_api_url(wiki.Domain_bry(), Bry_.new_u8(ttl_str), w, h);
Tfds.Eq(expd, actl);
}
public void Parse_size_tst(String xml_str, int expd_w, int expd_h) {
byte[] xml_bry = Bry_.new_u8(xml_str);
Xoapi_orig_wmf.Parse_xml(rv, app.Usr_dlg(), xml_bry);
Tfds.Eq(expd_w, rv.Orig_w());
Tfds.Eq(expd_h, rv.Orig_h());
}
public void Parse_reg_tst(String xml_str, String expd_wiki, String expd_page) {
byte[] xml_bry = Bry_.new_u8(xml_str);
Xoapi_orig_wmf.Parse_xml(rv, app.Usr_dlg(), xml_bry);
Tfds.Eq(expd_wiki, String_.new_u8(rv.Orig_wiki()));
Tfds.Eq(expd_page, String_.new_u8(rv.Orig_page()));
}
}

View File

@@ -1,36 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.wms.apis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wms.*;
import gplx.xowa.files.downloads.*;
public class Xoapi_orig_mok extends Xoapi_orig_base {
private String wiki_str = "", ttl_str = "", redirect_str = ""; private int orig_w, orig_h; private boolean fail = false;
public Xoapi_orig_mok Ini(String wiki_str, String ttl_str, String redirect_str, int orig_w, int orig_h, boolean pass) {
this.wiki_str = wiki_str; this.ttl_str = ttl_str; this.redirect_str = redirect_str; this.orig_w = orig_w; this.orig_h = orig_h; this.fail = !pass;
return this;
}
public void Clear() {wiki_str = ttl_str = redirect_str = ""; orig_w = orig_h = 0;}
@Override public boolean Api_query_size_exec(Xoapi_orig_rslts rv, Xof_download_wkr download_wkr, byte[] ttl, int width, int height, Gfo_usr_dlg usr_dlg, byte[] repo_wiki_key) {
if (!Bry_.Eq(ttl, Bry_.new_u8(ttl_str))) return false;
if (!String_.Eq(wiki_str, String_.new_a7(repo_wiki_key))) return false;
if (fail) return false;
byte[] orig_page = String_.Eq(redirect_str, "") ? ttl : Bry_.new_u8(redirect_str);
rv.Init_all(repo_wiki_key, orig_page, orig_w, orig_h);
return true;
}
public static final Xoapi_orig_mok Instance = new Xoapi_orig_mok(); Xoapi_orig_mok() {}
}

View File

@@ -1,31 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.wms.apis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wms.*;
public class Xoapi_orig_rslts {
public byte[] Orig_wiki() {return orig_wiki;} private byte[] orig_wiki;
public byte[] Orig_page() {return orig_page;} private byte[] orig_page;
public int Orig_w() {return orig_w;} private int orig_w;
public int Orig_h() {return orig_h;} private int orig_h;
public void Init_all(byte[] wiki, byte[] page, int w, int h) {
this.orig_wiki = wiki; this.orig_page = page; this.orig_w = w; this.orig_h = h;
}
public void Clear() {
orig_wiki = orig_page = null;
orig_w = orig_h = 0;
}
}

View File

@@ -1,96 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.wms.apis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wms.*;
import gplx.core.primitives.*; import gplx.core.net.*; import gplx.core.envs.*;
import gplx.langs.htmls.encoders.*;
import gplx.xowa.files.downloads.*;
import gplx.xowa.htmls.hrefs.*;
public class Xoapi_orig_wmf extends Xoapi_orig_base {
@Override public boolean Api_query_size_exec(Xoapi_orig_rslts rv, Xof_download_wkr download_wkr, byte[] ttl, int width, int height, Gfo_usr_dlg usr_dlg, byte[] repo_wiki_key) {
if (Env_.Mode_testing()) return false; // TEST: disable during tests else scrib_lib_title will try to call WMF API; DATE:2015-03-20
String src = Bld_api_url(repo_wiki_key, ttl, width, height);
// xrg.Prog_fmt_hdr_(); // TOMBSTONE: do not uncomment; api will reuse whatever's in place
byte[] xml = download_wkr.Download_xrg().Exec_as_bry(src);
return xml == null ? false : Parse_xml(rv, usr_dlg, xml);
}
public static boolean Parse_xml(Xoapi_orig_rslts rv, Gfo_usr_dlg usr_dlg, byte[] xml) {
synchronized (tmp_rng) {
rv.Clear();
int xml_len = xml.length;
int pos = 0;
pos = Bry_find_.Find_fwd(xml, Bry_xml_ii , pos, xml_len);
if (pos == Bry_find_.Not_found) {usr_dlg.Log_many(GRP_KEY, "api_failed", "api failed: ~{0}", String_.new_u8(xml)); return false;}
pos += Bry_xml_ii.length;
byte[] orig_wiki = null, orig_page = null; int orig_w = 0, orig_h = 0;
if (Parse_xml_val(tmp_rng, usr_dlg, xml, xml_len, pos, Bry_xml_width))
orig_w = Bry_.To_int_or(xml, tmp_rng.Val_0(), tmp_rng.Val_1(), 0);
if (Parse_xml_val(tmp_rng, usr_dlg, xml, xml_len, pos, Bry_xml_height))
orig_h = Bry_.To_int_or(xml, tmp_rng.Val_0(), tmp_rng.Val_1(), 0);
if (Parse_xml_val(tmp_rng, usr_dlg, xml, xml_len, pos, Bry_xml_descriptionurl)) {
byte[] file_url = Bry_.Mid(xml, tmp_rng.Val_0(), tmp_rng.Val_1());
orig_wiki = gplx.xowa.wikis.xwikis.Xow_xwiki_mgr.Get_domain_from_url(url_parser, url, file_url);
byte[] page = Xoa_ttl.Replace_spaces(url.Segs__get_at_nth());
int colon_pos = Bry_find_.Find_fwd(page, Byte_ascii.Colon, 0, page.length);
if (colon_pos != Bry_find_.Not_found)
page = Bry_.Mid(page, colon_pos + 1, page.length);
orig_page = page;
}
rv.Init_all(orig_wiki, orig_page, orig_w, orig_h);
return true;
}
}
private static Int_2_ref tmp_rng = new Int_2_ref();
private static Gfo_url_parser url_parser = new Gfo_url_parser(); private static Gfo_url url = new Gfo_url();
private static boolean Parse_xml_val(Int_2_ref rv, Gfo_usr_dlg usr_dlg, byte[] xml, int xml_len, int pos, byte[] key) {
int bgn = 0, end = 0;
bgn = Bry_find_.Find_fwd(xml, key, pos, xml_len); if (bgn == Bry_find_.Not_found) return false;
bgn += key.length;
end = Bry_find_.Find_fwd(xml, Byte_ascii.Quote , bgn, xml_len); if (end == Bry_find_.Not_found) return false;
rv.Val_all_(bgn, end);
return true;
}
public static String Bld_api_url(byte[] wiki_key, byte[] ttl, int width, int height) {
synchronized (tmp_bfr) {
tmp_bfr.Add(Xoh_href_.Bry__https) // "https://"
.Add(wiki_key) // "commons.wikimedia.org"
.Add(Bry_api) // "/w/api.php?action=query&format=xml&prop=imageinfo&iiprop=size|url&titles=File:"
.Add(tmp_encoder.Encode(ttl)) // "A%20B.png"
;
if (width > 0)
tmp_bfr.Add(Bry_width).Add_int_variable(width); // "&iiurlwidth=800"
if (height > 0 && width > 0) // NOTE: height cannot be used alone; width must also exist; "iiurlheight cannot be used without iiurlwidth"
tmp_bfr.Add(Bry_height).Add_int_variable(height); // "&iiurlheight=600"
return tmp_bfr.To_str_and_clear();
}
}
private static Gfo_url_encoder tmp_encoder = Gfo_url_encoder_.New__http_url().Init__diff__one(Byte_ascii.Space, Byte_ascii.Underline).Make();
private static final Bry_bfr tmp_bfr = Bry_bfr_.New();
private static final byte[]
Bry_api = Bry_.new_a7("/w/api.php?action=query&format=xml&prop=imageinfo&iiprop=size|url&redirects&titles=File:") // NOTE: using File b/c it should be canonical
, Bry_width = Bry_.new_a7("&iiurlwidth=")
, Bry_height = Bry_.new_a7("&iiurlheight=")
, Bry_xml_ii = Bry_.new_a7("<ii ")
, Bry_xml_width = Bry_.new_a7("width=\"")
, Bry_xml_height = Bry_.new_a7("height=\"")
, Bry_xml_descriptionurl = Bry_.new_a7("descriptionurl=\"")
;
public static final String GRP_KEY = "xowa.file.wmf.api";
}

View File

@@ -1,21 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.wms.apis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wms.*;
public class Xowmf_api_mgr {
public Xoapi_orig_base Api_orig() {return api_orig;} public void Api_orig_(Xoapi_orig_base v) {api_orig = v;} private Xoapi_orig_base api_orig = new Xoapi_orig_wmf();
}