Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.8.3.1

Browse Source
This commit is contained in:
gnosygnu
2015-08-17 02:09:16 -04:00
parent 34c34f227c
commit df10db140c
421 changed files with 4867 additions and 2429 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
400_xowa/src/gplx/xowa/files/bins/Io_download_mgr.java Normal file
View File

@@ -0,0 +1,22 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.ios.*;
public interface Io_download_mgr {
Io_stream_rdr Download_as_rdr(String src);
}

31
400_xowa/src/gplx/xowa/files/bins/Io_download_mgr_.java Normal file
View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.ios.*;
public class Io_download_mgr_ {
public static Io_download_mgr new_system() {return new Io_download_mgr__system();}
public static Io_download_mgr__memory new_memory() {return new Io_download_mgr__memory();}
}
class Io_download_mgr__system implements Io_download_mgr {
private final IoEngine_xrg_downloadFil download_wkr = IoEngine_xrg_downloadFil.new_("", Io_url_.Empty);
public void Upload_data(String url, byte[] data) {throw Err_.new_unimplemented();}
public Io_stream_rdr Download_as_rdr(String url) {
download_wkr.Init(url, Io_url_.Empty);
return download_wkr.Exec_as_rdr();
}
}

28
400_xowa/src/gplx/xowa/files/bins/Io_download_mgr__memory.java Normal file
View File

@@ -0,0 +1,28 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.ios.*;
public class Io_download_mgr__memory implements Io_download_mgr {
private final Ordered_hash hash = Ordered_hash_.new_();
public void Clear() {hash.Clear();}
public void Upload_data(String url, byte[] data) {hash.Add(url, data);}
public Io_stream_rdr Download_as_rdr(String url) {
byte[] data = (byte[])hash.Get_by(url); if (data == null) return Io_stream_rdr_.Noop;
return Io_stream_rdr_.mem_(data);
}
}

41
400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf.java
View File

@@ -16,21 +16,38 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.ios.*; import gplx.core.threads.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*;
import gplx.ios.*; import gplx.core.threads.*;
import gplx.xowa.apps.*;
import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*;
import gplx.xowa.wikis.*;
public class Xof_bin_wkr__http_wmf implements Xof_bin_wkr {
private final Xow_repo_mgr repo_mgr; private final IoEngine_xrg_downloadFil download_wkr;
private final Io_download_mgr download_mgr;
private final Xof_url_bldr url_bldr = new Xof_url_bldr();
public Xof_bin_wkr__http_wmf(Xow_repo_mgr repo_mgr, gplx.ios.IoEngine_xrg_downloadFil download_wkr) {this.repo_mgr = repo_mgr; this.download_wkr = download_wkr;}
public Xof_bin_wkr__http_wmf(Xow_repo_mgr repo_mgr, Io_download_mgr download_mgr, IoEngine_xrg_downloadFil download_wkr) {
this.repo_mgr = repo_mgr; this.download_mgr = download_mgr; this.download_wkr = download_wkr;
}
public byte Tid() {return Xof_bin_wkr_.Tid_http_wmf;}
public String Key() {return Xof_bin_wkr_.Key_http_wmf;}
public boolean Resize_allowed() {return bin_wkr_resize;} public void Resize_allowed_(boolean v) {bin_wkr_resize = v;} private boolean bin_wkr_resize = true;
public int Fail_timeout() {return fail_timeout;} public Xof_bin_wkr__http_wmf Fail_timeout_(int v) {fail_timeout = v; return this;} private int fail_timeout = 0; // NOTE: always default to 0; manually set to 1000 for fsdb_make only; DATE:2014-06-21
public Io_stream_rdr Get_as_rdr(Xof_fsdb_itm fsdb, boolean is_thumb, int w) {
Download_init(fsdb.Orig_repo_name(), fsdb.Orig_ttl(), fsdb.Orig_ttl_md5(), fsdb.Orig_ext(), is_thumb, w, fsdb.Lnki_time(), fsdb.Lnki_page(), Io_url_.Empty);
Io_stream_rdr rdr = download_wkr.Exec_as_rdr();
String src = Make_src(fsdb.Orig_repo_name(), fsdb.Orig_ttl(), fsdb.Orig_ttl_md5(), fsdb.Orig_ext(), is_thumb, w, fsdb.Lnki_time(), fsdb.Lnki_page(), Io_url_.Empty);
Io_stream_rdr rdr = download_mgr.Download_as_rdr(src);
boolean rv = rdr.Exists(); // NOTE: use Exists which detects for response_code 200, not content length > 0; DATE:2015-05-20
if (!rv) Handle_error();
fsdb.Fsdb_insert_y_();
if (!rv) {
Handle_error();
if (!rv && fsdb.Orig_repo_id() == Xof_repo_itm_.Repo_local) { // image is not found in local; check commons; occurs with bldr which relies on inaccurate data in image dumps; PAGE:en.w:Apollo_13; DATE:2015-08-05
src = Make_src(Xow_domain_.Domain_bry_commons, fsdb.Orig_ttl(), fsdb.Orig_ttl_md5(), fsdb.Orig_ext(), is_thumb, w, fsdb.Lnki_time(), fsdb.Lnki_page(), Io_url_.Empty);
rdr = download_mgr.Download_as_rdr(src);
rv = rdr.Exists();
if (rv)
fsdb.Change_repo(Xof_repo_itm_.Repo_remote, Xow_domain_.Domain_bry_commons); // set commons.wikimedia.org; DATE:2015-08-05
else
Handle_error();
}
}
if (rv) fsdb.Fsdb_insert_y_();
return rv ? rdr : Io_stream_rdr_.Noop;
}
public boolean Get_to_fsys(Xof_fsdb_itm fsdb, boolean is_thumb, int w, Io_url bin_url) {
@@ -39,7 +56,7 @@ public class Xof_bin_wkr__http_wmf implements Xof_bin_wkr {
return rv;
}
private boolean Get_to_fsys(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) {
Download_init(orig_repo, orig_ttl, orig_md5, orig_ext, lnki_is_thumb, file_w, lnki_time, lnki_page, file_url);
Init_download(orig_repo, orig_ttl, orig_md5, orig_ext, lnki_is_thumb, file_w, lnki_time, lnki_page, file_url);
boolean rv = download_wkr.Exec();
if (!rv) Handle_error();
return rv;
@@ -48,16 +65,22 @@ public class Xof_bin_wkr__http_wmf implements Xof_bin_wkr {
if (fail_timeout > 0)
Thread_adp_.Sleep(fail_timeout); // as per WMF policy, pause 1 second for every cache miss; http://lists.wikimedia.org/pipermail/wikitech-l/2013-September/071948.html
}
private void Download_init(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) {
private void Init_download(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) {
byte mode = lnki_is_thumb ? Xof_repo_itm_.Mode_thumb : Xof_repo_itm_.Mode_orig;
Xof_repo_pair repo_itm = repo_mgr.Repos_get_by_wiki(orig_repo);
String src = url_bldr.Init_for_src_file(mode, repo_itm.Src(), orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page).Xto_str();
download_wkr.Init(src, file_url);
}
private String Make_src(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) {
byte mode = lnki_is_thumb ? Xof_repo_itm_.Mode_thumb : Xof_repo_itm_.Mode_orig;
Xof_repo_pair repo_itm = repo_mgr.Repos_get_by_wiki(orig_repo);
return url_bldr.Init_for_src_file(mode, repo_itm.Src(), orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page).Xto_str();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_fail_timeout_)) fail_timeout = m.ReadInt("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_fail_timeout_ = "fail_timeout_";
public static Xof_bin_wkr__http_wmf new_(Xow_wiki wiki) {return new Xof_bin_wkr__http_wmf(wiki.File__repo_mgr(), wiki.App().Wmf_mgr().Download_wkr().Download_xrg());}
public static Xof_bin_wkr__http_wmf new_(Xow_wiki wiki) {return new_(wiki, Io_download_mgr_.new_system());}
public static Xof_bin_wkr__http_wmf new_(Xow_wiki wiki, Io_download_mgr download_mgr) {return new Xof_bin_wkr__http_wmf(wiki.File__repo_mgr(), download_mgr, wiki.App().Wmf_mgr().Download_wkr().Download_xrg());}
}

67
400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf__tst.java Normal file
View File

@@ -0,0 +1,67 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import org.junit.*; import gplx.ios.*; import gplx.xowa.files.repos.*;
public class Xof_bin_wkr__http_wmf__tst {
private final Xof_bin_wkr__http_wmf__fxt fxt = new Xof_bin_wkr__http_wmf__fxt();
@Before public void init() {fxt.Clear();}
@Test public void Basic() {
fxt.Init__Http("mem/http/commons.wikimedia.org/thumb/7/70/A.png/220px-A.png", "test_data");
fxt.Exec__Get_as_rdr(fxt.Fsdb_itm_mkr().Lnki__en_w("A.png").Orig__commons__lnki(), Bool_.Y, 220);
fxt.Test__Get_as_rdr__rdr("test_data");
}
@Test public void Enwiki_fails__fallback_to_commons() {
fxt.Init__Http("mem/http/commons.wikimedia.org/thumb/7/70/A.png/220px-A.png", "test_data"); // put file in commons
Xof_fsdb_itm fsdb_itm = fxt.Fsdb_itm_mkr().Lnki__en_w("A.png").Orig__enwiki__lnki().Make();
fxt.Exec__Get_as_rdr(fsdb_itm, Bool_.Y, 220); // look in enwiki
fxt.Test__Get_as_rdr__rdr("test_data"); // test that enwiki tries commons again
Tfds.Eq_str("commons.wikimedia.org", fsdb_itm.Orig_repo_name(), "repo_name"); // test that it's now commons
Tfds.Eq_byte(Xof_repo_itm_.Repo_remote, fsdb_itm.Orig_repo_id(), "repo_tid"); // test that it's now commons
}
@Test public void Long_filename_becomes_thumbnail() {
String filename = String_.Repeat("A", 200) + ".png";
fxt.Init__Http("mem/http/commons.wikimedia.org/thumb/1/14/" + filename + "/220px-thumbnail.png", "test_data"); // add file as "thumbnail.png"
Xof_fsdb_itm fsdb_itm = fxt.Fsdb_itm_mkr().Lnki__en_w(filename).Orig__enwiki__lnki().Make();
fxt.Exec__Get_as_rdr(fsdb_itm, Bool_.Y, 220); // look in enwiki
fxt.Test__Get_as_rdr__rdr("test_data"); // test that file is there
}
}
class Xof_bin_wkr__http_wmf__fxt {
private final Xof_bin_wkr__http_wmf wkr;
private final Io_download_mgr__memory download_mgr;
private Io_stream_rdr get_as_rdr__rdr;
public Xof_fsdb_itm_fxt Fsdb_itm_mkr() {return fsdb_itm_mkr;} private final Xof_fsdb_itm_fxt fsdb_itm_mkr = new Xof_fsdb_itm_fxt();
public Xof_bin_wkr__http_wmf__fxt() {
Xoae_app app = Xoa_app_fxt.app_();
Xowe_wiki wiki = Xoa_app_fxt.wiki_tst_(app);
Xoa_app_fxt.repo2_(app, wiki);
this.download_mgr = Io_download_mgr_.new_memory();
this.wkr = Xof_bin_wkr__http_wmf.new_(wiki, download_mgr);
}
public void Clear() {
download_mgr.Clear();
}
public void Init__Http(String url, String data) {download_mgr.Upload_data(url, Bry_.new_u8(data));}
public void Exec__Get_as_rdr(Xof_fsdb_itm_fxt fsdb_itm_mkr, boolean is_thumb, int w) {Exec__Get_as_rdr(fsdb_itm_mkr.Make(), is_thumb, w);}
public void Exec__Get_as_rdr(Xof_fsdb_itm fsdb_itm , boolean is_thumb, int w) {
this.get_as_rdr__rdr = wkr.Get_as_rdr(fsdb_itm, is_thumb, w);
}
public void Test__Get_as_rdr__rdr(String expd) {
Tfds.Eq_str(expd, Io_stream_rdr_.Load_all_as_str(get_as_rdr__rdr), "rdr_contents");
}
}