Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.7.2.1

Browse Source
This commit is contained in:
gnosygnu
2015-07-12 21:10:02 -04:00
commit 794b5a232f
3099 changed files with 238212 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

65
400_xowa/src/gplx/xowa/files/bins/Bin_fetcher.java Normal file
View File

@@ -0,0 +1,65 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.core.primitives.*;
interface Bin_fetcher {
boolean Save_as_url(Io_url trg);
boolean Save_as_bry(Bry_obj_ref bry);
}
class Bin_fetcher_fsys implements Bin_fetcher {
public void Init_src_url(Io_url src) {this.src = src;} private Io_url src;
public boolean Save_as_url(Io_url trg) {
try {Io_mgr.I.CopyFil(src, trg, true); return true;}
catch (Exception exc) {Exc_.Noop(exc); return false;}
}
public boolean Save_as_bry(Bry_obj_ref bry_ref) {
try {
byte[] bry = Io_mgr.I.LoadFilBry(src);
bry_ref.Val_(bry);
return true;
}
catch (Exception exc) {Exc_.Noop(exc); return false;}
}
}
class Bin_fetcher_http implements Bin_fetcher {
private gplx.ios.IoEngine_xrg_downloadFil download = gplx.ios.IoEngine_xrg_downloadFil.new_("", Io_url_.Empty);
public void Init_src_str(String src) {this.src = src;} private String src;
public boolean Save_as_url(Io_url trg) {
return download.Src_(src).Trg_(trg).Exec();
}
public boolean Save_as_bry(Bry_obj_ref bry_ref) {
try {
byte[] rv = download.Exec_as_bry(src);
bry_ref.Val_(rv);
return true;
}
catch (Exception exc) {Exc_.Noop(exc); return false;}
}
}
class Bin_fetcher_fsdb {
public void Init_id(int id) {}
public boolean Save_as_url(Io_url trg) {
// IoStream s = fsdb.Get_stream(id);
// s.Save(trg);
return true;
}
public byte[] Save_to_mem() {
// return db.Get_bry(id);
return null;
}
}

192
400_xowa/src/gplx/xowa/files/bins/Xof_bin_mgr.java Normal file
View File

@@ -0,0 +1,192 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.core.primitives.*; import gplx.ios.*;
import gplx.fsdb.meta.*;
import gplx.xowa.files.repos.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.files.cnvs.*; import gplx.xowa.files.caches.*;
import gplx.xowa.wmfs.*;
public class Xof_bin_mgr {
private final Fsm_mnt_mgr mnt_mgr;
private final Gfo_usr_dlg usr_dlg; private final Xow_repo_mgr repo_mgr; private final Xof_url_bldr url_bldr = Xof_url_bldr.new_v2();
private Xof_bin_wkr[] wkrs = Xof_bin_wkr_.Ary_empty; private int wkrs_len;
private final String_obj_ref resize_warning = String_obj_ref.null_(); private final Xof_img_size tmp_size = new Xof_img_size();
private final Io_download_fmt download_fmt;
private final Io_stream_rdr_wrapper rdr_wrapper = new Io_stream_rdr_wrapper();
public Xof_bin_mgr(Fsm_mnt_mgr mnt_mgr, Xow_repo_mgr repo_mgr, Xof_img_wkr_resize_img resize_wkr, Io_download_fmt download_fmt) {
this.mnt_mgr = mnt_mgr; this.repo_mgr = repo_mgr; this.download_fmt = download_fmt;
this.usr_dlg = Gfo_usr_dlg_.I;
this.Resizer_(resize_wkr);
}
public void Resizer_(Xof_img_wkr_resize_img v) {resizer = v;} private Xof_img_wkr_resize_img resizer;
public void Wkrs__del(String key) {
List_adp list = List_adp_.new_();
for (Xof_bin_wkr wkr : wkrs) {
if (String_.Eq(key, wkr.Key())) continue;
list.Add(wkr);
}
this.wkrs = (Xof_bin_wkr[])list.To_ary(Xof_bin_wkr.class);
this.wkrs_len = wkrs.length;
}
public void Wkrs__add(Xof_bin_wkr v) {
this.wkrs = (Xof_bin_wkr[])Array_.Resize_add_one(wkrs, wkrs_len, v);
++this.wkrs_len;
}
public Xof_bin_wkr Wkrs__get_or_null(String key) {
byte tid = Xof_bin_wkr_.X_key_to_tid(key);
for (int i = 0; i < wkrs_len; ++i) {
Xof_bin_wkr wkr = wkrs[i];
if (wkr.Tid() == tid) return wkr;
}
return null;
}
public boolean Find_to_url_as_bool(int exec_tid, Xof_fsdb_itm fsdb) {return Find_as(Bool_.Y, rdr_wrapper, exec_tid, fsdb);}
// public boolean Find_to_url_as_bool3(int exec_tid, Xof_fsdb_itm fsdb) {return Find_to_url(exec_tid, fsdb) != Io_url_.Empty;}
// private Io_url Find_to_url(int exec_tid, Xof_fsdb_itm fsdb) {
// Io_stream_rdr rdr = Find_as_rdr(exec_tid, fsdb);
// if (rdr == Io_stream_rdr_.Noop) return Io_url_.Empty;
// Io_url trg = fsdb.Html_view_url();
// fsdb.File_size_(rdr.Len());
// if (fsdb.File_resized()) return trg; // rdr is opened directly from trg; return its url; occurs when url goes through imageMagick / inkscape, or when thumb is already on disk;
// Io_stream_wtr_.Save_rdr(trg, rdr, download_fmt); // rdr is stream; either from http_wmf or fsdb; save to trg and return;
// return trg;
// }
public Io_stream_rdr Find_as_rdr(int exec_tid, Xof_fsdb_itm fsdb) {
rdr_wrapper.Rdr_(Io_stream_rdr_.Noop);
Find_as(Bool_.N, rdr_wrapper, exec_tid, fsdb);
return rdr_wrapper.Rdr();
}
public Io_stream_rdr Find_as_rdr3(int exec_tid, Xof_fsdb_itm fsdb) {
Io_stream_rdr rv = Io_stream_rdr_.Noop;
Xof_repo_itm repo = repo_mgr.Repos_get_by_wiki(fsdb.Orig_repo_name()).Trg();
boolean file_is_orig = fsdb.File_is_orig();
if (file_is_orig || exec_tid == Xof_exec_tid.Tid_viewer_app) { // orig or viewer_app; note that viewer_app always return orig
Io_url trg = url_bldr.To_url_trg(repo, fsdb, Bool_.Y);
fsdb.Html_view_url_(trg);
for (int i = 0; i < wkrs_len; i++) {
Xof_bin_wkr wkr = wkrs[i];
rv = wkr.Get_as_rdr(fsdb, Bool_.N, fsdb.Html_w());
if (rv == Io_stream_rdr_.Noop) continue; // orig not found; continue;
fsdb.File_exists_y_();
return rv;
}
}
else { // thumb
Io_url trg = url_bldr.To_url_trg(repo, fsdb, Bool_.N);
fsdb.Html_view_url_(trg);
for (int i = 0; i < wkrs_len; i++) {
Xof_bin_wkr wkr = wkrs[i];
rv = wkr.Get_as_rdr(fsdb, Bool_.Y, fsdb.Html_w()); // get thumb's bin
if (rv != Io_stream_rdr_.Noop) { // thumb's bin exists;
fsdb.File_exists_y_();
return rv;
}
if (fsdb.Orig_ext().Id_is_video()) continue; // item is video; don't download orig as imageMagick can't thumbnail it; DATE:2015-06-16
rv = wkr.Get_as_rdr(fsdb, Bool_.N, fsdb.Orig_w()); // thumb missing; get orig;
if (rv == Io_stream_rdr_.Noop) {
usr_dlg.Log_direct(String_.Format("bin_mgr:thumb not found; wkr={0} ttl={1} w={2}", wkr.Key(), fsdb.Orig_ttl(), fsdb.Lnki_w()));
continue; // nothing found; continue;
}
if (!wkr.Resize_allowed()) continue;
Io_url orig = url_bldr.To_url_trg(repo, fsdb, Bool_.Y); // get orig url
Io_stream_wtr_.Save_rdr(orig, rv, download_fmt);
boolean resized = Resize(exec_tid, fsdb, file_is_orig, orig, trg);
if (!resized) continue;
fsdb.File_exists_y_();
rv = Io_stream_rdr_.file_(trg); // return stream of resized url; (result of imageMagick / inkscape)
rv.Open();
return rv;
}
}
return Io_stream_rdr_.Noop;
}
private boolean Find_as(boolean save_to_fsys, Io_stream_rdr_wrapper rdr_wrapper, int exec_tid, Xof_fsdb_itm fsdb) {
Xof_repo_itm repo = repo_mgr.Repos_get_by_wiki(fsdb.Orig_repo_name()).Trg();
boolean file_is_orig = fsdb.File_is_orig();
if (file_is_orig || exec_tid == Xof_exec_tid.Tid_viewer_app) { // orig or viewer_app; note that viewer_app always return orig
Io_url trg = url_bldr.To_url_trg(repo, fsdb, Bool_.Y);
fsdb.Html_view_url_(trg);
for (int i = 0; i < wkrs_len; i++) {
Xof_bin_wkr wkr = wkrs[i];
boolean found = Get_bin(Bool_.N, fsdb.Html_w(), trg, save_to_fsys, rdr_wrapper, fsdb, wkr);
if (found) // orig found; return it;
return Set_found(save_to_fsys, fsdb, trg, rdr_wrapper);
}
}
else { // thumb
Io_url trg = url_bldr.To_url_trg(repo, fsdb, Bool_.N);
fsdb.Html_view_url_(trg);
for (int i = 0; i < wkrs_len; i++) {
Xof_bin_wkr wkr = wkrs[i];
boolean found = Get_bin(Bool_.Y, fsdb.Html_w(), trg, save_to_fsys, rdr_wrapper, fsdb, wkr);
if (found) // thumb found; return it;
return Set_found(save_to_fsys, fsdb, trg, rdr_wrapper);
if (fsdb.Orig_ext().Id_is_video()) continue; // item is video; don't download orig as imageMagick can't thumbnail it; DATE:2015-06-16
if (!wkr.Resize_allowed()) continue; // resize code below; exit early if wkr doesn't allow resize
Io_url orig = url_bldr.To_url_trg(repo, fsdb, Bool_.Y); // get orig url
found = Get_bin(Bool_.N, fsdb.Orig_w(), orig, Bool_.Y, rdr_wrapper, fsdb, wkr); // get orig; note: save_to_fsys set to true b/c imageMagick will need actual file to convert
if (!found) {
usr_dlg.Log_direct(String_.Format("bin_mgr:thumb not found; wkr={0} ttl={1} w={2}", wkr.Key(), fsdb.Orig_ttl(), fsdb.Lnki_w()));
continue; // orig not found; skip rest since resize can't happen;
}
boolean resized = Resize(exec_tid, fsdb, file_is_orig, orig, trg);
if (!resized) continue;
if (save_to_fsys) { // noop; already saved to trg
}
else {
Io_stream_rdr rdr = Io_stream_rdr_.file_(trg); // return stream of resized url; (result of imageMagick / inkscape)
rdr.Open();
rdr_wrapper.Rdr_(rdr);
}
return Set_found(save_to_fsys, fsdb, trg, rdr_wrapper);
}
}
return false;
}
private boolean Get_bin(boolean is_thumb, int w, Io_url trg_url, boolean save_to_fsys, Io_stream_rdr_wrapper rdr_wrapper, Xof_fsdb_itm fsdb, Xof_bin_wkr wkr) {
boolean found = false;
if (save_to_fsys)
found = wkr.Get_to_fsys(fsdb, is_thumb, w, trg_url);
else {
Io_stream_rdr rdr = wkr.Get_as_rdr(fsdb, is_thumb, w);
if (rdr != Io_stream_rdr_.Noop) {
found = true;
rdr_wrapper.Rdr_(rdr);
}
}
return found;
}
private boolean Set_found(boolean save_to_fsys, Xof_fsdb_itm fsdb, Io_url fsys_url, Io_stream_rdr_wrapper rdr_wrapper) {
long fsdb_len = -1;
if (save_to_fsys)
fsdb_len = Io_mgr.I.QueryFil(fsys_url).Size();
else
fsdb_len = rdr_wrapper.Rdr().Len();
fsdb.File_size_(fsdb_len);
fsdb.File_exists_y_();
return true;
}
private boolean Resize(int exec_tid, Xof_fsdb_itm itm, boolean file_is_orig, Io_url src, Io_url trg) {
tmp_size.Html_size_calc(exec_tid, itm.Lnki_w(), itm.Lnki_h(), itm.Lnki_type(), mnt_mgr.Patch_upright(), itm.Lnki_upright(), itm.Orig_ext().Id(), itm.Orig_w(), itm.Orig_h(), Xof_img_size.Thumb_width_img);
boolean rv = resizer.Resize_exec(src, trg, tmp_size.Html_w(), tmp_size.Html_h(), itm.Orig_ext().Id(), resize_warning);
itm.File_resized_y_();
return rv;
}
}
class Io_stream_rdr_wrapper {
public Io_stream_rdr Rdr() {return rdr;} public void Rdr_(Io_stream_rdr v) {rdr = v;} private Io_stream_rdr rdr;
}

89
400_xowa/src/gplx/xowa/files/bins/Xof_bin_skip_mgr.java Normal file
View File

@@ -0,0 +1,89 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.ios.*;
import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*;
public class Xof_bin_skip_mgr {
private Xof_bin_skip_wkr[] wkrs = new Xof_bin_skip_wkr[0]; private int wkrs_len;
public Xof_bin_skip_mgr(Fsm_cfg_mgr cfg_mgr, String[] wkr_keys) {
List_adp list = List_adp_.new_();
for (int i = 0; i < wkrs_len; ++i) {
Xof_bin_skip_wkr wkr = New_wkr(cfg_mgr, wkr_keys[i]);
if (wkr != null) list.Add(wkr);
}
this.wkrs = (Xof_bin_skip_wkr[])list.To_ary_and_clear(Xof_bin_skip_wkr.class);
this.wkrs_len = wkrs.length;
}
public boolean Skip(Xof_fsdb_itm fsdb, Io_stream_rdr src_rdr) {
for (int i = 0; i < wkrs_len; ++i) {
if (wkrs[i].Skip_exec(fsdb, src_rdr)) return true;
}
return false;
}
private Xof_bin_skip_wkr New_wkr(Fsm_cfg_mgr cfg_mgr, String key) {
Xof_bin_skip_wkr rv = null;
if (String_.Eq(key, Xof_bin_skip_wkr_.Key__page_gt_1)) rv = Xof_bin_skip_wkr__page_gt_1.I;
else if (String_.Eq(key, Xof_bin_skip_wkr_.Key__small_size)) rv = Xof_bin_skip_wkr__small_size.I;
else throw Exc_.new_unhandled(key);
if (!rv.Skip_init(cfg_mgr)) return null;
return rv;
}
public void Skip_term(Fsm_cfg_mgr cfg_mgr) {
for (int i = 0; i < wkrs_len; ++i)
wkrs[i].Skip_term(cfg_mgr);
}
}
interface Xof_bin_skip_wkr {
String Key();
boolean Skip_init(Fsm_cfg_mgr cfg_mgr);
boolean Skip_exec(Xof_fsdb_itm fsdb, Io_stream_rdr src_rdr);
void Skip_term(Fsm_cfg_mgr cfg_mgr);
}
class Xof_bin_skip_wkr_ {
public static final String Key__page_gt_1 = "page_gt_1", Key__small_size = "small_size";
}
class Xof_bin_skip_wkr__page_gt_1 implements Xof_bin_skip_wkr { // prior to v2.4.3; lnkis with page > 1 was mistakenly bringing down page 1; EX: [[A.pdf|page=5]] -> page1; DATE:2015-04-21
public String Key() {return Xof_bin_skip_wkr_.Key__page_gt_1;}
public boolean Skip_init(Fsm_cfg_mgr cfg_mgr) {return !cfg_mgr.Patch__page_gt_1();}
public boolean Skip_exec(Xof_fsdb_itm fsdb, Io_stream_rdr src_rdr) {
boolean rv = fsdb.Lnki_page() > 1;
if (rv)
Xoa_app_.Usr_dlg().Note_many("", "", "src_bin_mgr:skip page gt 1: file=~{0} width=~{1} page=~{2}", fsdb.Orig_ttl(), fsdb.Html_w(), fsdb.Lnki_page());
return rv;
}
public void Skip_term(Fsm_cfg_mgr cfg_mgr) {
cfg_mgr.Patch__save(Fsm_cfg_mgr.Key_patch__page_gt_1);
}
public static final Xof_bin_skip_wkr__page_gt_1 I = new Xof_bin_skip_wkr__page_gt_1(); Xof_bin_skip_wkr__page_gt_1() {}
}
class Xof_bin_skip_wkr__small_size implements Xof_bin_skip_wkr {// downloads can randomly be broken; assume that anything with a small size is broken and redownload again; DATE:2015-04-21
public String Key() {return Xof_bin_skip_wkr_.Key__small_size;}
public boolean Skip_init(Fsm_cfg_mgr cfg_mgr) {return true;}
public boolean Skip_exec(Xof_fsdb_itm fsdb, Io_stream_rdr src_rdr) {
boolean rv =
src_rdr.Len() < 500 // file is small (< 500 bytes)
&& fsdb.Html_w() > 50 // only apply to images larger than 50 px (arbitrarily chosen); needed to ignore 1x1 images as well as icon-sized images
&& Xof_ext_.Id_is_image_wo_svg(fsdb.Orig_ext().Id()) // only consider images; needed b/c of file_w check; note:ignore svg which can be small
;
if (rv)
Xoa_app_.Usr_dlg().Note_many("", "", "src_bin_mgr:skip small file: file=~{0} width=~{1} ext=~{2} len=~{3}", fsdb.Orig_ttl(), fsdb.Lnki_w(), fsdb.Orig_ext(), src_rdr.Len());
return rv;
}
public void Skip_term(Fsm_cfg_mgr cfg_mgr) {}
public static final Xof_bin_skip_wkr__small_size I = new Xof_bin_skip_wkr__small_size(); Xof_bin_skip_wkr__small_size() {}
}

27
400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr.java Normal file
View File

@@ -0,0 +1,27 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.ios.*;
import gplx.xowa.files.fsdb.*;
public interface Xof_bin_wkr {
byte Tid();
String Key();
boolean Resize_allowed(); void Resize_allowed_(boolean v);
Io_stream_rdr Get_as_rdr (Xof_fsdb_itm itm, boolean is_thumb, int w);
boolean Get_to_fsys (Xof_fsdb_itm itm, boolean is_thumb, int w, Io_url bin_url);
}

36
400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr_.java Normal file
View File

@@ -0,0 +1,36 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
public class Xof_bin_wkr_ {
public static final Xof_bin_wkr[] Ary_empty = new Xof_bin_wkr[0];
public static final byte Tid_null = Byte_.Max_value_127, Tid_noop = 1, Tid_not_found = 2
, Tid_fsdb_xowa = 3 , Tid_http_wmf = 5
, Tid_fsys_wmf = 6 , Tid_fsys_xowa = 7
;
public static final String
Key_fsdb_wiki = "xowa.fsdb.wiki" , Key_http_wmf = "xowa.http.wmf"
, Key_fsys_wmf = "xowa.fsys.wmf" , Key_fsys_xowa = "xowa.fsys.xowa"
;
public static byte X_key_to_tid(String key) {
if (String_.Eq(key, Key_fsdb_wiki)) return Tid_fsdb_xowa;
else if (String_.Eq(key, Key_http_wmf)) return Tid_http_wmf;
else if (String_.Eq(key, Key_fsys_wmf)) return Tid_fsys_wmf;
else if (String_.Eq(key, Key_fsys_xowa)) return Tid_fsys_xowa;
else return Tid_null;
}
}

130
400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__fsdb_sql.java Normal file
View File

@@ -0,0 +1,130 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.dbs.*; import gplx.ios.*; import gplx.cache.*; import gplx.xowa.files.fsdb.*;
import gplx.fsdb.*; import gplx.fsdb.data.*; import gplx.fsdb.meta.*;
public class Xof_bin_wkr__fsdb_sql implements Xof_bin_wkr {
private final Xof_bin_wkr_ids tmp_ids = new Xof_bin_wkr_ids();
private Xof_bin_skip_mgr skip_mgr;
Xof_bin_wkr__fsdb_sql(Fsm_mnt_mgr mnt_mgr) {this.mnt_mgr = mnt_mgr;}
public byte Tid() {return Xof_bin_wkr_.Tid_fsdb_xowa;}
public String Key() {return Xof_bin_wkr_.Key_fsdb_wiki;}
public Fsm_mnt_mgr Mnt_mgr() {return mnt_mgr;} private final Fsm_mnt_mgr mnt_mgr;
public boolean Resize_allowed() {return bin_wkr_resize;} public void Resize_allowed_(boolean v) {bin_wkr_resize = v;} private boolean bin_wkr_resize = false;
public Xof_bin_skip_mgr Skip_mgr() {return skip_mgr;}
public void Skip_mgr_init(Fsm_cfg_mgr cfg_mgr, String[] wkrs) {this.skip_mgr = new Xof_bin_skip_mgr(cfg_mgr, wkrs);}
public Io_stream_rdr Get_as_rdr(Xof_fsdb_itm fsdb, boolean is_thumb, int w) {
Find_ids(fsdb, is_thumb, w);
int bin_db_id = tmp_ids.Bin_db_id(); if (bin_db_id == Fsd_bin_tbl.Bin_db_id_null) return Io_stream_rdr_.Noop;
Fsm_bin_fil bin_db = mnt_mgr.Bins__at(tmp_ids.Mnt_id(), bin_db_id);
Io_stream_rdr rdr = bin_db.Select_as_rdr(tmp_ids.Itm_id());
if (skip_mgr != null && skip_mgr.Skip(fsdb, rdr)) return Io_stream_rdr_.Noop;
return rdr;
}
public boolean Get_to_fsys(Xof_fsdb_itm itm, boolean is_thumb, int w, Io_url bin_url) {return Get_to_fsys(itm.Orig_repo_name(), itm.Orig_ttl(), itm.Orig_ext(), is_thumb, w, itm.Lnki_time(), itm.Lnki_page(), bin_url);}
private boolean Get_to_fsys(byte[] orig_repo, byte[] orig_ttl, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) {
Find_ids(orig_repo, orig_ttl, orig_ext.Id(), lnki_time, lnki_page, lnki_is_thumb, file_w);
int bin_db_id = tmp_ids.Bin_db_id(); if (bin_db_id == Fsd_bin_tbl.Bin_db_id_null) return false;
Fsm_bin_fil bin_db = mnt_mgr.Bins__at(tmp_ids.Mnt_id(), bin_db_id);
return bin_db.Select_to_url(tmp_ids.Itm_id(), file_url);
}
public Io_stream_rdr Get_to_fsys_near(Xof_fsdb_itm rv, byte[] orig_repo, byte[] orig_ttl, Xof_ext orig_ext, double lnki_time, int lnki_page) {
Fsd_thm_itm thm_itm = Fsd_thm_itm.new_();
thm_itm.Init_by_req(Int_.MaxValue, lnki_time, lnki_page);
boolean found = Select_thm_bin(Bool_.N, thm_itm, orig_repo, orig_ttl);
if (found) {
tmp_ids.Init_by_thm(found, thm_itm);
rv.Init_by_fsdb_near(Bool_.N, thm_itm.W());
}
else {
Fsd_fil_itm fil_itm = Select_fil_bin(orig_repo, orig_ttl); // find orig
if (fil_itm == Fsd_fil_itm.Null) return Io_stream_rdr_.Noop;
tmp_ids.Init_by_fil(fil_itm);
rv.Init_by_fsdb_near(Bool_.Y, rv.Orig_w());
}
Fsm_bin_fil bin_db = mnt_mgr.Bins__at(tmp_ids.Mnt_id(), tmp_ids.Bin_db_id());
return bin_db.Select_as_rdr(tmp_ids.Itm_id());
}
private void Find_ids(Xof_fsdb_itm itm, boolean is_thumb, int w) {Find_ids(itm.Orig_repo_name(), itm.Orig_ttl(), itm.Orig_ext().Id(), itm.Lnki_time(), itm.Lnki_page(), is_thumb, w);}
private void Find_ids(byte[] orig_repo, byte[] orig_ttl, int orig_ext, double lnki_time, int lnki_page, boolean is_thumb, int w) {
synchronized (tmp_ids) {
byte[] dir = orig_repo, fil = orig_ttl;
if (is_thumb) {
Fsd_thm_itm thm_itm = Fsd_thm_itm.new_();
thm_itm.Init_by_req(w, lnki_time, lnki_page);
boolean found = Select_thm_bin(Bool_.Y, thm_itm, dir, fil);
tmp_ids.Init_by_thm(found, thm_itm);
}
else {
Fsd_fil_itm fil_itm = Select_fil_bin(dir, fil);
tmp_ids.Init_by_fil(fil_itm);
}
}
}
private Fsd_fil_itm Select_fil_bin(byte[] dir, byte[] fil) {
int len = mnt_mgr.Mnts__len();
for (int i = 0; i < len; i++) {
Fsd_fil_itm rv = mnt_mgr.Mnts__get_at(i).Select_fil_or_null(dir, fil);
if ( rv != Fsd_fil_itm.Null
&& rv.Bin_db_id() != Fsd_bin_tbl.Bin_db_id_null) { // NOTE: mnt_0 can have thumb, but mnt_1 can have itm; check for itm with Db_bin_id; DATE:2013-11-16
return rv;
}
}
return Fsd_fil_itm.Null;
}
private boolean Select_thm_bin(boolean exact, Fsd_thm_itm rv, byte[] dir, byte[] fil) {
int len = mnt_mgr.Mnts__len();
for (int i = 0; i < len; i++) {
boolean exists = mnt_mgr.Mnts__get_at(i).Select_thm(exact, rv, dir, fil);
if (exists) return true;
}
return false;
}
public void Txn_bgn() {mnt_mgr.Mnts__get_insert().Txn_bgn();}
public void Txn_end() {mnt_mgr.Mnts__get_insert().Txn_end();}
public static Xof_bin_wkr__fsdb_sql new_(Fsm_mnt_mgr mnt_mgr) {return new Xof_bin_wkr__fsdb_sql(mnt_mgr);}
}
class Xof_bin_wkr_ids {
public Xof_bin_wkr_ids() {this.Clear();}
public int Mnt_id() {return mnt_id;} private int mnt_id;
public int Bin_db_id() {return bin_db_id;} private int bin_db_id;
public int Itm_id() {return itm_id;} private int itm_id;
public void Init_by_thm(boolean found, Fsd_thm_itm thm) {
if (found) {
this.mnt_id = thm.Mnt_id();
this.bin_db_id = thm.Bin_db_id();
this.itm_id = thm.Thm_id();
}
else
this.Clear();
}
public void Init_by_fil(Fsd_fil_itm fil) {
if (fil == Fsd_fil_itm.Null)
this.Clear();
else {
this.mnt_id = fil.Mnt_id();
this.bin_db_id = fil.Bin_db_id();
this.itm_id = fil.Fil_id();
}
}
private void Clear() {
this.mnt_id = -1;
this.bin_db_id = Fsd_bin_tbl.Bin_db_id_null;
this.itm_id = -1;
}
}

68
400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__fsys_base.java Normal file
View File

@@ -0,0 +1,68 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.ios.*;
import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*;
public abstract class Xof_bin_wkr__fsys_base implements Xof_bin_wkr, GfoInvkAble {
public Xof_bin_wkr__fsys_base() {}
public abstract byte Tid();
public abstract String Key();
public boolean Resize_allowed() {return resize_allowed;} public void Resize_allowed_(boolean v) {resize_allowed = v;} private boolean resize_allowed = false;
public Io_stream_rdr Get_as_rdr(Xof_fsdb_itm itm, boolean is_thumb, int w) {
Io_url src_url = this.Get_src_url(Xof_repo_itm_.Mode_by_bool(is_thumb), String_.new_u8(itm.Orig_repo_name()), itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext(), w, itm.Lnki_time(), itm.Lnki_page());
return (src_url == Io_url_.Empty) ? gplx.ios.Io_stream_rdr_.Noop : gplx.ios.Io_stream_rdr_.file_(src_url);
}
public boolean Get_to_fsys(Xof_fsdb_itm itm, boolean is_thumb, int w, Io_url bin_url) {
return Get_to_fsys(itm.Orig_repo_name(), itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext(), is_thumb, w, itm.Lnki_time(), itm.Lnki_page(), bin_url);
}
private boolean Get_to_fsys(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) {
Io_url src_url = this.Get_src_url(Xof_repo_itm_.Mode_by_bool(lnki_is_thumb), String_.new_u8(orig_repo), orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page);
if (src_url == Io_url_.Empty) return false;
byte[] bin = Io_mgr.I.LoadFilBry(src_url);
return bin != Io_mgr.LoadFilBry_fail;
}
protected abstract Io_url Get_src_url(byte mode, String wiki, byte[] ttl_wo_ns, byte[] md5, Xof_ext ext, int w, double time, int page);
public abstract void Url_(Io_url v);
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_url_)) this.Url_(m.ReadIoUrl("v"));
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_url_ = "url_";
}
abstract class Xof_bin_wkr__fsys_wmf_base extends Xof_bin_wkr__fsys_base {
public Xof_url_bldr Url_bldr() {return url_bldr;} private Xof_url_bldr url_bldr = new Xof_url_bldr();
public abstract void Init_by_root();
@Override public void Url_(Io_url v) {url_bldr.Root_(Bry_.new_u8(v.Raw()));}
@Override protected Io_url Get_src_url(byte mode, String wiki, byte[] ttl_wo_ns, byte[] md5, Xof_ext ext, int w, double time, int page) {
return this.Url_bldr().Init_by_itm(mode, ttl_wo_ns, md5, ext, w, time, page).Xto_url();
}
}
class Xof_bin_wkr__fsys_wmf extends Xof_bin_wkr__fsys_wmf_base {
@Override public byte Tid() {return Xof_bin_wkr_.Tid_fsys_wmf;}
@Override public String Key() {return Xof_bin_wkr_.Key_fsys_wmf;}
@Override public void Init_by_root() {
this.Url_bldr().Init_by_root(Bry_.Empty, Op_sys.Cur().Fsys_dir_spr_byte(), Bool_.Y, Bool_.Y, Xof_repo_itm_.Dir_depth_wmf);
}
}
class Xof_bin_wkr__fsys_xowa extends Xof_bin_wkr__fsys_wmf_base {
@Override public byte Tid() {return Xof_bin_wkr_.Tid_fsys_xowa;}
@Override public String Key() {return Xof_bin_wkr_.Key_fsys_xowa;}
@Override public void Init_by_root() {
this.Url_bldr().Init_by_root(Bry_.Empty, Op_sys.Cur().Fsys_dir_spr_byte(), Bool_.N, Bool_.N, Xof_repo_itm_.Dir_depth_xowa);
}
}

63
400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf.java Normal file
View File

@@ -0,0 +1,63 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.ios.*; import gplx.core.threads.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*;
public class Xof_bin_wkr__http_wmf implements Xof_bin_wkr {
private final Xow_repo_mgr repo_mgr; private final IoEngine_xrg_downloadFil download_wkr;
private final Xof_url_bldr url_bldr = new Xof_url_bldr();
public Xof_bin_wkr__http_wmf(Xow_repo_mgr repo_mgr, gplx.ios.IoEngine_xrg_downloadFil download_wkr) {this.repo_mgr = repo_mgr; this.download_wkr = download_wkr;}
public byte Tid() {return Xof_bin_wkr_.Tid_http_wmf;}
public String Key() {return Xof_bin_wkr_.Key_http_wmf;}
public boolean Resize_allowed() {return bin_wkr_resize;} public void Resize_allowed_(boolean v) {bin_wkr_resize = v;} private boolean bin_wkr_resize = true;
public int Fail_timeout() {return fail_timeout;} public Xof_bin_wkr__http_wmf Fail_timeout_(int v) {fail_timeout = v; return this;} private int fail_timeout = 0; // NOTE: always default to 0; manually set to 1000 for fsdb_make only; DATE:2014-06-21
public Io_stream_rdr Get_as_rdr(Xof_fsdb_itm fsdb, boolean is_thumb, int w) {
Download_init(fsdb.Orig_repo_name(), fsdb.Orig_ttl(), fsdb.Orig_ttl_md5(), fsdb.Orig_ext(), is_thumb, w, fsdb.Lnki_time(), fsdb.Lnki_page(), Io_url_.Empty);
Io_stream_rdr rdr = download_wkr.Exec_as_rdr();
boolean rv = rdr.Exists(); // NOTE: use Exists which detects for response_code 200, not content length > 0; DATE:2015-05-20
if (!rv) Handle_error();
fsdb.Fsdb_insert_y_();
return rv ? rdr : Io_stream_rdr_.Noop;
}
public boolean Get_to_fsys(Xof_fsdb_itm fsdb, boolean is_thumb, int w, Io_url bin_url) {
boolean rv = Get_to_fsys(fsdb.Orig_repo_name(), fsdb.Orig_ttl(), fsdb.Orig_ttl_md5(), fsdb.Orig_ext(), is_thumb, w, fsdb.Lnki_time(), fsdb.Lnki_page(), bin_url);
if (rv) fsdb.Fsdb_insert_y_();
return rv;
}
private boolean Get_to_fsys(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) {
Download_init(orig_repo, orig_ttl, orig_md5, orig_ext, lnki_is_thumb, file_w, lnki_time, lnki_page, file_url);
boolean rv = download_wkr.Exec();
if (!rv) Handle_error();
return rv;
}
private void Handle_error() {
if (fail_timeout > 0)
Thread_adp_.Sleep(fail_timeout); // as per WMF policy, pause 1 second for every cache miss; http://lists.wikimedia.org/pipermail/wikitech-l/2013-September/071948.html
}
private void Download_init(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) {
byte mode = lnki_is_thumb ? Xof_repo_itm_.Mode_thumb : Xof_repo_itm_.Mode_orig;
Xof_repo_pair repo_itm = repo_mgr.Repos_get_by_wiki(orig_repo);
String src = url_bldr.Init_for_src_file(mode, repo_itm.Src(), orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page).Xto_str();
download_wkr.Init(src, file_url);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_fail_timeout_)) fail_timeout = m.ReadInt("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_fail_timeout_ = "fail_timeout_";
public static Xof_bin_wkr__http_wmf new_(Xow_wiki wiki) {return new Xof_bin_wkr__http_wmf(wiki.File__repo_mgr(), wiki.App().Wmf_mgr().Download_wkr().Download_xrg());}
}