Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.7.2.1

Browse Source
This commit is contained in:
gnosygnu
2015-07-12 21:10:02 -04:00
commit 794b5a232f
3099 changed files with 238212 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

120
400_xowa/src/gplx/xowa/files/origs/Xob_orig_tbl_bldr.java Normal file
View File

@@ -0,0 +1,120 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.bldrs.*; import gplx.fsdb.*;
public class Xob_orig_tbl_bldr extends Xob_itm_basic_base implements Xob_cmd {
private Db_conn conn; private boolean schema_1;
public Xob_orig_tbl_bldr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_file_orig_reg;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
Xof_fsdb_mode fsdb_mode = wiki.File__fsdb_mode();
fsdb_mode.Tid_v2_bld_y_();
wiki.Init_assert();
Fsdb_db_mgr db_core_mgr = Fsdb_db_mgr_.new_detect(wiki, wiki.Fsys_mgr().Root_dir(), wiki.Fsys_mgr().File_dir());
this.schema_1 = db_core_mgr.File__schema_is_1();
conn = db_core_mgr.File__orig_tbl_ary()[gplx.fsdb.meta.Fsm_mnt_mgr.Mnt_idx_main].Conn();
Io_url make_db_url = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Url();
Sqlite_engine_.Db_attach(conn, "make_db", make_db_url.Raw());
}
public void Cmd_run() {Exec();}
public void Cmd_end() {}
public void Cmd_term() {}
private void Exec() {
String tbl_name = "orig_reg", fld_status = "orig_status";
if (schema_1) {
tbl_name = "wiki_orig";
fld_status = "status";
}
conn.Exec_sql_plog_txn("orig_wkr.deleting orig_reg" , String_.Format(Sql_delete_wiki_orig, tbl_name)); // always delete orig_reg, else will not pick up changed sizes / moved repos; DATE:2014-07-21
conn.Exec_sql_plog_txn("orig_wkr.inserting xfer direct" , String_.Format(Sql_create_xfer_direct, tbl_name, fld_status));
conn.Exec_sql_plog_txn("orig_wkr.inserting xfer redirect" , String_.Format(Sql_create_xfer_redirect, tbl_name, fld_status));
conn.Exec_sql_plog_txn("orig_wkr.inserting orig direct" , String_.Format(Sql_create_orig_direct, tbl_name, fld_status));
conn.Exec_sql_plog_txn("orig_wkr.inserting orig redirect" , String_.Format(Sql_create_orig_redirect, tbl_name, fld_status));
}
private static final String
Sql_delete_wiki_orig = "DELETE FROM {0};"
, Sql_create_xfer_direct = String_.Concat_lines_nl
( "INSERT INTO {0} "
, "(orig_ttl, {1}, orig_repo, orig_ext, orig_w, orig_h, orig_redirect)"
, "SELECT DISTINCT"
, " xfer.lnki_ttl"
, ", 1 --pass"
, ", xfer.orig_repo"
, ", xfer.lnki_ext"
, ", xfer.orig_w"
, ", xfer.orig_h"
, ", ''"
, "FROM make_db.xfer_regy xfer"
, " LEFT JOIN {0} cur ON xfer.lnki_ttl = cur.orig_ttl"
, "WHERE cur.orig_ttl IS NULL"
)
, Sql_create_xfer_redirect = String_.Concat_lines_nl
( "INSERT INTO {0} "
, "(orig_ttl, {1}, orig_repo, orig_ext, orig_w, orig_h, orig_redirect)"
, "SELECT DISTINCT"
, " xfer.orig_redirect_src"
, ", 1 --pass"
, ", xfer.orig_repo"
, ", xfer.lnki_ext"
, ", xfer.orig_w"
, ", xfer.orig_h"
, ", xfer.lnki_ttl"
, "FROM make_db.xfer_regy xfer"
, " LEFT JOIN {0} cur ON xfer.orig_redirect_src = cur.orig_ttl"
, "WHERE cur.orig_ttl IS NULL"
, "AND Coalesce(xfer.orig_redirect_src, '') != ''"
)
, Sql_create_orig_direct = String_.Concat_lines_nl
( "INSERT INTO {0} "
, "(orig_ttl, {1}, orig_repo, orig_ext, orig_w, orig_h, orig_redirect)"
, "SELECT DISTINCT"
, " orig.lnki_ttl"
, ", 0 --unknown"
, ", orig.orig_repo"
, ", orig.lnki_ext"
, ", orig.orig_w"
, ", orig.orig_h"
, ", ''"
, "FROM make_db.orig_regy orig"
, " LEFT JOIN {0} cur ON orig.lnki_ttl = cur.orig_ttl"
, "WHERE cur.orig_ttl IS NULL" // not already in orig_reg
, "AND orig.orig_repo IS NOT NULL" // not found in oimg_image.sqlite3
, "AND Coalesce(orig.orig_w , -1) != -1" // ignore entries that are either ext_id = 0 ("File:1") or don't have any width / height info (makes it useless); need to try to get again from wmf_api
, "AND Coalesce(orig.orig_redirect_ttl, '') == ''" // direct
)
, Sql_create_orig_redirect = String_.Concat_lines_nl
( "INSERT INTO {0} "
, "(orig_ttl, {1}, orig_repo, orig_ext, orig_w, orig_h, orig_redirect)"
, "SELECT DISTINCT"
, " orig.orig_redirect_ttl"
, ", 0 --unknown"
, ", orig.orig_repo"
, ", orig.lnki_ext"
, ", orig.orig_w"
, ", orig.orig_h"
, ", ''"
, "FROM make_db.orig_regy orig"
, " LEFT JOIN {0} cur ON orig.orig_redirect_ttl = cur.orig_ttl"
, "WHERE cur.orig_ttl IS NULL" // not already in orig_reg
, "AND orig.orig_repo IS NOT NULL" // not found in oimg_image.sqlite3
, "AND Coalesce(orig.orig_w, -1) != -1" // ignore entries that are either ext_id = 0 ("File:1") or don't have any width / height info (makes it useless); need to try to get again from wmf_api
, "AND Coalesce(orig.orig_redirect_ttl, '') != ''" // redirect
)
;
}

56
400_xowa/src/gplx/xowa/files/origs/Xof_orig_itm.java Normal file
View File

@@ -0,0 +1,56 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.wikis.*; import gplx.xowa.files.*;
public class Xof_orig_itm {
public byte Repo() {return repo;} private byte repo;
public byte[] Ttl() {return ttl;} private byte[] ttl; // without file ns; EX: "A.png" not "File:A.png"
public int Ext_id() {return ext_id;} private int ext_id;
public Xof_ext Ext() {if (ext == null) ext = Xof_ext_.new_by_id_(ext_id); return ext;} private Xof_ext ext;
public int W() {return w;} private int w;
public int H() {return h;} private int h;
public byte[] Redirect() {return redirect;} private byte[] redirect; // redirect trg; EX: A.png is redirected to B.jpg; record will have A.png|jpg|220|200|B.jpg where jpg|220|200 are the attributes of B.jpg
public boolean Insert_new() {return insert_new;} public void Insert_new_y_() {insert_new = Bool_.Y;} private boolean insert_new;
public void Clear() {
this.repo = Repo_null;
this.ttl = this.redirect = null;
this.ext_id = Xof_ext_.Id_unknown;
this.w = this.h = Xof_img_size.Null;
this.ext = null;
}
public Xof_orig_itm Init(byte repo, byte[] ttl, int ext_id, int w, int h, byte[] redirect) {
this.repo = repo; this.ttl = ttl; this.ext_id = ext_id;
this.w = w; this.h = h; this.redirect = redirect;
return this;
}
public static final byte Repo_comm = 0, Repo_wiki = 1, Repo_null = Byte_.Max_value_127; // SERIALIZED: "wiki_orig.orig_repo"
public static final Xof_orig_itm Null = null;
public static final int File_len_null = -1; // file_len used for filters (EX: don't download ogg > 1 MB)
public static String dump(Xof_orig_itm itm) {
if (itm == null)
return "NULL";
Bry_bfr bfr = Bry_bfr.new_(255);
bfr.Add_str_a7("repo").Add_byte_eq().Add_int_variable((int)itm.repo).Add_byte_semic();
bfr.Add_str_a7("ttl").Add_byte_eq().Add(itm.ttl).Add_byte_semic();
bfr.Add_str_a7("ext_id").Add_byte_eq().Add_int_variable(itm.ext_id).Add_byte_semic();
bfr.Add_str_a7("w").Add_byte_eq().Add_int_variable(itm.w).Add_byte_semic();
bfr.Add_str_a7("h").Add_byte_eq().Add_int_variable(itm.h).Add_byte_semic();
bfr.Add_str_a7("redirect").Add_byte_eq().Add(itm.redirect).Add_byte_semic();
return bfr.Xto_str_and_clear();
}
}

94
400_xowa/src/gplx/xowa/files/origs/Xof_orig_mgr.java Normal file
View File

@@ -0,0 +1,94 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.core.primitives.*; import gplx.dbs.*;
import gplx.xowa.files.repos.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.wmfs.apis.*; import gplx.xowa.files.downloads.*;
public class Xof_orig_mgr {
private Xof_orig_wkr[] wkrs; private int wkrs_len;
private Xof_url_bldr url_bldr; private Xow_repo_mgr repo_mgr; private final Xof_img_size img_size = new Xof_img_size();
public Xof_orig_mgr() {this.Wkrs__clear();}
public void Init_by_wiki(Xow_wiki wiki, Xof_fsdb_mode fsdb_mode, Xof_orig_tbl[] orig_tbls, Xof_url_bldr url_bldr) {
this.repo_mgr = wiki.File__repo_mgr(); this.url_bldr = url_bldr;
// if (!fsdb_mode.Tid_v0()) { // add view,make; don't add if wmf
int orig_tbls_len = orig_tbls.length;
for (int i = 0; i < orig_tbls_len; ++i) {
Xof_orig_tbl orig_tbl = orig_tbls[i];
this.Wkrs__add_many(new Xof_orig_wkr__orig_db(orig_tbl, i == orig_tbls_len - 1));
}
// }
if (!fsdb_mode.Tid_v2_bld()) { // add if gui, but not if bld
Io_url wiki_meta_dir = wiki.App().Fsys_mgr().File_dir().GenSubDir_nest("#meta", wiki.Domain_str());
if (Io_mgr.I.ExistsDir(wiki_meta_dir)) {
Xof_orig_wkr__xo_meta xo_meta = new Xof_orig_wkr__xo_meta(wiki_meta_dir);
this.Wkrs__add_many(xo_meta);
}
this.Wkrs__add_many(new Xof_orig_wkr__wmf_api(new Xoapi_orig_wmf(), wiki.App().Wmf_mgr().Download_wkr(), repo_mgr, wiki.Domain_bry()));
}
}
public Xof_orig_itm Find_by_ttl_or_null(byte[] ttl) {return Find_by_ttl_or_null(ttl, 0, 1);}
public Xof_orig_itm Find_by_ttl_or_null(byte[] ttl, int list_idx, int list_len) {
for (int i = 0; i < wkrs_len; i++) {
Xof_orig_wkr wkr = wkrs[i];
Xof_orig_itm orig = wkr.Find_as_itm(ttl, list_idx, list_len); if (orig == Xof_orig_itm.Null) continue;
if (orig.Insert_new()) this.Insert(orig.Repo(), ttl, orig.Ext_id(), orig.W(), orig.H(), orig.Redirect()); // NOTE: orig_page must be same as find_arg not orig.Page() else will not be found for next call; DATE:2015-04-14
return orig;
}
return Xof_orig_itm.Null;
}
public void Find_by_list(Ordered_hash rv, List_adp itms, int exec_tid) {
for (int i = 0; i < wkrs_len; i++) {
Xof_orig_wkr wkr = wkrs[i];
wkr.Find_by_list(rv, itms);
}
int len = itms.Count();
for (int i = 0; i < len; i++) {
try {
Xof_fsdb_itm fsdb = (Xof_fsdb_itm)itms.Get_at(i);
fsdb.Orig_exists_n_(); // default to status = missing
Xof_orig_itm orig = (Xof_orig_itm)rv.Get_by(fsdb.Lnki_ttl()); if (orig == Xof_orig_itm.Null) continue;
if (orig.Insert_new()) this.Insert(orig.Repo(), fsdb.Lnki_ttl(), orig.Ext_id(), orig.W(), orig.H(), orig.Redirect()); // NOTE: orig_page must be same as find_arg not orig.Page() else will not be found for next call; DATE:2015-04-14
Xof_file_wkr.Eval_orig(orig, fsdb, url_bldr, repo_mgr, img_size);
if (!Io_mgr.I.ExistsFil(fsdb.Html_view_url()))
fsdb.File_exists_n_();
} catch (Exception e) {
Xoa_app_.Usr_dlg().Warn_many("", "", "orig: ~{0}", Err_.Message_gplx_brief(e));
}
}
}
public void Insert(byte repo, byte[] page, int ext, int w, int h, byte[] redirect) {
for (int i = 0; i < wkrs_len; i++) {
Xof_orig_wkr wkr = wkrs[i];
if (wkr.Add_orig(repo, page, ext, w, h, redirect)) break;
}
}
private void Wkrs__clear() {wkrs = Xof_orig_wkr_.Ary_empty; wkrs_len = 0;}
private void Wkrs__add_many(Xof_orig_wkr... v) {
wkrs = (Xof_orig_wkr[])Array_.Resize_add(wkrs, v);
wkrs_len += v.length;
}
public void Wkrs_del(byte tid) {
List_adp list = List_adp_.new_();
for (int i = 0; i < wkrs_len; ++i) {
Xof_orig_wkr wkr = wkrs[i];
if (wkr.Tid() == tid) continue; // do not add deleted wkr
list.Add(wkr);
}
wkrs = (Xof_orig_wkr[])list.To_ary_and_clear(Xof_orig_wkr.class);
wkrs_len = wkrs.length;
}
}

115
400_xowa/src/gplx/xowa/files/origs/Xof_orig_tbl.java Normal file
View File

@@ -0,0 +1,115 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.core.primitives.*;
import gplx.dbs.*; import gplx.dbs.utls.*;
import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*;
public class Xof_orig_tbl implements RlsAble {
private final String tbl_name; private final Db_meta_fld_list flds = Db_meta_fld_list.new_();
private final String fld_repo, fld_ttl, fld_status, fld_ext, fld_w, fld_h, fld_redirect;
private final Db_conn conn; private final Xof_orig_tbl__in_wkr select_in_wkr = new Xof_orig_tbl__in_wkr();
public Db_conn Conn() {return conn;}
public Xof_orig_tbl(Db_conn conn, boolean schema_is_1) {
this.conn = conn;
String fld_status_name = "orig_status";
if (schema_is_1) {tbl_name = "wiki_orig"; fld_status_name = "status";}
else {tbl_name = "orig_reg";}
fld_ttl = flds.Add_str("orig_ttl", 1024);
fld_repo = flds.Add_byte("orig_repo");
fld_status = flds.Add_byte(fld_status_name);
fld_ext = flds.Add_int("orig_ext");
fld_w = flds.Add_int("orig_w");
fld_h = flds.Add_int("orig_h");
fld_redirect = flds.Add_str("orig_redirect", 1024);
select_in_wkr.Ctor(this, tbl_name, flds, fld_ttl);
conn.Rls_reg(this);
}
public void Rls() {}
public void Create_tbl() {conn.Ddl_create_tbl(Db_meta_tbl.new_(tbl_name, flds, Db_meta_idx.new_normal_by_tbl(tbl_name, "main", fld_ttl)));}
public void Select_by_list(Ordered_hash rv, List_adp itms) {select_in_wkr.Init(rv, itms).Select_in(Cancelable_.Never, conn, 0, itms.Count());}
public Xof_orig_itm Select_itm(byte[] ttl) {
Xof_orig_itm rv = Xof_orig_itm.Null;
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_ttl).Clear().Crt_bry_as_str(fld_ttl, ttl).Exec_select__rls_auto();
try {
if (rdr.Move_next())
rv = Make_itm(rdr);
}
finally {rdr.Rls();}
return rv;
}
public boolean Exists__repo_ttl(byte repo, byte[] ttl) {
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, fld_repo, fld_ttl).Crt_byte(fld_repo, repo).Crt_bry_as_str(fld_ttl, ttl).Exec_select__rls_auto();
try {return rdr.Move_next();}
finally {rdr.Rls();}
}
public void Insert(byte repo, byte[] ttl, int ext, int w, int h, byte[] redirect) {
Db_stmt stmt = conn.Stmt_insert(tbl_name, flds);
stmt.Clear()
.Val_bry_as_str(fld_ttl, ttl).Val_byte(fld_repo, repo).Val_byte(fld_status, Status_found)
.Val_int(fld_ext, ext).Val_int(fld_w, w).Val_int(fld_h, h).Val_bry_as_str(fld_redirect, redirect)
.Exec_insert();
}
public void Update(byte repo, byte[] ttl, int ext, int w, int h, byte[] redirect) {
Db_stmt stmt = conn.Stmt_update_exclude(tbl_name, flds, String_.Ary(fld_repo, fld_ttl));
stmt.Clear()
.Val_byte(fld_status, Status_found)
.Val_int(fld_ext, ext).Val_int(fld_w, w).Val_int(fld_h, h).Val_bry_as_str(fld_redirect, redirect)
.Crt_byte(fld_repo, repo).Crt_bry_as_str(fld_ttl, ttl)
.Exec_update();
}
public Xof_orig_itm Make_itm(Db_rdr rdr) {
byte repo = rdr.Read_byte(fld_repo);
Xof_orig_itm rv = new Xof_orig_itm().Init
( repo
, rdr.Read_bry_by_str(fld_ttl)
, rdr.Read_int(fld_ext)
, rdr.Read_int(fld_w)
, rdr.Read_int(fld_h)
, rdr.Read_bry_by_str(fld_redirect)
);
return rv.W() == Xof_img_size.Null ? Xof_orig_itm.Null : rv;
}
private static final byte Status_found = 1;
}
class Xof_orig_tbl__in_wkr extends Db_in_wkr__base {
private Xof_orig_tbl tbl; private String tbl_name; private Db_meta_fld_list flds; private String fld_ttl;
private List_adp itms; private Ordered_hash rv;
public void Ctor(Xof_orig_tbl tbl, String tbl_name, Db_meta_fld_list flds, String fld_ttl) {
this.tbl = tbl; this.tbl_name = tbl_name; this.flds = flds; this.fld_ttl = fld_ttl;
}
public Xof_orig_tbl__in_wkr Init(Ordered_hash rv, List_adp itms) {this.itms = itms; this.rv = rv; return this;}
@Override protected Db_qry Make_qry(int bgn, int end) {
Object[] part_ary = In_ary(end - bgn);
return Db_qry_.select_cols_(tbl_name, Db_crt_.in_(fld_ttl, part_ary), flds.To_str_ary());
}
@Override protected void Fill_stmt(Db_stmt stmt, int bgn, int end) {
for (int i = bgn; i < end; i++) {
Xof_fsdb_itm fsdb_itm = (Xof_fsdb_itm)itms.Get_at(i);
stmt.Crt_bry_as_str(fld_ttl, fsdb_itm.Lnki_ttl());
}
}
@Override protected void Read_data(Cancelable cancelable, Db_rdr rdr) {
while (rdr.Move_next()) {
if (cancelable.Canceled()) return;
Xof_orig_itm itm = tbl.Make_itm(rdr);
if (itm == Xof_orig_itm.Null) continue;
byte[] itm_ttl = itm.Ttl();
rv.Add_if_dupe_use_1st(itm_ttl, itm); // guard against dupes; fails on en.w:Paris; DATE:2015-03-08
}
}
}

75
400_xowa/src/gplx/xowa/files/origs/Xof_orig_tbl_tst.java Normal file
View File

@@ -0,0 +1,75 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import org.junit.*;
import gplx.dbs.*; import gplx.xowa.*;
import gplx.xowa.files.*; import gplx.xowa.files.fsdb.*;
public class Xof_orig_tbl_tst {
@Before public void init() {fxt.Clear();} private Xof_orig_tbl_fxt fxt = new Xof_orig_tbl_fxt();
@Test public void Select_in() {
Xof_orig_itm itm_1 = fxt.Exec_insert("A.png", 220, 330);
fxt.Exec_insert("B.png", 220, 330);
Xof_orig_itm itm_3 = fxt.Exec_insert("C.png", 220, 330);
fxt.Test_select_in(String_.Ary("A.png", "C.png"), itm_1, itm_3);
}
}
class Xof_orig_tbl_fxt {
private Xof_orig_tbl tbl;
public void Clear() {
Io_url test_url = Io_url_.mem_fil_("mem/file/en.wikipedia.org/file/orig_regy");
Db_conn_bldr.I.Reg_default_mem();
Db_conn conn = Db_conn_bldr.I.New(test_url);
tbl = new Xof_orig_tbl(conn, Bool_.Y);
tbl.Create_tbl();
}
public Xof_orig_itm Exec_insert(String ttl, int w, int h) {
byte[] ttl_bry = Bry_.new_u8(ttl);
Xof_orig_itm rv = new Xof_orig_itm().Init(Xof_orig_itm.Repo_comm, ttl_bry, Xof_ext_.new_by_ttl_(ttl_bry).Id(), w, h, Bry_.Empty);
tbl.Insert(rv.Repo(), rv.Ttl(), rv.Ext_id(), rv.W(), rv.H(), rv.Redirect());
return rv;
}
public void Test_select_in(String[] itms, Xof_orig_itm... expd) {
Ordered_hash rv = Ordered_hash_.new_();
List_adp list = List_adp_.new_();
int itms_len = itms.length;
for (int i = 0; i < itms_len; ++i) {
String itm = itms[i];
Xof_fsdb_itm fsdb_itm = new Xof_fsdb_itm();
fsdb_itm.Init_at_lnki(Xof_exec_tid.Tid_wiki_page, Bry_.new_a7("en.w"), Bry_.new_u8(itm), Xop_lnki_type.Id_none, Xof_img_size.Upright_null, Xof_img_size.Null, Xof_img_size.Null, Xof_lnki_time.Null, Xof_lnki_page.Null, Xof_patch_upright_tid_.Tid_all);
list.Add(fsdb_itm);
}
tbl.Select_by_list(rv, list);
Tfds.Eq_str_lines(To_str_ary(expd), To_str_ary((Xof_orig_itm[])rv.To_ary(Xof_orig_itm.class)));
}
private static String To_str_ary(Xof_orig_itm... ary) {
Bry_bfr bfr = Bry_bfr.reset_(255);
int len = ary.length;
for (int i = 0; i < len; ++i) {
Xof_orig_itm itm = ary[i];
bfr .Add_byte(itm.Repo()).Add_byte_pipe()
.Add(itm.Ttl()).Add_byte_pipe()
.Add_int_variable(itm.Ext_id()).Add_byte_pipe()
.Add_int_variable(itm.W()).Add_byte_pipe()
.Add_int_variable(itm.H()).Add_byte_pipe()
.Add(itm.Redirect()).Add_byte_pipe()
;
bfr.Add_byte_nl();
}
return bfr.Xto_str_and_clear();
}
}

27
400_xowa/src/gplx/xowa/files/origs/Xof_orig_wkr.java Normal file
View File

@@ -0,0 +1,27 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.files.fsdb.*;
public interface Xof_orig_wkr {
byte Tid();
Xof_orig_itm Find_as_itm(byte[] ttl, int list_idx, int list_len);
void Find_by_list(Ordered_hash rv, List_adp itms);
boolean Add_orig(byte repo, byte[] page, int ext_id, int w, int h, byte[] redirect);
void Db_txn_save();
void Db_rls();
}

39
400_xowa/src/gplx/xowa/files/origs/Xof_orig_wkr_.java Normal file
View File

@@ -0,0 +1,39 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.wmfs.apis.*; import gplx.xowa.files.fsdb.*;
public class Xof_orig_wkr_ {
public static final Xof_orig_wkr[] Ary_empty = new Xof_orig_wkr[0];
public static void Find_by_list(Xof_orig_wkr wkr, Ordered_hash rv, List_adp itms) {
int len = itms.Count();
for (int i = 0; i < len; ++i) {
Xof_fsdb_itm fsdb = (Xof_fsdb_itm)itms.Get_at(i);
byte[] fsdb_ttl = fsdb.Lnki_ttl();
if (rv.Has(fsdb_ttl)) continue;
Xof_orig_itm orig = wkr.Find_as_itm(fsdb_ttl, i, len);
if (orig == Xof_orig_itm.Null) continue;
rv.Add(fsdb_ttl, orig);
}
}
public static final byte
Tid_xowa_db = 1
, Tid_wmf_api = 2
, Tid_xowa_meta = 3
;
}

37
400_xowa/src/gplx/xowa/files/origs/Xof_orig_wkr__orig_db.java Normal file
View File

@@ -0,0 +1,37 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.dbs.*;
public class Xof_orig_wkr__orig_db implements Xof_orig_wkr {
private final boolean addable;
public Xof_orig_wkr__orig_db(Xof_orig_tbl tbl, boolean addable) {this.tbl = tbl; this.addable = addable;}
public byte Tid() {return Xof_orig_wkr_.Tid_xowa_db;}
public Xof_orig_tbl Tbl() {return tbl;} private final Xof_orig_tbl tbl;
public void Find_by_list(Ordered_hash rv, List_adp itms) {tbl.Select_by_list(rv, itms);}
public Xof_orig_itm Find_as_itm(byte[] ttl, int list_idx, int list_len) {return tbl.Select_itm(ttl);}
public boolean Add_orig(byte repo, byte[] page, int ext_id, int w, int h, byte[] redirect) {
if (!addable) return false;
if (tbl.Exists__repo_ttl(repo, page))
tbl.Update(repo, page, ext_id, w, h, redirect);
else
tbl.Insert(repo, page, ext_id, w, h, redirect);
return true;
}
public void Db_txn_save() {tbl.Conn().Txn_end();}
public void Db_rls() {tbl.Conn().Rls_conn();}
}

55
400_xowa/src/gplx/xowa/files/origs/Xof_orig_wkr__wmf_api.java Normal file
View File

@@ -0,0 +1,55 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.files.repos.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.wmfs.apis.*; import gplx.xowa.files.downloads.*;
public class Xof_orig_wkr__wmf_api implements Xof_orig_wkr {
private final Xoapi_orig_base orig_api; private final Xof_download_wkr download_wkr; private final Xow_repo_mgr repo_mgr; private final byte[] wiki_domain;
private final Xoapi_orig_rslts api_rv = new Xoapi_orig_rslts();
public Xof_orig_wkr__wmf_api(Xoapi_orig_base orig_api, Xof_download_wkr download_wkr, Xow_repo_mgr repo_mgr, byte[] wiki_domain) {
this.orig_api = orig_api; this.download_wkr = download_wkr; this.repo_mgr = repo_mgr; this.wiki_domain = wiki_domain;
}
public byte Tid() {return Xof_orig_wkr_.Tid_wmf_api;}
public void Find_by_list(Ordered_hash rv, List_adp itms) {Xof_orig_wkr_.Find_by_list(this, rv, itms);}
public Xof_orig_itm Find_as_itm(byte[] ttl, int list_idx, int list_len) {
if (!gplx.ios.IoEngine_system.Web_access_enabled) return Xof_orig_itm.Null; // don't check api if download disabled, else prog messages; DATE:2015-06-17
Xoa_app_.Usr_dlg().Prog_none("", "", Prog_msg(list_idx, list_len, ttl));
boolean found = orig_api.Api_query_size(api_rv, download_wkr, repo_mgr, ttl, Xof_img_size.Null, Xof_img_size.Null); // pass in null size to look for orig; DATE:2015-02-10
if (!found) return Xof_orig_itm.Null; // ttl not found by api; return
byte api_repo = Bry_.Eq(api_rv.Orig_wiki(), wiki_domain) ? Xof_orig_itm.Repo_wiki : Xof_orig_itm.Repo_comm;
byte[] api_page = api_rv.Orig_page();
int api_w = api_rv.Orig_w(), api_h = api_rv.Orig_h();
Xof_ext api_ext = Xof_ext_.new_by_ttl_(api_page); api_ext = Ext__handle_ogg(api_ext, api_w, api_h);
byte[] api_redirect = Bry_.Eq(api_page, ttl) ? Bry_.Empty : api_page; // ttl is different; must be redirect
Xof_orig_itm rv = new Xof_orig_itm();
rv.Init(api_repo, api_page, api_ext.Id(), api_w, api_h, api_redirect);
rv.Insert_new_y_();
return rv;
}
public boolean Add_orig(byte repo, byte[] page, int ext_id, int w, int h, byte[] redirect) {return false;}
public void Db_txn_save() {}
public void Db_rls() {}
public static Xof_ext Ext__handle_ogg(Xof_ext ext, int w, int h) {
if (!ext.Id_is_ogg()) return ext;
boolean is_audio = w == 0 && h == 0; // wmf returns back w/h of 0 if audio; non-0 if video; DATE:2013-11-11
int actl_ext_id = is_audio ? Xof_ext_.Id_oga : Xof_ext_.Id_ogv;
return Xof_ext_.new_by_id_(actl_ext_id);
}
private static String Prog_msg(int idx, int len, byte[] ttl) {
return String_.Format("downloading file {0} of {1}: {2}", idx + List_adp_.Base1, len, ttl);
}
}

50
400_xowa/src/gplx/xowa/files/origs/Xof_orig_wkr__xo_meta.java Normal file
View File

@@ -0,0 +1,50 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.core.flds.*;
import gplx.dbs.*; import gplx.xowa.files.fsdb.*;
public class Xof_orig_wkr__xo_meta implements Xof_orig_wkr {
private final Io_url wiki_meta_dir; private final byte dir_spr_byte; private final Bry_bfr url_bfr = Bry_bfr.new_(255);
private final Gfo_fld_rdr meta_rdr = Gfo_fld_rdr.xowa_(); private final Xof_meta_thumb_parser parser = new Xof_meta_thumb_parser();
public Xof_orig_wkr__xo_meta(Io_url wiki_meta_dir) {this.wiki_meta_dir = wiki_meta_dir; this.dir_spr_byte = wiki_meta_dir.Info().DirSpr_byte();}
public byte Tid() {return Xof_orig_wkr_.Tid_xowa_meta;}
public void Find_by_list(Ordered_hash rv, List_adp itms) {Xof_orig_wkr_.Find_by_list(this, rv, itms);}
public Xof_orig_itm Find_as_itm(byte[] ttl, int list_idx, int list_len) {
byte[] md5 = Xof_file_wkr_.Md5_(ttl);
url_bfr.Add(wiki_meta_dir.RawBry()) // /xowa/file/#meta/simple.wikipedia.org/
.Add_byte(md5[0]).Add_byte(dir_spr_byte) // 0/
.Add_byte(md5[1]).Add_byte(dir_spr_byte) // 6/
.Add_mid(md5, 0, 3).Add_str_a7(".csv") // 061.csv
;
Io_url meta_url = Io_url_.new_fil_(url_bfr.Xto_str_and_clear());
byte[] meta_src = Io_mgr.I.LoadFilBry(meta_url); if (meta_src.length == 0) return Xof_orig_itm.Null;
meta_rdr.Ini(meta_src, 0);
Xof_meta_fil meta_fil = new Xof_meta_fil(null, md5); // NOTE: need to register file before loading it; defect wherein 2 files with same hash prefix would skip one b/c Loaded file was not registered; EX.WS: en.wikiquote.org/The Hitchhiker's Guide to the Galaxy; NMMP_dolphin_with_locator.jpeg, da6f95736ed249f371f30bf5f1205fbd; Hoags_object.jpg, daed4a54e48e4266bd2f2763b7c4018c
meta_fil.Load(meta_rdr, parser);
Xof_meta_itm meta_itm = meta_fil.Get_or_null(ttl); if (meta_itm == null) return Xof_orig_itm.Null;
Xof_orig_itm rv = new Xof_orig_itm();
rv.Init((byte)meta_itm.Vrtl_repo(), ttl, Xof_ext_.new_by_ttl_(ttl).Id(), meta_itm.Orig_w(), meta_itm.Orig_h(), meta_itm.Ptr_ttl());
rv.Insert_new_y_();
return rv;
}
public boolean Add_orig(byte repo, byte[] page, int ext_id, int w, int h, byte[] redirect) {
return false;
}
public void Db_txn_save() {}
public void Db_rls() {}
}

70
400_xowa/src/gplx/xowa/files/origs/Xof_wiki_finder.java Normal file
View File

@@ -0,0 +1,70 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.wikis.data.tbls.*;
class Xof_wiki_finder { // UNUSED
private Xowe_wiki wiki_0, wiki_1;
private Xowd_page_itm db_page = new Xowd_page_itm();
public Xof_wiki_finder(Xowe_wiki wiki_0, Xowe_wiki wiki_1) {
this.wiki_0 = wiki_0; this.wiki_1 = wiki_1;
}
public Xoae_page Get_page(int ns, byte[] ttl_bry) {
Xoae_page rv = Get_page__by_wiki(wiki_0, ns, ttl_bry);
if (rv.Missing())
rv = Get_page__by_wiki(wiki_1, ns, ttl_bry);
return rv;
}
private Xoae_page Get_page__by_wiki(Xowe_wiki wiki, int ns_id, byte[] ttl_bry) {
Xoa_ttl ttl = Xoa_ttl.parse_(wiki, ns_id, ttl_bry) ;
Xoa_url url = Xoa_url.new_(wiki.Domain_bry(), ttl_bry);
return wiki.Load_page_by_ttl(url, ttl);
}
private int qry_count, qry_count_max = 1000;
public boolean Find_page(Xof_wiki_finder_itm itm, int ns_id, byte[] ttl_bry) {
Xowe_wiki wiki = null;
if (Find_page__by_wiki(db_page, wiki_0, ns_id, ttl_bry)) {
wiki = wiki_0;
itm.Orig_repo_id_(Byte_.Zero);
}
else {
if (Find_page__by_wiki(db_page, wiki_1, ns_id, ttl_bry)) {
wiki = wiki_1;
itm.Orig_repo_id_((byte)1);
}
else
return false;
}
itm.Orig_ttl_(ttl_bry);
if (db_page.Redirected()) {
Xoae_page page = Get_page__by_wiki(wiki, ns_id, ttl_bry);
Xoa_ttl redirect_ttl = wiki.Redirect_mgr().Extract_redirect_loop(page.Data_raw());
itm.Orig_redirect_(redirect_ttl);
++qry_count;
if (qry_count >= qry_count_max) {
wiki.Appe().Reset_all();
qry_count = 0;
}
}
return true;
}
private boolean Find_page__by_wiki(Xowd_page_itm db_page, Xowe_wiki wiki, int ns_id, byte[] ttl_bry) {
Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
wiki.Db_mgr().Load_mgr().Load_page(db_page, ns, false);
return db_page.Exists();
}
}

28
400_xowa/src/gplx/xowa/files/origs/Xof_wiki_finder_itm.java Normal file
View File

@@ -0,0 +1,28 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.origs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
class Xof_wiki_finder_itm { // UNUSED
// public boolean Missing() {return missing;} public Xof_wiki_finder_itm Missing_(boolean v) {missing = v; return this;} private boolean missing;
public byte Orig_repo_id() {return orig_repo_id;} public Xof_wiki_finder_itm Orig_repo_id_(byte v) {orig_repo_id = v; return this;} private byte orig_repo_id;
public byte[] Orig_ttl() {return orig_ttl;} public Xof_wiki_finder_itm Orig_ttl_(byte[] v) {orig_ttl = v; return this;} private byte[] orig_ttl;
public Xowe_wiki Orig_wiki() {return orig_wiki;} public Xof_wiki_finder_itm Orig_wiki_(Xowe_wiki v) {orig_wiki = v; return this;} private Xowe_wiki orig_wiki;
public Xoa_ttl Orig_redirect() {return orig_redirect;} public Xof_wiki_finder_itm Orig_redirect_(Xoa_ttl v) {orig_redirect = v; return this;} private Xoa_ttl orig_redirect;
// public int Orig_w() {return orig_w;} private int orig_w;
// public int Orig_h() {return orig_h;} private int orig_h;
// public void Orig_size_(int w, int h) {this.orig_w = w; this.orig_h = h;}
}