Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Source: Restore broken commit

Browse Source
This commit is contained in:
gnosygnu
2017-02-06 22:14:55 -05:00
parent 938beac9f9
commit 3bfeb94b43
4380 changed files with 328018 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
400_xowa/src/gplx/xowa/files/repos/Xof_itm_ttl_.java Normal file
View File

@@ -0,0 +1,42 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
public class Xof_itm_ttl_ {
public static byte[] Remove_invalid(Bry_bfr bfr, byte[] ttl) {
int ttl_len = ttl.length;
for (int i = 0; i < ttl_len; i++) {
byte b = ttl[i];
if (gplx.core.envs.Op_sys_.Wnt_invalid_char(b)) b = Byte_ascii.Underline;
bfr.Add_byte(b);
}
return bfr.To_bry_and_clear();
}
public static byte[] Shorten(Bry_bfr bfr, byte[] ttl, int ttl_max, byte[] md5, byte[] ext_bry) {
byte[] rv = ttl;
int exceed_len = rv.length - ttl_max;
if (exceed_len > 0) {
bfr.Add_mid(rv, 0, ttl_max - 33); // add truncated title; 33=_.length + md5.length
bfr.Add_byte(Byte_ascii.Underline); // add underline; EX: "_"
bfr.Add(md5); // add md5; EX: "abcdefghijklmnopqrstuvwxyz0123456"
bfr.Add_byte(Byte_ascii.Dot); // add dot; EX: "."
bfr.Add(ext_bry); // add ext; EX: ".png"
rv = bfr.To_bry_and_clear();
}
return rv;
}
}

102
400_xowa/src/gplx/xowa/files/repos/Xof_repo_itm.java Normal file
View File

@@ -0,0 +1,102 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.core.consoles.*;
import gplx.xowa.apps.fsys.*; import gplx.xowa.files.exts.*; import gplx.xowa.files.imgs.*;
import gplx.xowa.wikis.domains.*;
public class Xof_repo_itm implements Gfo_invk {
private final Xoa_fsys_mgr app_fsys; private final Xof_rule_mgr ext_rule_mgr;
public Xof_repo_itm(byte[] key, Xoa_fsys_mgr app_fsys, Xof_rule_mgr ext_rule_mgr, byte[] wiki_domain) {
this.key = key; this.app_fsys = app_fsys; this.ext_rule_mgr = ext_rule_mgr;
Wiki_domain_(wiki_domain);
}
public byte[] Key() {return key;} private final byte[] key; // EX: "src_http_commons"
public byte Tid() {return tid;} private byte tid; // EX: Xof_repo_tid_.Tid__remote
public byte[] Wiki_domain() {return wiki_domain;} private byte[] wiki_domain; // EX: "commons.wikimedia.org"
public byte[] Wiki_abrv_xo() {return wiki_abrv_xo;} private byte[] wiki_abrv_xo; // EX: "c"
public byte Dir_spr() {return dir_spr;} private byte dir_spr; // EX: "/"
public int Dir_depth() {return dir_depth;} private int dir_depth = 4; // EX: "/1/2/3/4" vs "/1/2"
public byte[] Root_bry() {return root_bry;} private byte[] root_bry; // EX:
public byte[] Root_http() {return root_http;} private byte[] root_http = Bry_.Empty; // EX:
public String Root_str() {return root_str;} private String root_str;
public boolean Fsys_is_wnt() {return fsys_is_wnt;} private boolean fsys_is_wnt;
public boolean Shorten_ttl() {return shorten_ttl;} private boolean shorten_ttl = true;
public boolean Wmf_fsys() {return wmf_fsys;} private boolean wmf_fsys;
public boolean Wmf_api() {return wmf_api;} private boolean wmf_api;
public boolean Tarball() {return tarball;} private boolean tarball;
public byte[][] Mode_names() {return mode_names;} private byte[][] mode_names = new byte[][] {Xof_img_mode_.Names_ary[0], Xof_img_mode_.Names_ary[1]};
public Xof_rule_grp Ext_rules() {return ext_rules;} private Xof_rule_grp ext_rules;
public boolean Primary() {return primary;} private boolean primary;
public Xof_repo_itm Fsys_is_wnt_(boolean v) {fsys_is_wnt = v; return this;}
public Xof_repo_itm Shorten_ttl_(boolean v) {shorten_ttl = v; return this;}
public Xof_repo_itm Wmf_fsys_(boolean v) {wmf_fsys = v; return this;}
public Xof_repo_itm Wmf_api_(boolean v) {wmf_api = v; return this;}
public Xof_repo_itm Tarball_(boolean v) {tarball = v; return this;}
public Xof_repo_itm Ext_rules_(byte[] ext_rules_key) {ext_rules = ext_rule_mgr.Get_or_new(ext_rules_key); return this;}
public Xof_repo_itm Dir_depth_(int v) {dir_depth = v; return this;}
public Xof_repo_itm Primary_(boolean v) {primary = v; return this;}
public void Wiki_domain_(byte[] v) {
this.wiki_domain = v;
Xow_domain_itm domain_itm = Xow_domain_itm_.parse(v);
if (domain_itm == null) {
Xoa_app_.Usr_dlg().Warn_many("", "", "repo:invalid domain; raw=~{0}", v);
this.wiki_abrv_xo = Bry_.Empty;
}
else
this.wiki_abrv_xo = Xow_abrv_xo_.To_bry(v, domain_itm.Lang_actl_key(), domain_itm.Domain_type());
}
public Xof_repo_itm Root_str_(String root_str) {
this.wmf_fsys = String_.Has_at_bgn(root_str, "http") || String_.Has_at_bgn(root_str, "ftp");
if (wmf_fsys) {
this.root_bry = Bry_.new_u8(root_str);
this.dir_spr = Byte_ascii.Slash;
this.wmf_api = true;
}
else {
Io_url root_url = Gfo_cmd_arg_itm_.Val_as_url__rel_url_or(root_str, Bool_.Y, app_fsys.File_dir(), Io_url_.new_dir_(root_str));
this.root_bry = root_url.RawBry();
this.dir_spr = root_url.Info().DirSpr_byte();
this.root_http = root_url.To_http_file_bry();
}
this.root_str = root_str;
return this;
}
public byte[] Gen_name_src(Bry_bfr tmp_bfr, byte[] name) {
if (!fsys_is_wnt || wmf_fsys) return name;
return Xof_itm_ttl_.Remove_invalid(tmp_bfr, name);
}
public byte[] Gen_name_trg(Bry_bfr tmp_bfr, byte[] bry, byte[] md5, Xof_ext ext) {
byte[] rv = Gen_name_src(tmp_bfr, bry);
return shorten_ttl ? Xof_itm_ttl_.Shorten(tmp_bfr, rv, Ttl__max_len, md5, ext.Ext()) : rv;
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_owner)) throw Err_.new_unimplemented_w_msg("deprecated repo_itm.owner");
else if (ctx.Match(k, Invk_fsys_)) fsys_is_wnt = String_.Eq(m.ReadStr("v"), "wnt");
else if (ctx.Match(k, Invk_primary_)) primary = m.ReadYn("v");
else if (ctx.Match(k, Invk_ext_rules_)) Ext_rules_(m.ReadBry("v"));
else if (ctx.Match(k, Invk_wmf_api_)) wmf_api = m.ReadYn("v");
else if (ctx.Match(k, Invk_tarball_)) tarball = m.ReadYn("v");
else if (ctx.Match(k, Invk_tid_)) tid = Xof_repo_tid_.By_str(m.ReadStr("v"));
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk_owner = "owner", Invk_fsys_ = "fsys_", Invk_ext_rules_ = "ext_rules_", Invk_primary_ = "primary_", Invk_wmf_api_ = "wmf_api_", Invk_tarball_ = "tarball_", Invk_tid_ = "tid_";
public static final int Ttl__max_len = 180;
}

33
400_xowa/src/gplx/xowa/files/repos/Xof_repo_pair.java Normal file
View File

@@ -0,0 +1,33 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
public class Xof_repo_pair implements Gfo_invk {
public Xof_repo_pair(byte id, byte[] wiki_domain, Xof_repo_itm src, Xof_repo_itm trg) {
this.id = id; this.wiki_domain = wiki_domain; this.src = src; this.trg = trg;
}
public byte Id() {return id;} private byte id;
public byte[] Wiki_domain() {return wiki_domain;} private final byte[] wiki_domain;
public Xof_repo_itm Src() {return src;} private final Xof_repo_itm src;
public Xof_repo_itm Trg() {return trg;} private final Xof_repo_itm trg;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_repo_id_)) id = m.ReadByte("v");
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk_repo_id_ = "repo_id_";
}

36
400_xowa/src/gplx/xowa/files/repos/Xof_repo_tid_.java Normal file
View File

@@ -0,0 +1,36 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
public class Xof_repo_tid_ {
public static final byte
Tid__remote = 0 // EX: "https://commons.wikimedia.org/wiki/File:A.png"
, Tid__local = 1 // EX: "https://en.wikipedia.org/wiki/File:A.png"
, Tid__xtns = 2 // EX: "https://en.wikipedia.org/w/extensions/ImageMap/desc-20.png?15600"
, Tid__math = 3 // EX: "https://wikimedia.org/api/rest_v1/media/math/render/svg/596f8baf206a81478afd4194b44138715dc1a05c"
, Tid__null = Byte_.Max_value_127
;
public static byte By_str(String v) {
if (String_.Eq(v, "self")) return Tid__local;
else if (String_.Eq(v, "comm")) return Tid__remote;
else if (String_.Eq(v, "math")) return Tid__math;
else throw Err_.new_unhandled_default(v);
}
public static final byte[]
Bry__math = Bry_.new_a7("wikimedia.org/math") // EX: https://wikimedia.org/api/rest_v1/media/math/render/svg/596f8baf206a81478afd4194b44138715dc1a05c
;
}

31
400_xowa/src/gplx/xowa/files/repos/Xofw_file_finder_rslt.java Normal file
View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.parsers.utils.*;
public class Xofw_file_finder_rslt {
public byte[] Ttl() {return ttl;} private byte[] ttl;
public byte[] Redirect() {return redirect;} private byte[] redirect;
public int Repo_idx() {return repo_idx;} private int repo_idx;
public byte[] Repo_wiki_key() {return repo_wiki_key;} private byte[] repo_wiki_key;
public void Init(byte[] ttl) {
this.ttl = ttl; redirect = Xop_redirect_mgr.Redirect_null_bry; repo_wiki_key = null; repo_idx = Byte_.Max_value_127;
}
public void Done(int repo_idx, byte[] repo_wiki_key, byte[] redirect) {
this.repo_idx = repo_idx; this.repo_wiki_key = repo_wiki_key; this.redirect = redirect;
}
}

23
400_xowa/src/gplx/xowa/files/repos/Xofw_wiki_finder.java Normal file
View File

@@ -0,0 +1,23 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.wikis.*; import gplx.xowa.files.*; import gplx.xowa.files.repos.*;
public interface Xofw_wiki_finder {
void Find(List_adp repo_pairs, Xof_xfer_itm file);
boolean Locate(Xofw_file_finder_rslt rv, List_adp repo_pairs, byte[] ttl_bry);
}

72
400_xowa/src/gplx/xowa/files/repos/Xofw_wiki_wkr_base.java Normal file
View File

@@ -0,0 +1,72 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.*; import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.parsers.utils.*;
public class Xofw_wiki_wkr_base implements Xofw_wiki_finder {
public Xofw_wiki_wkr_base(Xowe_wiki wiki, Xoae_wiki_mgr wiki_mgr) {this.wiki = wiki; this.wiki_mgr = wiki_mgr;} private Xowe_wiki wiki; Xoae_wiki_mgr wiki_mgr;
public void Find(List_adp repo_pairs, Xof_xfer_itm file) {
byte[] ttl_bry = file.Lnki_ttl();
int repo_pairs_len = repo_pairs.Count();
for (int i = 0; i < repo_pairs_len; i++) {
Xof_repo_pair repo_pair = (Xof_repo_pair)repo_pairs.Get_at(i);
byte[] wiki_key = repo_pair.Src().Wiki_domain();
if (repo_pair.Src().Wmf_api()) continue;
Xowe_wiki repo_wiki = (Xowe_wiki)wiki_mgr.Get_by_or_null(wiki_key); if (repo_wiki == null) {continue;}
Xoa_ttl ttl = Xoa_ttl.Parse(repo_wiki, ttl_bry);
Xow_ns file_ns = repo_wiki.Ns_mgr().Ns_file();
boolean found = repo_wiki.Db_mgr().Load_mgr().Load_by_ttl(tmp_db_page, file_ns, ttl.Page_db());
if (!found) {continue;}
byte[] redirect = Get_redirect(repo_wiki, file_ns, tmp_db_page);
file.Orig_ttl_and_redirect_(ttl.Page_txt(), redirect);
file.Orig_repo_id_(i);
return;
}
file.Orig_repo_id_(-1);
}
public boolean Locate(Xofw_file_finder_rslt rv, List_adp repo_pairs, byte[] ttl_bry) {
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, ttl_bry); // NOTE: parse(ttl_bry) should be the same across all wikis; i.e.: there should be no aliases/namespaces
Xow_ns file_ns = wiki.Ns_mgr().Ns_file(); // NOTE: file_ns should also be the same across all wikis; being used for data_mgr.Parse below
byte[] ttl_db_key = ttl.Page_db();
rv.Init(ttl_db_key);
int repo_pairs_len = repo_pairs.Count();
for (int i = 0; i < repo_pairs_len; i++) {
Xof_repo_pair repo_pair = (Xof_repo_pair)repo_pairs.Get_at(i);
byte[] src_wiki_key = repo_pair.Src().Wiki_domain();
Xowe_wiki src_wiki = (Xowe_wiki)wiki_mgr.Get_by_or_null(src_wiki_key); if (src_wiki == null) continue; // src_wiki defined as repo_pair in cfg, but it has not been downloaded; continue; EX: commons set up but not downloaded
boolean found = src_wiki.Db_mgr().Load_mgr().Load_by_ttl(tmp_db_page, file_ns, ttl_db_key);
if (!found) continue; // ttl does not exist in src_wiki; continue; EX: file does not exist in commons, but exists in en_wiki
byte[] redirect = Get_redirect(src_wiki, file_ns, tmp_db_page);
rv.Done(i, src_wiki_key, redirect);
return true;
}
return false;
}
private byte[] Get_redirect(Xowe_wiki wiki, Xow_ns file_ns, Xowd_page_itm db_page) {
if (db_page.Redirected()) {
wiki.Db_mgr().Load_mgr().Load_page(db_page, file_ns);
byte[] src = db_page.Text();
Xoa_ttl redirect_ttl = wiki.Redirect_mgr().Extract_redirect(src);
return redirect_ttl == Xop_redirect_mgr.Redirect_null_ttl ? Xop_redirect_mgr.Redirect_null_bry : redirect_ttl.Page_db();
}
else
return Xop_redirect_mgr.Redirect_null_bry;
}
private static final Xowd_page_itm tmp_db_page = Xowd_page_itm.new_tmp();
}

43
400_xowa/src/gplx/xowa/files/repos/Xofw_wiki_wkr_mock.java Normal file
View File

@@ -0,0 +1,43 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.files.*; import gplx.xowa.wikis.tdbs.metas.*;
public class Xofw_wiki_wkr_mock implements Xofw_wiki_finder {
int repo_idx; byte[] repo_wiki_key;
public Xofw_wiki_wkr_mock Clear_en_wiki() {return Clear(1, Bry_en_wiki);}
public Xofw_wiki_wkr_mock Clear_commons() {return Clear(0, Bry_commons);}
Xofw_wiki_wkr_mock Clear(int repo_idx, byte[] repo_wiki_key) {
this.repo_idx = repo_idx; this.repo_wiki_key = repo_wiki_key;
if_ttl = then_redirect = Bry_.Empty;
return this;
}
private static final byte[] Bry_commons = Bry_.new_a7("commons.wikimedia.org"), Bry_en_wiki = Bry_.new_a7("en.wikipedia.org");
public Xofw_wiki_wkr_mock Repo_idx_(int v) {this.repo_idx = v; return this;}
public Xofw_wiki_wkr_mock Redirect_(String if_ttl_str, String then_redirect_str) {this.if_ttl = Bry_.new_u8(if_ttl_str); this.then_redirect = Bry_.new_u8(then_redirect_str); return this;} private byte[] if_ttl, then_redirect;
public void Find(List_adp repo_pairs, Xof_xfer_itm file) {
byte[] ttl = file.Lnki_ttl();
if (Bry_.Eq(ttl, if_ttl) && repo_idx != -1) {file.Orig_ttl_and_redirect_(ttl, then_redirect); file.Orig_repo_id_(repo_idx);}
else {file.Orig_ttl_and_redirect_(ttl, Bry_.Empty) ; file.Orig_repo_id_(Xof_meta_itm.Repo_unknown);} // FUTURE: this should be missing, but haven't implemented unknown yet
}
public boolean Locate(Xofw_file_finder_rslt rv, List_adp repo_pairs, byte[] ttl) {
rv.Init(ttl);
byte[] redirect = Bry_.Eq(ttl, if_ttl) ? then_redirect : null;
rv.Done(repo_idx, repo_wiki_key, redirect);
return true;
}
}

25
400_xowa/src/gplx/xowa/files/repos/Xow_repo_mgr.java Normal file
View File

@@ -0,0 +1,25 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
public interface Xow_repo_mgr {
Xof_repo_pair Repos_get_at(int i);
Xof_repo_pair Repos_get_by_wiki(byte[] wiki);
Xof_repo_pair[] Repos_ary();
Xof_repo_itm Get_trg_by_id_or_null(int id, byte[] lnki_ttl, byte[] page_url);
Xof_repo_itm Get_src_by_id_or_null(int id, byte[] lnki_ttl, byte[] page_url);
}

55
400_xowa/src/gplx/xowa/files/repos/Xow_repo_mgr_.java Normal file
View File

@@ -0,0 +1,55 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.apps.fsys.*; import gplx.xowa.files.exts.*;
import gplx.xowa.wikis.domains.*;
public class Xow_repo_mgr_ {
public static void Assert_repos(Xoae_app app, Xowe_wiki wiki) {
synchronized (app) { // LOCK:app-level; DATE:2016-07-07
Xoa_repo_mgr repo_mgr = app.File_mgr().Repo_mgr();
Xoa_fsys_mgr app_fsys_mgr = app.Fsys_mgr();
Xof_rule_mgr ext_rule_mgr = app.File_mgr().Ext_rules();
byte[] domain_bry = wiki.Domain_bry();
Xof_repo_itm repo_itm = repo_mgr.Get_by(domain_bry);
if (repo_itm == null) { // no repo for wiki exists; create it;
repo_itm = new Xof_repo_itm(domain_bry, app_fsys_mgr, ext_rule_mgr, domain_bry);
repo_itm.Root_str_(wiki.Fsys_mgr().Root_dir().Raw()); // NOTE: needed for mass_parse; ordinarily called by xowa.gfs; DATE:2016-07-07
repo_mgr.Add(repo_itm);
}
Xowe_repo_mgr pair_mgr = wiki.File_mgr().Repo_mgr();
if (pair_mgr.Repos_len() == 0) { // no pairs defined; add at least 1
Xof_repo_itm repo_src = repo_mgr.Get_by(File_repo_xowa_null);
if (repo_src == null) {
repo_itm = new Xof_repo_itm(File_repo_xowa_null, app_fsys_mgr, ext_rule_mgr, Xow_domain_tid_.Bry__home);
repo_mgr.Add(repo_itm);
}
pair_mgr.Add_repo(File_repo_xowa_null, domain_bry);
// add commons; needed for mass_parse, else multiple "repo_mgr.invalid_repo" when common files exist in user_cache; DATE:2016-07-07
Xof_repo_itm commons_repo = repo_mgr.Get_by(Xow_domain_itm_.Bry__commons);
if (commons_repo == null) {
commons_repo = new Xof_repo_itm(Xow_domain_itm_.Bry__commons, app_fsys_mgr, ext_rule_mgr, domain_bry);
commons_repo.Root_str_(app.Fsys_mgr().Wiki_dir().GenSubDir(Xow_domain_itm_.Str__commons).Raw()); // NOTE: needed for mass_parse; ordinarily called by xowa.gfs; DATE:2016-07-07
repo_mgr.Add(commons_repo);
}
pair_mgr.Add_repo(Xow_domain_itm_.Bry__commons, Xow_domain_itm_.Bry__commons);
}
}
}
private static byte[] File_repo_xowa_null = Bry_.new_a7("xowa_repo_null");
}

276
400_xowa/src/gplx/xowa/files/repos/Xowe_repo_mgr.java Normal file
View File

@@ -0,0 +1,276 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*;
import gplx.xowa.files.xfers.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.parsers.utils.*;
import gplx.xowa.wikis.tdbs.metas.*;
public class Xowe_repo_mgr implements Xow_repo_mgr, Gfo_invk {
private Xofw_file_finder_rslt tmp_rslt = new Xofw_file_finder_rslt();
private Xowe_wiki wiki; private final List_adp repos = List_adp_.New();
public Xowe_repo_mgr(Xowe_wiki wiki) {
this.wiki = wiki;
Xoae_app app = wiki.Appe();
xfer_mgr = new Xof_xfer_mgr(app.File_mgr(), app.Wmf_mgr());
page_finder = new Xofw_wiki_wkr_base(wiki, app.Wiki_mgr());
}
public Xof_xfer_mgr Xfer_mgr() {return xfer_mgr;} private Xof_xfer_mgr xfer_mgr;
public Xowe_repo_mgr Page_finder_(Xofw_wiki_finder v) {page_finder = v; return this;} private Xofw_wiki_finder page_finder;
public int Repos_len() {return repos.Count();}
public Xof_repo_pair Repos_get_by_wiki(byte[] wiki) {
int len = repos.Count();
for (int i = 0; i < len; i++) {
Xof_repo_pair pair = (Xof_repo_pair)repos.Get_at(i);
if (Bry_.Eq(wiki, pair.Wiki_domain()))
return pair;
}
return null;
}
public void Repos_clear() {repos.Clear();}
public void Clone(Xowe_repo_mgr src) {
this.Repos_clear();
int len = src.Repos_len();
for (int i = 0; i < len; ++i) {
Xof_repo_pair repo_pair = src.Repos_get_at(i);
this.Add_repo(repo_pair.Src().Key(), repo_pair.Trg().Key());
}
}
public Xof_repo_pair Repos_get_at(int i) {return (Xof_repo_pair)repos.Get_at(i);}
private Xof_repo_pair Repos_get_by_id(int id) {
int len = repos.Count();
for (int i = 0; i < len; i++) {
Xof_repo_pair pair = (Xof_repo_pair)repos.Get_at(i);
if (pair.Id() == id) return pair;
}
return null;
}
public Xof_repo_itm Get_trg_by_id_or_null(int id, byte[] lnki_ttl, byte[] page_url) {
Xof_repo_pair pair = Repos_get_by_id(id);
if (pair == null) {
Xoa_app_.Usr_dlg().Warn_many("", "", "repo_mgr.invalid_repo: repo=~{0} lnki_ttl=~{1} page_url=~{2}", id, lnki_ttl, page_url);
return null;
}
else
return pair.Trg();
}
public Xof_repo_itm Get_src_by_id_or_null(int id, byte[] lnki_ttl, byte[] page_url) {
Xof_repo_pair pair = Repos_get_by_id(id);
if (pair == null) {
Xoa_app_.Usr_dlg().Warn_many("", "", "repo_mgr.invalid_repo: repo=~{0} lnki_ttl=~{1} page_url=~{2}", id, lnki_ttl, page_url);
return null;
}
else
return pair.Src();
}
public Xof_repo_itm Get_trg_by_tid_or_null(byte[] tid) {
return null;
}
public Xof_repo_pair[] Repos_ary() {if (repos_ary == null) repos_ary = (Xof_repo_pair[])repos.To_ary(Xof_repo_pair.class); return repos_ary;} private Xof_repo_pair[] repos_ary;
public boolean Xfer_by_meta(Xof_xfer_itm xfer_itm, Xof_xfer_queue queue) {
byte[] ttl = xfer_itm.Lnki_ttl();
Xof_meta_itm meta_itm = xfer_itm.Dbmeta_itm();
boolean chk_all = false;
byte[] src_wiki_key = wiki.Domain_bry();
if (meta_itm.State_new()) {
byte rslt = Xfer_by_meta__find_file(ttl, meta_itm, wiki.Domain_bry());
switch (rslt) {
case Xof_meta_itm.Tid_null: xfer_itm.Orig_repo_id_(0); chk_all = true; break; // NOTE: src_wiki_key becomes wiki.Key_bry() for sake of simplicity
case Xof_meta_itm.Tid_main: xfer_itm.Orig_repo_id_(Xof_meta_itm.Repo_same); break;
case Xof_meta_itm.Tid_ptr: src_wiki_key = Xfer_by_meta__find_main_ptr(meta_itm, xfer_itm); break;
case Xof_meta_itm.Tid_vrtl: src_wiki_key = Xfer_by_meta__find_main_vrtl(meta_itm, xfer_itm); break;
}
}
else {
Xof_repo_itm src_repo = meta_itm.Repo_itm(wiki);
src_wiki_key = src_repo.Wiki_domain();
xfer_itm.Orig_repo_id_(meta_itm.Vrtl_repo()); // NOTE: set trg_repo_idx b/c xfer_mgr will always set meta_itm.Vrtl_repo() with trg_repo_idx
}
if (meta_itm.Ptr_ttl_exists())
xfer_itm.Orig_ttl_and_redirect_(meta_itm.Ttl(), meta_itm.Ptr_ttl());
boolean main_exists_unknown = src_wiki_key == null; // WORKAROUND/HACK: SEE:NOTE_1:reset_main_exists
boolean rv = Xfer_by_meta__exec(chk_all, xfer_itm, meta_itm, src_wiki_key, queue, false);
if (!rv && (!chk_all && !main_exists_unknown)) { // xfer failed even with page found in wiki; try again, but chk all
rv = Xfer_by_meta__exec(true, xfer_itm, meta_itm, src_wiki_key, queue, true);
}
return rv;
}
byte[] Xfer_by_meta__find_main_ptr(Xof_meta_itm meta_itm, Xof_xfer_itm xfer_itm) {
byte[] redirect = meta_itm.Ptr_ttl(); int redirect_tries = 0;
byte[] md5 = Xof_file_wkr_.Md5(redirect);
while (true) {
boolean found = page_finder.Locate(tmp_rslt, repos, redirect);
if (!found) return null;
Xowe_wiki trg_wiki = wiki;
int repo_idx = tmp_rslt.Repo_idx();
byte[] trg_wiki_key = Bry_.Empty;
if (repo_idx != Xof_meta_itm.Repo_unknown) {
trg_wiki_key = wiki.File_mgr().Repo_mgr().Repos_get_at(repo_idx).Wiki_domain();
trg_wiki = wiki.Appe().Wiki_mgr().Get_by_or_make(trg_wiki_key);
}
Xof_meta_itm redirect_meta = trg_wiki.File_mgr().Dbmeta_mgr().Get_itm_or_new(redirect, md5);
if (tmp_rslt.Redirect() == Xop_redirect_mgr.Redirect_null_bry) {
if (redirect_meta.State_new()) {
if (repo_idx == Xof_meta_itm.Repo_unknown) {
// meta_itm.Vrtl_repo_(Xof_meta_itm.Repo_same);
xfer_itm.Orig_repo_id_(Xof_meta_itm.Repo_same);
}
else {
if (!Bry_.Eq(trg_wiki_key, wiki.Domain_bry())) {
// meta_itm.Vrtl_repo_(tmp_rslt.Repo_idx());
xfer_itm.Orig_repo_id_(tmp_rslt.Repo_idx());
}
else {
// meta_itm.Vrtl_repo_(Xof_meta_itm.Repo_same);
xfer_itm.Orig_repo_id_(Xof_meta_itm.Repo_same);
}
}
}
return trg_wiki_key;
}
redirect = tmp_rslt.Redirect();
if (++redirect_tries > Xop_redirect_mgr.Redirect_max_allowed) return null;
}
}
byte[] Xfer_by_meta__find_main_vrtl(Xof_meta_itm meta_itm, Xof_xfer_itm xfer_itm) {
int repo_idx = meta_itm.Vrtl_repo();
if (repo_idx == Xof_meta_itm.Repo_unknown) return null;;
Xof_repo_itm trg_repo = wiki.File_mgr().Repo_mgr().Repos_get_at(repo_idx).Trg();
xfer_itm.Orig_repo_id_(repo_idx);
return trg_repo.Wiki_domain();
}
boolean Xfer_by_meta__exec(boolean chk_all, Xof_xfer_itm xfer_itm, Xof_meta_itm meta_itm, byte[] main_wiki_key, Xof_xfer_queue queue, boolean second_chance) {
int repos_len = repos.Count();
for (int i = 0; i < repos_len; i++) { // iterate over all repo pairs
Xof_repo_pair pair = (Xof_repo_pair)repos.Get_at(i);
Xof_repo_itm pair_src = pair.Src();
boolean main_wiki_key_is_pair_src = Bry_.Eq(main_wiki_key, pair_src.Wiki_domain());
if ( (chk_all && !main_wiki_key_is_pair_src) // only do chk_all if main_wiki is not pair_src; note that chk_all will only be called in two ways (1) with main_wiki_key as null; (2) with main_key_as val
|| (!chk_all && main_wiki_key_is_pair_src)) { // pair.Src.Wiki == main.Wiki; note that there can be multiple pairs with same src; EX: have 2 pairs for commons: one for file and another for http
xfer_mgr.Atrs_by_itm(xfer_itm, pair_src, pair.Trg());
boolean make = xfer_mgr.Make_file(wiki);
if (make) {
xfer_itm.Trg_repo_itm_(pair.Trg());
if (second_chance && xfer_itm.Dbmeta_itm().Vrtl_repo() == 0) // second_chance and item found; change vrtl_repo from commons back to same; EX: tarball and [[Image:Rembrandt De aartsengel verlaat Tobias en zijn gezin. 1637.jpg|120px]]
xfer_itm.Dbmeta_itm().Vrtl_repo_(Xof_meta_itm.Repo_same);
return true; // file was made; return; if not continue looking at other repos
}
}
}
return false;
}
public Xofw_file_finder_rslt Page_finder_locate(byte[] ttl_bry) {page_finder.Locate(tmp_rslt, repos, ttl_bry); return tmp_rslt;}
byte Xfer_by_meta__find_file(byte[] ttl_bry, Xof_meta_itm meta_itm, byte[] cur_wiki_key) {
byte new_tid = Byte_.Max_value_127;
boolean found = page_finder.Locate(tmp_rslt, repos, ttl_bry);
if (found) {
if (Bry_.Eq(cur_wiki_key, tmp_rslt.Repo_wiki_key())) { // itm is in same repo as cur wiki
new_tid = Xof_meta_itm.Tid_main;
byte[] redirect = tmp_rslt.Redirect();
if (redirect != Xop_redirect_mgr.Redirect_null_bry) { // redirect found
meta_itm.Ptr_ttl_(redirect);
}
meta_itm.Vrtl_repo_(Xof_meta_itm.Repo_same);
}
else { // itm is in diff repo
new_tid = Xof_meta_itm.Tid_vrtl;
meta_itm.Vrtl_repo_(tmp_rslt.Repo_idx());
meta_itm.Ptr_ttl_(tmp_rslt.Redirect());
}
}
else { // itm not found; has to be vrtl, but mark repo as unknown
meta_itm.Vrtl_repo_(Xof_meta_itm.Repo_unknown);
}
return new_tid;
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_add)) return Add_repo(m.ReadBry("src"), m.ReadBry("trg"));
else if (ctx.Match(k, Invk_clear)) {repos.Clear(); repos_ary = null;} // reset repos_ary variable
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk_add = "add", Invk_clear = "clear";
public Xof_repo_pair Add_repo(byte[] src_repo_key, byte[] trg_repo_key) {
repos_ary = null; // reset repos_ary variable
Xoa_repo_mgr repo_mgr = wiki.Appe().File_mgr().Repo_mgr();
Xof_repo_itm src_repo = repo_mgr.Get_by(src_repo_key), trg_repo = repo_mgr.Get_by(trg_repo_key);
byte[] src_wiki_key = src_repo.Wiki_domain(), trg_wiki_key = trg_repo.Wiki_domain();
if (!Bry_.Eq(src_wiki_key, trg_wiki_key) && !Bry_.Eq(src_wiki_key, Xow_domain_tid_.Bry__home)) throw Err_.new_wo_type("wiki keys do not match", "src", String_.new_u8(src_wiki_key), "trg", String_.new_u8(trg_wiki_key));
Xof_repo_pair pair = new Xof_repo_pair((byte)repos.Count(), src_wiki_key, src_repo, trg_repo);
repos.Add(pair);
return pair;
}
public boolean Xfer_file(Xof_xfer_itm file) {
int repo_idx = file.Orig_repo_id();
boolean wiki_is_unknown = repo_idx == Xof_meta_itm.Repo_unknown;
boolean make_pass = false;
int len = repos.Count();
if (!wiki_is_unknown) make_pass = Xfer_file_exec(file, this.Repos_get_at(repo_idx), repo_idx);
if (make_pass) return true;
for (int i = 0; i < len; i++) {
Xof_repo_pair pair = (Xof_repo_pair)repos.Get_at(i);
if (i != repo_idx) { // try other wikis
file.Dbmeta_itm().Orig_exists_(Xof_meta_itm.Exists_unknown); // always reset orig exists; this may have been flagged to missing above and should be cleared
make_pass = Xfer_file_exec(file, pair, i);
if (make_pass) break;
}
}
return make_pass;
}
private boolean Xfer_file_exec(Xof_xfer_itm file, Xof_repo_pair pair, int repo_idx) {
xfer_mgr.Atrs_by_itm(file, pair.Src(), pair.Trg());
Xof_meta_itm meta_itm = xfer_mgr.Dbmeta_itm();
boolean rv = xfer_mgr.Make_file(wiki);
if (rv) {
meta_itm.Vrtl_repo_(repo_idx); // update repo_idx to whatever is found
}
return rv;
}
}
/*
NOTE_1:reset_main_exists
this is primarily for Img_missing_wiki_1 and [[Image:Alcott-L.jpg|thumb|right|Louisa May Alcott]]
first some details:
. file pages exist in both en_wiki and in commons_wiki; w:File:Alcott-L.jpg; c:File:Alcott-L.jpg
. images are slightly different (w: is darker than c:)
. c was recently updated to redirect to File:Louisa May Alcott (1881 illustration).jpg
the actual effect:
. [[Image:Alcott-L.jpg|thumb|right|Louisa May Alcott]] in en.wikipedia should pull the w: one (not the c:) one
. this behavior seems contrary to all other wiki behavior where c: is given primacy over w:
.. EX: [[File:Tanks of WWI.ogg|thumb|thumbtime=12|alt=Primitive]] exists in both en_wiki and in commons, but the file only exists in commons
. moreover this page has a special note about the image deliberately remainining in en_wiki
the workaround/hack (described via intended sequence):
1: commons is defined as first repo_pair
2: image found in commons page (note that en_wiki is not checked for PERF reasons)
3: image is searched for in commons, but not found; note that both thumb and orig are missing
4: image will be searched for in en_wiki and found
note that (4) however reuses the same meta which is marked as "not found" from the commons attempt
. item will then have orig marked falsely as not-found
one approach would be to create a new meta and send that into the chk_all function. this turned out to be problematic
so, the workaround:
if going into chk_all
...and orig was previous unknown
...but is now marked no
then reset to unknown to give it a "clean" slate for the chk all
in theory, the above statement seems fine, but it does seem hackish...
*/