Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.10.3.1

Browse Source
This commit is contained in:
gnosygnu
2015-10-18 22:17:57 -04:00
parent 8e18af05b6
commit 4f43f51b18
1935 changed files with 12500 additions and 12889 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
400_xowa/src/gplx/xowa/bldrs/css/Xoa_css_extractor.java
View File

@@ -16,14 +16,14 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.css; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.ios.*; import gplx.xowa.html.*;
import gplx.ios.*; import gplx.xowa.htmls.*;
import gplx.langs.htmls.encoders.*;
import gplx.xowa.nss.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.data.*;
import gplx.xowa.files.downloads.*;
import gplx.core.net.*;
public class Xoa_css_extractor {
public IoEngine_xrg_downloadFil Download_xrg() {return download_xrg;} private IoEngine_xrg_downloadFil download_xrg = Io_mgr.I.DownloadFil_args("", Io_url_.Empty);
public IoEngine_xrg_downloadFil Download_xrg() {return download_xrg;} private IoEngine_xrg_downloadFil download_xrg = Io_mgr.Instance.DownloadFil_args("", Io_url_.Empty);
public Xoa_css_extractor Wiki_domain_(byte[] v) {wiki_domain = v; return this;} private byte[] wiki_domain;
public Xoa_css_extractor Usr_dlg_(Gfo_usr_dlg v) {usr_dlg = v; return this;} private Gfo_usr_dlg usr_dlg;
public Xoa_css_extractor Failover_dir_(Io_url v) {failover_dir = v; return this;} private Io_url failover_dir;
@@ -50,9 +50,9 @@ public class Xoa_css_extractor {
this.wiki_html_dir = wiki.App().Fsys_mgr().Wiki_css_dir(wiki.Domain_str()); // EX: /xowa/user/anonymous/wiki/en.wikipedia.org
Io_url css_comm_fil = wiki_html_dir.GenSubFil(Css_common_name);
Io_url css_wiki_fil = wiki_html_dir.GenSubFil(Css_wiki_name);
wiki.Html__page_wtr_mgr().Init_css_urls(css_comm_fil, css_wiki_fil);
if (wiki.Domain_tid() == Xow_domain_type_.Int__home || Env_.Mode_testing()) return; // NOTE: do not download if home_wiki; also needed for TEST
if (Io_mgr.I.ExistsFil(css_wiki_fil)) return; // css file exists; nothing to generate
wiki.Html__wtr_mgr().Init_css_urls(css_comm_fil, css_wiki_fil);
if (wiki.Domain_tid() == Xow_domain_tid_.Int__home || Env_.Mode_testing()) return; // NOTE: do not download if home_wiki; also needed for TEST
if (Io_mgr.Instance.ExistsFil(css_wiki_fil)) return; // css file exists; nothing to generate
if (wiki.Html__css_installing()) return;
wiki.Html__css_installing_(true);
wiki.App().Usr_dlg().Log_many("", "", "generating css for '~{0}'", wiki.Domain_str());
@@ -95,7 +95,7 @@ public class Xoa_css_extractor {
return false;
}
Xowd_db_file core_db = core_db_mgr.Db__core();
return gplx.xowa.html.css.Xowd_css_core_mgr.Get(core_db.Tbl__css_core(), core_db.Tbl__css_file(), wiki_html_dir, css_key);
return gplx.xowa.htmls.css.Xowd_css_core_mgr.Get(core_db.Tbl__css_core(), core_db.Tbl__css_file(), wiki_html_dir, css_key);
}
public void Css_common_setup() {
if (opt_download_css_common)
@@ -105,7 +105,7 @@ public class Xoa_css_extractor {
}
private void Css_common_failover() {
Io_url trg_fil = wiki_html_dir.GenSubFil(Css_common_name);
Io_mgr.I.CopyFil(Css_common_failover_url(), trg_fil, true);
Io_mgr.Instance.CopyFil(Css_common_failover_url(), trg_fil, true);
css_img_downloader.Chk(wiki_domain, trg_fil);
}
private void Css_common_download() {
@@ -113,19 +113,19 @@ public class Xoa_css_extractor {
Io_url trg_fil = wiki_html_dir.GenSubFil(Css_common_name);
css_stylesheet_common_missing = !Css_scrape_setup();
if (css_stylesheet_common_missing)
Io_mgr.I.CopyFil(Css_common_failover_url(), trg_fil, true);
Io_mgr.Instance.CopyFil(Css_common_failover_url(), trg_fil, true);
else
css_img_downloader.Chk(wiki_domain, trg_fil);
}
private Io_url Css_common_failover_url() {
Io_url css_commons_url = failover_dir.GenSubDir("xowa_common_override").GenSubFil_ary("xowa_common_", String_.new_u8(wiki_code), ".css");
if (Io_mgr.I.ExistsFil(css_commons_url)) return css_commons_url; // specific css exists for wiki; use it; EX: xowa_common_wiki_mediawikiwiki.css
if (Io_mgr.Instance.ExistsFil(css_commons_url)) return css_commons_url; // specific css exists for wiki; use it; EX: xowa_common_wiki_mediawikiwiki.css
return failover_dir.GenSubFil(lang_is_ltr ? Css_common_name_ltr : Css_common_name_rtl);
}
public void Css_wiki_setup() {
boolean css_stylesheet_wiki_missing = true;
Io_url trg_fil = wiki_html_dir.GenSubFil(Css_wiki_name);
if (Io_mgr.I.ExistsFil(trg_fil)) return; // don't download if already there
if (Io_mgr.Instance.ExistsFil(trg_fil)) return; // don't download if already there
css_stylesheet_wiki_missing = !Css_wiki_generate(trg_fil);
if (css_stylesheet_wiki_missing)
Failover(trg_fil);
@@ -136,9 +136,9 @@ public class Xoa_css_extractor {
Bry_bfr bfr = Bry_bfr.new_();
Css_wiki_generate_section(bfr, Ttl_common_css);
Css_wiki_generate_section(bfr, Ttl_vector_css);
byte[] bry = bfr.Xto_bry_and_clear();
byte[] bry = bfr.To_bry_and_clear();
bry = Bry_.Replace(bry, gplx.xowa.bldrs.xmls.Xob_xml_parser_.Bry_tab_ent, gplx.xowa.bldrs.xmls.Xob_xml_parser_.Bry_tab);
Io_mgr.I.SaveFilBry(trg_fil, bry);
Io_mgr.Instance.SaveFilBry(trg_fil, bry);
return true;
} private static final byte[] Ttl_common_css = Bry_.new_a7("Common.css"), Ttl_vector_css = Bry_.new_a7("Vector.css");
private boolean Css_wiki_generate_section(Bry_bfr bfr, byte[] ttl) {
@@ -152,7 +152,7 @@ public class Xoa_css_extractor {
public void Logo_setup() {
boolean logo_missing = true;
Io_url logo_url = wiki_html_dir.GenSubFil("logo.png");
if (Io_mgr.I.ExistsFil(logo_url)) return; // don't download if already there
if (Io_mgr.Instance.ExistsFil(logo_url)) return; // don't download if already there
logo_missing = !Logo_download(logo_url);
if (logo_missing)
Failover(logo_url);
@@ -172,7 +172,7 @@ public class Xoa_css_extractor {
}
private boolean Logo_copy_from_css(Io_url trg_fil) {
Io_url commons_file = wiki_html_dir.GenSubFil(Css_common_name);
byte[] commons_src = Io_mgr.I.LoadFilBry(commons_file);
byte[] commons_src = Io_mgr.Instance.LoadFilBry(commons_file);
int bgn_pos = Bry_find_.Find_fwd(commons_src, Bry_mw_wiki_logo); if (bgn_pos == Bry_find_.Not_found) return false;
bgn_pos += Bry_mw_wiki_logo.length;
int end_pos = Bry_find_.Find_fwd(commons_src, Byte_ascii.Quote, bgn_pos + 1); if (end_pos == Bry_find_.Not_found) return false;
@@ -181,7 +181,7 @@ public class Xoa_css_extractor {
if (wiki_html_dir.Info().DirSpr_byte() == Byte_ascii.Backslash)
src_bry = Bry_.Replace(src_bry, Byte_ascii.Slash, Byte_ascii.Backslash);
Io_url src_fil = wiki_html_dir.GenSubFil(String_.new_u8(src_bry));
Io_mgr.I.CopyFil(src_fil, trg_fil, true);
Io_mgr.Instance.CopyFil(src_fil, trg_fil, true);
return true;
} private static final byte[] Bry_mw_wiki_logo = Bry_.new_a7(".mw-wiki-logo{background-image:url(\"");
private String Logo_find_src() {
@@ -211,18 +211,18 @@ public class Xoa_css_extractor {
}
private void Failover(Io_url trg_fil) {
usr_dlg.Note_many("", "", "copying failover file: trg_fil=~{0};", trg_fil.Raw());
Io_mgr.I.CopyFil(failover_dir.GenSubFil(trg_fil.NameAndExt()), trg_fil, true);
Io_mgr.Instance.CopyFil(failover_dir.GenSubFil(trg_fil.NameAndExt()), trg_fil, true);
}
public boolean Css_scrape_setup() {
Io_url trg_fil = wiki_html_dir.GenSubFil(Css_common_name);
// if (Io_mgr.I.ExistsFil(trg_fil)) return; // don't download if already there; DELETED: else main_page is not scraped for all stylesheet links; simple.d: fails; DATE:2014-02-11
// if (Io_mgr.Instance.ExistsFil(trg_fil)) return; // don't download if already there; DELETED: else main_page is not scraped for all stylesheet links; simple.d: fails; DATE:2014-02-11
byte[] css_url = Css_scrape();
if (css_url == null) {
Css_common_failover();
return false;
}
else {
Io_mgr.I.SaveFilBry(trg_fil, css_url);
Io_mgr.Instance.SaveFilBry(trg_fil, css_url);
css_img_downloader.Chk(wiki_domain, trg_fil);
return true;
}
@@ -267,14 +267,14 @@ public class Xoa_css_extractor {
tmp_bfr.Add(Xoa_css_img_downloader.Bry_comment_bgn).Add_str(css_url).Add(Xoa_css_img_downloader.Bry_comment_end).Add_byte_nl();
tmp_bfr.Add(css_bry).Add_byte_nl().Add_byte_nl();
}
return tmp_bfr.Xto_bry_and_clear();
return tmp_bfr.To_bry_and_clear();
}
private static byte[] Url_root_fix(byte[] domain, byte[] url) {// DATE:2015-09-20
if (url.length < 3) return url; // need at least 2 chars
if ( url[0] == Byte_ascii.Slash // starts with "/" EX: "/w/api.php"
&& url[1] != Byte_ascii.Slash // but not "//"; EX: "//en.wikipedia.org"
)
return Bry_.Add(gplx.xowa.html.hrefs.Xoh_href_.Bry__https, domain, url);
return Bry_.Add(gplx.xowa.htmls.hrefs.Xoh_href_.Bry__https, domain, url);
else
return url;
}

6
400_xowa/src/gplx/xowa/bldrs/css/Xoa_css_extractor_basic_tst.java
View File

@@ -78,7 +78,7 @@ public class Xoa_css_extractor_basic_tst {
}
class Xoa_css_extractor_fxt {
public void Clear() {
Io_mgr.I.InitEngine_mem();
Io_mgr.Instance.InitEngine_mem();
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Test();
css_installer = new Xoa_css_extractor();
css_installer.Download_xrg().Trg_engine_key_(IoEngine_.MemKey);
@@ -101,8 +101,8 @@ class Xoa_css_extractor_fxt {
page_fetcher.Add(ns_id, Bry_.new_a7(ttl), Bry_.new_a7(text));
}
public void Init_fil_empty(String url) {Init_fil(url, "");}
public void Init_fil(String url, String text) {Io_mgr.I.SaveFilStr(url, text);}
public void Test_fil(String url, String expd) {Tfds.Eq_str_lines(expd, Io_mgr.I.LoadFilStr(Io_url_.new_any_(url)));}
public void Init_fil(String url, String text) {Io_mgr.Instance.SaveFilStr(url, text);}
public void Test_fil(String url, String expd) {Tfds.Eq_str_lines(expd, Io_mgr.Instance.LoadFilStr(Io_url_.new_any_(url)));}
public void Exec_logo_setup() {
css_installer.Mainpage_download();
css_installer.Logo_setup();

2
400_xowa/src/gplx/xowa/bldrs/css/Xoa_css_extractor_wiki_tst.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.css; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*; import gplx.ios.*; import gplx.xowa.nss.*;
import org.junit.*; import gplx.ios.*; import gplx.xowa.wikis.nss.*;
public class Xoa_css_extractor_wiki_tst {
@Before public void init() {fxt.Clear();} private Xoa_css_extractor_fxt fxt = new Xoa_css_extractor_fxt();
@Test public void Css_wiki_generate() {

10
400_xowa/src/gplx/xowa/bldrs/css/Xoa_css_img_downloader.java
View File

@@ -27,12 +27,12 @@ public class Xoa_css_img_downloader {
public void Chk(byte[] wiki_domain, Io_url css_fil) {
this.wiki_domain = wiki_domain;
List_adp img_list = List_adp_.new_();
byte[] old_bry = Io_mgr.I.LoadFilBry(css_fil);
byte[] old_bry = Io_mgr.Instance.LoadFilBry(css_fil);
byte[] rel_url_prefix = Bry_.Add(Bry_fwd_slashes, wiki_domain);
byte[] new_bry = Convert_to_local_urls(rel_url_prefix, old_bry, img_list);
Io_url img_dir = css_fil.OwnerDir();
Download_fils(img_dir, img_list.To_str_ary());
Io_mgr.I.SaveFilBry(css_fil, new_bry);
Io_mgr.Instance.SaveFilBry(css_fil, new_bry);
}
public byte[] Convert_to_local_urls(byte[] rel_url_prefix, byte[] src, List_adp list) {
try {
@@ -96,7 +96,7 @@ public class Xoa_css_img_downloader {
if (!quoted) bfr.Add_byte(Byte_ascii.Quote);
prv_pos = end_pos;
}
return bfr.Xto_bry_and_clear();
return bfr.To_bry_and_clear();
}
catch (Exception e) {
usr_dlg.Warn_many("", "", "failed to convert local_urls: ~{0} ~{1}", String_.new_u8(rel_url_prefix), Err_.Message_gplx_full(e));
@@ -157,9 +157,9 @@ public class Xoa_css_img_downloader {
for (int i = 0; i < ary_len; i++) {
String src = ary[i];
Io_url trg = css_dir.GenSubFil_nest(Op_sys.Cur().Fsys_http_frag_to_url_str(Replace_invalid_chars_str(src)));
if (Io_mgr.I.ExistsFil(trg)) continue;
if (Io_mgr.Instance.ExistsFil(trg)) continue;
download_wkr.Download(true, "https://" + src, trg, "download: " + src); // ILN
if (Io_mgr.I.QueryFil(trg).Size() == 0) { // warn if 0 byte files downloaded; DATE:2015-07-06
if (Io_mgr.Instance.QueryFil(trg).Size() == 0) { // warn if 0 byte files downloaded; DATE:2015-07-06
Xoa_app_.Usr_dlg().Warn_many("", "", "css.download; 0 byte file downloaded; file=~{0}", trg.Raw());
}
}

20
400_xowa/src/gplx/xowa/bldrs/css/Xoa_css_img_downloader_tst.java
View File

@@ -82,8 +82,8 @@ public class Xoa_css_img_downloader_tst {
@Test public void Clean_exc_site_only() {fxt.Test_clean_img_url("//site" , null);}
@Test public void Clean_exc_site_only_2() {fxt.Test_clean_img_url("//site/" , null);}
@Test public void Import_url() {
Io_mgr.I.InitEngine_mem();
Io_mgr.I.SaveFilStr("mem/www/b.css", "imported_css");
Io_mgr.Instance.InitEngine_mem();
Io_mgr.Instance.SaveFilStr("mem/www/b.css", "imported_css");
fxt.Test_css_convert
( "x @import url(\"mem/www/b.css\") screen; z"
, String_.Concat_lines_nl
@@ -101,8 +101,8 @@ public class Xoa_css_img_downloader_tst {
fxt.Test_import_url("https://a.org" , "https://a.org"); // unless starts with https EX:: handle @import(https://...); PAGE:tr.n:Main_Page; DATE:2014-06-04
}
@Test public void Import_url_relative() { // PURPOSE: if directory, add domain; "/a/b.css" -> "//domain/a/b.css"; DATE:2014-02-03
Io_mgr.I.InitEngine_mem();
Io_mgr.I.SaveFilStr("mem/en.wikipedia.org/www/b.css", "imported_css");
Io_mgr.Instance.InitEngine_mem();
Io_mgr.Instance.SaveFilStr("mem/en.wikipedia.org/www/b.css", "imported_css");
fxt.Test_css_convert
( "x @import url(\"/www/b.css\") screen; z" // starts with "/"
, String_.Concat_lines_nl
@@ -116,8 +116,8 @@ public class Xoa_css_img_downloader_tst {
}
@Test public void Import_url_relative_skip() { // PURPOSE: if rel path, skip; "//site/a/b.css"; DATE:2014-02-03
fxt.Downloader().Stylesheet_prefix_(Bry_.new_a7("mem")); // stylesheet prefix prefix defaults to ""; set to "mem", else test will try to retrieve "//url" which will fail
Io_mgr.I.InitEngine_mem();
Io_mgr.I.SaveFilStr("mem//en.wikipedia.org/a/b.css", "imported_css");
Io_mgr.Instance.InitEngine_mem();
Io_mgr.Instance.SaveFilStr("mem//en.wikipedia.org/a/b.css", "imported_css");
fxt.Test_css_convert
( "x @import url(\"//en.wikipedia.org/a/b.css\") screen; z" // starts with "//"
, String_.Concat_lines_nl
@@ -130,8 +130,8 @@ public class Xoa_css_img_downloader_tst {
);
}
@Test public void Import_url_space() { // PURPOSE: some css has spaces; replace with underlines else fails when downloaded; EX: https://it.wikivoyage.org/w/index.php?title=MediaWiki:Container e Infobox.css&action=raw&ctype=text/css; DATE:2015-03-08
Io_mgr.I.InitEngine_mem();
Io_mgr.I.SaveFilStr("mem/www/b_c.css", "imported_css");
Io_mgr.Instance.InitEngine_mem();
Io_mgr.Instance.SaveFilStr("mem/www/b_c.css", "imported_css");
fxt.Test_css_convert
( "x @import url(\"mem/www/b c.css\") screen; z"
, String_.Concat_lines_nl
@@ -145,8 +145,8 @@ public class Xoa_css_img_downloader_tst {
}
@Test public void Wikisource_freedimg() { // PURPOSE: check that "wikimedia" is replaced for FreedImg hack; PAGE:en.s:Page:Notes_on_Osteology_of_Baptanodon._With_a_Description_of_a_New_Species.pdf/3 DATE:2014-09-06
fxt.Downloader().Stylesheet_prefix_(Bry_.new_a7("mem")); // stylesheet prefix prefix defaults to ""; set to "mem", else test will try to retrieve "//url" which will fail
Io_mgr.I.InitEngine_mem();
Io_mgr.I.SaveFilStr("mem//en.wikisource.org/w/index.php?title=MediaWiki:Dynimg.css", ".freedImg img[src*=\"wikipedia\"], .freedImg img[src*=\"wikisource\"], .freedImg img[src*=\"score\"], .freedImg img[src*=\"math\"] {");
Io_mgr.Instance.InitEngine_mem();
Io_mgr.Instance.SaveFilStr("mem//en.wikisource.org/w/index.php?title=MediaWiki:Dynimg.css", ".freedImg img[src*=\"wikipedia\"], .freedImg img[src*=\"wikisource\"], .freedImg img[src*=\"score\"], .freedImg img[src*=\"math\"] {");
fxt.Test_css_convert
( "x @import url(\"//en.wikisource.org/w/index.php?title=MediaWiki:Dynimg.css\") screen; z" // starts with "//"
, String_.Concat_lines_nl

2
400_xowa/src/gplx/xowa/bldrs/css/Xob_css_parser__url_tst.java
View File

@@ -39,7 +39,7 @@ class Xob_css_parser__url_fxt {
this.src_bry = Bry_.new_u8(src_str);
this.Exec_parse_hook();
cur_frag.Write(bfr, src_bry);
String actl_str = bfr.Xto_str_and_clear();
String actl_str = bfr.To_str_and_clear();
Tfds.Eq(expd_tid, cur_frag.Tid(), "wrong tid; expd={0}, actl={1}", expd_tid, cur_frag.Tid());
Tfds.Eq(expd_str, actl_str);
}

6
400_xowa/src/gplx/xowa/bldrs/css/Xob_mirror_mgr.java
View File

@@ -31,8 +31,8 @@ public class Xob_mirror_mgr {
byte[] trg_url = Xob_css_tkn__url.To_fsys(src_url);
code_hash.Add_if_dupe_use_1st(src_url, new Xobc_download_itm(Xobc_download_itm.Tid_css, String_.new_u8(src_url), trg_url));
}
public Ordered_hash Code_hash() {return code_hash;} private final Ordered_hash code_hash = Ordered_hash_.new_();
public Ordered_hash File_hash() {return file_hash;} private final Ordered_hash file_hash = Ordered_hash_.new_();
public Ordered_hash Code_hash() {return code_hash;} private final Ordered_hash code_hash = Ordered_hash_.New();
public Ordered_hash File_hash() {return file_hash;} private final Ordered_hash file_hash = Ordered_hash_.New();
public void Exec() {
usr_dlg.Plog_many("", "", "html_mirror:download.root_page; url=~{0}", page_url);
IoEngine_xrg_downloadFil download_xrg = download_wkr.Download_xrg();
@@ -44,7 +44,7 @@ public class Xob_mirror_mgr {
for (int i = 0; i < code_ary_len; ++i) {
Xobc_download_itm code = code_ary[i];
byte[] code_src = download_xrg.Exec_as_bry(code.Http_str());
Io_mgr.I.SaveFilBry(fsys_root.Gen_sub_path_for_os(String_.new_u8(code.Fsys_url())), code_src);
Io_mgr.Instance.SaveFilBry(fsys_root.Gen_sub_path_for_os(String_.new_u8(code.Fsys_url())), code_src);
css_parser.Parse(code_src);
}
}

6
400_xowa/src/gplx/xowa/bldrs/css/Xob_mirror_mgr_tst.java
View File

@@ -42,7 +42,7 @@ class Xob_mirror_mgr_fxt {
}
class Io_fsys_fxt {
public void Clear() {
Io_mgr.I.InitEngine_mem();
Io_mgr.Instance.InitEngine_mem();
}
public void Init_fil(String url_str) {
Io_url url = Io_url_.new_fil_(url_str);
@@ -50,7 +50,7 @@ class Io_fsys_fxt {
}
public void Init_fil(String url_str, String text) {Init_fil(Io_url_.new_fil_(url_str), text);}
public void Init_fil(Io_url url, String text) {
Io_mgr.I.SaveFilStr(url, text);
Io_mgr.Instance.SaveFilStr(url, text);
}
public void Test_fil(String url_str) {
Io_url url = Io_url_.new_fil_(url_str);
@@ -58,6 +58,6 @@ class Io_fsys_fxt {
}
public void Test_fil(String url, String expd) {Test_fil(Io_url_.new_fil_(url), expd);}
public void Test_fil(Io_url url, String expd) {
Tfds.Eq_str_lines(expd, Io_mgr.I.LoadFilStr(url));
Tfds.Eq_str_lines(expd, Io_mgr.Instance.LoadFilStr(url));
}
}