mirror of
https://github.com/gnosygnu/xowa.git
synced 2026-03-02 03:49:30 +00:00
v1.6.5.1
This commit is contained in:
232
400_xowa/src_240_install/gplx/xowa/Xoa_css_extractor.java
Normal file
232
400_xowa/src_240_install/gplx/xowa/Xoa_css_extractor.java
Normal file
@@ -0,0 +1,232 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import gplx.ios.*; import gplx.xowa.wikis.*; import gplx.xowa.html.*;
|
||||
public class Xoa_css_extractor {
|
||||
public IoEngine_xrg_downloadFil Download_xrg() {return download_xrg;} private IoEngine_xrg_downloadFil download_xrg = Io_mgr._.DownloadFil_args("", Io_url_.Null);
|
||||
public Xoa_css_extractor Wiki_domain_(byte[] v) {wiki_domain = v; return this;} private byte[] wiki_domain;
|
||||
public Xoa_css_extractor Usr_dlg_(Gfo_usr_dlg v) {usr_dlg = v; return this;} private Gfo_usr_dlg usr_dlg;
|
||||
public Xoa_css_extractor Failover_dir_(Io_url v) {failover_dir = v; return this;} private Io_url failover_dir;
|
||||
public Xoa_css_extractor Wiki_html_dir_(Io_url v) {wiki_html_dir = v; return this;} private Io_url wiki_html_dir;
|
||||
public Xoa_css_extractor Mainpage_url_(String v) {mainpage_url = v; return this;} private String mainpage_url;
|
||||
public Xoa_css_extractor Protocol_prefix_(String v) {protocol_prefix = v; return this;} private String protocol_prefix = "http:";
|
||||
public Xoa_css_extractor Page_fetcher_(Xow_page_fetcher v) {page_fetcher = v; return this;} private Xow_page_fetcher page_fetcher;
|
||||
public Xoa_css_extractor Css_img_downloader_(Xoa_css_img_downloader v) {this.css_img_downloader = v; return this;} private Xoa_css_img_downloader css_img_downloader;
|
||||
public Xoa_css_extractor Opt_download_css_common_(boolean v) {opt_download_css_common = v; return this;} private boolean opt_download_css_common;
|
||||
public Xoa_css_extractor Url_encoder_(Url_encoder v) {url_encoder = v; return this;} private Url_encoder url_encoder;
|
||||
public Xoa_css_extractor Wiki_code_(byte[] v) {this.wiki_code = v; return this;} private byte[] wiki_code = null;
|
||||
private byte[] mainpage_html; private boolean lang_is_ltr = true;
|
||||
public void Init_by_app(Xoa_app app) {
|
||||
this.usr_dlg = app.Usr_dlg();
|
||||
Xof_download_wkr download_wkr = app.File_mgr().Download_mgr().Download_wkr();
|
||||
this.download_xrg = download_wkr.Download_xrg();
|
||||
css_img_downloader = new Xoa_css_img_downloader().Ctor(usr_dlg, download_wkr, Bry_.new_utf8_(protocol_prefix));
|
||||
failover_dir = app.Fsys_mgr().Bin_any_dir().GenSubDir_nest("html", "xowa", "import");
|
||||
url_encoder = app.Url_converter_url();
|
||||
}
|
||||
public void Install_assert(Xow_wiki wiki, Io_url wiki_html_dir) {
|
||||
try {
|
||||
Io_url css_common_url = wiki_html_dir.GenSubFil(Css_common_name);
|
||||
Io_url css_wiki_url = wiki_html_dir.GenSubFil(Css_wiki_name);
|
||||
Xoh_page_wtr_mgr wiki_article = wiki.Html_mgr().Page_wtr_mgr();
|
||||
wiki_article.Css_common_bry_(css_common_url).Css_wiki_bry_(css_wiki_url);
|
||||
if (wiki.Domain_tid() == Xow_wiki_domain_.Tid_home || Env_.Mode_testing()) return; // NOTE: do not download if home_wiki; also needed for TEST
|
||||
if (Io_mgr._.ExistsFil(css_wiki_url)) return; // css file exists; nothing to generate
|
||||
wiki.App().Usr_dlg().Log_many("", "", "generating css for '~{0}'", wiki.Domain_str());
|
||||
this.Install(wiki, wiki_html_dir);
|
||||
}
|
||||
catch (Exception e) { // if error, failover; paranoia catch for outliers like bad network connectivity fail, or MediaWiki: message not existing; DATE:2013-11-21
|
||||
wiki.App().Usr_dlg().Warn_many("", "", "failed while trying to generate css; failing over; wiki='~{0}' err=~{1}", wiki.Domain_str(), Err_.Message_gplx(e));
|
||||
Css_common_failover(); // only failover xowa_common.css; xowa_wiki.css comes from MediaWiki:Common.css / Vector.css
|
||||
}
|
||||
}
|
||||
public void Install(Xow_wiki wiki, Io_url wiki_html_dir) {
|
||||
opt_download_css_common = wiki.App().Setup_mgr().Dump_mgr().Css_commons_download();
|
||||
if (!wiki.App().User().Cfg_mgr().Security_mgr().Web_access_enabled()) opt_download_css_common = false; // if !web_access_enabled, don't download
|
||||
this.wiki_domain = wiki.Domain_bry();
|
||||
mainpage_url = "http://" + wiki.Domain_str(); // NOTE: cannot reuse protocol_prefix b/c "//" needs to be added manually; protocol_prefix is used for logo and images which have form of "//domain/image.png"
|
||||
if (page_fetcher == null) page_fetcher = new Xow_page_fetcher_wiki();
|
||||
page_fetcher.Wiki_(wiki);
|
||||
this.wiki_html_dir = wiki_html_dir;
|
||||
this.lang_is_ltr = wiki.Lang().Dir_ltr();
|
||||
this.wiki_code = wiki.Domain_abrv();
|
||||
mainpage_html = Mainpage_download_html();
|
||||
Logo_setup();
|
||||
Css_common_setup();
|
||||
Css_wiki_setup();
|
||||
}
|
||||
public void Css_common_setup() {
|
||||
if (opt_download_css_common)
|
||||
Css_common_download();
|
||||
else
|
||||
Css_common_failover();
|
||||
}
|
||||
private void Css_common_failover() {
|
||||
Io_url trg_fil = wiki_html_dir.GenSubFil(Css_common_name);
|
||||
Io_mgr._.CopyFil(Css_common_failover_url(), trg_fil, true);
|
||||
css_img_downloader.Chk(wiki_domain, trg_fil);
|
||||
}
|
||||
private void Css_common_download() {
|
||||
boolean css_stylesheet_common_missing = true;
|
||||
Io_url trg_fil = wiki_html_dir.GenSubFil(Css_common_name);
|
||||
css_stylesheet_common_missing = !Css_scrape_setup();
|
||||
if (css_stylesheet_common_missing)
|
||||
Io_mgr._.CopyFil(Css_common_failover_url(), trg_fil, true);
|
||||
else
|
||||
css_img_downloader.Chk(wiki_domain, trg_fil);
|
||||
}
|
||||
private Io_url Css_common_failover_url() {
|
||||
Io_url css_commons_url = failover_dir.GenSubDir("xowa_common_override").GenSubFil_ary("xowa_common_", String_.new_utf8_(wiki_code), ".css");
|
||||
if (Io_mgr._.ExistsFil(css_commons_url)) return css_commons_url; // specific css exists for wiki; use it; EX: xowa_common_wiki_mediawikiwiki.css
|
||||
return failover_dir.GenSubFil(lang_is_ltr ? Css_common_name_ltr : Css_common_name_rtl);
|
||||
}
|
||||
// private boolean Css_common_download(Io_url trg_fil) { // DELETE: replaced with Css_scrape; DATE:2014-02-11
|
||||
// String src_fil = String_.Format(css_stylesheet_common_src_fmt, String_.new_utf8_(wiki_domain));
|
||||
// String log_msg = usr_dlg.Prog_many("", "", "downloading css common: '~{0}'", src_fil);
|
||||
// boolean rv = download_xrg.Prog_fmt_hdr_(log_msg).Src_(src_fil).Trg_(trg_fil).Exec();
|
||||
// if (!rv)
|
||||
// usr_dlg.Warn_many("", "", "failed to download css_common: src_url=~{0};", src_fil);
|
||||
// return rv;
|
||||
// }
|
||||
public void Css_wiki_setup() {
|
||||
boolean css_stylesheet_wiki_missing = true;
|
||||
Io_url trg_fil = wiki_html_dir.GenSubFil(Css_wiki_name);
|
||||
if (Io_mgr._.ExistsFil(trg_fil)) return; // don't download if already there
|
||||
css_stylesheet_wiki_missing = !Css_wiki_generate(trg_fil);
|
||||
if (css_stylesheet_wiki_missing)
|
||||
Failover(trg_fil);
|
||||
else
|
||||
css_img_downloader.Chk(wiki_domain, trg_fil);
|
||||
}
|
||||
private boolean Css_wiki_generate(Io_url trg_fil) {
|
||||
Bry_bfr bfr = Bry_bfr.new_();
|
||||
Css_wiki_generate_section(bfr, Ttl_common_css);
|
||||
Css_wiki_generate_section(bfr, Ttl_vector_css);
|
||||
byte[] bry = bfr.XtoAryAndClear();
|
||||
bry = Bry_.Replace(bry, gplx.xowa.bldrs.xmls.Xob_xml_parser_.Bry_tab_ent, gplx.xowa.bldrs.xmls.Xob_xml_parser_.Bry_tab);
|
||||
Io_mgr._.SaveFilBry(trg_fil, bry);
|
||||
return true;
|
||||
} private static final byte[] Ttl_common_css = Bry_.new_ascii_("Common.css"), Ttl_vector_css = Bry_.new_ascii_("Vector.css");
|
||||
private boolean Css_wiki_generate_section(Bry_bfr bfr, byte[] ttl) {
|
||||
byte[] page = page_fetcher.Fetch(Xow_ns_.Id_mediaWiki, ttl);
|
||||
if (page == null) return false;
|
||||
if (bfr.Len() != 0) bfr.Add_byte_nl().Add_byte_nl(); // add "\n\n" between sections; !=0 checks against first
|
||||
Css_wiki_section_hdr.Bld_bfr_many(bfr, ttl); // add "/*XOWA:MediaWiki:Common.css*/\n"
|
||||
bfr.Add(page); // add page
|
||||
return true;
|
||||
} static final Bry_fmtr Css_wiki_section_hdr = Bry_fmtr.new_("/*XOWA:MediaWiki:~{ttl}*/\n", "ttl");
|
||||
public void Logo_setup() {
|
||||
boolean logo_missing = true;
|
||||
Io_url logo_url = wiki_html_dir.GenSubFil("logo.png");
|
||||
if (Io_mgr._.ExistsFil(logo_url)) return; // don't download if already there
|
||||
logo_missing = !Logo_download(logo_url);
|
||||
if (logo_missing)
|
||||
Failover(logo_url);
|
||||
}
|
||||
private boolean Logo_download(Io_url trg_fil) {
|
||||
String src_fil = Logo_find_src();
|
||||
if (src_fil == null) {
|
||||
usr_dlg.Warn_many("", "", "failed to extract logo: trg_fil=~{0};", trg_fil.Raw());
|
||||
return false;
|
||||
}
|
||||
String log_msg = usr_dlg.Prog_many("", "", "downloading logo: '~{0}'", src_fil);
|
||||
boolean rv = download_xrg.Prog_fmt_hdr_(log_msg).Src_(src_fil).Trg_(trg_fil).Exec();
|
||||
if (!rv)
|
||||
usr_dlg.Warn_many("", "", "failed to download logo: src_url=~{0};", src_fil);
|
||||
return rv;
|
||||
}
|
||||
private String Logo_find_src() {
|
||||
if (mainpage_html == null) return null;
|
||||
int main_page_html_len = mainpage_html.length;
|
||||
int logo_bgn = Bry_finder.Find_fwd(mainpage_html, Logo_find_bgn, 0); if (logo_bgn == Bry_.NotFound) return null;
|
||||
logo_bgn += Logo_find_bgn.length;
|
||||
logo_bgn = Bry_finder.Find_fwd(mainpage_html, Logo_find_end, logo_bgn); if (logo_bgn == Bry_.NotFound) return null;
|
||||
logo_bgn += Logo_find_end.length;
|
||||
int logo_end = Bry_finder.Find_fwd(mainpage_html, Byte_ascii.Paren_end, logo_bgn, main_page_html_len); if (logo_bgn == Bry_.NotFound) return null;
|
||||
byte[] logo_bry = Bry_.Mid(mainpage_html, logo_bgn, logo_end);
|
||||
return protocol_prefix + String_.new_utf8_(logo_bry);
|
||||
}
|
||||
private static final byte[] Logo_find_bgn = Bry_.new_ascii_("<div id=\"p-logo\""), Logo_find_end = Bry_.new_ascii_("background-image: url(");
|
||||
public boolean Mainpage_download() {
|
||||
mainpage_html = Mainpage_download_html();
|
||||
return mainpage_html != null;
|
||||
}
|
||||
private byte[] Mainpage_download_html() {
|
||||
String log_msg = usr_dlg.Prog_many("", "main_page.download", "downloading main page for '~{0}'", wiki_domain);
|
||||
byte[] main_page_html = download_xrg.Prog_fmt_hdr_(log_msg).Exec_as_bry(mainpage_url);
|
||||
if (main_page_html == null) usr_dlg.Warn_many("", "", "failed to download main_page: src_url=~{0};", mainpage_url);
|
||||
return main_page_html;
|
||||
}
|
||||
private void Failover(Io_url trg_fil) {
|
||||
usr_dlg.Note_many("", "", "copying failover file: trg_fil=~{0};", trg_fil.Raw());
|
||||
Io_mgr._.CopyFil(failover_dir.GenSubFil(trg_fil.NameAndExt()), trg_fil, true);
|
||||
}
|
||||
public boolean Css_scrape_setup() {
|
||||
Io_url trg_fil = wiki_html_dir.GenSubFil(Css_common_name);
|
||||
// if (Io_mgr._.ExistsFil(trg_fil)) return; // don't download if already there; DELETED: else main_page is not scraped for all stylesheet links; simple.d: fails; DATE:2014-02-11
|
||||
byte[] css_url = Css_scrape();
|
||||
if (css_url == null) {
|
||||
Css_common_failover();
|
||||
return false;
|
||||
}
|
||||
else {
|
||||
Io_mgr._.SaveFilBry(trg_fil, css_url);
|
||||
css_img_downloader.Chk(wiki_domain, trg_fil);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
private byte[] Css_scrape() {
|
||||
if (mainpage_html == null) return null;
|
||||
String[] css_urls = Css_scrape_urls(mainpage_html); if (css_urls.length == 0) return null;
|
||||
return Css_scrape_download(css_urls);
|
||||
}
|
||||
private String[] Css_scrape_urls(byte[] raw) {
|
||||
ListAdp rv = ListAdp_.new_();
|
||||
int raw_len = raw.length;
|
||||
int prv_pos = 0;
|
||||
int css_find_bgn_len = Css_find_bgn.length;
|
||||
byte[] protocol_prefix_bry = Bry_.new_utf8_(protocol_prefix);
|
||||
while (true) {
|
||||
int url_bgn = Bry_finder.Find_fwd(raw, Css_find_bgn, prv_pos); if (url_bgn == Bry_.NotFound) break; // nothing left; stop
|
||||
url_bgn += css_find_bgn_len;
|
||||
int url_end = Bry_finder.Find_fwd(raw, Byte_ascii.Quote, url_bgn, raw_len); if (url_end == Bry_.NotFound) {usr_dlg.Warn_many("", "main_page.css_parse", "could not find css; pos='~{0}' text='~{1}'", url_bgn, String_.new_utf8_len_safe_(raw, url_bgn, url_bgn + 32)); break;}
|
||||
byte[] css_url_bry = Bry_.Mid(raw, url_bgn, url_end);
|
||||
css_url_bry = Bry_.Replace(css_url_bry, Css_amp_find, Css_amp_repl); // & -> &
|
||||
css_url_bry = url_encoder.Decode(css_url_bry); // %2C -> %7C -> |
|
||||
css_url_bry = Bry_.Add(protocol_prefix_bry, css_url_bry);
|
||||
rv.Add(String_.new_utf8_(css_url_bry));
|
||||
prv_pos = url_end;
|
||||
}
|
||||
return rv.XtoStrAry();
|
||||
} private static final byte[] Css_find_bgn = Bry_.new_ascii_("<link rel=\"stylesheet\" href=\""), Css_amp_find = Bry_.new_ascii_("&"), Css_amp_repl = Bry_.new_ascii_("&");
|
||||
private byte[] Css_scrape_download(String[] css_urls) {
|
||||
int css_urls_len = css_urls.length;
|
||||
Bry_bfr tmp_bfr = Bry_bfr.new_();
|
||||
for (int i = 0; i < css_urls_len; i++) {
|
||||
String css_url = css_urls[i];
|
||||
usr_dlg.Prog_many("", "main_page.css_download", "downloading css for '~{0}'", css_url);
|
||||
download_xrg.Prog_fmt_hdr_(css_url);
|
||||
byte[] css_bry = download_xrg.Exec_as_bry(css_url); if (css_bry == null) continue; // css not found; continue
|
||||
tmp_bfr.Add(Xoa_css_img_downloader.Bry_comment_bgn).Add_str(css_url).Add(Xoa_css_img_downloader.Bry_comment_end).Add_byte_nl();
|
||||
tmp_bfr.Add(css_bry).Add_byte_nl().Add_byte_nl();
|
||||
}
|
||||
return tmp_bfr.XtoAryAndClear();
|
||||
}
|
||||
public static final String Css_common_name = "xowa_common.css", Css_wiki_name = "xowa_wiki.css"
|
||||
, Css_common_name_ltr = "xowa_common_ltr.css", Css_common_name_rtl = "xowa_common_rtl.css";
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import org.junit.*; import gplx.ios.*;
|
||||
public class Xoa_css_extractor_basic_tst {
|
||||
@Before public void init() {fxt.Clear();} private Xoa_css_extractor_fxt fxt = new Xoa_css_extractor_fxt();
|
||||
@Test public void Logo_download() {
|
||||
fxt.Init_fil("mem/http/en.wikipedia.org" , Xoa_css_extractor_fxt.Main_page_html);
|
||||
fxt.Init_fil("mem/http/wiki.png" , "download");
|
||||
fxt.Exec_logo_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/logo.png", "download");
|
||||
}
|
||||
@Test public void Logo_failover() {
|
||||
fxt.Init_fil("mem/xowa/bin/any/html/xowa/import/logo.png" , "failover");
|
||||
fxt.Exec_logo_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/logo.png", "failover");
|
||||
}
|
||||
@Test public void Css_common_download_failover() {
|
||||
fxt.Css_installer().Opt_download_css_common_(true);
|
||||
fxt.Init_fil("mem/xowa/bin/any/html/xowa/import/xowa_common_ltr.css", "failover");
|
||||
fxt.Exec_css_common_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/xowa_common.css", "failover");
|
||||
}
|
||||
@Test public void Css_common_copy() {
|
||||
fxt.Css_installer().Opt_download_css_common_(false);
|
||||
fxt.Init_fil("mem/xowa/bin/any/html/xowa/import/xowa_common_ltr.css", "failover");
|
||||
fxt.Exec_css_common_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/xowa_common.css", "failover");
|
||||
}
|
||||
@Test public void Css_common_copy_specific_wiki() { // PURPOSE: css for specific wiki
|
||||
fxt.Css_installer().Opt_download_css_common_(false).Wiki_code_(Bry_.new_ascii_("enwiki"));
|
||||
fxt.Init_fil("mem/xowa/bin/any/html/xowa/import/xowa_common_override/xowa_common_enwiki.css", "failover");
|
||||
fxt.Exec_css_common_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/xowa_common.css", "failover");
|
||||
}
|
||||
@Test public void Css_scrape_download() {
|
||||
fxt.Css_installer().Url_encoder_(Url_encoder.new_http_url_());
|
||||
fxt.Init_fil("mem/http/en.wikipedia.org" , Xoa_css_extractor_fxt.Main_page_html);
|
||||
fxt.Init_fil("mem/http/en.wikipedia.org/common.css" , "download");
|
||||
fxt.Init_fil("mem/http/www/a&0|b,c" , "data=css_0");
|
||||
fxt.Init_fil("mem/http/www/a&1|b,c" , "data=css_1");
|
||||
fxt.Exec_css_mainpage_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/xowa_common.css", String_.Concat_lines_nl
|
||||
( "/*XOWA:mem/http/www/a&0|b,c*/"
|
||||
, "data=css_0"
|
||||
, ""
|
||||
, "/*XOWA:mem/http/www/a&1|b,c*/"
|
||||
, "data=css_1"
|
||||
));
|
||||
}
|
||||
@Test public void Css_scrape_failover() {
|
||||
fxt.Init_fil("mem/xowa/bin/any/html/xowa/import/xowa_common_ltr.css", "failover");
|
||||
fxt.Exec_css_mainpage_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/xowa_common.css", "failover");
|
||||
}
|
||||
}
|
||||
class Xoa_css_extractor_fxt {
|
||||
public void Clear() {
|
||||
Io_mgr._.InitEngine_mem();
|
||||
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_base.test_();
|
||||
css_installer = new Xoa_css_extractor();
|
||||
css_installer.Download_xrg().Trg_engine_key_(IoEngine_.MemKey);
|
||||
css_installer
|
||||
.Usr_dlg_(usr_dlg)
|
||||
.Wiki_domain_(Bry_.new_ascii_("en.wikipedia.org"))
|
||||
.Protocol_prefix_("mem/http/")
|
||||
.Mainpage_url_("mem/http/en.wikipedia.org")
|
||||
.Failover_dir_(Io_url_.new_any_("mem/xowa/bin/any/html/xowa/import/")) // "mem/xowa/user/anonymous/wiki/home/html/"
|
||||
.Wiki_html_dir_(Io_url_.new_any_("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/"))
|
||||
;
|
||||
page_fetcher = new Xow_page_fetcher_mok();
|
||||
css_installer.Page_fetcher_(page_fetcher);
|
||||
Xoa_css_img_downloader css_img_downloader = new Xoa_css_img_downloader();
|
||||
css_img_downloader.Ctor(usr_dlg, new Xof_download_wkr_test(), Bry_.new_ascii_("mem/http/"));
|
||||
css_installer.Css_img_downloader_(css_img_downloader);
|
||||
} private Xow_page_fetcher_mok page_fetcher;
|
||||
public Xoa_css_extractor Css_installer() {return css_installer;} private Xoa_css_extractor css_installer;
|
||||
public void Init_page(int ns_id, String ttl, String text) {
|
||||
page_fetcher.Add(ns_id, Bry_.new_ascii_(ttl), Bry_.new_ascii_(text));
|
||||
}
|
||||
public void Init_fil_empty(String url) {Init_fil(url, "");}
|
||||
public void Init_fil(String url, String text) {Io_mgr._.SaveFilStr(url, text);}
|
||||
public void Test_fil(String url, String expd) {Tfds.Eq_str_lines(expd, Io_mgr._.LoadFilStr(Io_url_.new_any_(url)));}
|
||||
public void Exec_logo_setup() {
|
||||
css_installer.Mainpage_download();
|
||||
css_installer.Logo_setup();
|
||||
}
|
||||
public void Exec_css_common_setup() {
|
||||
css_installer.Mainpage_download();
|
||||
css_installer.Css_common_setup();
|
||||
}
|
||||
public void Exec_css_wiki_setup() {css_installer.Css_wiki_setup();}
|
||||
public void Exec_css_mainpage_setup() {
|
||||
css_installer.Mainpage_download();
|
||||
css_installer.Css_scrape_setup();
|
||||
}
|
||||
public static String Main_page_html = String_.Concat_lines_nl
|
||||
( "<html>"
|
||||
, " <head>"
|
||||
, " <link rel=\"stylesheet\" href=\"www/a&0%7Cb%2Cc\" />"
|
||||
, " <link rel=\"stylesheet\" href=\"www/a&1%7Cb%2Cc\" />"
|
||||
, " </head>"
|
||||
, " <body>"
|
||||
, " <div id=\"p-logo\" role=\"banner\"><a style=\"background-image: url(wiki.png);\""
|
||||
, " </body>"
|
||||
, "</html>"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import org.junit.*; import gplx.ios.*;
|
||||
public class Xoa_css_extractor_wiki_tst {
|
||||
@Before public void init() {fxt.Clear();} private Xoa_css_extractor_fxt fxt = new Xoa_css_extractor_fxt();
|
||||
@Test public void Css_wiki_generate() {
|
||||
fxt.Init_page(Xow_ns_.Id_mediaWiki, "Common.css" , "css_0");
|
||||
fxt.Init_page(Xow_ns_.Id_mediaWiki, "Vector.css" , "css_1");
|
||||
fxt.Exec_css_wiki_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/xowa_wiki.css", String_.Concat_lines_nl
|
||||
( "/*XOWA:MediaWiki:Common.css*/"
|
||||
, "css_0"
|
||||
, ""
|
||||
, "/*XOWA:MediaWiki:Vector.css*/"
|
||||
, "css_1"
|
||||
));
|
||||
}
|
||||
@Test public void Css_wiki_missing() {
|
||||
fxt.Exec_css_wiki_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/xowa_wiki.css", "");
|
||||
}
|
||||
@Test public void Css_wiki_tab() { // PURPOSE: swap out 	 for xdat files
|
||||
fxt.Init_page(Xow_ns_.Id_mediaWiki, "Common.css" , "a	b");
|
||||
fxt.Exec_css_wiki_setup();
|
||||
fxt.Test_fil("mem/xowa/user/anonymous/wiki/en.wikipedia.org/html/xowa_wiki.css", String_.Concat_lines_nl
|
||||
( "/*XOWA:MediaWiki:Common.css*/"
|
||||
, "a\tb"
|
||||
));
|
||||
}
|
||||
}
|
||||
178
400_xowa/src_240_install/gplx/xowa/Xoa_css_img_downloader.java
Normal file
178
400_xowa/src_240_install/gplx/xowa/Xoa_css_img_downloader.java
Normal file
@@ -0,0 +1,178 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
public class Xoa_css_img_downloader {
|
||||
public Xoa_css_img_downloader Ctor(Gfo_usr_dlg usr_dlg, Xof_download_wkr download_wkr, byte[] stylesheet_prefix) {
|
||||
this.usr_dlg = usr_dlg; this.download_wkr = download_wkr; this.stylesheet_prefix = stylesheet_prefix;
|
||||
return this;
|
||||
} private Gfo_usr_dlg usr_dlg; private Xof_download_wkr download_wkr;
|
||||
public Xoa_css_img_downloader Stylesheet_prefix_(byte[] v) {stylesheet_prefix = v; return this;} private byte[] stylesheet_prefix; // TEST: setter exposed b/c tests can handle "mem/" but not "//mem"
|
||||
public void Chk(byte[] wiki_domain, Io_url css_fil) {
|
||||
ListAdp img_list = ListAdp_.new_();
|
||||
byte[] old_bry = Io_mgr._.LoadFilBry(css_fil);
|
||||
byte[] rel_url_prefix = Bry_.Add(Bry_fwd_slashes, wiki_domain);
|
||||
byte[] new_bry = Convert_to_local_urls(rel_url_prefix, old_bry, img_list);
|
||||
Io_url img_dir = css_fil.OwnerDir();
|
||||
Download_fils(img_dir, img_list.XtoStrAry());
|
||||
Io_mgr._.SaveFilBry(css_fil, new_bry);
|
||||
}
|
||||
public byte[] Convert_to_local_urls(byte[] rel_url_prefix, byte[] src, ListAdp list) {
|
||||
try {
|
||||
int src_len = src.length;
|
||||
int prv_pos = 0;
|
||||
Bry_bfr bfr = Bry_bfr.new_(src_len);
|
||||
HashAdp img_hash = HashAdp_.new_bry_();
|
||||
while (true) {
|
||||
int url_pos = Bry_finder.Find_fwd(src, Bry_url, prv_pos);
|
||||
if (url_pos == Bry_.NotFound) {bfr.Add_mid(src, prv_pos, src_len); break;} // no more "url("; exit;
|
||||
int bgn_pos = url_pos + Bry_url_len; // set bgn_pos after "url("
|
||||
byte bgn_byte = src[bgn_pos];
|
||||
byte end_byte = Byte_ascii.Nil;
|
||||
boolean quoted = true;
|
||||
switch (bgn_byte) { // find end_byte
|
||||
case Byte_ascii.Quote: case Byte_ascii.Apos: // quoted; end_byte is ' or "
|
||||
end_byte = bgn_byte;
|
||||
++bgn_pos;
|
||||
break;
|
||||
default: // not quoted; end byte is ")"
|
||||
end_byte = Byte_ascii.Paren_end;
|
||||
quoted = false;
|
||||
break;
|
||||
}
|
||||
int end_pos = Bry_finder.Find_fwd(src, end_byte, bgn_pos, src_len);
|
||||
if (end_pos == Bry_.NotFound) { // unclosed "url("; exit since nothing else will be found
|
||||
usr_dlg.Warn_many(GRP_KEY, "parse.invalid_url.end_missing", "could not find end_sequence for 'url(': bgn='~{0}' end='~{1}'", prv_pos, String_.new_utf8_len_safe_(src, prv_pos, prv_pos + 25));
|
||||
bfr.Add_mid(src, prv_pos, src_len);
|
||||
break;
|
||||
}
|
||||
if (end_pos - bgn_pos == 0) { // empty; "url()"; ignore
|
||||
usr_dlg.Warn_many(GRP_KEY, "parse.invalid_url.empty", "'url(' is empty: bgn='~{0}' end='~{1}'", prv_pos, String_.new_utf8_len_safe_(src, prv_pos, prv_pos + 25));
|
||||
bfr.Add_mid(src, prv_pos, bgn_pos);
|
||||
prv_pos = bgn_pos;
|
||||
continue;
|
||||
}
|
||||
byte[] img_raw = Bry_.Mid(src, bgn_pos, end_pos); int img_raw_len = img_raw.length;
|
||||
if (Bry_.HasAtBgn(img_raw, Bry_data_image, 0, img_raw_len)) { // base64
|
||||
bfr.Add_mid(src, prv_pos, end_pos); // nothing to download; just add entire String
|
||||
prv_pos = end_pos;
|
||||
continue;
|
||||
}
|
||||
int import_url_end = Import_url_chk(rel_url_prefix, src, src_len, prv_pos, url_pos, img_raw, bfr); // check for embedded stylesheets via @import tag
|
||||
if (import_url_end != Bry_.NotFound) {
|
||||
prv_pos = import_url_end;
|
||||
continue;
|
||||
}
|
||||
byte[] img_cleaned = Clean_img_url(img_raw, img_raw_len);
|
||||
if (img_cleaned == null) { // could not clean img
|
||||
usr_dlg.Warn_many(GRP_KEY, "parse.invalid_url.clean_failed", "could not extract valid http src: bgn='~{0}' end='~{1}'", prv_pos, String_.new_utf8_(img_raw));
|
||||
bfr.Add_mid(src, prv_pos, bgn_pos); prv_pos = bgn_pos; continue;
|
||||
}
|
||||
if (!img_hash.Has(img_cleaned)) {// only add unique items for download;
|
||||
img_hash.AddKeyVal(img_cleaned);
|
||||
list.Add(String_.new_utf8_(img_cleaned));
|
||||
}
|
||||
img_cleaned = Replace_invalid_chars(Bry_.Copy(img_cleaned)); // NOTE: must call ByteAry.Copy else img_cleaned will change *inside* hash
|
||||
bfr.Add_mid(src, prv_pos, bgn_pos);
|
||||
if (!quoted) bfr.Add_byte(Byte_ascii.Quote);
|
||||
bfr.Add(img_cleaned);
|
||||
if (!quoted) bfr.Add_byte(Byte_ascii.Quote);
|
||||
prv_pos = end_pos;
|
||||
}
|
||||
return bfr.XtoAryAndClear();
|
||||
}
|
||||
catch (Exception e) {
|
||||
usr_dlg.Warn_many("", "", "failed to convert local_urls: ~{0} ~{1}", String_.new_utf8_(rel_url_prefix), Err_.Message_gplx(e));
|
||||
return src;
|
||||
}
|
||||
}
|
||||
public static byte[] Import_url_build(byte[] stylesheet_prefix, byte[] rel_url_prefix, byte[] css_url) {
|
||||
return Bry_.HasAtBgn(css_url, Bry_http_protocol) // css_url already starts with "http"; return self; PAGE:tr.n:Main_Page; DATE:2014-06-04
|
||||
? css_url
|
||||
: Bry_.Add(stylesheet_prefix, css_url)
|
||||
;
|
||||
}
|
||||
private int Import_url_chk(byte[] rel_url_prefix, byte[] src, int src_len, int old_pos, int find_bgn, byte[] url_raw, Bry_bfr bfr) {
|
||||
if (find_bgn < Bry_import_len) return Bry_.NotFound;
|
||||
if (!Bry_.Match(src, find_bgn - Bry_import_len, find_bgn, Bry_import)) return Bry_.NotFound;
|
||||
byte[] css_url = url_raw; int css_url_len = css_url.length;
|
||||
if (css_url_len > 0 && css_url[0] == Byte_ascii.Slash) { // css_url starts with "/"; EX: "/page" or "//site/page" DATE:2014-02-03
|
||||
if (css_url_len > 1 && css_url[1] != Byte_ascii.Slash) // skip if css_url starts with "//"; EX: "//site/page"
|
||||
css_url = Bry_.Add(rel_url_prefix, css_url); // "/w/a.css" -> "//en.wikipedia.org/w/a.css"
|
||||
}
|
||||
byte[] css_src_bry = Import_url_build(stylesheet_prefix, rel_url_prefix, css_url);
|
||||
String css_src_str = String_.new_utf8_(css_src_bry);
|
||||
download_wkr.Download_xrg().Prog_fmt_hdr_(usr_dlg.Log_many(GRP_KEY, "logo.download", "downloading import for '~{0}'", css_src_str));
|
||||
byte[] css_trg_bry = download_wkr.Download_xrg().Exec_as_bry(css_src_str);
|
||||
if (css_trg_bry == null) {
|
||||
usr_dlg.Warn_many("", "", "could not import css: url=~{0}", css_src_str);
|
||||
return Bry_.NotFound; // css not found
|
||||
}
|
||||
bfr.Add_mid(src, old_pos, find_bgn - Bry_import_len).Add_byte_nl();
|
||||
bfr.Add(Bry_comment_bgn).Add(css_url).Add(Bry_comment_end).Add_byte_nl();
|
||||
bfr.Add(css_trg_bry).Add_byte_nl();
|
||||
bfr.Add_byte_nl();
|
||||
int semic_pos = Bry_finder.Find_fwd(src, Byte_ascii.Semic, find_bgn + url_raw.length, src_len);
|
||||
return semic_pos + Int_.Const_dlm_len;
|
||||
}
|
||||
public byte[] Clean_img_url(byte[] raw, int raw_len) {
|
||||
int pos_bgn = 0;
|
||||
if (Bry_.HasAtBgn(raw, Bry_fwd_slashes, 0, raw_len)) pos_bgn = Bry_fwd_slashes.length;
|
||||
if (Bry_.HasAtBgn(raw, Bry_http, 0, raw_len)) pos_bgn = Bry_http.length;
|
||||
int pos_slash = Bry_finder.Find_fwd(raw, Byte_ascii.Slash, pos_bgn, raw_len);
|
||||
if (pos_slash == Bry_.NotFound) return null; // first segment is site_name; at least one slash must be present for image name; EX: site.org/img_name.jpg
|
||||
if (pos_slash == raw_len - 1) return null; // "site.org/" is invalid
|
||||
int pos_end = raw_len;
|
||||
int pos_question = Bry_finder.Find_bwd(raw, Byte_ascii.Question);
|
||||
if (pos_question != Bry_.NotFound)
|
||||
pos_end = pos_question; // remove query params; EX: img_name?key=val
|
||||
return Bry_.Mid(raw, pos_bgn, pos_end);
|
||||
}
|
||||
private void Download_fils(Io_url css_dir, String[] ary) {
|
||||
int ary_len = ary.length;
|
||||
for (int i = 0; i < ary_len; i++) {
|
||||
String src = ary[i];
|
||||
Io_url trg = css_dir.GenSubFil_nest(Op_sys.Cur().Fsys_http_frag_to_url_str(Replace_invalid_chars_str(src)));
|
||||
if (Io_mgr._.ExistsFil(trg)) continue;
|
||||
download_wkr.Download(true, "http://" + src, trg, "download: " + src); // ILN
|
||||
}
|
||||
}
|
||||
String Replace_invalid_chars_str(String raw_str) {return String_.new_utf8_(Replace_invalid_chars(Bry_.new_utf8_(raw_str)));}
|
||||
byte[] Replace_invalid_chars(byte[] raw_bry) {
|
||||
int raw_len = raw_bry.length;
|
||||
for (int i = 0; i < raw_len; i++) { // convert invalid wnt chars to underscores
|
||||
byte b = raw_bry[i];
|
||||
switch (b) {
|
||||
//case Byte_ascii.Slash:
|
||||
case Byte_ascii.Backslash: case Byte_ascii.Colon: case Byte_ascii.Asterisk: case Byte_ascii.Question:
|
||||
case Byte_ascii.Quote: case Byte_ascii.Lt: case Byte_ascii.Gt: case Byte_ascii.Pipe:
|
||||
raw_bry[i] = Byte_ascii.Underline;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return raw_bry;
|
||||
}
|
||||
private static final byte[]
|
||||
Bry_url = Bry_.new_ascii_("url("), Bry_data_image = Bry_.new_ascii_("data:image/")
|
||||
, Bry_http = Bry_.new_ascii_("http://"), Bry_fwd_slashes = Bry_.new_ascii_("//"), Bry_import = Bry_.new_ascii_("@import ")
|
||||
, Bry_http_protocol = Bry_.new_ascii_("http")
|
||||
;
|
||||
public static final byte[]
|
||||
Bry_comment_bgn = Bry_.new_ascii_("/*XOWA:"), Bry_comment_end = Bry_.new_ascii_("*/");
|
||||
private static final int Bry_url_len = Bry_url.length, Bry_import_len = Bry_import.length;
|
||||
static final String GRP_KEY = "xowa.wikis.init.css";
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import org.junit.*;
|
||||
public class Xoa_css_img_downloader_tst {
|
||||
@Before public void init() {fxt.Clear();} private Xoa_css_img_downloader_fxt fxt = new Xoa_css_img_downloader_fxt();
|
||||
@Test public void Basic() {
|
||||
fxt.Test_css_convert
|
||||
( "x {url(\"//site/a.jpg\")} y {url(\"//site/b.jpg\")}"
|
||||
, "x {url(\"site/a.jpg\")} y {url(\"site/b.jpg\")}"
|
||||
, "site/a.jpg"
|
||||
, "site/b.jpg"
|
||||
);
|
||||
}
|
||||
@Test public void Unquoted() {
|
||||
fxt.Test_css_convert
|
||||
( "x {url(//site/a.jpg)}"
|
||||
, "x {url(\"site/a.jpg\")}"
|
||||
, "site/a.jpg"
|
||||
);
|
||||
}
|
||||
@Test public void Http() {
|
||||
fxt.Test_css_convert
|
||||
( "x {url(http://site/a.jpg)}"
|
||||
, "x {url(\"site/a.jpg\")}"
|
||||
, "site/a.jpg"
|
||||
);
|
||||
}
|
||||
@Test public void Base64() {
|
||||
fxt.Test_css_convert
|
||||
( "x {url(\"//site/a.jpg\")} y {url(\"data:image/png;base64,BASE64DATA;ABC=\")} z {}"
|
||||
, "x {url(\"site/a.jpg\")} y {url(\"data:image/png;base64,BASE64DATA;ABC=\")} z {}"
|
||||
, "site/a.jpg"
|
||||
);
|
||||
}
|
||||
@Test public void Exc_missing_quote() {
|
||||
fxt.Test_css_convert
|
||||
( "x {url(\"//site/a.jpg\")} y {url(\"//site/b.jpg} z {}"
|
||||
, "x {url(\"site/a.jpg\")} y {url(\"//site/b.jpg} z {}"
|
||||
, "site/a.jpg"
|
||||
);
|
||||
}
|
||||
@Test public void Exc_empty() {
|
||||
fxt.Test_css_convert
|
||||
( "x {url(\"//site/a.jpg\")} y {url(\"\"} z {}"
|
||||
, "x {url(\"site/a.jpg\")} y {url(\"\"} z {}"
|
||||
, "site/a.jpg"
|
||||
);
|
||||
}
|
||||
@Test public void Exc_name_only() {
|
||||
fxt.Test_css_convert
|
||||
( "x {url(\"//site/a.jpg\")} y {url(\"b.jpg\"} z {}"
|
||||
, "x {url(\"site/a.jpg\")} y {url(\"b.jpg\"} z {}"
|
||||
, "site/a.jpg"
|
||||
);
|
||||
}
|
||||
@Test public void Repeat() {// PURPOSE.fix: exact same item was being added literally
|
||||
fxt.Test_css_convert
|
||||
( "x {url(\"//site/a.jpg?a=b\")} y {url(\"//site/a.jpg?a=b\"}"
|
||||
, "x {url(\"site/a.jpg\")} y {url(\"site/a.jpg\"}"
|
||||
, "site/a.jpg"
|
||||
);
|
||||
}
|
||||
@Test public void Clean_basic() {fxt.Test_clean_img_url("//site/a.jpg" , "site/a.jpg");}
|
||||
@Test public void Clean_query() {fxt.Test_clean_img_url("//site/a.jpg?key=val" , "site/a.jpg");}
|
||||
@Test public void Clean_dir() {fxt.Test_clean_img_url("//site/a/b/c.jpg?key=val" , "site/a/b/c.jpg");}
|
||||
@Test public void Clean_exc_site_only() {fxt.Test_clean_img_url("//site" , null);}
|
||||
@Test public void Clean_exc_site_only_2() {fxt.Test_clean_img_url("//site/" , null);}
|
||||
@Test public void Import_url() {
|
||||
Io_mgr._.InitEngine_mem();
|
||||
Io_mgr._.SaveFilStr("mem/www/b.css", "imported_css");
|
||||
fxt.Test_css_convert
|
||||
( "x @import url(\"mem/www/b.css\") screen; z"
|
||||
, String_.Concat_lines_nl
|
||||
( "x "
|
||||
, "/*XOWA:mem/www/b.css*/"
|
||||
, "imported_css"
|
||||
, ""
|
||||
, " z"
|
||||
)
|
||||
);
|
||||
}
|
||||
@Test public void Import_url_make() {
|
||||
fxt.Test_import_url("a.org/b" , "http:a.org/b"); // add "stylesheet_prefix"
|
||||
fxt.Test_import_url("http://a.org" , "http://a.org"); // unless it starts with http
|
||||
fxt.Test_import_url("https://a.org" , "https://a.org"); // unless starts with https EX:: handle @import(https://...); PAGE:tr.n:Main_Page; DATE:2014-06-04
|
||||
}
|
||||
@Test public void Import_url_relative() { // PURPOSE: if directory, add domain; "/a/b.css" -> "//domain/a/b.css"; DATE:2014-02-03
|
||||
Io_mgr._.InitEngine_mem();
|
||||
Io_mgr._.SaveFilStr("mem/en.wikipedia.org/www/b.css", "imported_css");
|
||||
fxt.Test_css_convert
|
||||
( "x @import url(\"/www/b.css\") screen; z" // starts with "/"
|
||||
, String_.Concat_lines_nl
|
||||
( "x "
|
||||
, "/*XOWA:mem/en.wikipedia.org/www/b.css*/"
|
||||
, "imported_css"
|
||||
, ""
|
||||
, " z"
|
||||
)
|
||||
);
|
||||
}
|
||||
@Test public void Import_url_relative_skip() { // PURPOSE: if rel path, skip; "//site/a/b.css"; DATE:2014-02-03
|
||||
fxt.Downloader().Stylesheet_prefix_(Bry_.new_utf8_("mem")); // stylesheet prefix prefix defaults to ""; set to "mem", else test will try to retrieve "//url" which will fail
|
||||
Io_mgr._.InitEngine_mem();
|
||||
Io_mgr._.SaveFilStr("mem//en.wikipedia.org/a/b.css", "imported_css");
|
||||
fxt.Test_css_convert
|
||||
( "x @import url(\"//en.wikipedia.org/a/b.css\") screen; z" // starts with "//"
|
||||
, String_.Concat_lines_nl
|
||||
( "x "
|
||||
, "/*XOWA://en.wikipedia.org/a/b.css*/"
|
||||
, "imported_css"
|
||||
, ""
|
||||
, " z"
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
class Xoa_css_img_downloader_fxt {
|
||||
public Xoa_css_img_downloader Downloader() {return downloader;} private Xoa_css_img_downloader downloader;
|
||||
public void Clear() {
|
||||
downloader = new Xoa_css_img_downloader();
|
||||
downloader.Ctor(Gfo_usr_dlg_base.test_(), new Xof_download_wkr_test(), Bry_.Empty);
|
||||
}
|
||||
public void Test_css_convert(String raw, String expd, String... expd_img_ary) {
|
||||
ListAdp actl_img_list = ListAdp_.new_();
|
||||
byte[] actl_bry = downloader.Convert_to_local_urls(Bry_.new_ascii_("mem/en.wikipedia.org"), Bry_.new_utf8_(raw), actl_img_list);
|
||||
Tfds.Eq_str_lines(expd, String_.new_utf8_(actl_bry));
|
||||
Tfds.Eq_ary_str(expd_img_ary, actl_img_list.XtoStrAry());
|
||||
}
|
||||
public void Test_clean_img_url(String raw_str, String expd) {
|
||||
byte[] raw = Bry_.new_ascii_(raw_str);
|
||||
byte[] actl = downloader.Clean_img_url(raw, raw.length);
|
||||
Tfds.Eq(expd, actl == null ? null : String_.new_ascii_(actl));
|
||||
}
|
||||
public void Test_import_url(String raw, String expd) {
|
||||
byte[] actl = Xoa_css_img_downloader.Import_url_build(Bry_.new_ascii_("http:"), Bry_.new_ascii_("//en.wikipedia.org"), Bry_.new_utf8_(raw));
|
||||
Tfds.Eq(expd, String_.new_utf8_(actl));
|
||||
}
|
||||
}
|
||||
61
400_xowa/src_240_install/gplx/xowa/Xob_dump_file.java
Normal file
61
400_xowa/src_240_install/gplx/xowa/Xob_dump_file.java
Normal file
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import gplx.xowa.wikis.*;
|
||||
public class Xob_dump_file {
|
||||
public Xow_wiki_domain Wiki_type() {return wiki_type;} private Xow_wiki_domain wiki_type;
|
||||
public String Dump_date() {return dump_date;} public void Dump_date_(String v) {dump_date = v;} String dump_date;
|
||||
public String Dump_file_type() {return dump_file_type;} private String dump_file_type;
|
||||
public String Server_url() {return server_url;} private String server_url;
|
||||
public String File_url() {return file_url;} private String file_url;
|
||||
public String File_name() {return file_name;} private String file_name;
|
||||
public long File_len() {return file_len;} long file_len;
|
||||
public DateAdp File_modified() {return file_modified;} DateAdp file_modified;
|
||||
public byte[] Wiki_alias() {return wiki_alias;} private byte[] wiki_alias;
|
||||
public Xob_dump_file Ctor(String wiki_domain, String dump_date, String dump_file_type) {
|
||||
this.dump_date = dump_date; this.dump_file_type = dump_file_type;
|
||||
this.wiki_type = Xow_wiki_domain_.parse_by_domain(Bry_.new_ascii_(wiki_domain));
|
||||
this.wiki_alias = Xob_bz2_file.Build_alias(wiki_type);
|
||||
byte[] dump_file_bry = Bry_.new_utf8_(dump_file_type);
|
||||
byte dump_file_tid = Xob_bz2_file.Parse__tid(dump_file_bry);
|
||||
byte[] ext = Xob_dump_file_.Ext_xml_bz2;
|
||||
switch (dump_file_tid) {
|
||||
case Xob_bz2_file.Tid_page_props: case Xob_bz2_file.Tid_categorylinks: case Xob_bz2_file.Tid_image:
|
||||
ext = Xob_dump_file_.Ext_sql_gz;
|
||||
break;
|
||||
}
|
||||
this.file_name = String_.new_utf8_(Xob_dump_file_.Bld_dump_file_name(wiki_alias, Bry_.new_utf8_(dump_date), dump_file_bry, ext));
|
||||
return this;
|
||||
}
|
||||
public void Server_url_(String server_url) {
|
||||
this.server_url = server_url;
|
||||
String dump_dir_url = String_.new_utf8_(Xob_dump_file_.Bld_dump_dir_url(Bry_.new_utf8_(server_url), wiki_alias, Bry_.new_utf8_(dump_date)));
|
||||
this.file_url = dump_dir_url + file_name;
|
||||
}
|
||||
public boolean Connect() {
|
||||
gplx.ios.IoEngine_xrg_downloadFil args = Io_mgr._.DownloadFil_args("", Io_url_.Null);
|
||||
boolean rv = args.Src_last_modified_query_(true).Exec_meta(file_url);
|
||||
if (rv) {
|
||||
file_len = args.Src_content_length();
|
||||
file_modified = args.Src_last_modified();
|
||||
if (file_modified.Timestamp_unix() <= 0) return false;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public static Xob_dump_file new_(String wiki_domain, String dump_date, String dump_type) {return new Xob_dump_file().Ctor(wiki_domain, dump_date, dump_type);}
|
||||
}
|
||||
71
400_xowa/src_240_install/gplx/xowa/Xob_dump_file_.java
Normal file
71
400_xowa/src_240_install/gplx/xowa/Xob_dump_file_.java
Normal file
@@ -0,0 +1,71 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
public class Xob_dump_file_ {
|
||||
public static boolean Connect_first(Xob_dump_file rv, String[] server_urls) {
|
||||
int len = server_urls.length;
|
||||
for (int i = 0; i < len; ++i) {
|
||||
String server_url = server_urls[i];
|
||||
rv.Server_url_(server_url);
|
||||
Override_dump_date(rv, server_url);
|
||||
if (rv.Connect()) return true;
|
||||
}
|
||||
return false;
|
||||
}
|
||||
private static void Override_dump_date(Xob_dump_file rv, String dump_server) {
|
||||
String dump_date = rv.Dump_date();
|
||||
if ( String_.Eq(dump_date, Xob_dump_file_.Date_latest)
|
||||
&& ( String_.Eq(dump_server, Xob_dump_file_.Server_c3sl)
|
||||
|| String_.Eq(dump_server, Xob_dump_file_.Server_masaryk)
|
||||
)
|
||||
){
|
||||
Xoi_mirror_parser mirror_parser = new Xoi_mirror_parser();
|
||||
String dump_wiki_url = dump_server + String_.new_ascii_(rv.Wiki_alias()) + "/";
|
||||
byte[] dump_url_wiki_html = gplx.ios.IoEngine_xrg_downloadFil.new_("", Io_url_.Null).Exec_as_bry(dump_wiki_url); if (Bry_.Len_eq_0(dump_url_wiki_html)) return;
|
||||
String[] dump_available_dates = mirror_parser.Parse(String_.new_utf8_(dump_url_wiki_html));
|
||||
String dump_dates_latest = Xoi_mirror_parser.Find_last_lte(dump_available_dates, dump_date);
|
||||
if (String_.Eq(dump_dates_latest, "")) return; // nothing found
|
||||
rv.Dump_date_(dump_dates_latest);
|
||||
}
|
||||
}
|
||||
public static byte[] Bld_dump_dir_url(byte[] server_url, byte[] alias, byte[] date) {
|
||||
return Bry_.Add
|
||||
( server_url // "http://dumps.wikimedia.org/"
|
||||
, Bry_.Replace(alias, Byte_ascii.Dash, Byte_ascii.Underline), Bry_slash // "simplewiki/"
|
||||
, date, Bry_slash // "latest/"
|
||||
);
|
||||
}
|
||||
public static byte[] Bld_dump_file_name(byte[] alias, byte[] date, byte[] dump_file_type, byte[] ext) {
|
||||
return Bry_.Add
|
||||
( Bry_.Replace(alias, Byte_ascii.Dash, Byte_ascii.Underline), Bry_dash // "simplewiki-"
|
||||
, date, Bry_dash // "latest-"
|
||||
, dump_file_type // "pages-articles"
|
||||
, ext // ".xml.bz2"
|
||||
);
|
||||
}
|
||||
private static final byte[] Bry_dash = new byte[] {Byte_ascii.Dash}, Bry_slash = new byte[] {Byte_ascii.Slash};
|
||||
public static final byte[] Ext_xml_bz2 = Bry_.new_ascii_(".xml.bz2");
|
||||
public static final byte[] Ext_sql_gz = Bry_.new_ascii_(".sql.gz");
|
||||
public static final String
|
||||
Server_wmf = "http://dumps.wikimedia.org/"
|
||||
, Server_your_org = "http://dumps.wikimedia.your.org/"
|
||||
, Server_c3sl = "http://wikipedia.c3sl.ufpr.br/"
|
||||
, Server_masaryk = "http://ftp.fi.muni.cz/pub/wikimedia/"
|
||||
, Date_latest = "latest"
|
||||
;
|
||||
}
|
||||
106
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_base.java
Normal file
106
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_base.java
Normal file
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import gplx.threads.*;
|
||||
abstract class Xoi_cmd_base implements Gfo_thread_cmd {
|
||||
public void Ctor(Xoi_setup_mgr install_mgr, String wiki_key) {
|
||||
this.install_mgr = install_mgr; this.wiki_key = wiki_key;
|
||||
this.Owner_(install_mgr);
|
||||
} private Xoi_setup_mgr install_mgr; String wiki_key;
|
||||
@gplx.Virtual public void Cmd_ctor() {}
|
||||
public abstract String Async_key();
|
||||
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
|
||||
public boolean Async_prog_enabled() {return false;}
|
||||
public void Async_prog_run(int async_sleep_sum) {}
|
||||
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
|
||||
public boolean Async_term() {
|
||||
// install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.end", "import.end ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
|
||||
return true;
|
||||
}
|
||||
public GfoInvkAble Owner() {return owner;} public Xoi_cmd_base Owner_(GfoInvkAble v) {owner = v; return this;} GfoInvkAble owner;
|
||||
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
|
||||
public void Async_run() {
|
||||
running = true;
|
||||
// install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.bgn", "import.bgn ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
|
||||
ThreadAdp_.invk_(this, Invk_process_async).Start();
|
||||
}
|
||||
public boolean Async_running() {return running;} private boolean running;
|
||||
public void Process_async() {
|
||||
Xoa_app app = install_mgr.App();
|
||||
Xob_bldr bldr = app.Bldr();
|
||||
Xow_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_ascii_(wiki_key));
|
||||
wiki.Init_assert();
|
||||
bldr.Cmd_mgr().Clear();
|
||||
Process_async_init(app, wiki, bldr);
|
||||
bldr.Pause_at_end_(false);
|
||||
bldr.Run();
|
||||
app.Gui_wtr().Prog_none("", "clear", "");
|
||||
app.Gui_wtr().Note_none("", "clear", "");
|
||||
Process_async_done(app, wiki, bldr);
|
||||
running = false;
|
||||
}
|
||||
public abstract void Process_async_init(Xoa_app app, Xow_wiki wiki, Xob_bldr bldr);
|
||||
public abstract void Process_async_done(Xoa_app app, Xow_wiki wiki, Xob_bldr bldr);
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_process_async)) Process_async();
|
||||
else if (ctx.Match(k, Invk_owner)) return owner;
|
||||
else return GfoInvkAble_.Rv_unhandled;
|
||||
return this;
|
||||
} private static final String Invk_process_async = "run_async", Invk_owner = "owner";
|
||||
}
|
||||
class Xoi_cmd_category2_page_props extends Xoi_cmd_wiki_download { public Xoi_cmd_category2_page_props(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date) {this.Ctor_download_(install_mgr, wiki_key, dump_date, Xob_bz2_file.Key_page_props);}
|
||||
@Override public String Download_file_ext() {return ".sql.gz2";}
|
||||
public static final String KEY_category2 = "wiki.category2.download.page_props";
|
||||
}
|
||||
class Xoi_cmd_category2_categorylinks extends Xoi_cmd_wiki_download { public Xoi_cmd_category2_categorylinks(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date) {this.Ctor_download_(install_mgr, wiki_key, dump_date, Xob_bz2_file.Key_categorylinks);}
|
||||
@Override public String Download_file_ext() {return ".sql.gz2";}
|
||||
public static final String KEY_category2 = "wiki.category2.download.categorylinks";
|
||||
}
|
||||
class Xoi_cmd_category2_build extends Xoi_cmd_base {
|
||||
public Xoi_cmd_category2_build(Xoi_setup_mgr install_mgr, String wiki_key) {this.Ctor(install_mgr, wiki_key); this.app = install_mgr.App(); this.wiki_key = wiki_key;} private Xoa_app app; private String wiki_key;
|
||||
@Override public void Cmd_ctor() {
|
||||
Xow_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_utf8_(wiki_key));
|
||||
wiki.Import_cfg().Category_version_(gplx.xowa.ctgs.Xoa_ctg_mgr.Version_2);
|
||||
}
|
||||
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.category2.build";
|
||||
@Override public void Process_async_init(Xoa_app app, Xow_wiki wiki, Xob_bldr bldr) {
|
||||
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
|
||||
wiki.Db_mgr_as_sql().Category_version_update(false);
|
||||
bldr.Cmd_mgr().Add_many(wiki, "import.sql.category_registry", "import.sql.categorylinks", "import.sql.hiddencat");
|
||||
}
|
||||
else
|
||||
bldr.Cmd_mgr().Add_many(wiki, "ctg.hiddencat_sql", "ctg.hiddencat_ttl", "ctg.link_sql", "ctg.link_idx");
|
||||
}
|
||||
@Override public void Process_async_done(Xoa_app app, Xow_wiki wiki, Xob_bldr bldr) {
|
||||
app.Gui_wtr().Prog_many("", "", "category2 setup done");
|
||||
}
|
||||
}
|
||||
class Xoi_cmd_search2_build extends Xoi_cmd_base {
|
||||
public Xoi_cmd_search2_build(Xoi_setup_mgr install_mgr, String wiki_key) {this.Ctor(install_mgr, wiki_key);}
|
||||
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.search2.build";
|
||||
@Override public void Process_async_init(Xoa_app app, Xow_wiki wiki, Xob_bldr bldr) {
|
||||
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
|
||||
wiki.Db_mgr_as_sql().Category_version_update(false);
|
||||
bldr.Cmd_mgr().Add_many(wiki, "import.sql.search_title.cmd");
|
||||
}
|
||||
}
|
||||
@Override public void Process_async_done(Xoa_app app, Xow_wiki wiki, Xob_bldr bldr) {
|
||||
app.Gui_wtr().Prog_many("", "", "search2 setup done");
|
||||
wiki.Db_mgr().Search_version_refresh();
|
||||
}
|
||||
}
|
||||
60
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_dumpfile.java
Normal file
60
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_dumpfile.java
Normal file
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import gplx.threads.*;
|
||||
class Xoi_cmd_dumpfile {
|
||||
public byte[] Domain() {return domain;} private byte[] domain;
|
||||
public Io_url Bz2_url() {return bz2_url;} Io_url bz2_url;
|
||||
public Io_url Xml_url() {return xml_url;} Io_url xml_url;
|
||||
public boolean Bz2_unzip() {return bz2_unzip;} private boolean bz2_unzip;
|
||||
public void Clear() {domain = null; bz2_url = xml_url = null; bz2_unzip = false;}
|
||||
public Xoi_cmd_dumpfile Parse_msg(GfoMsg m) {
|
||||
Io_url dump_url = m.ReadIoUrl("url");
|
||||
domain = m.ReadBry("domain");
|
||||
if (Bry_.Len_eq_0(domain)) domain = Bry_.new_utf8_(dump_url.OwnerDir().NameOnly());
|
||||
bz2_unzip = String_.Eq(m.ReadStr("args"), "unzip");
|
||||
String dump_ext = dump_url.Ext();
|
||||
if (String_.Eq(dump_ext, ".bz2")) {
|
||||
bz2_url = dump_url;
|
||||
if (bz2_unzip) {
|
||||
xml_url = bz2_url.GenNewExt(""); // remove .bz2 extension (new file path should be .xml)
|
||||
if (!String_.Eq(xml_url.Ext(), ".xml"))
|
||||
xml_url = xml_url.GenNewExt(".xml");
|
||||
}
|
||||
}
|
||||
else if (String_.Eq(dump_ext, ".xml")) { // user selected xml file;
|
||||
bz2_url = null;
|
||||
xml_url = dump_url;
|
||||
bz2_unzip = false; // ignore unzip arge
|
||||
}
|
||||
return this;
|
||||
}
|
||||
public Gfo_thread_cmd Exec(Xoi_cmd_mgr cmd_mgr) {
|
||||
Xow_wiki wiki = cmd_mgr.App().Wiki_mgr().Get_by_key_or_make(domain);
|
||||
if (bz2_unzip) { // unzip requested; add unzip cmd
|
||||
GfoMsg unzip_msg = GfoMsg_.new_parse_(Gfo_thread_cmd_unzip.KEY).Add("v", Gfo_thread_cmd_unzip.KEY).Add("src", bz2_url.Raw()).Add("trg", xml_url.Raw());
|
||||
Gfo_thread_cmd_unzip unzip_cmd = (Gfo_thread_cmd_unzip)cmd_mgr.Cmd_add(unzip_msg);
|
||||
unzip_cmd.Term_cmd_for_src_(Gfo_thread_cmd_unzip.Term_cmd_for_src_noop); // don't do anything with bz2 after unzip
|
||||
}
|
||||
if (xml_url == null)
|
||||
wiki.Import_cfg().Src_fil_bz2_(bz2_url);
|
||||
else
|
||||
wiki.Import_cfg().Src_fil_xml_(xml_url);
|
||||
return cmd_mgr.Dump_add_many_custom(String_.new_utf8_(domain), "", "", true);
|
||||
}
|
||||
}
|
||||
76
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_dumpfile_tst.java
Normal file
76
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_dumpfile_tst.java
Normal file
@@ -0,0 +1,76 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import org.junit.*;
|
||||
public class Xoi_cmd_dumpfile_tst {
|
||||
@Before public void init() {fxt.Clear();} private Xoi_cmd_dumpfile_fxt fxt = new Xoi_cmd_dumpfile_fxt();
|
||||
@Test public void Bz2__unzip() {
|
||||
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml.bz2", "", "unzip")
|
||||
.Test_domain("en.wikipedia.org")
|
||||
.Test_vals("mem/en.wikipedia.org/fil.xml.bz2", "mem/en.wikipedia.org/fil.xml", true)
|
||||
;
|
||||
}
|
||||
@Test public void Bz2__unzip__assert_xml_ext() { // xml ext relies on removing ".bz2" from ".xml.bz2"; if just ".bz2" add an ".xml"
|
||||
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.bz2", "", "unzip")
|
||||
.Test_vals("mem/en.wikipedia.org/fil.bz2", "mem/en.wikipedia.org/fil.xml", true)
|
||||
;
|
||||
}
|
||||
@Test public void Bz2__direct() {
|
||||
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.bz2", "", "")
|
||||
.Test_vals("mem/en.wikipedia.org/fil.bz2", null, false)
|
||||
;
|
||||
}
|
||||
@Test public void Xml__unzip_n() {
|
||||
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml", "", "")
|
||||
.Test_vals(null, "mem/en.wikipedia.org/fil.xml", false)
|
||||
;
|
||||
}
|
||||
@Test public void Xml__unzip_y() {
|
||||
fxt .Exec_parse_msg("mem/en.wikipedia.org/fil.xml", "", "")
|
||||
.Test_vals(null, "mem/en.wikipedia.org/fil.xml", false)
|
||||
;
|
||||
}
|
||||
}
|
||||
class Xoi_cmd_dumpfile_fxt {
|
||||
public void Clear() {
|
||||
dumpfile.Clear();
|
||||
} private Xoi_cmd_dumpfile dumpfile = new Xoi_cmd_dumpfile();
|
||||
public Xoi_cmd_dumpfile_fxt Exec_parse_msg(String url, String domain, String args) {
|
||||
GfoMsg m = GfoMsg_.new_parse_("").Add("url", url).Add("domain", domain).Add("args", args);
|
||||
dumpfile.Parse_msg(m);
|
||||
return this;
|
||||
}
|
||||
public Xoi_cmd_dumpfile_fxt Test_vals(String expd_bz2, String expd_xml, boolean expd_unzip) {
|
||||
Eq_url(expd_bz2, dumpfile.Bz2_url());
|
||||
Eq_url(expd_xml, dumpfile.Xml_url());
|
||||
Tfds.Eq(expd_unzip, dumpfile.Bz2_unzip());
|
||||
return this;
|
||||
}
|
||||
public Xoi_cmd_dumpfile_fxt Test_domain(String expd_domain) {
|
||||
Tfds.Eq(expd_domain, String_.new_utf8_(dumpfile.Domain()));
|
||||
return this;
|
||||
}
|
||||
private void Eq_url(String expd, Io_url actl) {
|
||||
if (expd == null && actl == null) return;
|
||||
else if (expd != null && actl != null) {
|
||||
Tfds.Eq(expd, actl.Raw());
|
||||
}
|
||||
else if (expd == null) throw Err_.new_fmt_("actl should be null: {0}", expd);
|
||||
else if (actl == null) throw Err_.new_fmt_("actl should not be null: {0}", expd);
|
||||
}
|
||||
}
|
||||
144
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_mgr.java
Normal file
144
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_mgr.java
Normal file
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import gplx.threads.*;
|
||||
public class Xoi_cmd_mgr implements GfoInvkAble {
|
||||
ListAdp cmds = ListAdp_.new_();
|
||||
public Xoi_cmd_mgr(Xoi_setup_mgr install_mgr) {this.app = install_mgr.App(); this.install_mgr = install_mgr;} private Xoa_app app; Xoi_setup_mgr install_mgr;
|
||||
public Xoa_app App() {return app;}
|
||||
public void Canceled_y_() {canceled = true;} private boolean canceled = false;
|
||||
public boolean Working() {return working;} private boolean working;
|
||||
private void Process_async(Gfo_thread_cmd cmd) {
|
||||
byte init_rslt = cmd.Async_init();
|
||||
if (init_rslt == Gfo_thread_cmd_.Init_ok) {
|
||||
cmd.Async_run();
|
||||
int async_sleep_interval = cmd.Async_sleep_interval();
|
||||
boolean async_prog_enabled = cmd.Async_prog_enabled();
|
||||
int async_sleep_sum = 0;
|
||||
while (cmd.Async_running()) {
|
||||
if (canceled) {working = false; return;}
|
||||
if (async_prog_enabled) cmd.Async_prog_run(async_sleep_sum);
|
||||
ThreadAdp_.Sleep(async_sleep_interval);
|
||||
async_sleep_sum += async_sleep_interval; // NOTE: this is not exact
|
||||
}
|
||||
}
|
||||
boolean term_pass = cmd.Async_term();
|
||||
if (cmd.Async_next_cmd() != null && init_rslt != Gfo_thread_cmd_.Init_cancel_all && term_pass)
|
||||
Run_async(cmd.Async_next_cmd());
|
||||
else
|
||||
working = false;
|
||||
}
|
||||
private void Run_async(Gfo_thread_cmd cmd) {ThreadAdp_.invk_msg_(this, GfoMsg_.new_cast_(Invk_process_async).Add("v", cmd)).Start();}
|
||||
private void Cmds_run() {
|
||||
if (working) {
|
||||
app.Gui_mgr().Kit().Ask_ok("", "", "An import is in progress. Please wait for it to complete. If you want to do multiple imports at once, see Help:Import/Script.");
|
||||
return;
|
||||
}
|
||||
int cmds_len = cmds.Count();
|
||||
if (cmds_len == 0) return;
|
||||
for (int i = 0; i < cmds_len - 1; i++) {
|
||||
Gfo_thread_cmd cur_cmd = (Gfo_thread_cmd)cmds.FetchAt(i);
|
||||
Gfo_thread_cmd nxt_cmd = (Gfo_thread_cmd)cmds.FetchAt(i + 1);
|
||||
cur_cmd.Cmd_ctor();
|
||||
cur_cmd.Async_next_cmd_(nxt_cmd);
|
||||
}
|
||||
Gfo_thread_cmd cmd = (Gfo_thread_cmd)cmds.FetchAt(0);
|
||||
cmds.Clear();
|
||||
working = true;
|
||||
this.Run_async(cmd);
|
||||
}
|
||||
Object Dump_add_many(GfoMsg m) {
|
||||
int args_len = m.Args_count();
|
||||
if (args_len < 4) throw Err_mgr._.fmt_(GRP_KEY, "dump_add_many", "Please provide the following: wiki name, wiki date, dump_type, and one command; EX: ('simple.wikipedia.org', 'latest', 'pages-articles', 'wiki.download')");
|
||||
String wiki_key = m.Args_getAt(0).Val_to_str_or_empty();
|
||||
String wiki_date = m.Args_getAt(1).Val_to_str_or_empty();
|
||||
String dump_type = m.Args_getAt(2).Val_to_str_or_empty();
|
||||
Gfo_thread_cmd cmd = null;
|
||||
for (int i = 3; i < args_len; i++) {
|
||||
KeyVal kv = m.Args_getAt(i);
|
||||
String kv_val = kv.Val_to_str_or_empty();
|
||||
if (String_.Eq(kv_val, Wiki_cmd_custom))
|
||||
return Dump_add_many_custom(wiki_key, wiki_date, dump_type, false);
|
||||
else {
|
||||
cmd = Dump_cmd_new(wiki_key, wiki_date, dump_type, kv.Val_to_str_or_empty());
|
||||
cmds.Add(cmd);
|
||||
}
|
||||
}
|
||||
return cmd; // return last cmd
|
||||
}
|
||||
public Gfo_thread_cmd Dump_add_many_custom(String wiki_key, String wiki_date, String dump_type, boolean dumpfile_cmd) {
|
||||
String[] custom_cmds = app.Setup_mgr().Dump_mgr().Custom_cmds();
|
||||
int custom_cmds_len = custom_cmds.length;
|
||||
Gfo_thread_cmd cmd = null;
|
||||
for (int j = 0; j < custom_cmds_len; j++) {
|
||||
cmd = Dump_cmd_new(wiki_key, wiki_date, dump_type, custom_cmds[j]);
|
||||
if (dumpfile_cmd) {
|
||||
if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_download.Key_wiki_download)) continue; // skip download if wiki.dump_file
|
||||
else if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_unzip.KEY_dump)) {
|
||||
Xow_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_utf8_(wiki_key));
|
||||
if (wiki.Import_cfg().Src_fil_xml() != null) continue; // skip unzip if xml exists
|
||||
}
|
||||
else if (String_.Eq(cmd.Async_key(), Xoi_cmd_wiki_import.KEY)) {
|
||||
((Xoi_cmd_wiki_import)cmd).Import_move_bz2_to_done_(false);
|
||||
}
|
||||
}
|
||||
cmds.Add(cmd);
|
||||
}
|
||||
return cmd;
|
||||
}
|
||||
Gfo_thread_cmd Dump_cmd_new(String wiki_key, String wiki_date, String dump_type, String cmd_key) {
|
||||
if (String_.Eq(cmd_key, Xoi_cmd_wiki_download.Key_wiki_download)) return new Xoi_cmd_wiki_download().Ctor_download_(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_unzip.KEY_dump)) return new Xoi_cmd_wiki_unzip(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_import.KEY)) return new Xoi_cmd_wiki_import(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_zip.KEY)) return new Xoi_cmd_wiki_zip(install_mgr, wiki_key, wiki_date, dump_type).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_category2_build.KEY)) return new Xoi_cmd_category2_build(install_mgr, wiki_key).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_category2_page_props.KEY_category2)) return new Xoi_cmd_category2_page_props(install_mgr, wiki_key, wiki_date).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_category2_categorylinks.KEY_category2)) return new Xoi_cmd_category2_categorylinks(install_mgr, wiki_key, wiki_date).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_search2_build.KEY)) return new Xoi_cmd_search2_build(install_mgr, wiki_key).Owner_(this);
|
||||
else throw Err_mgr._.unhandled_(cmd_key);
|
||||
}
|
||||
public static final String[] Wiki_cmds_valid = new String[] {Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_unzip.KEY_dump, Xoi_cmd_wiki_import.KEY, Xoi_cmd_wiki_zip.KEY, Xoi_cmd_category2_build.KEY, Xoi_cmd_category2_page_props.KEY_category2, Xoi_cmd_category2_categorylinks.KEY_category2};
|
||||
public static final String Wiki_cmd_custom = "wiki.custom", Wiki_cmd_dump_file = "wiki.dump_file";
|
||||
public Gfo_thread_cmd Cmd_add(GfoMsg m) {Gfo_thread_cmd rv = Cmd_new(m); cmds.Add(rv); return rv;}
|
||||
Gfo_thread_cmd Cmd_new(GfoMsg m) {
|
||||
String cmd_key = m.ReadStr("v");
|
||||
if (String_.Eq(cmd_key, Gfo_thread_cmd_download.KEY)) return new Gfo_thread_cmd_download().Init("downloading", m.ReadStr("src"), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg"))).Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());
|
||||
else if (String_.Eq(cmd_key, Gfo_thread_cmd_unzip.KEY)) return new Gfo_thread_cmd_unzip().Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Init(app.Usr_dlg(), app.Gui_mgr().Kit(), app.Fsys_mgr().App_mgr().App_decompress_bz2(), app.Fsys_mgr().App_mgr().App_decompress_zip(), app.Fsys_mgr().App_mgr().App_decompress_gz(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("src")), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg")));
|
||||
else if (String_.Eq(cmd_key, Gfo_thread_cmd_replace.KEY)) return new Gfo_thread_cmd_replace().Url_eval_mgr_(app.Url_cmd_eval()).Owner_(this).Init(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("fil")));
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_image_cfg.KEY_dump)) return new Xoi_cmd_wiki_image_cfg(app, Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("fil"))).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_wiki_goto_page.KEY)) return new Xoi_cmd_wiki_goto_page(app, m.ReadStr("v")).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_msg_ok.KEY)) return new Xoi_cmd_msg_ok(app.Usr_dlg(), app.Gui_mgr().Kit(), m.ReadStr("v")).Owner_(this);
|
||||
// else if (String_.Eq(cmd_key, Gfo_thread_exec_sync.KEY)) return new Gfo_thread_exec_sync(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("url")), m.ReadStr("args")).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Xoi_cmd_imageMagick_download.KEY_imageMagick)) return new Xoi_cmd_imageMagick_download(app.Usr_dlg(), app.Gui_mgr().Kit(), Bry_fmtr_eval_mgr_.Eval_url(app.Url_cmd_eval(), m.ReadBry("trg"))).Owner_(this);
|
||||
else if (String_.Eq(cmd_key, Wiki_cmd_dump_file)) return Wiki_cmd_dump_file_make(m);
|
||||
else throw Err_mgr._.unhandled_(cmd_key);
|
||||
}
|
||||
Gfo_thread_cmd Wiki_cmd_dump_file_make(GfoMsg m) {
|
||||
Xoi_cmd_dumpfile dumpfile = new Xoi_cmd_dumpfile().Parse_msg(m);
|
||||
return dumpfile.Exec(this);
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_process_async)) Process_async((Gfo_thread_cmd)m.CastObj("v"));
|
||||
else if (ctx.Match(k, Invk_dump_add_many)) return Dump_add_many(m);
|
||||
else if (ctx.Match(k, Invk_cmd_add)) return Cmd_add(m);
|
||||
else if (ctx.Match(k, Invk_run)) Cmds_run();
|
||||
else return GfoInvkAble_.Rv_unhandled;
|
||||
return this;
|
||||
} private static final String Invk_process_async = "process_async", Invk_dump_add_many = "dump_add_many", Invk_run = "run", Invk_cmd_add = "cmd_add";
|
||||
static final String GRP_KEY = "xowa.install_mgr.cmd_mgr";
|
||||
}
|
||||
199
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_wiki.java
Normal file
199
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_wiki.java
Normal file
@@ -0,0 +1,199 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import gplx.gfui.*;
|
||||
import gplx.threads.*;
|
||||
class Xoi_cmd_wiki_download extends Gfo_thread_cmd_download implements Gfo_thread_cmd { private Xoi_setup_mgr install_mgr; private String wiki_key, dump_date, dump_type;
|
||||
public Xoi_cmd_wiki_download Ctor_download_(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date, String dump_type) {
|
||||
this.install_mgr = install_mgr;
|
||||
this.wiki_key = wiki_key;
|
||||
this.dump_date = dump_date;
|
||||
this.dump_type = dump_type;
|
||||
this.Owner_(install_mgr);
|
||||
return this;
|
||||
}
|
||||
@gplx.Virtual public String Download_file_ext() {return ".xml.bz2";} // wiki.download is primarily used for dump files; default to .xml.bz2; NOTE: changed from ".xml"; DATE:2013-11-07
|
||||
@Override public String Async_key() {return Key_wiki_download;} public static final String Key_wiki_download = "wiki.download";
|
||||
@Override public byte Async_init() {
|
||||
Xoa_app app = install_mgr.App();
|
||||
Xob_dump_file dump_file = Xob_dump_file.new_(wiki_key, dump_date, dump_type);
|
||||
boolean connected = Xob_dump_file_.Connect_first(dump_file, install_mgr.Dump_mgr().Server_urls());
|
||||
if (connected)
|
||||
app.Usr_dlg().Note_many("", "", "url: ~{0}", dump_file.File_url());
|
||||
else {
|
||||
if (!Dump_servers_offline_msg_shown) {
|
||||
app.Gui_mgr().Kit().Ask_ok("", "", "all dump servers are offline: ~{0}", String_.AryXtoStr(install_mgr.Dump_mgr().Server_urls()));
|
||||
Dump_servers_offline_msg_shown = true;
|
||||
}
|
||||
}
|
||||
Xow_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(dump_file.Wiki_type().Raw());
|
||||
Io_url root_dir = wiki.Fsys_mgr().Root_dir();
|
||||
Io_url[] trg_fil_ary = Io_mgr._.QueryDir_args(root_dir).FilPath_("*." + dump_type + Download_file_ext() + "*").ExecAsUrlAry();
|
||||
Io_url trg = trg_fil_ary.length == 0 ? root_dir.GenSubFil(dump_file.File_name()) : trg_fil_ary[0];
|
||||
this.Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());
|
||||
this.Init("download", dump_file.File_url(), trg);
|
||||
return super.Async_init();
|
||||
}
|
||||
private static boolean Dump_servers_offline_msg_shown = false;
|
||||
}
|
||||
class Xoi_cmd_wiki_unzip extends Gfo_thread_cmd_unzip implements Gfo_thread_cmd { public static final String KEY_dump = "wiki.unzip";
|
||||
public Xoi_cmd_wiki_unzip(Xoi_setup_mgr install_mgr, String wiki_key, String dump_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.dump_date = dump_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, dump_date, dump_type;
|
||||
@Override public String Async_key() {return KEY_dump;}
|
||||
@Override public byte Async_init() {
|
||||
Xoa_app app = install_mgr.App(); Gfui_kit kit = app.Gui_mgr().Kit();
|
||||
Xow_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_utf8_(wiki_key));
|
||||
Io_url wiki_dir = wiki.Import_cfg().Src_dir();
|
||||
Io_url[] urls = Io_mgr._.QueryDir_args(wiki_dir).Recur_(false).FilPath_("*.xml.bz2").ExecAsUrlAry();
|
||||
if (urls.length == 0) {
|
||||
kit.Ask_ok(GRP_KEY, "dump.unzip_latest.file_missing", "Could not find a dump file for ~{0} in ~{1}", wiki_key, wiki_dir.Raw());
|
||||
return Gfo_thread_cmd_.Init_cancel_step;
|
||||
}
|
||||
Io_url src = urls[urls.length - 1];
|
||||
Io_url trg = app.Fsys_mgr().Wiki_dir().GenSubFil_nest(wiki_key, src.NameOnly()); // NOTE: NameOnly() will strip trailing .bz2; EX: a.xml.bz2 -> a.xml
|
||||
super.Init(app.Usr_dlg(), app.Gui_mgr().Kit(), app.Fsys_mgr().App_mgr().App_decompress_bz2(), app.Fsys_mgr().App_mgr().App_decompress_zip(), app.Fsys_mgr().App_mgr().App_decompress_gz(), src, trg);
|
||||
this.Term_cmd_for_src_(Term_cmd_for_src_move);
|
||||
this.Term_cmd_for_src_url_(app.Fsys_mgr().Wiki_dir().GenSubFil_nest("#dump", "done", src.NameAndExt()));
|
||||
if (Io_mgr._.ExistsFil(trg)) {
|
||||
int rslt = kit.Ask_yes_no_cancel(GRP_KEY, "target_exists", "Target file already exists: '~{0}'.\nDo you want to delete it?", trg.Raw());
|
||||
switch (rslt) {
|
||||
case Gfui_dlg_msg_.Btn_yes: Io_mgr._.DeleteFil(trg); break;
|
||||
case Gfui_dlg_msg_.Btn_no: return Gfo_thread_cmd_.Init_cancel_step;
|
||||
case Gfui_dlg_msg_.Btn_cancel: return Gfo_thread_cmd_.Init_cancel_all;
|
||||
default: throw Err_mgr._.unhandled_(rslt);
|
||||
}
|
||||
}
|
||||
return Gfo_thread_cmd_.Init_ok;
|
||||
}
|
||||
static final String GRP_KEY = "xowa.thread.dump.unzip";
|
||||
}
|
||||
class Xoi_cmd_wiki_image_cfg extends Gfo_thread_cmd_replace implements Gfo_thread_cmd { public Xoi_cmd_wiki_image_cfg(Xoa_app app, Io_url url) {this.app = app; super.Init(app.Usr_dlg(), app.Gui_mgr().Kit(), url);} private Xoa_app app;
|
||||
@Override public void Async_run() {
|
||||
super.Async_run();
|
||||
app.Cfg_mgr().Set_by_app("app.files.download.enabled", "y");
|
||||
app.Cfg_mgr().Db_save_txt();
|
||||
}
|
||||
static final String GRP_KEY = "xowa.thread.dump.image_cfg";
|
||||
public static final String KEY_dump = "wiki.image_cfg";
|
||||
}
|
||||
class Xoi_cmd_wiki_goto_page extends Gfo_thread_cmd_base implements Gfo_thread_cmd {
|
||||
public Xoi_cmd_wiki_goto_page(Xoa_app app, String page) {this.app = app; this.page = page; this.Ctor(app.Usr_dlg(), app.Gui_mgr().Kit());} private Xoa_app app; String page;
|
||||
@Override public void Async_run() {kit.New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_goto_page, GfoMsg_.Null);}
|
||||
private void Goto_page(String page) {app.Gui_mgr().Browser_win().Page__navigate_by_url_bar(page);}
|
||||
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_goto_page)) Goto_page(page);
|
||||
else return super.Invk(ctx, ikey, k, m);
|
||||
return this;
|
||||
} private static final String Invk_goto_page = "goto_page";
|
||||
public static final String KEY = "wiki.goto_page";
|
||||
}
|
||||
class Xoi_cmd_imageMagick_download extends Gfo_thread_cmd_download implements Gfo_thread_cmd {// private static final byte[] Bry_windows_zip = Bry_.new_ascii_("-windows.zip");
|
||||
// static final String Src_imageMagick = "ftp://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/binaries/";
|
||||
public Xoi_cmd_imageMagick_download(Gfo_usr_dlg usr_dlg, Gfui_kit kit, Io_url trg) {this.Ctor(usr_dlg, kit); this.trg = trg;} Io_url trg;
|
||||
@Override public byte Async_init() { // <a href="ImageMagick-6.8.1-9-Q16-x86-windows.zip">
|
||||
// byte[] raw = xrg.Exec_as_bry(Src_imageMagick);
|
||||
// int find_pos = Bry_finder.Find_fwd(raw, Bry_windows_zip); if (find_pos == Bry_.NotFound) return Fail();
|
||||
// int bgn_pos = Bry_finder.Find_bwd(raw, Byte_ascii.Quote, find_pos); if (bgn_pos == Bry_.NotFound) return Fail();
|
||||
// ++bgn_pos;
|
||||
// int end_pos = Bry_finder.Find_fwd(raw, Byte_ascii.Quote, bgn_pos); if (end_pos == Bry_.NotFound) return Fail();
|
||||
// String src = Src_imageMagick + String_.new_ascii_(Bry_.Mid(raw, bgn_pos, end_pos));
|
||||
String src = "http://ftp.sunet.se/pub/multimedia/graphics/ImageMagick/binaries/ImageMagick-6.8.8-1-Q16-x86-windows.zip";
|
||||
this.Init("downloading", src, trg);
|
||||
return super.Async_init();
|
||||
}
|
||||
byte Fail() {
|
||||
kit.Ask_ok(GRP_KEY, "windows_not_found", "Could not find Windows binary. Please download ImageMagick directly from the site.");
|
||||
return Gfo_thread_cmd_.Init_cancel_step;
|
||||
}
|
||||
public static final String KEY_imageMagick = "download.imageMagick";
|
||||
static final String GRP_KEY = "xowa.install.cmds.download.imageMagick";
|
||||
}
|
||||
class Xoi_cmd_msg_ok extends Gfo_thread_cmd_base implements Gfo_thread_cmd {
|
||||
public Xoi_cmd_msg_ok(Gfo_usr_dlg usr_dlg, Gfui_kit kit, String msg) {this.msg = msg; this.Ctor(usr_dlg, kit);} private String msg;
|
||||
@Override public boolean Async_term() {
|
||||
kit.Ask_ok("msg_ok", "msg", msg);
|
||||
return true;
|
||||
}
|
||||
public static final String KEY = "msg.ok";
|
||||
}
|
||||
// class Gfo_thread_exec_sync : Gfo_thread_cmd_base, Gfo_thread_cmd {
|
||||
// public Gfo_thread_exec_sync(Gfo_usr_dlg usr_dlg, Gfui_kit kit, Io_url exec_url, String exec_args) {this.Ctor(usr_dlg, kit); this.exec_url = exec_url; this.exec_args = exec_args;} Io_url exec_url; String exec_args;
|
||||
// public override byte Async_init() {
|
||||
// if (!kit.Ask_yes_no(GRP_KEY, "confirm", "You are about to run '~{0}'. Please confirm that XOWA requested you to run '~{0}' and that you trust it to run on your machine.", exec_url.NameAndExt())) {
|
||||
// usr_dlg.Warn_many(GRP_KEY, "confirm.fail", "program was untrusted: ~{0} ~{1}", exec_url.Raw(), exec_args);
|
||||
// return Gfo_thread_cmd_.Init_cancel_all;
|
||||
// }
|
||||
// usr_dlg.Prog_many("exec_sync", "bgn", "running process. please wait; ~{0} ~{1}", exec_url.Raw(), exec_args);
|
||||
// return Gfo_thread_cmd_.Init_ok;
|
||||
// }
|
||||
// public override void Async_run() {
|
||||
// ProcessAdp process = new ProcessAdp().Exe_url_(exec_url).Args_str_(exec_args).Prog_dlg_(usr_dlg);
|
||||
// process.Run_mode_(ProcessAdp.Run_mode_sync_block);
|
||||
// process.Run();
|
||||
// }
|
||||
// public static final String KEY = "exec.sync";
|
||||
// static final String GRP_KEY = "exec.sync";
|
||||
// }
|
||||
class Xoi_cmd_wiki_zip implements Gfo_thread_cmd {
|
||||
public Xoi_cmd_wiki_zip(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
|
||||
public static final String KEY = "wiki.zip";
|
||||
public void Cmd_ctor() {}
|
||||
public String Async_key() {return KEY;}
|
||||
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
|
||||
public boolean Async_prog_enabled() {return false;}
|
||||
public void Async_prog_run(int async_sleep_sum) {}
|
||||
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
|
||||
public boolean Async_term() {
|
||||
wiki.Fsys_mgr().Scan_dirs();
|
||||
install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "zip.end", "zip.end ~{0}", wiki_key);
|
||||
install_mgr.App().Gui_wtr().Prog_many(GRP_KEY, "zip.done", "zip done");
|
||||
return true;
|
||||
}
|
||||
public GfoInvkAble Owner() {return owner;} public Xoi_cmd_wiki_zip Owner_(GfoInvkAble v) {owner = v; return this;} GfoInvkAble owner;
|
||||
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
|
||||
public void Async_run() {
|
||||
running = true;
|
||||
install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "zip.bgn", "zip.bgn ~{0}", wiki_key);
|
||||
ThreadAdp_.invk_(this, Invk_process_async).Start();
|
||||
}
|
||||
public boolean Async_running() {
|
||||
return running;
|
||||
}
|
||||
boolean running, delete_dirs_page = true, notify_done = true;
|
||||
private void Process_async() {
|
||||
Xoa_app app = install_mgr.App();
|
||||
Xob_bldr bldr = app.Bldr();
|
||||
wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_ascii_(wiki_key));
|
||||
wiki.Init_assert();
|
||||
bldr.Cmd_mgr().Clear();
|
||||
bldr.Pause_at_end_(false);
|
||||
((Xobc_deploy_zip)bldr.Cmd_mgr().Add_cmd(wiki, "deploy.zip")).Delete_dirs_page_(delete_dirs_page);
|
||||
bldr.Run();
|
||||
app.Gui_wtr().Prog_none(GRP_KEY, "clear", "");
|
||||
app.Gui_wtr().Note_none(GRP_KEY, "clear", "");
|
||||
running = false;
|
||||
} private Xow_wiki wiki;
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_process_async)) Process_async();
|
||||
else if (ctx.Match(k, Invk_owner)) return owner;
|
||||
else if (ctx.Match(k, Invk_delete_dirs_page_)) delete_dirs_page = m.ReadYn("v");
|
||||
else if (ctx.Match(k, Invk_notify_done_)) notify_done = m.ReadYn("v");
|
||||
else return GfoInvkAble_.Rv_unhandled;
|
||||
return this;
|
||||
} private static final String Invk_process_async = "run_async", Invk_owner = "owner", Invk_delete_dirs_page_ = "delete_dirs_page_", Invk_notify_done_ = "notify_done_";
|
||||
private static final String GRP_KEY = "xowa.thread.op.wiki.zip";
|
||||
}
|
||||
116
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_wiki_import.java
Normal file
116
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_wiki_import.java
Normal file
@@ -0,0 +1,116 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import gplx.threads.*; import gplx.xowa.gui.views.*;
|
||||
class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
|
||||
public Xoi_cmd_wiki_import(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
|
||||
public static final String KEY = "wiki.import";
|
||||
public void Cmd_ctor() {}
|
||||
public String Async_key() {return KEY;}
|
||||
public int Async_sleep_interval() {return Gfo_thread_cmd_.Async_sleep_interval_1_second;}
|
||||
public boolean Async_prog_enabled() {return false;}
|
||||
public void Async_prog_run(int async_sleep_sum) {}
|
||||
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
|
||||
public boolean Async_term() {
|
||||
install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.end", "import.end ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
|
||||
return true;
|
||||
}
|
||||
public GfoInvkAble Owner() {return owner;} public Xoi_cmd_wiki_import Owner_(GfoInvkAble v) {owner = v; return this;} GfoInvkAble owner;
|
||||
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
|
||||
public void Async_run() {
|
||||
running = true;
|
||||
install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.bgn", "import.bgn ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
|
||||
ThreadAdp_.invk_(this, Invk_process_async).Start();
|
||||
}
|
||||
public boolean Async_running() {
|
||||
return running;
|
||||
}
|
||||
boolean running;
|
||||
public boolean Import_move_bz2_to_done() {return import_move_bz2_to_done;} public Xoi_cmd_wiki_import Import_move_bz2_to_done_(boolean v) {import_move_bz2_to_done = v; return this;} private boolean import_move_bz2_to_done = true;
|
||||
private void Process_txt(Xob_bldr bldr) {
|
||||
((gplx.xowa.bldrs.imports.Xobc_core_cleanup)bldr.Cmd_mgr().Add_cmd(wiki, "core.cleanup")).Delete_wiki_(true).Delete_sqlite3_(true);
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "core.init");
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "core.make_page");
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "core.make_id");
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "core.make_search_title");
|
||||
if (wiki.Import_cfg().Category_version() == gplx.xowa.ctgs.Xoa_ctg_mgr.Version_1)
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "core.make_category");
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "core.calc_stats");
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "core.term");
|
||||
}
|
||||
private void Process_sql(Xob_bldr bldr) {
|
||||
((gplx.xowa.bldrs.imports.Xobc_core_cleanup)bldr.Cmd_mgr().Add_cmd(wiki, "core.cleanup")).Delete_wiki_(true).Delete_sqlite3_(true);
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.init");
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.page");
|
||||
if (wiki.Import_cfg().Category_version() == gplx.xowa.ctgs.Xoa_ctg_mgr.Version_1) {
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.category_v1");
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.category_registry");
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.categorylinks");
|
||||
}
|
||||
if (wiki.App().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.specials.search.Xosrh_core.Version_2)
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.search_title.wkr");
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, "import.sql.term");
|
||||
}
|
||||
private void Process_async() {
|
||||
Xoa_app app = install_mgr.App();
|
||||
app.Usr_dlg().Prog_one("", "", "preparing import: ~{0}", wiki_key);
|
||||
Xob_bldr bldr = app.Bldr();
|
||||
wiki = app.Wiki_mgr().Get_by_key_or_make(Bry_.new_ascii_(wiki_key));
|
||||
wiki.Init_assert();
|
||||
bldr.Cmd_mgr().Clear();
|
||||
bldr.Pause_at_end_(false);
|
||||
Io_url src_url = wiki.Import_cfg().Src_rdr().Url();
|
||||
if (install_mgr.Dump_mgr().Wiki_storage_type_is_sql())
|
||||
Process_sql(bldr);
|
||||
else
|
||||
Process_txt(bldr);
|
||||
bldr.Run();
|
||||
app.Gui_wtr().Prog_none(GRP_KEY, "clear", "");
|
||||
app.Gui_wtr().Note_none(GRP_KEY, "clear", "");
|
||||
app.User().Available_from_fsys();
|
||||
wiki.Init_needed_(true);
|
||||
wiki.Html_mgr().Page_wtr_mgr().Init_(true);
|
||||
wiki.Init_assert();
|
||||
if (String_.Eq(src_url.Ext(), ".xml")) {
|
||||
if (app.Setup_mgr().Dump_mgr().Delete_xml_file())
|
||||
Io_mgr._.DeleteFil(src_url);
|
||||
}
|
||||
else if (String_.Eq(src_url.Ext(), ".bz2")) {
|
||||
Io_url trg_fil = app.Fsys_mgr().Wiki_dir().GenSubFil_nest("#dump", "done", src_url.NameAndExt());
|
||||
if (import_move_bz2_to_done)
|
||||
Io_mgr._.MoveFil_args(src_url, trg_fil, true).Exec();
|
||||
}
|
||||
running = false;
|
||||
wiki.Import_cfg().Src_fil_xml_(null).Src_fil_bz2_(null); // reset file else error when going from Import/Script to Import/List
|
||||
app.Gui_mgr().Kit().New_cmd_sync(this).Invk(GfsCtx.new_(), 0, Invk_open_wiki, GfoMsg_.Null);
|
||||
} private Xow_wiki wiki;
|
||||
private void Open_wiki(String wiki_key) {
|
||||
Xog_win_itm main_win = install_mgr.App().Gui_mgr().Browser_win();
|
||||
if (main_win.Active_page() == null) return; // will be null when invoked through cmd-line
|
||||
byte[] url = Bry_.Add(wiki.Domain_bry(), Xoh_href_parser.Href_wiki_bry, wiki.Props().Main_page());
|
||||
main_win.Page__navigate_by_url_bar(String_.new_utf8_(url));
|
||||
}
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_process_async)) Process_async();
|
||||
else if (ctx.Match(k, Invk_owner)) return owner;
|
||||
else if (ctx.Match(k, Invk_open_wiki)) Open_wiki(wiki_key);
|
||||
else return GfoInvkAble_.Rv_unhandled;
|
||||
return this;
|
||||
} private static final String Invk_process_async = "run_async", Invk_owner = "owner", Invk_open_wiki = "open_wiki";
|
||||
static final String GRP_KEY = "xowa.thread.op.build";
|
||||
}
|
||||
901
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_wiki_tst.java
Normal file
901
400_xowa/src_240_install/gplx/xowa/Xoi_cmd_wiki_tst.java
Normal file
@@ -0,0 +1,901 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import org.junit.*;
|
||||
import gplx.brys.*; import gplx.threads.*; import gplx.xowa.wikis.*;
|
||||
public class Xoi_cmd_wiki_tst {
|
||||
@Test public void Run() {
|
||||
// Bld_import_list(Wikis);
|
||||
// Bld_cfg_files(Wikis); // NOTE: remember to carry over the wikisource / page / index commands from the existing xowa_build_cfg.gfs; also, only run the xowa_build_cfg.gfs once; DATE:2013-10-15
|
||||
}
|
||||
public void Bld_import_list(String... ary) {
|
||||
int ary_len = ary.length;
|
||||
Bry_bfr bfr = Bry_bfr.reset_(255);
|
||||
Bry_fmtr_arg_time time_fmtr = new Bry_fmtr_arg_time();
|
||||
Xob_dump_file dump_file = new Xob_dump_file();
|
||||
for (int i = 0; i < ary_len; i++) {
|
||||
String itm = ary[i];
|
||||
dump_file.Ctor(itm, "latest", Xob_bz2_file.Key_pages_articles);
|
||||
int count = 0;
|
||||
while (count++ < 10) {
|
||||
dump_file.Server_url_(Xob_dump_file_.Server_wmf);
|
||||
if (dump_file.Connect()) break;
|
||||
Tfds.WriteText(String_.Format("retrying: {0} {1}\n", count, dump_file.File_modified()));
|
||||
ThreadAdp_.Sleep(2000); // wait for connection to reset
|
||||
}
|
||||
if (count == 10) {
|
||||
Tfds.WriteText(String_.Format("failed: {0}\n", dump_file.File_url()));
|
||||
continue;
|
||||
}
|
||||
else
|
||||
Tfds.WriteText(String_.Format("passed: {0}\n", itm));
|
||||
bfr.Add_str(itm).Add_byte_pipe();
|
||||
bfr.Add_str(dump_file.File_url()).Add_byte_pipe();
|
||||
bfr.Add(Xow_wiki_domain_.Key_by_tid(dump_file.Wiki_type().Tid())).Add_byte_pipe();
|
||||
// Xol_lang_itm lang_itm = Xol_lang_itm_.Get_by_key(wiki_type.Lang_key());
|
||||
// if (lang_itm == null) lang_itm = Xol_lang_itm_.Get_by_key(Xol_lang_.Key_en); // commons, species, meta, etc will have no lang
|
||||
// bfr.Add(lang_itm.Local_name()).Add_byte_pipe();
|
||||
// bfr.Add(lang_itm.Canonical_name()).Add_byte_pipe();
|
||||
long src_size = dump_file.File_len();
|
||||
bfr.Add_long_variable(src_size).Add_byte_pipe();
|
||||
bfr.Add_str(gplx.ios.Io_size_.Xto_str(src_size)).Add_byte_pipe();
|
||||
time_fmtr.Seconds_(Math_.Div_safe_as_long(src_size, 1000000)).XferAry(bfr, 0);
|
||||
bfr.Add_byte_pipe();
|
||||
bfr.Add_str(dump_file.File_modified().XtoStr_fmt_yyyy_MM_dd_HH_mm());
|
||||
bfr.Add_byte_pipe();
|
||||
// bfr.Add_str(String_.ConcatWith_any(",", (Object[])dump_file.Dump_available_dates()));
|
||||
// bfr.Add_byte_pipe();
|
||||
bfr.Add_str(dump_file.Dump_date());
|
||||
bfr.Add_byte_nl();
|
||||
}
|
||||
Io_mgr._.SaveFilStr("C:\\temp.txt", bfr.XtoStr());
|
||||
}
|
||||
public void Bld_cfg_files(String... ary) {
|
||||
Bry_bfr bfr = Bry_bfr.reset_(255);
|
||||
gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_api api = new gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_api();
|
||||
gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_wiki wiki = new gplx.xowa.bldrs.wiki_cfgs.Xoi_wiki_props_wiki();
|
||||
int ary_len = ary.length;
|
||||
for (int i = 0; i < ary_len; i++) {
|
||||
String wiki_domain = ary[i];
|
||||
try {
|
||||
byte[] xml = api.Exec_api(api.Api_src(wiki_domain));
|
||||
wiki.Wiki_domain_(Bry_.new_ascii_(wiki_domain));
|
||||
api.Parse(wiki, String_.new_utf8_(xml));
|
||||
api.Build_cfg(bfr, wiki);
|
||||
}
|
||||
catch (Exception e) {
|
||||
ConsoleAdp._.WriteLine(Err_.Message_gplx_brief(e));
|
||||
}
|
||||
}
|
||||
bfr.Add_str("app.wiki_cfg_bldr.run;").Add_byte_nl();
|
||||
Io_mgr._.SaveFilStr("C:\\xowa_build_cfg.gfs", bfr.XtoStr());
|
||||
}
|
||||
public static String[] Wikis = new String[]
|
||||
{ "simple.wikipedia.org"
|
||||
};
|
||||
//{ "aa.wikipedia.org"
|
||||
//, "aa.wikibooks.org"
|
||||
//, "aa.wiktionary.org"
|
||||
//, "ab.wikipedia.org"
|
||||
//, "ab.wiktionary.org"
|
||||
//, "ace.wikipedia.org"
|
||||
//, "af.wikipedia.org"
|
||||
//, "af.wikibooks.org"
|
||||
//, "af.wikiquote.org"
|
||||
//, "af.wiktionary.org"
|
||||
//, "ak.wikipedia.org"
|
||||
//, "ak.wikibooks.org"
|
||||
//, "ak.wiktionary.org"
|
||||
//, "als.wikipedia.org"
|
||||
//, "als.wikibooks.org"
|
||||
//, "als.wikiquote.org"
|
||||
//, "als.wiktionary.org"
|
||||
//, "am.wikipedia.org"
|
||||
//, "am.wikiquote.org"
|
||||
//, "am.wiktionary.org"
|
||||
//, "ang.wikipedia.org"
|
||||
//, "ang.wikibooks.org"
|
||||
//, "ang.wikiquote.org"
|
||||
//, "ang.wikisource.org"
|
||||
//, "ang.wiktionary.org"
|
||||
//, "an.wikipedia.org"
|
||||
//, "an.wiktionary.org"
|
||||
//, "arc.wikipedia.org"
|
||||
//, "ar.wikipedia.org"
|
||||
//, "ar.wikibooks.org"
|
||||
//, "ar.wikinews.org"
|
||||
//, "ar.wikiquote.org"
|
||||
//, "ar.wikisource.org"
|
||||
//, "ar.wikiversity.org"
|
||||
//, "ar.wiktionary.org"
|
||||
//, "arz.wikipedia.org"
|
||||
//, "ast.wikipedia.org"
|
||||
//, "ast.wikibooks.org"
|
||||
//, "ast.wikiquote.org"
|
||||
//, "ast.wiktionary.org"
|
||||
//, "as.wikipedia.org"
|
||||
//, "as.wikibooks.org"
|
||||
//, "as.wikisource.org"
|
||||
//, "as.wiktionary.org"
|
||||
//, "av.wikipedia.org"
|
||||
//, "av.wiktionary.org"
|
||||
//, "ay.wikipedia.org"
|
||||
//, "ay.wikibooks.org"
|
||||
//, "ay.wiktionary.org"
|
||||
//, "az.wikipedia.org"
|
||||
//, "az.wikibooks.org"
|
||||
//, "az.wikiquote.org"
|
||||
//, "az.wikisource.org"
|
||||
//, "az.wiktionary.org"
|
||||
//, "bar.wikipedia.org"
|
||||
//, "bat-smg.wikipedia.org"
|
||||
//, "ba.wikipedia.org"
|
||||
//, "ba.wikibooks.org"
|
||||
//, "bcl.wikipedia.org"
|
||||
//, "be-x-old.wikipedia.org"
|
||||
//, "be.wikipedia.org"
|
||||
//, "be.wikibooks.org"
|
||||
//, "be.wikiquote.org"
|
||||
//, "be.wikisource.org"
|
||||
//, "be.wiktionary.org"
|
||||
//, "bg.wikipedia.org"
|
||||
//, "bg.wikibooks.org"
|
||||
//, "bg.wikinews.org"
|
||||
//, "bg.wikiquote.org"
|
||||
//, "bg.wikisource.org"
|
||||
//, "bg.wiktionary.org"
|
||||
//, "bh.wikipedia.org"
|
||||
//, "bh.wiktionary.org"
|
||||
//, "bi.wikipedia.org"
|
||||
//, "bi.wikibooks.org"
|
||||
//, "bi.wiktionary.org"
|
||||
//, "bjn.wikipedia.org"
|
||||
//, "bm.wikipedia.org"
|
||||
//, "bm.wikibooks.org"
|
||||
//, "bm.wikiquote.org"
|
||||
//, "bm.wiktionary.org"
|
||||
//, "bn.wikipedia.org"
|
||||
//, "bn.wikibooks.org"
|
||||
//, "bn.wikisource.org"
|
||||
//, "bn.wiktionary.org"
|
||||
//, "bo.wikipedia.org"
|
||||
//, "bo.wikibooks.org"
|
||||
//, "bo.wiktionary.org"
|
||||
//, "bpy.wikipedia.org"
|
||||
//, "br.wikipedia.org"
|
||||
//, "br.wikiquote.org"
|
||||
//, "br.wikisource.org"
|
||||
//, "br.wiktionary.org"
|
||||
//, "bs.wikipedia.org"
|
||||
//, "bs.wikibooks.org"
|
||||
//, "bs.wikinews.org"
|
||||
//, "bs.wikiquote.org"
|
||||
//, "bs.wikisource.org"
|
||||
//, "bs.wiktionary.org"
|
||||
//, "bug.wikipedia.org"
|
||||
//, "bxr.wikipedia.org"
|
||||
//, "ca.wikipedia.org"
|
||||
//, "ca.wikibooks.org"
|
||||
//, "ca.wikinews.org"
|
||||
//, "ca.wikiquote.org"
|
||||
//, "ca.wikisource.org"
|
||||
//, "ca.wiktionary.org"
|
||||
//, "cbk-zam.wikipedia.org"
|
||||
//, "cdo.wikipedia.org"
|
||||
//, "ceb.wikipedia.org"
|
||||
//, "ce.wikipedia.org"
|
||||
//, "cho.wikipedia.org"
|
||||
//, "chr.wikipedia.org"
|
||||
//, "chr.wiktionary.org"
|
||||
//, "ch.wikipedia.org"
|
||||
//, "ch.wikibooks.org"
|
||||
//, "ch.wiktionary.org"
|
||||
//, "chy.wikipedia.org"
|
||||
//, "ckb.wikipedia.org"
|
||||
//, "commons.wikimedia.org"
|
||||
//, "co.wikipedia.org"
|
||||
//, "co.wikibooks.org"
|
||||
//, "co.wikiquote.org"
|
||||
//, "co.wiktionary.org"
|
||||
//, "crh.wikipedia.org"
|
||||
//, "cr.wikipedia.org"
|
||||
//, "cr.wikiquote.org"
|
||||
//, "cr.wiktionary.org"
|
||||
//, "csb.wikipedia.org"
|
||||
//, "csb.wiktionary.org"
|
||||
//, "cs.wikipedia.org"
|
||||
//, "cs.wikibooks.org"
|
||||
//, "cs.wikinews.org"
|
||||
//, "cs.wikiquote.org"
|
||||
//, "cs.wikisource.org"
|
||||
//, "cs.wikiversity.org"
|
||||
//, "cs.wiktionary.org"
|
||||
//, "cu.wikipedia.org"
|
||||
//, "cv.wikipedia.org"
|
||||
//, "cv.wikibooks.org"
|
||||
//, "cy.wikipedia.org"
|
||||
//, "cy.wikibooks.org"
|
||||
//, "cy.wikiquote.org"
|
||||
//, "cy.wikisource.org"
|
||||
//, "cy.wiktionary.org"
|
||||
//, "da.wikipedia.org"
|
||||
//, "da.wikibooks.org"
|
||||
//, "da.wikiquote.org"
|
||||
//, "da.wikisource.org"
|
||||
//, "da.wiktionary.org"
|
||||
//, "de.wikipedia.org"
|
||||
//, "de.wikibooks.org"
|
||||
//, "de.wikinews.org"
|
||||
//, "de.wikiquote.org"
|
||||
//, "de.wikisource.org"
|
||||
//, "de.wikiversity.org"
|
||||
//, "de.wikivoyage.org"
|
||||
//, "de.wiktionary.org"
|
||||
//, "diq.wikipedia.org"
|
||||
//, "dsb.wikipedia.org"
|
||||
//, "dv.wikipedia.org"
|
||||
//, "dv.wiktionary.org"
|
||||
//, "dz.wikipedia.org"
|
||||
//, "dz.wiktionary.org"
|
||||
//, "ee.wikipedia.org"
|
||||
//, "el.wikipedia.org"
|
||||
//, "el.wikibooks.org"
|
||||
//, "el.wikinews.org"
|
||||
//, "el.wikiquote.org"
|
||||
//, "el.wikisource.org"
|
||||
//, "el.wikiversity.org"
|
||||
//, "el.wikivoyage.org"
|
||||
//, "el.wiktionary.org"
|
||||
//, "eml.wikipedia.org"
|
||||
//, "en.wikipedia.org"
|
||||
//, "en.wikibooks.org"
|
||||
//, "en.wikinews.org"
|
||||
//, "en.wikiquote.org"
|
||||
//, "en.wikisource.org"
|
||||
//, "en.wikiversity.org"
|
||||
//, "en.wikivoyage.org"
|
||||
//, "en.wiktionary.org"
|
||||
//, "eo.wikipedia.org"
|
||||
//, "eo.wikibooks.org"
|
||||
//, "eo.wikinews.org"
|
||||
//, "eo.wikiquote.org"
|
||||
//, "eo.wikisource.org"
|
||||
//, "eo.wiktionary.org"
|
||||
//, "es.wikipedia.org"
|
||||
//, "es.wikibooks.org"
|
||||
//, "es.wikinews.org"
|
||||
//, "es.wikiquote.org"
|
||||
//, "es.wikisource.org"
|
||||
//, "es.wikiversity.org"
|
||||
//, "es.wikivoyage.org"
|
||||
//, "es.wiktionary.org"
|
||||
//, "et.wikipedia.org"
|
||||
//, "et.wikibooks.org"
|
||||
//, "et.wikiquote.org"
|
||||
//, "et.wikisource.org"
|
||||
//, "et.wiktionary.org"
|
||||
//, "eu.wikipedia.org"
|
||||
//, "eu.wikibooks.org"
|
||||
//, "eu.wikiquote.org"
|
||||
//, "eu.wiktionary.org"
|
||||
//, "ext.wikipedia.org"
|
||||
//, "fa.wikipedia.org"
|
||||
//, "fa.wikibooks.org"
|
||||
//, "fa.wikinews.org"
|
||||
//, "fa.wikiquote.org"
|
||||
//, "fa.wikisource.org"
|
||||
//, "fa.wiktionary.org"
|
||||
//, "ff.wikipedia.org"
|
||||
//, "fiu-vro.wikipedia.org"
|
||||
//, "fi.wikipedia.org"
|
||||
//, "fi.wikibooks.org"
|
||||
//, "fi.wikinews.org"
|
||||
//, "fi.wikiquote.org"
|
||||
//, "fi.wikisource.org"
|
||||
//, "fi.wikiversity.org"
|
||||
//, "fi.wiktionary.org"
|
||||
//, "fj.wikipedia.org"
|
||||
//, "fj.wiktionary.org"
|
||||
//, "wikimediafoundation.org"
|
||||
//, "fo.wikipedia.org"
|
||||
//, "fo.wikisource.org"
|
||||
//, "fo.wiktionary.org"
|
||||
//, "frp.wikipedia.org"
|
||||
//, "frr.wikipedia.org"
|
||||
//, "fr.wikipedia.org"
|
||||
//, "fr.wikibooks.org"
|
||||
//, "fr.wikinews.org"
|
||||
//, "fr.wikiquote.org"
|
||||
//, "fr.wikisource.org"
|
||||
//, "fr.wikiversity.org"
|
||||
//, "fr.wikivoyage.org"
|
||||
//, "fr.wiktionary.org"
|
||||
//, "fur.wikipedia.org"
|
||||
//, "fy.wikipedia.org"
|
||||
//, "fy.wikibooks.org"
|
||||
//, "fy.wiktionary.org"
|
||||
//, "gag.wikipedia.org"
|
||||
//, "gan.wikipedia.org"
|
||||
//, "ga.wikipedia.org"
|
||||
//, "ga.wikibooks.org"
|
||||
//, "ga.wikiquote.org"
|
||||
//, "ga.wiktionary.org"
|
||||
//, "gd.wikipedia.org"
|
||||
//, "gd.wiktionary.org"
|
||||
//, "glk.wikipedia.org"
|
||||
//, "gl.wikipedia.org"
|
||||
//, "gl.wikibooks.org"
|
||||
//, "gl.wikiquote.org"
|
||||
//, "gl.wikisource.org"
|
||||
//, "gl.wiktionary.org"
|
||||
//, "gn.wikipedia.org"
|
||||
//, "gn.wikibooks.org"
|
||||
//, "gn.wiktionary.org"
|
||||
//, "got.wikipedia.org"
|
||||
//, "got.wikibooks.org"
|
||||
//, "gu.wikipedia.org"
|
||||
//, "gu.wikibooks.org"
|
||||
//, "gu.wikiquote.org"
|
||||
//, "gu.wikisource.org"
|
||||
//, "gu.wiktionary.org"
|
||||
//, "gv.wikipedia.org"
|
||||
//, "gv.wiktionary.org"
|
||||
//, "hak.wikipedia.org"
|
||||
//, "ha.wikipedia.org"
|
||||
//, "ha.wiktionary.org"
|
||||
//, "haw.wikipedia.org"
|
||||
//, "he.wikipedia.org"
|
||||
//, "he.wikibooks.org"
|
||||
//, "he.wikinews.org"
|
||||
//, "he.wikiquote.org"
|
||||
//, "he.wikisource.org"
|
||||
//, "he.wikivoyage.org"
|
||||
//, "he.wiktionary.org"
|
||||
//, "hif.wikipedia.org"
|
||||
//, "hi.wikipedia.org"
|
||||
//, "hi.wikibooks.org"
|
||||
//, "hi.wikiquote.org"
|
||||
//, "hi.wiktionary.org"
|
||||
//, "ho.wikipedia.org"
|
||||
//, "hr.wikipedia.org"
|
||||
//, "hr.wikibooks.org"
|
||||
//, "hr.wikiquote.org"
|
||||
//, "hr.wikisource.org"
|
||||
//, "hr.wiktionary.org"
|
||||
//, "hsb.wikipedia.org"
|
||||
//, "hsb.wiktionary.org"
|
||||
//, "ht.wikipedia.org"
|
||||
//, "ht.wikisource.org"
|
||||
//, "hu.wikipedia.org"
|
||||
//, "hu.wikibooks.org"
|
||||
//, "hu.wikinews.org"
|
||||
//, "hu.wikiquote.org"
|
||||
//, "hu.wikisource.org"
|
||||
//, "hu.wiktionary.org"
|
||||
//, "hy.wikipedia.org"
|
||||
//, "hy.wikibooks.org"
|
||||
//, "hy.wikiquote.org"
|
||||
//, "hy.wikisource.org"
|
||||
//, "hy.wiktionary.org"
|
||||
//, "hz.wikipedia.org"
|
||||
//, "ia.wikipedia.org"
|
||||
//, "ia.wikibooks.org"
|
||||
//, "ia.wiktionary.org"
|
||||
//, "id.wikipedia.org"
|
||||
//, "id.wikibooks.org"
|
||||
//, "id.wikiquote.org"
|
||||
//, "id.wikisource.org"
|
||||
//, "id.wiktionary.org"
|
||||
//, "ie.wikipedia.org"
|
||||
//, "ie.wikibooks.org"
|
||||
//, "ie.wiktionary.org"
|
||||
//, "ig.wikipedia.org"
|
||||
//, "ii.wikipedia.org"
|
||||
//, "ik.wikipedia.org"
|
||||
//, "ik.wiktionary.org"
|
||||
//, "ilo.wikipedia.org"
|
||||
//, "incubator.wikimedia.org"
|
||||
//, "io.wikipedia.org"
|
||||
//, "io.wiktionary.org"
|
||||
//, "is.wikipedia.org"
|
||||
//, "is.wikibooks.org"
|
||||
//, "is.wikiquote.org"
|
||||
//, "is.wikisource.org"
|
||||
//, "is.wiktionary.org"
|
||||
//, "it.wikipedia.org"
|
||||
//, "it.wikibooks.org"
|
||||
//, "it.wikinews.org"
|
||||
//, "it.wikiquote.org"
|
||||
//, "it.wikisource.org"
|
||||
//, "it.wikiversity.org"
|
||||
//, "it.wikivoyage.org"
|
||||
//, "it.wiktionary.org"
|
||||
//, "iu.wikipedia.org"
|
||||
//, "iu.wiktionary.org"
|
||||
//, "ja.wikipedia.org"
|
||||
//, "ja.wikibooks.org"
|
||||
//, "ja.wikinews.org"
|
||||
//, "ja.wikiquote.org"
|
||||
//, "ja.wikisource.org"
|
||||
//, "ja.wikiversity.org"
|
||||
//, "ja.wiktionary.org"
|
||||
//, "jbo.wikipedia.org"
|
||||
//, "jbo.wiktionary.org"
|
||||
//, "jv.wikipedia.org"
|
||||
//, "jv.wiktionary.org"
|
||||
//, "kaa.wikipedia.org"
|
||||
//, "kab.wikipedia.org"
|
||||
//, "ka.wikipedia.org"
|
||||
//, "ka.wikibooks.org"
|
||||
//, "ka.wikiquote.org"
|
||||
//, "ka.wiktionary.org"
|
||||
//, "kbd.wikipedia.org"
|
||||
//, "kg.wikipedia.org"
|
||||
//, "ki.wikipedia.org"
|
||||
//, "kj.wikipedia.org"
|
||||
//, "kk.wikipedia.org"
|
||||
//, "kk.wikibooks.org"
|
||||
//, "kk.wikiquote.org"
|
||||
//, "kk.wiktionary.org"
|
||||
//, "kl.wikipedia.org"
|
||||
//, "kl.wiktionary.org"
|
||||
//, "km.wikipedia.org"
|
||||
//, "km.wikibooks.org"
|
||||
//, "km.wiktionary.org"
|
||||
//, "kn.wikipedia.org"
|
||||
//, "kn.wikibooks.org"
|
||||
//, "kn.wikiquote.org"
|
||||
//, "kn.wikisource.org"
|
||||
//, "kn.wiktionary.org"
|
||||
//, "koi.wikipedia.org"
|
||||
//, "ko.wikipedia.org"
|
||||
//, "ko.wikibooks.org"
|
||||
//, "ko.wikinews.org"
|
||||
//, "ko.wikiquote.org"
|
||||
//, "ko.wikisource.org"
|
||||
//, "ko.wikiversity.org"
|
||||
//, "ko.wiktionary.org"
|
||||
//, "krc.wikipedia.org"
|
||||
//, "kr.wikipedia.org"
|
||||
//, "kr.wikiquote.org"
|
||||
//, "ksh.wikipedia.org"
|
||||
//, "ks.wikipedia.org"
|
||||
//, "ks.wikibooks.org"
|
||||
//, "ks.wikiquote.org"
|
||||
//, "ks.wiktionary.org"
|
||||
//, "ku.wikipedia.org"
|
||||
//, "ku.wikibooks.org"
|
||||
//, "ku.wikiquote.org"
|
||||
//, "ku.wiktionary.org"
|
||||
//, "kv.wikipedia.org"
|
||||
//, "kw.wikipedia.org"
|
||||
//, "kw.wikiquote.org"
|
||||
//, "kw.wiktionary.org"
|
||||
//, "ky.wikipedia.org"
|
||||
//, "ky.wikibooks.org"
|
||||
//, "ky.wikiquote.org"
|
||||
//, "ky.wiktionary.org"
|
||||
//, "lad.wikipedia.org"
|
||||
//, "la.wikipedia.org"
|
||||
//, "la.wikibooks.org"
|
||||
//, "la.wikiquote.org"
|
||||
//, "la.wikisource.org"
|
||||
//, "la.wiktionary.org"
|
||||
//, "lbe.wikipedia.org"
|
||||
//, "lb.wikipedia.org"
|
||||
//, "lb.wikibooks.org"
|
||||
//, "lb.wikiquote.org"
|
||||
//, "lb.wiktionary.org"
|
||||
//, "lez.wikipedia.org"
|
||||
//, "lg.wikipedia.org"
|
||||
//, "lij.wikipedia.org"
|
||||
//, "li.wikipedia.org"
|
||||
//, "li.wikibooks.org"
|
||||
//, "li.wikiquote.org"
|
||||
//, "li.wikisource.org"
|
||||
//, "li.wiktionary.org"
|
||||
//, "lmo.wikipedia.org"
|
||||
//, "ln.wikipedia.org"
|
||||
//, "ln.wikibooks.org"
|
||||
//, "ln.wiktionary.org"
|
||||
//, "lo.wikipedia.org"
|
||||
//, "lo.wiktionary.org"
|
||||
//, "ltg.wikipedia.org"
|
||||
//, "lt.wikipedia.org"
|
||||
//, "lt.wikibooks.org"
|
||||
//, "lt.wikiquote.org"
|
||||
//, "lt.wikisource.org"
|
||||
//, "lt.wiktionary.org"
|
||||
//, "lv.wikipedia.org"
|
||||
//, "lv.wikibooks.org"
|
||||
//, "lv.wiktionary.org"
|
||||
//, "map-bms.wikipedia.org"
|
||||
//, "mdf.wikipedia.org"
|
||||
//, "www.mediawiki.org"
|
||||
//, "meta.wikimedia.org"
|
||||
//, "mg.wikipedia.org"
|
||||
//, "mg.wikibooks.org"
|
||||
//, "mg.wiktionary.org"
|
||||
//, "mhr.wikipedia.org"
|
||||
//, "mh.wikipedia.org"
|
||||
//, "mh.wiktionary.org"
|
||||
//, "min.wikipedia.org"
|
||||
//, "mi.wikipedia.org"
|
||||
//, "mi.wikibooks.org"
|
||||
//, "mi.wiktionary.org"
|
||||
//, "mk.wikipedia.org"
|
||||
//, "mk.wikibooks.org"
|
||||
//, "mk.wikisource.org"
|
||||
//, "mk.wiktionary.org"
|
||||
//, "ml.wikipedia.org"
|
||||
//, "ml.wikibooks.org"
|
||||
//, "ml.wikiquote.org"
|
||||
//, "ml.wikisource.org"
|
||||
//, "ml.wiktionary.org"
|
||||
//, "mn.wikipedia.org"
|
||||
//, "mn.wikibooks.org"
|
||||
//, "mn.wiktionary.org"
|
||||
//, "mo.wikipedia.org"
|
||||
//, "mo.wiktionary.org"
|
||||
//, "mrj.wikipedia.org"
|
||||
//, "mr.wikipedia.org"
|
||||
//, "mr.wikibooks.org"
|
||||
//, "mr.wikiquote.org"
|
||||
//, "mr.wikisource.org"
|
||||
//, "mr.wiktionary.org"
|
||||
//, "ms.wikipedia.org"
|
||||
//, "ms.wikibooks.org"
|
||||
//, "ms.wiktionary.org"
|
||||
//, "mt.wikipedia.org"
|
||||
//, "mt.wiktionary.org"
|
||||
//, "mus.wikipedia.org"
|
||||
//, "mwl.wikipedia.org"
|
||||
//, "myv.wikipedia.org"
|
||||
//, "my.wikipedia.org"
|
||||
//, "my.wikibooks.org"
|
||||
//, "my.wiktionary.org"
|
||||
//, "mzn.wikipedia.org"
|
||||
//, "nah.wikipedia.org"
|
||||
//, "nah.wikibooks.org"
|
||||
//, "nah.wiktionary.org"
|
||||
//, "nap.wikipedia.org"
|
||||
//, "na.wikipedia.org"
|
||||
//, "na.wikibooks.org"
|
||||
//, "na.wikiquote.org"
|
||||
//, "na.wiktionary.org"
|
||||
//, "nds-nl.wikipedia.org"
|
||||
//, "nds.wikipedia.org"
|
||||
//, "nds.wikibooks.org"
|
||||
//, "nds.wikiquote.org"
|
||||
//, "nds.wiktionary.org"
|
||||
//, "ne.wikipedia.org"
|
||||
//, "ne.wikibooks.org"
|
||||
//, "ne.wiktionary.org"
|
||||
//, "new.wikipedia.org"
|
||||
//, "ng.wikipedia.org"
|
||||
//, "nl.wikipedia.org"
|
||||
//, "nl.wikibooks.org"
|
||||
//, "nl.wikinews.org"
|
||||
//, "nl.wikiquote.org"
|
||||
//, "nl.wikisource.org"
|
||||
//, "nl.wikivoyage.org"
|
||||
//, "nl.wiktionary.org"
|
||||
//, "nn.wikipedia.org"
|
||||
//, "nn.wikiquote.org"
|
||||
//, "nn.wiktionary.org"
|
||||
//, "nov.wikipedia.org"
|
||||
//, "no.wikipedia.org"
|
||||
//, "no.wikibooks.org"
|
||||
//, "no.wikinews.org"
|
||||
//, "no.wikiquote.org"
|
||||
//, "no.wikisource.org"
|
||||
//, "no.wiktionary.org"
|
||||
//, "nrm.wikipedia.org"
|
||||
//, "nso.wikipedia.org"
|
||||
//, "nv.wikipedia.org"
|
||||
//, "ny.wikipedia.org"
|
||||
//, "oc.wikipedia.org"
|
||||
//, "oc.wikibooks.org"
|
||||
//, "oc.wiktionary.org"
|
||||
//, "om.wikipedia.org"
|
||||
//, "om.wiktionary.org"
|
||||
//, "or.wikipedia.org"
|
||||
//, "or.wiktionary.org"
|
||||
//, "os.wikipedia.org"
|
||||
//, "pag.wikipedia.org"
|
||||
//, "pam.wikipedia.org"
|
||||
//, "pap.wikipedia.org"
|
||||
//, "pa.wikipedia.org"
|
||||
//, "pa.wikibooks.org"
|
||||
//, "pa.wiktionary.org"
|
||||
//, "pcd.wikipedia.org"
|
||||
//, "pdc.wikipedia.org"
|
||||
//, "pfl.wikipedia.org"
|
||||
//, "pih.wikipedia.org"
|
||||
//, "pi.wikipedia.org"
|
||||
//, "pi.wiktionary.org"
|
||||
//, "pl.wikipedia.org"
|
||||
//, "pl.wikibooks.org"
|
||||
//, "pl.wikinews.org"
|
||||
//, "pl.wikiquote.org"
|
||||
//, "pl.wikisource.org"
|
||||
//, "pl.wikivoyage.org"
|
||||
//, "pl.wiktionary.org"
|
||||
//, "pms.wikipedia.org"
|
||||
//, "pnb.wikipedia.org"
|
||||
//, "pnb.wiktionary.org"
|
||||
//, "pnt.wikipedia.org"
|
||||
//, "ps.wikipedia.org"
|
||||
//, "ps.wikibooks.org"
|
||||
//, "ps.wiktionary.org"
|
||||
//, "pt.wikipedia.org"
|
||||
//, "pt.wikibooks.org"
|
||||
//, "pt.wikinews.org"
|
||||
//, "pt.wikiquote.org"
|
||||
//, "pt.wikisource.org"
|
||||
//, "pt.wikiversity.org"
|
||||
//, "pt.wikivoyage.org"
|
||||
//, "pt.wiktionary.org"
|
||||
//, "qu.wikipedia.org"
|
||||
//, "qu.wikibooks.org"
|
||||
//, "qu.wikiquote.org"
|
||||
//, "qu.wiktionary.org"
|
||||
//, "rm.wikipedia.org"
|
||||
//, "rm.wikibooks.org"
|
||||
//, "rm.wiktionary.org"
|
||||
//, "rmy.wikipedia.org"
|
||||
//, "rn.wikipedia.org"
|
||||
//, "rn.wiktionary.org"
|
||||
//, "roa-rup.wikipedia.org"
|
||||
//, "roa-rup.wiktionary.org"
|
||||
//, "roa-tara.wikipedia.org"
|
||||
//, "ro.wikipedia.org"
|
||||
//, "ro.wikibooks.org"
|
||||
//, "ro.wikinews.org"
|
||||
//, "ro.wikiquote.org"
|
||||
//, "ro.wikisource.org"
|
||||
//, "ro.wikivoyage.org"
|
||||
//, "ro.wiktionary.org"
|
||||
//, "rue.wikipedia.org"
|
||||
//, "ru.wikipedia.org"
|
||||
//, "ru.wikibooks.org"
|
||||
//, "ru.wikinews.org"
|
||||
//, "ru.wikiquote.org"
|
||||
//, "ru.wikisource.org"
|
||||
//, "ru.wikiversity.org"
|
||||
//, "ru.wikivoyage.org"
|
||||
//, "ru.wiktionary.org"
|
||||
//, "rw.wikipedia.org"
|
||||
//, "rw.wiktionary.org"
|
||||
//, "sah.wikipedia.org"
|
||||
//, "sah.wikisource.org"
|
||||
//, "sa.wikipedia.org"
|
||||
//, "sa.wikibooks.org"
|
||||
//, "sa.wikiquote.org"
|
||||
//, "sa.wikisource.org"
|
||||
//, "sa.wiktionary.org"
|
||||
//, "scn.wikipedia.org"
|
||||
//, "scn.wiktionary.org"
|
||||
//, "sco.wikipedia.org"
|
||||
//, "sc.wikipedia.org"
|
||||
//, "sc.wiktionary.org"
|
||||
//, "sd.wikipedia.org"
|
||||
//, "sd.wikinews.org"
|
||||
//, "sd.wiktionary.org"
|
||||
//, "se.wikipedia.org"
|
||||
//, "se.wikibooks.org"
|
||||
//, "sg.wikipedia.org"
|
||||
//, "sg.wiktionary.org"
|
||||
//, "sh.wikipedia.org"
|
||||
//, "sh.wiktionary.org"
|
||||
//, "simple.wikipedia.org"
|
||||
//, "simple.wikibooks.org"
|
||||
//, "simple.wikiquote.org"
|
||||
//, "simple.wiktionary.org"
|
||||
//, "si.wikipedia.org"
|
||||
//, "si.wikibooks.org"
|
||||
//, "si.wiktionary.org"
|
||||
//, "sk.wikipedia.org"
|
||||
//, "sk.wikibooks.org"
|
||||
//, "sk.wikiquote.org"
|
||||
//, "sk.wikisource.org"
|
||||
//, "sk.wiktionary.org"
|
||||
//, "sl.wikipedia.org"
|
||||
//, "sl.wikibooks.org"
|
||||
//, "sl.wikiquote.org"
|
||||
//, "sl.wikisource.org"
|
||||
//, "sl.wikiversity.org"
|
||||
//, "sl.wiktionary.org"
|
||||
//, "sm.wikipedia.org"
|
||||
//, "sm.wiktionary.org"
|
||||
//, "sn.wikipedia.org"
|
||||
//, "sn.wiktionary.org"
|
||||
//, "so.wikipedia.org"
|
||||
//, "so.wiktionary.org"
|
||||
//, "species.wikimedia.org"
|
||||
//, "sq.wikipedia.org"
|
||||
//, "sq.wikibooks.org"
|
||||
//, "sq.wikinews.org"
|
||||
//, "sq.wikiquote.org"
|
||||
//, "sq.wiktionary.org"
|
||||
//, "srn.wikipedia.org"
|
||||
//, "sr.wikipedia.org"
|
||||
//, "sr.wikibooks.org"
|
||||
//, "sr.wikinews.org"
|
||||
//, "sr.wikiquote.org"
|
||||
//, "sr.wikisource.org"
|
||||
//, "sr.wiktionary.org"
|
||||
//, "ss.wikipedia.org"
|
||||
//, "ss.wiktionary.org"
|
||||
//, "stq.wikipedia.org"
|
||||
//, "st.wikipedia.org"
|
||||
//, "st.wiktionary.org"
|
||||
//, "su.wikipedia.org"
|
||||
//, "su.wikibooks.org"
|
||||
//, "su.wikiquote.org"
|
||||
//, "su.wiktionary.org"
|
||||
//, "sv.wikipedia.org"
|
||||
//, "sv.wikibooks.org"
|
||||
//, "sv.wikinews.org"
|
||||
//, "sv.wikiquote.org"
|
||||
//, "sv.wikisource.org"
|
||||
//, "sv.wikiversity.org"
|
||||
//, "sv.wikivoyage.org"
|
||||
//, "sv.wiktionary.org"
|
||||
//, "sw.wikipedia.org"
|
||||
//, "sw.wikibooks.org"
|
||||
//, "sw.wiktionary.org"
|
||||
//, "szl.wikipedia.org"
|
||||
//, "ta.wikipedia.org"
|
||||
//, "ta.wikibooks.org"
|
||||
//, "ta.wikinews.org"
|
||||
//, "ta.wikiquote.org"
|
||||
//, "ta.wikisource.org"
|
||||
//, "ta.wiktionary.org"
|
||||
//, "tet.wikipedia.org"
|
||||
//, "te.wikipedia.org"
|
||||
//, "te.wikibooks.org"
|
||||
//, "te.wikiquote.org"
|
||||
//, "te.wikisource.org"
|
||||
//, "te.wiktionary.org"
|
||||
//, "tg.wikipedia.org"
|
||||
//, "tg.wikibooks.org"
|
||||
//, "tg.wiktionary.org"
|
||||
//, "th.wikipedia.org"
|
||||
//, "th.wikibooks.org"
|
||||
//, "th.wikinews.org"
|
||||
//, "th.wikiquote.org"
|
||||
//, "th.wikisource.org"
|
||||
//, "th.wiktionary.org"
|
||||
//, "ti.wikipedia.org"
|
||||
//, "ti.wiktionary.org"
|
||||
//, "tk.wikipedia.org"
|
||||
//, "tk.wikibooks.org"
|
||||
//, "tk.wikiquote.org"
|
||||
//, "tk.wiktionary.org"
|
||||
//, "tl.wikipedia.org"
|
||||
//, "tl.wikibooks.org"
|
||||
//, "tl.wiktionary.org"
|
||||
//, "tn.wikipedia.org"
|
||||
//, "tn.wiktionary.org"
|
||||
//, "to.wikipedia.org"
|
||||
//, "to.wiktionary.org"
|
||||
//, "tpi.wikipedia.org"
|
||||
//, "tpi.wiktionary.org"
|
||||
//, "tr.wikipedia.org"
|
||||
//, "tr.wikibooks.org"
|
||||
//, "tr.wikinews.org"
|
||||
//, "tr.wikiquote.org"
|
||||
//, "tr.wikisource.org"
|
||||
//, "tr.wiktionary.org"
|
||||
//, "ts.wikipedia.org"
|
||||
//, "ts.wiktionary.org"
|
||||
//, "tt.wikipedia.org"
|
||||
//, "tt.wikibooks.org"
|
||||
//, "tt.wikiquote.org"
|
||||
//, "tt.wiktionary.org"
|
||||
//, "tum.wikipedia.org"
|
||||
//, "tw.wikipedia.org"
|
||||
//, "tw.wiktionary.org"
|
||||
//, "tyv.wikipedia.org"
|
||||
//, "ty.wikipedia.org"
|
||||
//, "udm.wikipedia.org"
|
||||
//, "ug.wikipedia.org"
|
||||
//, "ug.wikibooks.org"
|
||||
//, "ug.wikiquote.org"
|
||||
//, "ug.wiktionary.org"
|
||||
//, "uk.wikipedia.org"
|
||||
//, "uk.wikibooks.org"
|
||||
//, "uk.wikinews.org"
|
||||
//, "uk.wikiquote.org"
|
||||
//, "uk.wikisource.org"
|
||||
//, "uk.wikivoyage.org"
|
||||
//, "uk.wiktionary.org"
|
||||
//, "ur.wikipedia.org"
|
||||
//, "ur.wikibooks.org"
|
||||
//, "ur.wikiquote.org"
|
||||
//, "ur.wiktionary.org"
|
||||
//, "uz.wikipedia.org"
|
||||
//, "uz.wikibooks.org"
|
||||
//, "uz.wikiquote.org"
|
||||
//, "uz.wiktionary.org"
|
||||
//, "vec.wikipedia.org"
|
||||
//, "vec.wikisource.org"
|
||||
//, "vec.wiktionary.org"
|
||||
//, "vep.wikipedia.org"
|
||||
//, "ve.wikipedia.org"
|
||||
//, "vi.wikipedia.org"
|
||||
//, "vi.wikibooks.org"
|
||||
//, "vi.wikiquote.org"
|
||||
//, "vi.wikisource.org"
|
||||
//, "vi.wikivoyage.org"
|
||||
//, "vi.wiktionary.org"
|
||||
//, "vls.wikipedia.org"
|
||||
//, "vo.wikipedia.org"
|
||||
//, "vo.wikibooks.org"
|
||||
//, "vo.wikiquote.org"
|
||||
//, "vo.wiktionary.org"
|
||||
//, "war.wikipedia.org"
|
||||
//, "wa.wikipedia.org"
|
||||
//, "wa.wikibooks.org"
|
||||
//, "wa.wiktionary.org"
|
||||
//, "www.wikidata.org"
|
||||
//, "wo.wikipedia.org"
|
||||
//, "wo.wikiquote.org"
|
||||
//, "wo.wiktionary.org"
|
||||
//, "wuu.wikipedia.org"
|
||||
//, "xal.wikipedia.org"
|
||||
//, "xh.wikipedia.org"
|
||||
//, "xh.wikibooks.org"
|
||||
//, "xh.wiktionary.org"
|
||||
//, "xmf.wikipedia.org"
|
||||
//, "yi.wikipedia.org"
|
||||
//, "yi.wikisource.org"
|
||||
//, "yi.wiktionary.org"
|
||||
//, "yo.wikipedia.org"
|
||||
//, "yo.wikibooks.org"
|
||||
//, "yo.wiktionary.org"
|
||||
//, "za.wikipedia.org"
|
||||
//, "za.wikibooks.org"
|
||||
//, "za.wikiquote.org"
|
||||
//, "za.wiktionary.org"
|
||||
//, "zea.wikipedia.org"
|
||||
//, "zh-classical.wikipedia.org"
|
||||
//, "zh-min-nan.wikipedia.org"
|
||||
//, "zh-min-nan.wikibooks.org"
|
||||
//, "zh-min-nan.wikiquote.org"
|
||||
//, "zh-min-nan.wikisource.org"
|
||||
//, "zh-min-nan.wiktionary.org"
|
||||
//, "zh-yue.wikipedia.org"
|
||||
//, "zh.wikipedia.org"
|
||||
//, "zh.wikibooks.org"
|
||||
//, "zh.wikinews.org"
|
||||
//, "zh.wikiquote.org"
|
||||
//, "zh.wikisource.org"
|
||||
//, "zh.wikivoyage.org"
|
||||
//, "zh.wiktionary.org"
|
||||
//, "zu.wikipedia.org"
|
||||
//, "zu.wikibooks.org"
|
||||
//, "zu.wiktionary.org"
|
||||
//};
|
||||
}
|
||||
//, "als.wikisource.org"
|
||||
//, "als.wikinews.org"
|
||||
//, "nds.wikinews.org"
|
||||
//, "ba.wiktionary.org"
|
||||
//, "tokipona.wikibooks.org"
|
||||
241
400_xowa/src_240_install/gplx/xowa/Xoi_css_offline_mgr.java
Normal file
241
400_xowa/src_240_install/gplx/xowa/Xoi_css_offline_mgr.java
Normal file
@@ -0,0 +1,241 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
class Xoi_css_offline_itm {
|
||||
public Xoi_css_offline_itm(byte[] http_url) {this.http_url = http_url;}
|
||||
public byte[] Http_url() {return http_url;} private byte[] http_url;
|
||||
public byte[] File_url() {return file_url;} public void File_url_(byte[] v) {file_url = v;} private byte[] file_url;
|
||||
}
|
||||
class Xoi_css_url_info {
|
||||
// private Gfo_usr_dlg usr_dlg;
|
||||
public int Bgn_pos() {return bgn_pos;} public void Bgn_pos_(int v) {bgn_pos = v;} private int bgn_pos;
|
||||
public int End_pos() {return end_pos;} public void End_pos_(int v) {end_pos = v;} private int end_pos;
|
||||
public boolean Found() {return found;} public void Found_(boolean v) {found = v;} private boolean found;
|
||||
public void Init(Gfo_usr_dlg usr_dlg) {
|
||||
// this.usr_dlg = usr_dlg;
|
||||
}
|
||||
public void Clear() {
|
||||
bgn_pos = end_pos = -1;
|
||||
found = false;
|
||||
}
|
||||
public void Quote_data(byte end_byte, boolean quoted) {
|
||||
}
|
||||
public Xoi_css_url_info Rslt_fail(int end_pos, String fmt, Object... args) {
|
||||
this.end_pos = end_pos;
|
||||
// if (bgn_pos == src_len) {usr_dlg.Warn_many("", "", "eos after 'url(': bgn=~{bgn}", tkn_bgn); return Bry_finder.Not_found;}
|
||||
return this;
|
||||
}
|
||||
public Xoi_css_url_info Rslt_pass(int end_pos) {
|
||||
this.end_pos = end_pos;
|
||||
return this;
|
||||
}
|
||||
public Xoi_css_url_info Rslt_pass(int end_pos, byte[] url_clean) {
|
||||
this.end_pos = end_pos;
|
||||
return this;
|
||||
}
|
||||
}
|
||||
class Xoi_css_offline_mgr {
|
||||
private Bry_bfr bfr;
|
||||
private byte[] src;
|
||||
private int src_len, pos;
|
||||
private OrderedHash download_queue;
|
||||
private Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Null;
|
||||
public void Offline(Bry_bfr bfr, OrderedHash download_queue, Object download_wkr, byte[] src) {
|
||||
this.bfr = bfr;
|
||||
this.download_queue = download_queue;
|
||||
this.src = src; this.src_len = src.length;
|
||||
this.pos = 0;
|
||||
while (true) {
|
||||
boolean last = pos == src_len;
|
||||
byte b = last ? Byte_ascii.NewLine : src[pos];
|
||||
Object o = tkns_trie.Match(b, src, pos, src_len);
|
||||
if (o == null) {
|
||||
bfr.Add_byte(b);
|
||||
++pos;
|
||||
}
|
||||
else {
|
||||
byte tkn_tid = ((Byte_obj_val)o).Val();
|
||||
int match_pos = tkns_trie.Match_pos();
|
||||
int nxt_pos = -1;
|
||||
switch (tkn_tid) {
|
||||
case Tkn_url: nxt_pos = Process_url(pos, match_pos); break;
|
||||
case Tkn_import: nxt_pos = Process_import(pos, match_pos); break;
|
||||
}
|
||||
pos = nxt_pos;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
// "//id.wikibooks.org/w/index.php?title=MediaWiki:Common.css&oldid=43393&action=raw&ctype=text/css";
|
||||
private int Process_import(int tkn_bgn, int tkn_end) { // @import
|
||||
// get url
|
||||
// if null, add to bfr and exit;
|
||||
// else download, and recursively call self
|
||||
// int bgn_pos = Bry_finder.Find_fwd(src, end_byte, bgn_pos, src_len);
|
||||
// int end_pos = Bry_finder.Find_fwd(src, end_byte, bgn_pos, src_len);
|
||||
return -1;
|
||||
}
|
||||
private int Process_url(int tkn_bgn, int tkn_end) { // " url"
|
||||
int bgn_pos = Bry_finder.Find_fwd_while_ws(src, tkn_end, src_len); // skip any ws after " url("
|
||||
if (bgn_pos == src_len) {usr_dlg.Warn_many("", "", "eos after 'url(': bgn=~{bgn}", tkn_bgn); return Bry_finder.Not_found;}
|
||||
byte end_byte = src[bgn_pos]; boolean quoted = true;
|
||||
switch (end_byte) {
|
||||
case Byte_ascii.Quote: case Byte_ascii.Apos: // quoted; increment position
|
||||
++bgn_pos;
|
||||
break;
|
||||
default: // not quoted; end byte is ")"
|
||||
end_byte = Byte_ascii.Paren_end;
|
||||
quoted = false;
|
||||
break;
|
||||
}
|
||||
int end_pos = Bry_finder.Find_fwd(src, end_byte, bgn_pos, src_len);
|
||||
if (end_pos == Bry_.NotFound) { // unclosed "url("; exit since nothing else will be found
|
||||
usr_dlg.Warn_many("", "", "could not end_byte for 'url(': bgn='~{0}' end='~{1}'", bgn_pos, String_.new_utf8_len_safe_(src, tkn_bgn, tkn_bgn + 25));
|
||||
bfr.Add_mid(src, tkn_bgn, src_len);
|
||||
return Bry_finder.Not_found;
|
||||
}
|
||||
if (end_pos - bgn_pos == 0) { // empty; "url()"; ignore
|
||||
usr_dlg.Warn_many("", "", "'url(' is empty: bgn='~{0}' end='~{1}'", tkn_bgn, String_.new_utf8_len_safe_(src, tkn_bgn, tkn_bgn + 25));
|
||||
return end_pos;
|
||||
}
|
||||
byte[] url_raw = Bry_.Mid(src, bgn_pos, end_pos); int url_raw_len = url_raw.length;
|
||||
if (Bry_.HasAtBgn(url_raw, Bry_data_image, 0, url_raw_len)) { // base64
|
||||
++end_pos; // include end_byte;
|
||||
bfr.Add_mid(src, tkn_bgn, end_pos); // nothing to download; just add entire String
|
||||
return end_pos;
|
||||
}
|
||||
byte[] url_cleaned = Clean_url(url_raw, url_raw_len);
|
||||
if (url_cleaned == null) { // could not clean url
|
||||
usr_dlg.Warn_many("", "", "could not extract valid url: bgn='~{0}' end='~{1}'", tkn_bgn, String_.new_utf8_(url_raw));
|
||||
bfr.Add_mid(src, tkn_bgn, bgn_pos);
|
||||
return bgn_pos;
|
||||
}
|
||||
Xoi_css_offline_itm url_itm = (Xoi_css_offline_itm)download_queue.Fetch(url_cleaned);
|
||||
if (url_itm == null) { // only add unique items for download;
|
||||
url_itm = new Xoi_css_offline_itm(url_cleaned);
|
||||
download_queue.Add(url_cleaned, url_itm);
|
||||
}
|
||||
byte[] file_url = Replace_invalid_chars(Bry_.Copy(url_cleaned)); // NOTE: must call ByteAry.Copy else url_cleaned will change *inside* bry
|
||||
url_itm.File_url_(file_url);
|
||||
bfr.Add_mid(src, tkn_bgn, tkn_end);
|
||||
if (!quoted) bfr.Add_byte(Byte_ascii.Quote);
|
||||
bfr.Add(file_url);
|
||||
if (!quoted) bfr.Add_byte(Byte_ascii.Quote);
|
||||
return end_pos;
|
||||
}
|
||||
public static Xoi_css_url_info Process_url(byte[] src, int src_len, int tkn_bgn, int tkn_end, Xoi_css_url_info inf) { // " url"
|
||||
inf.Clear();
|
||||
int bgn_pos = Bry_finder.Find_fwd_while_ws(src, tkn_end, src_len); // skip any ws after " url("
|
||||
if (bgn_pos == src_len) return inf.Rslt_fail(src_len, "eos after 'url(': bgn=~{bgn}");
|
||||
byte end_byte = src[bgn_pos]; boolean quoted = true;
|
||||
switch (end_byte) {
|
||||
case Byte_ascii.Quote: case Byte_ascii.Apos: // quoted; increment position
|
||||
++bgn_pos;
|
||||
break;
|
||||
default: // not quoted; end byte is ")"
|
||||
end_byte = Byte_ascii.Paren_end;
|
||||
quoted = false;
|
||||
break;
|
||||
}
|
||||
inf.Quote_data(end_byte, quoted);
|
||||
int end_pos = Bry_finder.Find_fwd(src, end_byte, bgn_pos, src_len);
|
||||
if (end_pos == Bry_.NotFound) { // unclosed "url("; exit since nothing else will be found
|
||||
return inf.Rslt_fail(src_len, "could not end_byte for 'url(': bgn='~{0}' end='~{1}'");
|
||||
}
|
||||
if (end_pos - bgn_pos == 0) { // empty; "url()"; ignore
|
||||
return inf.Rslt_fail(end_pos + 1, "'url(' is empty: bgn='~{0}' end='~{1}'");
|
||||
}
|
||||
byte[] url_raw = Bry_.Mid(src, bgn_pos, end_pos); int url_raw_len = url_raw.length;
|
||||
if (Bry_.HasAtBgn(url_raw, Bry_data_image, 0, url_raw_len)) { // base64
|
||||
return inf.Rslt_pass(end_pos + 1); // nothing to download; just add entire String
|
||||
}
|
||||
byte[] url_cleaned = Clean_url(url_raw, url_raw_len);
|
||||
if (url_cleaned == null) // could not clean url
|
||||
return inf.Rslt_fail(bgn_pos, "could not extract valid url: bgn='~{0}' end='~{1}'");
|
||||
return inf.Rslt_pass(end_pos, url_cleaned);
|
||||
// Xoi_css_offline_itm url_itm = (Xoi_css_offline_itm)download_queue.Fetch(url_cleaned);
|
||||
// if (url_itm == null) { // only add unique items for download;
|
||||
// url_itm = new Xoi_css_offline_itm(url_cleaned);
|
||||
// download_queue.Add(url_cleaned, url_itm);
|
||||
// }
|
||||
// byte[] file_url = Replace_invalid_chars(Bry_.Copy(url_cleaned)); // NOTE: must call ByteAry.Copy else url_cleaned will change *inside* bry
|
||||
// url_itm.File_url_(file_url);
|
||||
// bfr.Add_mid(src, tkn_bgn, tkn_end);
|
||||
// if (!quoted) bfr.Add_byte(Byte_ascii.Quote);
|
||||
// bfr.Add(file_url);
|
||||
// if (!quoted) bfr.Add_byte(Byte_ascii.Quote);
|
||||
// return inf;
|
||||
}
|
||||
public static byte[] Clean_url(byte[] raw, int len) { // return "site/img.png" if "//site/img.png" or "http://site/img.png", "img.png?key=val"
|
||||
int bgn = 0;
|
||||
if (Bry_.HasAtBgn(raw, Bry_fwd_slashes , 0, len)) bgn = Bry_fwd_slashes.length; // skip if starts with "//"
|
||||
else if (Bry_.HasAtBgn(raw, Bry_http , 0, len)) bgn = Bry_http.length; // skip if starts with "http://"
|
||||
else if (Bry_.HasAtBgn(raw, Bry_https , 0, len)) bgn = Bry_https.length; // skip if starts with "https://"
|
||||
int slash_pos = Bry_finder.Find_fwd(raw, Byte_ascii.Slash, bgn, len); // find 1st slash
|
||||
if ( slash_pos == Bry_finder.Not_found // no slashes; must have at least 1 slash to have 2 segments; EX: site.org/img.png
|
||||
|| slash_pos == len - 1 // first slash is last char; EX: "site.org/"
|
||||
)
|
||||
return null; // invalid
|
||||
int end = len;
|
||||
int question_pos = Bry_finder.Find_bwd(raw, Byte_ascii.Question);
|
||||
if (question_pos != Bry_finder.Not_found) // url has query String; EX:site.org/img.png?key=val
|
||||
end = question_pos; // remove query String
|
||||
return Bry_.Mid(raw, bgn, end);
|
||||
}
|
||||
public static byte[] Replace_invalid_chars(byte[] src) {
|
||||
int len = src.length;
|
||||
for (int i = 0; i < len; i++) { // convert invalid wnt chars to underscores
|
||||
byte b = src[i];
|
||||
switch (b) {
|
||||
//case Byte_ascii.Slash:
|
||||
case Byte_ascii.Backslash: case Byte_ascii.Colon: case Byte_ascii.Asterisk: case Byte_ascii.Question:
|
||||
case Byte_ascii.Quote: case Byte_ascii.Lt: case Byte_ascii.Gt: case Byte_ascii.Pipe:
|
||||
src[i] = Byte_ascii.Underline;
|
||||
break;
|
||||
}
|
||||
}
|
||||
return src;
|
||||
}
|
||||
|
||||
public static final byte[] Tkn_url_bry = Bry_.new_ascii_(" url(");
|
||||
private static final byte Tkn_import = 1, Tkn_url = 2;
|
||||
private static final ByteTrieMgr_slim tkns_trie = ByteTrieMgr_slim.ci_ascii_()
|
||||
.Add_str_byte("@import" , Tkn_import)
|
||||
.Add_bry_bval(Tkn_url_bry , Tkn_url)
|
||||
;
|
||||
private static final byte[]
|
||||
Bry_data_image = Bry_.new_ascii_("data:image/")
|
||||
, Bry_http = Bry_.new_ascii_("http://")
|
||||
, Bry_https = Bry_.new_ascii_("https://")
|
||||
, Bry_fwd_slashes = Bry_.new_ascii_("//")
|
||||
// , Bry_http_protocol = Bry_.new_ascii_("http"), Bry_url = Bry_.new_ascii_("url("), Bry_import = Bry_.new_ascii_("@import ")
|
||||
;
|
||||
public static final byte[]
|
||||
Bry_comment_bgn = Bry_.new_ascii_("/*XOWA:")
|
||||
, Bry_comment_end = Bry_.new_ascii_("*/")
|
||||
;
|
||||
// private static final int Bry_url_len = Bry_url.length, Bry_import_len = Bry_import.length;
|
||||
}
|
||||
// class Io_download_itm {
|
||||
// public byte[] Src_url() {return src_url;} public void Src_url_(byte[] v) {src_url = v;} private byte[] src_url;
|
||||
// public Io_url Trg_url() {return trg_url;} public void Trg_url_(Io_url v) {trg_url = v;} private Io_url trg_url;]
|
||||
// public String Download_err() {return download_err;} public void Download_err_(String v) {download_err = v;} private String download_err;
|
||||
// }
|
||||
// interface Io_download_mgr {
|
||||
// void Download(Io_download_itm itm);
|
||||
// }
|
||||
@@ -0,0 +1,52 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import org.junit.*;
|
||||
public class Xoi_css_offline_mgr_tst {
|
||||
@Before public void init() {fxt.Clear();} private Xoi_css_offline_mgr_fxt fxt = new Xoi_css_offline_mgr_fxt();
|
||||
@Test public void Basic() {
|
||||
// fxt.Test_extract_url_warn("url(" , "end of stream");
|
||||
// fxt.Test_extract_url_warn("url(abc" , "end not found");
|
||||
// fxt.Test_extract_url_warn("url()" , "url is empty");
|
||||
fxt.Test_extract_url_pass("url('a/b')" , "url is empty");
|
||||
}
|
||||
}
|
||||
class Xoi_css_offline_mgr_fxt {
|
||||
private Xoi_css_url_info info = new Xoi_css_url_info();
|
||||
private Gfo_usr_dlg usr_dlg = null;
|
||||
// private Xoi_css_offline_mgr mgr;
|
||||
public void Clear() {
|
||||
info.Init(usr_dlg);
|
||||
// mgr = new Xoi_css_offline_mgr();
|
||||
}
|
||||
public void Test_extract_url_warn(String raw, String err) {
|
||||
}
|
||||
public void Test_extract_url_pass(String src_str, String expd) {
|
||||
byte[] src = Bry_.new_utf8_(src_str);
|
||||
Test_extract_url(src);
|
||||
// String actl = String_.new_ascii_(src, info.Bgn_pos(), info.End_pos());
|
||||
// Tfds.Eq(expd, actl);
|
||||
}
|
||||
private void Test_extract_url(byte[] src) {
|
||||
int src_len = src.length;
|
||||
info.Clear();
|
||||
int tkn_bgn = Bry_finder.Find_fwd(src, Xoi_css_offline_mgr.Tkn_url_bry, 0, src_len);
|
||||
int tkn_end = tkn_bgn + Xoi_css_offline_mgr.Tkn_url_bry.length;
|
||||
Xoi_css_offline_mgr.Process_url(src, src_len, tkn_bgn, tkn_end, info);
|
||||
}
|
||||
}
|
||||
112
400_xowa/src_240_install/gplx/xowa/Xoi_dump_mgr.java
Normal file
112
400_xowa/src_240_install/gplx/xowa/Xoi_dump_mgr.java
Normal file
@@ -0,0 +1,112 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
public class Xoi_dump_mgr implements GfoInvkAble {
|
||||
public String[] Server_urls() {return server_urls;} private String[] server_urls = String_.Ary(Xob_dump_file_.Server_wmf, Xob_dump_file_.Server_your_org, Xob_dump_file_.Server_c3sl, Xob_dump_file_.Server_masaryk);
|
||||
public String[] Custom_cmds() {return custom_cmds;} private String[] custom_cmds = String_.Ary(Xoi_cmd_wiki_download.Key_wiki_download, Xoi_cmd_wiki_import.KEY);
|
||||
public byte Data_storage_format() {return data_storage_format;} public Xoi_dump_mgr Data_storage_format_(byte v) {data_storage_format = v; return this;} private byte data_storage_format = gplx.ios.Io_stream_.Tid_gzip;
|
||||
public long Db_text_max() {return db_text_max;} long db_text_max = (long)3000 * Io_mgr.Len_mb;
|
||||
public long Db_categorylinks_max() {return db_categorylinks_max;} long db_categorylinks_max = (long)3600 * Io_mgr.Len_mb;
|
||||
public long Db_wikidata_max() {return db_wikidata_max;} long db_wikidata_max = (long)3600 * Io_mgr.Len_mb;
|
||||
public byte Wiki_storage_type() {return wiki_storage_type;} private byte wiki_storage_type = Wiki_storage_type_sqlite;
|
||||
public boolean Wiki_storage_type_is_sql() {return wiki_storage_type == Wiki_storage_type_sqlite;}
|
||||
public String Db_ns_map() {return db_ns_map;} private String db_ns_map = "Template~Module";
|
||||
public boolean Css_wiki_update() {return css_wiki_update;} private boolean css_wiki_update = true;
|
||||
public boolean Css_commons_download() {return css_commons_download;} private boolean css_commons_download = false;
|
||||
public boolean Delete_xml_file() {return delete_xml_file;} private boolean delete_xml_file = true;
|
||||
public byte Search_version() {return search_version;} private byte search_version = gplx.xowa.specials.search.Xosrh_core.Version_2;
|
||||
public boolean Import_bz2_by_stdout() {return import_bz2_by_stdout;} private boolean import_bz2_by_stdout = true;
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_server_urls)) return String_.Concat_with_str(",\n", server_urls);
|
||||
else if (ctx.Match(k, Invk_server_urls_)) server_urls = m.ReadStrAryIgnore("v", ",", "\n");
|
||||
else if (ctx.Match(k, Invk_custom_cmds)) return String_.Concat_with_str(",", custom_cmds);
|
||||
else if (ctx.Match(k, Invk_custom_cmds_)) custom_cmds = String_.Ary_filter(m.ReadStrAry("v", ","), Xoi_cmd_mgr.Wiki_cmds_valid);
|
||||
else if (ctx.Match(k, Invk_data_storage_format)) return Wtr_tid_to_str(data_storage_format);
|
||||
else if (ctx.Match(k, Invk_data_storage_format_)) data_storage_format = Wtr_tid_parse(m.ReadStr("v"));
|
||||
else if (ctx.Match(k, Invk_data_storage_format_list)) return Options_data_storage_format_list;
|
||||
else if (ctx.Match(k, Invk_db_text_max)) return db_text_max / Io_mgr.Len_mb;
|
||||
else if (ctx.Match(k, Invk_db_text_max_)) db_text_max = m.ReadLong("v") * Io_mgr.Len_mb;
|
||||
else if (ctx.Match(k, Invk_db_categorylinks_max)) return db_categorylinks_max / Io_mgr.Len_mb;
|
||||
else if (ctx.Match(k, Invk_db_categorylinks_max_)) db_categorylinks_max = m.ReadLong("v") * Io_mgr.Len_mb;
|
||||
else if (ctx.Match(k, Invk_db_wikidata_max)) return db_wikidata_max / Io_mgr.Len_mb;
|
||||
else if (ctx.Match(k, Invk_db_wikidata_max_)) db_wikidata_max = m.ReadLong("v") * Io_mgr.Len_mb;
|
||||
else if (ctx.Match(k, Invk_db_ns_map)) return db_ns_map;
|
||||
else if (ctx.Match(k, Invk_db_ns_map_)) db_ns_map = m.ReadStr("v");
|
||||
else if (ctx.Match(k, Invk_wiki_storage_type)) return Wiki_storage_type_str(wiki_storage_type);
|
||||
else if (ctx.Match(k, Invk_wiki_storage_type_)) wiki_storage_type = Wiki_storage_type_parse(m.ReadStr("v"));
|
||||
else if (ctx.Match(k, Invk_wiki_storage_type_list)) return Options_storage_type_list;
|
||||
else if (ctx.Match(k, Invk_css_wiki_update)) return Yn.X_to_str(css_wiki_update);
|
||||
else if (ctx.Match(k, Invk_css_wiki_update_)) css_wiki_update = m.ReadYn("v");
|
||||
else if (ctx.Match(k, Invk_css_commons_download)) return Yn.X_to_str(css_commons_download);
|
||||
else if (ctx.Match(k, Invk_css_commons_download_)) css_commons_download = m.ReadYn("v");
|
||||
else if (ctx.Match(k, Invk_delete_xml_file)) return Yn.X_to_str(delete_xml_file);
|
||||
else if (ctx.Match(k, Invk_delete_xml_file_)) delete_xml_file = m.ReadYn("v");
|
||||
else if (ctx.Match(k, Invk_search_version)) return Options_search_version_str(search_version);
|
||||
else if (ctx.Match(k, Invk_search_version_)) search_version = Options_search_version_parse(m.ReadStr("v"));
|
||||
else if (ctx.Match(k, Invk_search_version_list)) return Options_search_version_list;
|
||||
else if (ctx.Match(k, Invk_import_bz2_by_stdout)) return Yn.X_to_str(import_bz2_by_stdout);
|
||||
else if (ctx.Match(k, Invk_import_bz2_by_stdout_)) import_bz2_by_stdout = m.ReadYn("v");
|
||||
else return GfoInvkAble_.Rv_unhandled;
|
||||
return this;
|
||||
}
|
||||
static final String Invk_server_urls = "server_urls", Invk_server_urls_ = "server_urls_", Invk_custom_cmds = "custom_cmds", Invk_custom_cmds_ = "custom_cmds_"
|
||||
, Invk_data_storage_format = "data_storage_format", Invk_data_storage_format_ = "data_storage_format_", Invk_data_storage_format_list = "data_storage_format_list"
|
||||
, Invk_db_text_max = "db_text_max", Invk_db_text_max_ = "db_text_max_", Invk_db_categorylinks_max = "db_categorylinks_max", Invk_db_categorylinks_max_ = "db_categorylinks_max_", Invk_db_wikidata_max = "db_wikidata_max", Invk_db_wikidata_max_ = "db_wikidata_max_"
|
||||
, Invk_db_ns_map = "db_ns_map", Invk_db_ns_map_ = "db_ns_map_"
|
||||
, Invk_wiki_storage_type = "wiki_storage_type", Invk_wiki_storage_type_ = "wiki_storage_type_", Invk_wiki_storage_type_list = "wiki_storage_type_list"
|
||||
, Invk_css_wiki_update = "css_wiki_update", Invk_css_wiki_update_ = "css_wiki_update_"
|
||||
, Invk_css_commons_download = "css_commons_download", Invk_css_commons_download_ = "css_commons_download_"
|
||||
, Invk_delete_xml_file = "delete_xml_file", Invk_delete_xml_file_ = "delete_xml_file_"
|
||||
, Invk_search_version = "search_version", Invk_search_version_ = "search_version_", Invk_search_version_list = "search_version_list"
|
||||
, Invk_import_bz2_by_stdout = "import_bz2_by_stdout", Invk_import_bz2_by_stdout_ = "import_bz2_by_stdout_"
|
||||
;
|
||||
private static KeyVal[] Options_data_storage_format_list = KeyVal_.Ary(KeyVal_.new_(".xdat", "text"), KeyVal_.new_(".gz", "gzip"), KeyVal_.new_(".bz2", "bzip2")); // removed .zip; DATE:2014-05-13; updated aliases; DATE:2014-06-20
|
||||
public static String Wtr_tid_to_str(byte v) {
|
||||
switch (v) {
|
||||
case gplx.ios.Io_stream_.Tid_file : return ".xdat";
|
||||
case gplx.ios.Io_stream_.Tid_zip : return ".zip";
|
||||
case gplx.ios.Io_stream_.Tid_gzip : return ".gz";
|
||||
case gplx.ios.Io_stream_.Tid_bzip2 : return ".bz2";
|
||||
default : throw Err_.unhandled(v);
|
||||
}
|
||||
}
|
||||
public static byte Wtr_tid_parse(String v) {
|
||||
if (String_.Eq(v, ".xdat")) return gplx.ios.Io_stream_.Tid_file;
|
||||
else if (String_.Eq(v, ".zip")) return gplx.ios.Io_stream_.Tid_zip;
|
||||
else if (String_.Eq(v, ".gz")) return gplx.ios.Io_stream_.Tid_gzip;
|
||||
else if (String_.Eq(v, ".bz2")) return gplx.ios.Io_stream_.Tid_bzip2;
|
||||
else throw Err_.unhandled(v);
|
||||
}
|
||||
static final byte Wiki_storage_type_xdat = 1, Wiki_storage_type_sqlite = 2;
|
||||
private static final KeyVal[] Options_storage_type_list = KeyVal_.Ary(KeyVal_.new_("sqlite"), KeyVal_.new_("xdat"));
|
||||
public static String Wiki_storage_type_str(byte v) {
|
||||
switch (v) {
|
||||
case Xoi_dump_mgr.Wiki_storage_type_xdat : return "xdat";
|
||||
case Xoi_dump_mgr.Wiki_storage_type_sqlite : return "sqlite";
|
||||
default : throw Err_.unhandled(v);
|
||||
}
|
||||
}
|
||||
public static byte Wiki_storage_type_parse(String v) {
|
||||
if (String_.Eq(v, "xdat")) return Xoi_dump_mgr.Wiki_storage_type_xdat;
|
||||
else if (String_.Eq(v, "sqlite")) return Xoi_dump_mgr.Wiki_storage_type_sqlite;
|
||||
else throw Err_.unhandled(v);
|
||||
}
|
||||
private static final KeyVal[] Options_search_version_list = KeyVal_.Ary(KeyVal_.new_("1"), KeyVal_.new_("2"));
|
||||
public static String Options_search_version_str(byte v) {return Byte_.XtoStr(v);}
|
||||
public static byte Options_search_version_parse(String v) {return Byte_.parse_(v);}
|
||||
}
|
||||
47
400_xowa/src_240_install/gplx/xowa/Xoi_mirror_parser.java
Normal file
47
400_xowa/src_240_install/gplx/xowa/Xoi_mirror_parser.java
Normal file
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
public class Xoi_mirror_parser {
|
||||
public String[] Parse(String raw_str) {
|
||||
if (String_.Len_eq_0(raw_str)) return String_.Ary_empty;
|
||||
byte[] raw = Bry_.new_utf8_(raw_str);
|
||||
ListAdp rv = ListAdp_.new_();
|
||||
int pos = 0;
|
||||
while (true) {
|
||||
int bgn = Bry_finder.Find_fwd(raw, CONST_href_bgn, pos); if (bgn == Bry_.NotFound) break;
|
||||
bgn += CONST_href_bgn.length;
|
||||
int end = Bry_finder.Find_fwd(raw, CONST_href_end, bgn); if (end == Bry_.NotFound) return String_.Ary_empty;
|
||||
byte[] date = Bry_.Mid(raw, bgn, end);
|
||||
pos = end + CONST_href_end.length;
|
||||
if (Bry_.Match(date, CONST_date_parent_dir)) continue;
|
||||
int date_pos_last = date.length - 1;
|
||||
if (date_pos_last == -1) return String_.Ary_empty;
|
||||
if (date[date_pos_last] == Byte_ascii.Slash) date = Bry_.Mid(date, 0, date_pos_last); // trim trailing /; EX: "20130101/" -> "20130101"
|
||||
rv.Add(String_.new_utf8_(date));
|
||||
}
|
||||
return rv.XtoStrAry();
|
||||
} static final byte[] CONST_href_bgn = Bry_.new_ascii_("<a href=\""), CONST_href_end = Bry_.new_ascii_("\""), CONST_date_parent_dir = Bry_.new_ascii_("../");
|
||||
public static String Find_last_lte(String[] ary, String comp) { // assuming sorted ary, find last entry that is lte comp
|
||||
int len = ary.length;
|
||||
for (int i = len - 1; i > -1; i--) {
|
||||
String itm = ary[i];
|
||||
if (CompareAble_.Is_lessOrSame(itm, comp)) return itm;
|
||||
}
|
||||
return "";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import org.junit.*;
|
||||
public class Xoi_mirror_parser_tst {
|
||||
@Test public void Basic() {
|
||||
Tst_parse(String_.Concat_lines_nl
|
||||
( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
|
||||
, "<!DOCTYPE html PUBLIC \"-//W3C//DTD XHTML 1.1//EN\" \"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd\">"
|
||||
, "<html xmlns=\"http://www.w3.org/1999/xhtml\" xml:lang=\"en\">"
|
||||
, "<head>"
|
||||
, "<title>Index of /simplewiki/</title>"
|
||||
, "<link rel=\"stylesheet\" type=\"text/css\" href=\"/pub/misc/lighttpd-white-dir.css\" />"
|
||||
, "</head>"
|
||||
, "<body>"
|
||||
, "<h2>Index of /simplewiki/</h2>"
|
||||
, "<div class=\"list\">"
|
||||
, "<table summary=\"Directory Listing\" cellpadding=\"0\" cellspacing=\"0\">"
|
||||
, "<thead><tr><th class=\"n\">Name</th><th class=\"m\">Last Modified</th><th class=\"s\">Size</th><th class=\"t\">Type</th></tr></thead>"
|
||||
, "<tbody>"
|
||||
, "<tr><td class=\"n\"><a href=\"../\">Parent Directory</a>/</td><td class=\"m\"> </td><td class=\"s\">- </td><td class=\"t\">Directory</td></tr>"
|
||||
, "<tr><td class=\"n\"><a href=\"20120516/\">20120516</a>/</td><td class=\"m\">2012-May-17 01:04:39</td><td class=\"s\">- </td><td class=\"t\">Directory</td></tr>"
|
||||
, "<tr><td class=\"n\"><a href=\"20121220/\">20121220</a>/</td><td class=\"m\">2012-Dec-20 20:15:55</td><td class=\"s\">- </td><td class=\"t\">Directory</td></tr>"
|
||||
, "<tr><td class=\"n\"><a href=\"20130214/\">20130214</a>/</td><td class=\"m\">2013-Feb-14 06:28:41</td><td class=\"s\">- </td><td class=\"t\">Directory</td></tr>"
|
||||
, "<tr><td class=\"n\"><a href=\"latest/\">latest</a>/</td><td class=\"m\">2013-Feb-14 06:28:41</td><td class=\"s\">- </td><td class=\"t\">Directory</td></tr>"
|
||||
, "</tbody>"
|
||||
, "</table>"
|
||||
, "</div>"
|
||||
, "<div class=\"foot\">lighttpd</div>"
|
||||
, "</body>"
|
||||
, "</html>"
|
||||
), String_.Ary("20120516", "20121220", "20130214", "latest"));
|
||||
}
|
||||
@Test public void Find_last_lte() {
|
||||
Tst_find_last_lte(String_.Ary("20120516", "20121220", "20130214", "latest"), "20130101", "20121220");
|
||||
Tst_find_last_lte(String_.Ary("20120516", "20121220", "20130214", "latest"), "20120101", "");
|
||||
}
|
||||
private void Tst_parse(String raw, String[] expd) {
|
||||
Xoi_mirror_parser parser = new Xoi_mirror_parser();
|
||||
Tfds.Eq_ary_str(expd, parser.Parse(raw));
|
||||
}
|
||||
private void Tst_find_last_lte(String[] ary, String comp, String expd) {
|
||||
Tfds.Eq(expd, Xoi_mirror_parser.Find_last_lte(ary, comp));
|
||||
}
|
||||
}
|
||||
44
400_xowa/src_240_install/gplx/xowa/Xoi_setup_mgr.java
Normal file
44
400_xowa/src_240_install/gplx/xowa/Xoi_setup_mgr.java
Normal file
@@ -0,0 +1,44 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import gplx.gfui.*; import gplx.xowa.setup.addons.*;
|
||||
import gplx.xowa.setup.maints.*;
|
||||
public class Xoi_setup_mgr implements GfoInvkAble {
|
||||
public Xoi_setup_mgr(Xoa_app app) {
|
||||
this.app = app;
|
||||
cmd_mgr = new Xoi_cmd_mgr(this);
|
||||
maint_mgr = new Xoa_maint_mgr(app);
|
||||
}
|
||||
public void Init_by_app(Xoa_app app) {
|
||||
addon_mgr.Init_by_app(app);
|
||||
}
|
||||
public Xoa_app App() {return app;} private Xoa_app app;
|
||||
public Xoi_cmd_mgr Cmd_mgr() {return cmd_mgr;} private Xoi_cmd_mgr cmd_mgr;
|
||||
public Xoi_dump_mgr Dump_mgr() {return dump_mgr;} private Xoi_dump_mgr dump_mgr = new Xoi_dump_mgr();
|
||||
public Xoi_addon_mgr Addon_mgr() {return addon_mgr;} private Xoi_addon_mgr addon_mgr = new Xoi_addon_mgr();
|
||||
public Xoa_maint_mgr Maint_mgr() {return maint_mgr;} private Xoa_maint_mgr maint_mgr;
|
||||
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
|
||||
if (ctx.Match(k, Invk_cmds)) return cmd_mgr;
|
||||
else if (ctx.Match(k, Invk_dumps)) return dump_mgr;
|
||||
else if (ctx.Match(k, Invk_addons)) return addon_mgr;
|
||||
else if (ctx.Match(k, Invk_maint)) return maint_mgr;
|
||||
else return GfoInvkAble_.Rv_unhandled;
|
||||
}
|
||||
static final String Invk_cmds = "cmds", Invk_dumps = "dumps", Invk_addons = "addons", Invk_maint = "maint";
|
||||
static final String GRP_KEY = "xowa.setup";
|
||||
}
|
||||
86
400_xowa/src_240_install/gplx/xowa/Xow_cfg_wiki_core.java
Normal file
86
400_xowa/src_240_install/gplx/xowa/Xow_cfg_wiki_core.java
Normal file
@@ -0,0 +1,86 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
public class Xow_cfg_wiki_core {
|
||||
public Xow_cfg_wiki_core(Xow_wiki wiki) {this.wiki = wiki;} private Xow_wiki wiki;
|
||||
public void Save() {
|
||||
}
|
||||
public byte[] Build_gfs() {
|
||||
Gfs_bldr wtr = new Gfs_bldr();
|
||||
Xow_wiki_props props = wiki.Props();
|
||||
wtr.Add_proc_init_many(Xow_wiki.Invk_props).Add_nl();
|
||||
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_bldr_version_).Add_parens_str(props.Bldr_version()).Add_nl();
|
||||
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_main_page_).Add_parens_str(props.Main_page()).Add_nl();
|
||||
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_siteinfo_misc_).Add_parens_str(props.Siteinfo_misc()).Add_nl();
|
||||
wtr.Add_proc_cont_one(Xow_wiki_props.Invk_siteinfo_mainpage_).Add_parens_str(props.Siteinfo_mainpage()).Add_nl();
|
||||
wtr.Add_term_nl();
|
||||
wtr.Add_proc_init_many(Xow_wiki.Invk_ns_mgr).Add_nl();
|
||||
wtr.Add_proc_cont_one(Xow_ns_mgr.Invk_clear).Add_nl();
|
||||
wtr.Add_proc_cont_one(Xow_ns_mgr.Invk_load).Add_paren_bgn().Add_nl();
|
||||
wtr.Add_quote_xtn_bgn();
|
||||
Xol_csv_parser csv_parser = Xol_csv_parser._;
|
||||
int nms_len = wiki.Ns_mgr().Count();
|
||||
for (int i = 0; i < nms_len; i++) {
|
||||
Xow_ns ns = wiki.Ns_mgr().Ords_get_at(i);
|
||||
wtr.Bfr().Add_int_variable(ns.Id()).Add_byte_pipe().Add_int_fixed(ns.Case_match(), 1).Add_byte_pipe();
|
||||
csv_parser.Save(wtr.Bfr(), ns.Name_txt());
|
||||
wtr.Add_nl();
|
||||
}
|
||||
wtr.Add_quote_xtn_end();
|
||||
wtr.Add_paren_end().Add_term_nl();
|
||||
return wtr.Xto_bry();
|
||||
}
|
||||
public void Load(String raw) {
|
||||
wiki.App().Gfs_mgr().Run_str_for(wiki, raw);
|
||||
}
|
||||
public static void Load_ns_(Xow_ns_mgr ns_mgr, byte[] src) {// 10|1|Template
|
||||
int len = src.length; int pos = 0, fld_bgn = 0, fld_idx = 0, row_bgn = 0;
|
||||
int cur_id = Int_.MinValue; byte cur_case_match = Byte_.MaxValue_127; byte[] cur_name = Bry_.Empty;
|
||||
Xol_csv_parser csv_parser = Xol_csv_parser._;
|
||||
while (true) {
|
||||
boolean last = pos == len;
|
||||
byte b = last ? Byte_ascii.NewLine : src[pos];
|
||||
switch (b) {
|
||||
case Byte_ascii.Pipe:
|
||||
switch (fld_idx) {
|
||||
case 0: cur_id = Bry_.X_to_int_or(src, fld_bgn, pos, Int_.MinValue); if (cur_id == Int_.MinValue) throw Err_mgr._.fmt_(GRP_KEY, "fail.load.id", "failed to load id: ~{0}", String_.new_utf8_(src, fld_bgn, pos)); break;
|
||||
case 1: cur_case_match = Bry_.X_to_byte_by_int(src, fld_bgn, pos, Byte_.MaxValue_127); if (cur_id == Byte_.MaxValue_127) throw Err_mgr._.fmt_(GRP_KEY, "fail.load.case_match", "failed to load match: ~{0}", String_.new_utf8_(src, fld_bgn, pos)); break;
|
||||
default: throw Err_.unhandled(fld_idx);
|
||||
}
|
||||
fld_bgn = pos + 1;
|
||||
++fld_idx;
|
||||
break;
|
||||
case Byte_ascii.NewLine:
|
||||
if (pos > row_bgn) { // guard against blank lines
|
||||
cur_name = csv_parser.Load(src, fld_bgn, pos);
|
||||
ns_mgr.Add_new(cur_id, cur_name, cur_case_match, false);
|
||||
}
|
||||
cur_id = Int_.MinValue; cur_case_match = Byte_.MaxValue_127;
|
||||
fld_bgn = pos + 1;
|
||||
fld_idx = 0;
|
||||
row_bgn = fld_bgn;
|
||||
break;
|
||||
}
|
||||
if (last) break;
|
||||
++pos;
|
||||
}
|
||||
ns_mgr.Init_w_defaults();
|
||||
}
|
||||
static final String Url_wiki_core_gfs = "wiki_core.gfs";
|
||||
static final String GRP_KEY = "xowa.wiki.cfg.wiki_core";
|
||||
}
|
||||
111
400_xowa/src_240_install/gplx/xowa/Xow_cfg_wiki_core_tst.java
Normal file
111
400_xowa/src_240_install/gplx/xowa/Xow_cfg_wiki_core_tst.java
Normal file
@@ -0,0 +1,111 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
import org.junit.*;
|
||||
public class Xow_cfg_wiki_core_tst {
|
||||
Xow_cfg_wiki_core_fxt fxt = new Xow_cfg_wiki_core_fxt();
|
||||
@Before public void init() {fxt.Clear();}
|
||||
@Test public void Save() {
|
||||
fxt.Save_tst(Xoa_app_.Version, "Main_Page", "Wikipedia|MediaWiki 1.21wmf5|first-letter|", 0, "User test", Const_wiki_core_cfg);
|
||||
}
|
||||
@Test public void Load_and_save() {
|
||||
fxt.Load_and_save_tst(Const_wiki_core_cfg);
|
||||
}
|
||||
@Test public void Load() {
|
||||
fxt.Load_tst(String_.Concat_lines_nl
|
||||
( "ns_mgr.clear.load("
|
||||
, "<:['"
|
||||
, "0|0|" // DEFECT: test that 0 sets case_match to case-sensitive; empty name was causing it to "default" to 1; DATE:2013-01-30
|
||||
, "" // test to make sure blank line doesn't fail
|
||||
, "4|1|Wikipedia" // test to make sure that values are updated after blank line
|
||||
, ""
|
||||
, "']:>"
|
||||
, ");"
|
||||
)
|
||||
, fxt.ns_(Xow_ns_.Id_main , true, "")
|
||||
, fxt.ns_(Xow_ns_.Id_project , false, "Wikipedia")
|
||||
);
|
||||
}
|
||||
public static final String Const_wiki_core_cfg = String_.Concat_lines_nl
|
||||
( "props"
|
||||
, ".bldr_version_('" + Xoa_app_.Version + "')"
|
||||
, ".main_page_('Main_Page')"
|
||||
, ".siteinfo_misc_('Wikipedia|MediaWiki 1.21wmf5|first-letter|')"
|
||||
, ".siteinfo_mainpage_('')"
|
||||
, ";"
|
||||
, "ns_mgr"
|
||||
, ".clear"
|
||||
, ".load("
|
||||
, "<:['"
|
||||
, "-2|1|Media"
|
||||
, "-1|1|Special"
|
||||
, "0|1|"
|
||||
, "1|1|Talk"
|
||||
, "2|0|User test" // NOTE: intentionally changing this to "0|User test" to differ from existing
|
||||
, "3|1|User talk"
|
||||
, "4|1|Wikipedia"
|
||||
, "5|1|Wikipedia talk"
|
||||
, "6|1|File"
|
||||
, "7|1|File talk"
|
||||
, "8|1|MediaWiki"
|
||||
, "9|1|MediaWiki talk"
|
||||
, "10|1|Template"
|
||||
, "11|1|Template talk"
|
||||
, "12|1|Help"
|
||||
, "13|1|Help talk"
|
||||
, "14|1|Category"
|
||||
, "15|1|Category talk"
|
||||
, "100|1|Portal"
|
||||
, "101|1|Portal talk"
|
||||
, "108|1|Book"
|
||||
, "109|1|Book talk"
|
||||
, "828|1|Module"
|
||||
, "829|1|Module talk"
|
||||
, "']:>"
|
||||
, ");"
|
||||
);
|
||||
}
|
||||
class Xow_cfg_wiki_core_fxt {
|
||||
Xoa_app app; Xow_wiki wiki;
|
||||
public Xow_wiki Wiki() {return wiki;}
|
||||
public void Clear() {
|
||||
app = Xoa_app_fxt.app_();
|
||||
wiki = Xoa_app_fxt.wiki_tst_(app);
|
||||
}
|
||||
public void Save_tst(String bldr_version, String main_page, String siteinfo_misc, int ns_user_case_match, String ns_user_name, String expd) {
|
||||
wiki.Props().Bldr_version_(Bry_.new_ascii_(bldr_version)).Main_page_(Bry_.new_ascii_(main_page)).Siteinfo_misc_(Bry_.new_ascii_(siteinfo_misc));
|
||||
Xow_ns ns_user = wiki.Ns_mgr().Ids_get_or_null(Xow_ns_.Id_user);
|
||||
ns_user.Case_match_((byte)ns_user_case_match); ns_user.Name_bry_(Bry_.new_ascii_(ns_user_name));
|
||||
Tfds.Eq_str_lines(expd, String_.new_ascii_(wiki.Cfg_wiki_core().Build_gfs()));
|
||||
}
|
||||
public void Load_and_save_tst(String raw) {
|
||||
wiki.Cfg_wiki_core().Load(raw);
|
||||
Tfds.Eq_str_lines(raw, String_.new_ascii_(wiki.Cfg_wiki_core().Build_gfs()));
|
||||
}
|
||||
public Xow_ns ns_(int id, boolean case_match, String name) {return new Xow_ns(id, case_match ? Xow_ns_case_.Id_all : Xow_ns_case_.Id_1st, Bry_.new_utf8_(name), false);}
|
||||
public void Load_tst(String raw, Xow_ns... expd_ary) {
|
||||
wiki.Cfg_wiki_core().Load(raw);
|
||||
int expd_len = expd_ary.length;
|
||||
for (int i = 0; i < expd_len; i++) {
|
||||
Xow_ns expd = expd_ary[i];
|
||||
Xow_ns actl = wiki.Ns_mgr().Ids_get_or_null(expd.Id());
|
||||
Tfds.Eq(expd.Case_match(), actl.Case_match(), Int_.XtoStr(expd.Id()));
|
||||
Tfds.Eq(expd.Name_str(), actl.Name_str(), Int_.XtoStr(expd.Id()));
|
||||
}
|
||||
}
|
||||
}
|
||||
43
400_xowa/src_240_install/gplx/xowa/Xow_page_fetcher.java
Normal file
43
400_xowa/src_240_install/gplx/xowa/Xow_page_fetcher.java
Normal file
@@ -0,0 +1,43 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012 gnosygnu@gmail.com
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU Affero General Public License as
|
||||
published by the Free Software Foundation, either version 3 of the
|
||||
License, or (at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU Affero General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU Affero General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
package gplx.xowa; import gplx.*;
|
||||
public interface Xow_page_fetcher {
|
||||
Xow_page_fetcher Wiki_(Xow_wiki v);
|
||||
byte[] Fetch(int ns_id, byte[] ttl);
|
||||
}
|
||||
class Xow_page_fetcher_wiki implements Xow_page_fetcher {
|
||||
public Xow_page_fetcher Wiki_(Xow_wiki v) {this.wiki = v; return this;} private Xow_wiki wiki;
|
||||
public byte[] Fetch(int ns_id, byte[] ttl_bry) {
|
||||
Xoa_ttl ttl = Xoa_ttl.parse_(wiki, ns_id, ttl_bry);
|
||||
Xoa_page page = wiki.Data_mgr().Get_page(ttl, false); // go through data_mgr in case of redirects
|
||||
return page.Missing() ? null : page.Data_raw();
|
||||
}
|
||||
}
|
||||
class Xow_page_fetcher_mok implements Xow_page_fetcher {
|
||||
public Xow_page_fetcher Wiki_(Xow_wiki v) {return this;}
|
||||
public void Clear() {pages.Clear();} private HashAdp pages = HashAdp_.new_();
|
||||
public void Add(int ns_id, byte[] ttl, byte[] text) {
|
||||
Xodb_page page = new Xodb_page().Ns_id_(ns_id).Ttl_wo_ns_(ttl).Text_(text);
|
||||
pages.Add(Make_key(ns_id, ttl), page);
|
||||
}
|
||||
public byte[] Fetch(int ns_id, byte[] ttl) {
|
||||
Xodb_page rv = (Xodb_page)pages.Fetch(Make_key(ns_id, ttl));
|
||||
return rv == null ? null : rv.Text();
|
||||
}
|
||||
String Make_key(int ns_id, byte[] ttl) {return Int_.XtoStr(ns_id) + "|" + String_.new_utf8_(ttl);}
|
||||
}
|
||||
Reference in New Issue
Block a user