Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.10.3.1

Browse Source
This commit is contained in:
gnosygnu
2015-10-18 22:17:57 -04:00
parent 8e18af05b6
commit 4f43f51b18
1935 changed files with 12500 additions and 12889 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
400_xowa/src/gplx/xowa/bldrs/setups/maints/Wmf_dump_itm.java Normal file
View File

@@ -0,0 +1,37 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
public class Wmf_dump_itm implements gplx.CompareAble {
public byte[] Wiki_abrv() {return wiki_abrv;} public void Wiki_abrv_(byte[] v) {this.wiki_abrv = v;} private byte[] wiki_abrv; // EX: enwiki
public DateAdp Dump_date() {return dump_date;} public void Dump_date_(DateAdp v) {this.dump_date = v;} private DateAdp dump_date; // EX: 20140304
public DateAdp Status_time() {return status_time;} public void Status_time_(DateAdp v) {this.status_time = v;} private DateAdp status_time; // EX: 2014-03-15 23:22:06
public byte[] Status_msg() {return status_msg;} // EX: Dump in progress / Dump complete
public void Status_msg_(byte[] v) {
this.status_msg = v;
if (Bry_.Eq(status_msg, Status_msg_dump_complete))
status_tid = Status_tid_complete;
else if (Bry_.Eq(status_msg, Status_msg_dump_in_progress))
status_tid = Status_tid_working;
else
status_tid = Status_tid_error;
} private byte[] status_msg;
public byte Status_tid() {return status_tid;} private byte status_tid;
public int compareTo(Object obj) {Wmf_dump_itm comp = (Wmf_dump_itm)obj; return Bry_.Compare(wiki_abrv, comp.wiki_abrv);}
private static byte[] Status_msg_dump_complete = Bry_.new_a7("Dump complete"), Status_msg_dump_in_progress = Bry_.new_a7("Dump in progress");
public static final byte Status_tid_complete = 0, Status_tid_working = 1, Status_tid_error = 2;
}

72
400_xowa/src/gplx/xowa/bldrs/setups/maints/Wmf_dump_list_parser.java Normal file
View File

@@ -0,0 +1,72 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
public class Wmf_dump_list_parser {
public Wmf_dump_itm[] Parse(byte[] src) {
Ordered_hash itms = Ordered_hash_.New_bry();
int pos = 0;
while (true) {
int a_pos = Bry_find_.Find_fwd(src, Find_anchor, pos); if (a_pos == Bry_.NotFound) break; // no more anchors found
pos = a_pos + Find_anchor.length;
try {
Wmf_dump_itm itm = new Wmf_dump_itm();
if (!Parse_href(itm, src, a_pos)) continue; // anchor not parseable; not a link to a wmf dump
if (itms.Has(itm.Wiki_abrv())) continue; // ignore dupes
itms.Add(itm.Wiki_abrv(), itm);
itm.Status_time_(Parse_status_time(src, a_pos));
itm.Status_msg_(Parse_status_msg(src, a_pos));
} catch (Exception e) {Err_.Noop(e);}
}
return (Wmf_dump_itm[])itms.To_ary(Wmf_dump_itm.class);
}
private boolean Parse_href(Wmf_dump_itm itm, byte[] src, int a_pos) { // EX: http://dumps.wikimedia.org/enwiki/20130807
int href_pos = Bry_find_.Find_fwd(src, Find_href, a_pos); if (href_pos == Bry_.NotFound) return false; // no <li>; something bad happened
int href_bgn_pos = Bry_find_.Find_fwd(src, Byte_ascii.Quote, href_pos + Find_href.length);
int href_end_pos = Bry_find_.Find_fwd(src, Byte_ascii.Quote, href_bgn_pos + 1); if (href_end_pos == Bry_.NotFound) return false;
byte[] href_bry = Bry_.Mid(src, href_bgn_pos + 1, href_end_pos);
int date_end = href_bry.length;
int date_bgn = Bry_find_.Find_bwd(href_bry, Byte_ascii.Slash); if (date_bgn == Bry_.NotFound) return false;
byte[] date_bry = Bry_.Mid(href_bry, date_bgn + 1, date_end);
DateAdp date = DateAdp_.parse_fmt(String_.new_a7(date_bry), "yyyyMMdd");
itm.Dump_date_(date);
int abrv_end = date_bgn;
int abrv_bgn = Bry_find_.Find_bwd(href_bry, Byte_ascii.Slash, abrv_end); if (abrv_bgn == Bry_.NotFound) abrv_bgn = -1; // "enwiki/20130708"
byte[] abrv_bry = Bry_.Mid(href_bry, abrv_bgn + 1, abrv_end);
itm.Wiki_abrv_(Bry_.Replace(abrv_bry, Byte_ascii.Underline, Byte_ascii.Dash));
return true;
}
private DateAdp Parse_status_time(byte[] src, int a_pos) {
int li_pos = Bry_find_.Find_bwd(src, Find_li, a_pos); if (li_pos == Bry_.NotFound) return null;
int bgn = Bry_find_.Find_fwd(src, Byte_ascii.Gt, li_pos + Find_li.length); if (bgn == Bry_.NotFound) return null;
byte[] rv_bry = Bry_.Mid(src, bgn + 1, a_pos);
return DateAdp_.parse_fmt(String_.Trim(String_.new_a7(rv_bry)), "yyyy-MM-dd HH:mm:ss");
}
private byte[] Parse_status_msg(byte[] src, int a_pos) {
int span_pos = Bry_find_.Find_fwd(src, Find_span_bgn, a_pos); if (span_pos == Bry_.NotFound) return null;
int bgn = Bry_find_.Find_fwd(src, Byte_ascii.Gt, span_pos + Find_span_bgn.length); if (bgn == Bry_.NotFound) return null;
int end = Bry_find_.Find_fwd(src, Find_span_end, bgn); if (end == Bry_.NotFound) return null;
return Bry_.Mid(src, bgn + 1, end);
}
private static byte[]
Find_anchor = Bry_.new_a7("<a")
, Find_href = Bry_.new_a7(" href=")
, Find_li = Bry_.new_a7("<li")
, Find_span_bgn = Bry_.new_a7("<span")
, Find_span_end = Bry_.new_a7("</span>")
;
}

137
400_xowa/src/gplx/xowa/bldrs/setups/maints/Wmf_dump_list_parser_tst.java Normal file
View File

@@ -0,0 +1,137 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
import org.junit.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*;
public class Wmf_dump_list_parser_tst {
@Before public void init() {fxt.Clear();} private Wmf_dump_list_parser_fxt fxt = new Wmf_dump_list_parser_fxt();
@Test public void Parse() {
fxt.Test_parse
( "<li>2013-07-17 00:32:33 <a href=\"http://dumps.wikimedia.org/enwiki/20130708\">enwiki</a>: <span class=\"done\">Dump complete</span></li>"
, fxt.itm("enwiki", "20130708", Wmf_dump_itm.Status_tid_complete, "Dump complete", "2013-07-17 00:32:33")
);
fxt.Test_parse(String_.Concat_lines_nl
( "<li>2013-07-24 02:02:13 <a href=\"http://dumps.wikimedia.org/kawiki/20130724\">kawiki</a>: <span class=\"in-progress\">Dump in progress</span></li>"
, "<ul><li class=\"in-progress\"><span class=\"updates\">2013-07-24 00:54:55</span> <span class=\"status\">in-progress</span> <span class=\"title\">All pages with complete page edit history (.bz2)</span><div class=\"progress\">2013-07-24 02:02:13: kawiki (ID 18587) 22046 pages (5.5|11140.9/sec all|curr), 869000 revs (215.2|505.3/sec all|curr), 99.9%|99.9% prefetched (all|curr), ETA 2013-07-24 04:09:41 [max 2514872]</div>"
, "<ul><li class=\"file\">kawiki-20130724-pages-meta-history.xml.bz2 245.2 MB (written) </li></ul></li></ul>"
)
, fxt.itm("kawiki", "20130724", Wmf_dump_itm.Status_tid_working, "Dump in progress", "2013-07-24 02:02:13")
);
fxt.Test_parse
( "<li>2013-07-17 00:32:33 <a href=\"http://dumps.wikimedia.org/enwiki/20130708\">enwiki</a>: <span class=\"done\">Error</span></li>"
, fxt.itm("enwiki", "20130708", Wmf_dump_itm.Status_tid_error, "Error", "2013-07-17 00:32:33")
);
fxt.Test_parse
( "<li>2013-11-28 06:08:56 <a href=\"zh_classicalwiki/20131128\">zh_classicalwiki</a>: <span class='done'>Dump complete</span></li>"
, fxt.itm("zh-classicalwiki", "20131128", Wmf_dump_itm.Status_tid_complete, "Dump complete", "2013-11-28 06:08:56")
);
}
// @Test public void Update() { // MAINT:QUARTERLY:2015-08-23; must run C:\xowa\ and update dump status
// Hash_adp_bry excluded_domains = Hash_adp_bry.cs().Add_many_str
// ( "advisory.wikipedia.org", "beta.wikiversity.org", "donate.wikipedia.org", "login.wikipedia.org"
// , "nostalgia.wikipedia.org", "outreach.wikipedia.org", "quality.wikipedia.org", "sources.wikipedia.org"
// , "strategy.wikipedia.org", "ten.wikipedia.org", "test2.wikipedia.org", "test.wikipedia.org"
// , "usability.wikipedia.org", "vote.wikipedia.org"
// , "bd.wikimedia.org", "dk.wikimedia.org", "mx.wikimedia.org", "nyc.wikimedia.org", "nz.wikimedia.org", "pa-us.wikimedia.org", "rs.wikimedia.org", "ua.wikimedia.org"
// );
// Wmf_dump_itm[] itms = new Wmf_dump_list_parser().Parse(Io_mgr.Instance.LoadFilBry("C:\\xowa\\bin\\any\\xowa\\xtns\\xowa\\maintenance\\backup-index.html"));
// Array_.Sort(itms);
// Bry_bfr sql_bfr = Bry_bfr.new_();
// Bry_bfr bld_bfr = Bry_bfr.new_();
// int itms_len = itms.length;
// int counter = 1;
// for (int i = 0; i < itms_len; i++) {
// Wmf_dump_itm itm = itms[i];
// byte[] abrv = itm.Wiki_abrv();
// if (Bry_.Eq(abrv, Bry_.new_a7("testwikidatawiki"))) continue;
// byte[] domain_bry = Xow_abrv_wm_.Parse_to_domain_bry(abrv);
// if (domain_bry == null) continue; // not a standard WMF wiki; ignore
// if (Bry_find_.Find_fwd(domain_bry, Bry_.new_a7("wikimania")) != Bry_.NotFound) continue;
// if (excluded_domains.Has(domain_bry)) continue;
// Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry);
// byte[] tid_name = Xto_display_name(Xow_domain_tid_.Get_type_as_bry(domain_itm.Domain_type_id()));
// sql_bfr
// .Add_byte(Byte_ascii.Paren_bgn)
// .Add_int_variable(counter++)
// .Add_byte(Byte_ascii.Comma)
// .Add_int_variable(1)
// .Add_byte(Byte_ascii.Comma)
// .Add_byte(Byte_ascii.Apos)
// .Add(domain_itm.Lang_orig_key())
// .Add_byte(Byte_ascii.Apos)
// .Add_byte(Byte_ascii.Comma)
// .Add_byte(Byte_ascii.Apos)
// .Add(tid_name)
// .Add_byte(Byte_ascii.Apos)
// .Add_byte(Byte_ascii.Paren_end)
// .Add_byte(Byte_ascii.Comma)
// .Add_str("--" + String_.new_u8(abrv))
// .Add_byte_nl()
// ;
// bld_bfr
// .Add_byte(Byte_ascii.Comma)
// .Add_byte(Byte_ascii.Space)
// .Add_byte(Byte_ascii.Quote)
// .Add(domain_bry)
// .Add_byte(Byte_ascii.Quote)
// .Add_byte_nl()
// ;
// }
// Io_url temp = Io_url_.new_fil_("C:\\import_update.txt");
// Io_mgr.Instance.SaveFilBfr(temp, sql_bfr);
//// Io_mgr.Instance.AppendFilBfr(temp, bld_bfr);
// }
// private static byte[] Xto_display_name(byte[] v) {
// if (Bry_.Eq(v, Xow_domain_tid_.Bry__wmforg)) return Bry_.new_a7("Wikimedia Foundation");
// else if (Bry_.Eq(v, Xow_domain_tid_.Bry__species)) return Bry_.new_a7("Wikispecies");
// else if (Bry_.Eq(v, Xow_domain_tid_.Bry__mediawiki)) return Bry_.new_a7("MediaWiki");
// else return Bry_.Add(Byte_ascii.Case_upper(v[0]), Bry_.Mid(v, 1, v.length));
// }
}
class Wmf_dump_list_parser_fxt {
public void Clear() {}
private Wmf_dump_list_parser parser = new Wmf_dump_list_parser();
public String itm(String wiki_abrv, String dump_date, byte status_done, String status_msg, String status_time) {
return String_.Concat_with_str("\n", wiki_abrv, dump_date
, Byte_.To_str(status_done)
, status_msg
, status_time
);
}
public void Test_parse(String raw, String... expd) {
Wmf_dump_itm[] actl = parser.Parse(Bry_.new_a7(raw));
Tfds.Eq_str_lines(String_.Concat_lines_nl(expd), String_.Concat_lines_nl(Xto_str(actl)));
}
public String[] Xto_str(Wmf_dump_itm[] ary) {
int len = ary.length;
String[] rv = new String[len];
for (int i = 0; i < len; i++)
rv[i] = Xto_str(ary[i]);
return rv;
}
public static String Xto_str(Wmf_dump_itm itm) {
DateAdp status_time = itm.Status_time();
String status_time_str = status_time == null ? "" : status_time.XtoStr_fmt(DateAdp_.Fmt_iso8561_date_time);
return String_.Concat_with_str("\n", String_.new_a7(itm.Wiki_abrv()), itm.Dump_date().XtoStr_fmt("yyyyMMdd")
, Byte_.To_str(itm.Status_tid())
, String_.new_a7(itm.Status_msg())
, status_time_str
);
}
}

26
400_xowa/src/gplx/xowa/bldrs/setups/maints/Wmf_latest_itm.java Normal file
View File

@@ -0,0 +1,26 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
public class Wmf_latest_itm {
public Wmf_latest_itm(byte[] name, DateAdp date, long size) {
this.name = name; this.date = date; this.size = size;
}
public byte[] Name() {return name;} private final byte[] name;
public DateAdp Date() {return date;} private final DateAdp date;
public long Size() {return size;} private final long size;
}

67
400_xowa/src/gplx/xowa/bldrs/setups/maints/Wmf_latest_parser.java Normal file
View File

@@ -0,0 +1,67 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
import gplx.core.btries.*; import gplx.ios.*;
public class Wmf_latest_parser {
private Ordered_hash hash = Ordered_hash_.New_bry();
public int Count() {return hash.Count();}
public Wmf_latest_itm Get_at(int i) {return (Wmf_latest_itm)hash.Get_at(i);}
public Wmf_latest_itm Get_by(byte[] k) {return (Wmf_latest_itm)hash.Get_by(k);}
public Wmf_latest_itm[] To_ary() {return (Wmf_latest_itm[])hash.To_ary(Wmf_latest_itm.class);}
public void Parse(byte[] src) {
hash.Clear();
Bry_bfr tmp_bfr = Bry_bfr.reset_(255);
byte[] name_bgn_bry = Bry_.new_a7("\n<a href=\"");
byte[] date_bgn_bry = Bry_.new_a7("</a>");
byte[] date_end_bry = Bry_.new_a7(" ");
// byte[] size_bgn_bry = Bry_.new_a7("</td><td class=\"s\">");
Btrie_slim_mgr date_trie = Btrie_slim_mgr.cs()
.Add_bry("Jan", "01").Add_bry("Feb", "02").Add_bry("Mar", "03").Add_bry("Apr", "04").Add_bry("May", "05").Add_bry("Jun", "06")
.Add_bry("Jul", "07").Add_bry("Aug", "08").Add_bry("Sep", "09").Add_bry("Oct", "10").Add_bry("Nov", "11").Add_bry("Dec", "12")
;
// Btrie_slim_mgr size_trie = Btrie_slim_mgr.cs()
// .Add_bry("B", " B").Add_bry("K", " KB").Add_bry("M", " MB").Add_bry("G", " GB");
byte[] date_or = Bry_.new_a7("1970-01-01 00:00:00");
// byte[] size_or = Bry_.new_a7("0 B");
int size_end = 0; int src_len = src.length;
while (true) {
int name_bgn = Bry_find_.Move_fwd(src, name_bgn_bry, size_end, src_len); if (name_bgn == Bry_find_.Not_found) break;
int name_end = Bry_find_.Find_fwd(src, Byte_ascii.Quote, name_bgn, src_len);
byte[] name = Bry_.Mid(src, name_bgn, name_end);
int date_bgn = Bry_find_.Move_fwd(src, date_bgn_bry, name_end, src_len); if (date_bgn == Bry_find_.Not_found) {Gfo_usr_dlg_.Instance.Warn_many("", "", "date_bgn not found"); break;}
date_bgn = Bry_find_.Find_fwd_while_space_or_tab(src, date_bgn, src_len); if (date_bgn == Bry_find_.Not_found) {Gfo_usr_dlg_.Instance.Warn_many("", "", "date_bgn not found"); break;}
int date_end = Bry_find_.Find_fwd(src, date_end_bry, date_bgn, src_len);
byte[] date_bry = Bry_.Mid(src, date_bgn, date_end);
DateAdp date = DateAdp_.parse_fmt(String_.new_a7(Replace_or(tmp_bfr, date_trie, date_bry, 3, date_or)), "dd-MM-yyyy HH:mm");
int size_bgn = Bry_find_.Find_fwd_while_space_or_tab(src, date_end, src_len); if (size_bgn == Bry_find_.Not_found) {Gfo_usr_dlg_.Instance.Warn_many("", "", "size_bgn not found"); break;}
size_end = Bry_find_.Find_fwd(src, Byte_ascii.Cr, size_bgn, src_len);
byte[] size_bry = Bry_.Mid(src, size_bgn, size_end);
long size = Long_.parse_or(String_.new_u8(size_bry), -1);
Wmf_latest_itm itm = new Wmf_latest_itm(name, date, size);
hash.Add(name, itm);
}
}
private static byte[] Replace_or(Bry_bfr tmp_bfr, Btrie_slim_mgr trie, byte[] src, int pos, byte[] or) {
int src_len = src.length;
Object o = trie.Match_bgn(src, pos, src_len); if (o == null) return or;
tmp_bfr.Add_mid(src, 0, pos);
tmp_bfr.Add((byte[])o);
tmp_bfr.Add_mid(src, trie.Match_pos(), src_len);
return tmp_bfr.To_bry_and_clear();
}
}

53
400_xowa/src/gplx/xowa/bldrs/setups/maints/Wmf_latest_parser_tst.java Normal file
View File

@@ -0,0 +1,53 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
import org.junit.*; import gplx.ios.*;
public class Wmf_latest_parser_tst {
@Before public void init() {fxt.Clear();} private Wmf_latest_parser_fxt fxt = new Wmf_latest_parser_fxt();
@Test public void Parse() {
fxt.Test_parse
( "\n<a href=\"enwiki-latest-pages-articles.xml.bz2\">enwiki-latest-pages-articles.xml.bz2</a> 15-Jan-2015 05:43 11575640561\r\n"
, fxt.itm("enwiki-latest-pages-articles.xml.bz2", "2015-01-15 05:43", "10.781 GB")
);
}
// @Test public void Smoke() {
// Wmf_latest_parser parser = new Wmf_latest_parser();
// parser.Parse(Io_mgr.Instance.LoadFilBry("C:\\wmf_latest.html"));
// Tfds.Write(String_.Concat_lines_nl(Wmf_latest_parser_fxt.Xto_str_ary(parser.To_ary())));
// }
}
class Wmf_latest_parser_fxt {
public void Clear() {}
private Wmf_latest_parser parser = new Wmf_latest_parser();
public Wmf_latest_itm itm(String name, String date, String size) {return new Wmf_latest_itm(Bry_.new_a7(name), DateAdp_.parse_iso8561(date), Io_size_.parse_or(size, 0));}
public void Test_parse(String raw, Wmf_latest_itm... expd) {
parser.Parse(Bry_.new_a7(raw));
Wmf_latest_itm[] actl = parser.To_ary();
Tfds.Eq_str_lines(String_.Concat_lines_nl(Xto_str_ary(expd)), String_.Concat_lines_nl(Xto_str_ary(actl)));
}
public static String[] Xto_str_ary(Wmf_latest_itm[] ary) {
int len = ary.length;
String[] rv = new String[len];
for (int i = 0; i < len; i++)
rv[i] = Xto_str(ary[i]);
return rv;
}
public static String Xto_str(Wmf_latest_itm itm) {
return String_.Concat_with_str("\n", String_.new_a7(itm.Name()), itm.Date().XtoStr_fmt_iso_8561(), Io_size_.To_str(itm.Size()));
}
}

83
400_xowa/src/gplx/xowa/bldrs/setups/maints/Xoa_maint_mgr.java Normal file
View File

@@ -0,0 +1,83 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
import gplx.ios.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.files.downloads.*;
public class Xoa_maint_mgr implements GfoInvkAble {
public Xoa_maint_mgr(Xoae_app app) {
this.app = app;
wmf_dump_status_url = Wmf_dump_status_url(app);
wiki_mgr = new Xoa_maint_wikis_mgr(app);
} private Xoae_app app; private Io_url wmf_dump_status_url;
public Xoa_maint_wikis_mgr Wiki_mgr() {return wiki_mgr;} private Xoa_maint_wikis_mgr wiki_mgr;
public boolean Wmf_dump_status_loaded() {return wmf_dump_status_loaded;} private boolean wmf_dump_status_loaded;
public void Wmf_dump_status_loaded_assert() {
if (!wmf_dump_status_loaded) {
Wmf_status_parse();
wmf_dump_status_loaded = true;
}
}
public void Wmf_status_update() {
Wmf_status_download();
Wmf_status_parse();
}
public boolean Wmf_status_download() {
String[] server_urls = app.Setup_mgr().Dump_mgr().Server_urls();
int len = server_urls.length;
Xof_download_wkr download_wkr = app.Wmf_mgr().Download_wkr();
for (int i = 0; i < len; i++) {
String server_url = server_urls[i] + "backup-index.html";
byte rslt = download_wkr.Download(true, server_url, wmf_dump_status_url, "downloading wmf status");
if (rslt == IoEngine_xrg_downloadFil.Rslt_pass) return true;
}
app.Usr_dlg().Prog_many("", "", "could not download latest status");
return false;
}
public boolean Wmf_status_parse() {
Wmf_dump_list_parser parser = new Wmf_dump_list_parser();
Hash_adp_bry itms_hash = Hash_adp_bry.cs();
Wmf_dump_itm[] itms = parser.Parse(Io_mgr.Instance.LoadFilBry(wmf_dump_status_url));
int len = itms.length;
Xoa_app_.Usr_dlg().Log_many("", "", "maint.html count; count=~{0}", len);
for (int i = 0; i < len; i++) {
Wmf_dump_itm itm = itms[i];
byte[] wiki_abrv = itm.Wiki_abrv();
Xoa_app_.Usr_dlg().Log_many("", "", "maint.html itm; itm=~{0}", wiki_abrv);
byte[] wiki_domain = Xow_abrv_wm_.Parse_to_domain_bry(wiki_abrv);
if (wiki_domain == null) continue; // invalid wiki-name; ex: nycwikimedia
itms_hash.Add(wiki_domain, itm);
}
len = app.Wiki_mgr().Count();
Xoa_app_.Usr_dlg().Log_many("", "", "maint.wiki_count; count=~{0}", len);
for (int i = 0; i < len; i++) {
Xowe_wiki wiki = app.Wiki_mgr().Get_at(i);
Xoa_app_.Usr_dlg().Log_many("", "", "maint.wiki_itm; wiki=~{0}", wiki.Domain_str());
Wmf_dump_itm itm = (Wmf_dump_itm)itms_hash.Get_by_bry(wiki.Domain_bry()); if (itm == null) continue;
wiki.Maint_mgr().Wmf_dump_date_(itm.Dump_date()).Wmf_dump_done_(itm.Status_tid() == Wmf_dump_itm.Status_tid_complete).Wmf_dump_status_(itm.Status_msg());
}
return true;
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_wmf_status_update)) Wmf_status_update();
else if (ctx.Match(k, Invk_wikis)) return wiki_mgr;
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_wmf_status_update = "wmf_status_update", Invk_wikis = "wikis";
public static Io_url Wmf_dump_status_url(Xoae_app app) {return app.Fsys_mgr().Bin_xowa_dir().GenSubDir_nest("xtns", "xowa", "maintenance", "backup-index.html");}
}

47
400_xowa/src/gplx/xowa/bldrs/setups/maints/Xoa_maint_wikis_mgr.java Normal file
View File

@@ -0,0 +1,47 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
public class Xoa_maint_wikis_mgr implements GfoInvkAble {
public Xoa_maint_wikis_mgr(Xoae_app app) {this.app = app;} private Xoae_app app;
private Ordered_hash hash = Ordered_hash_.New_bry();
public int Len() {return hash.Count();}
public Xowe_wiki Get_at(int i) {
if (init) Init();
byte[] domain = (byte[])hash.Get_at(i);
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(domain);
wiki.Init_assert();
return wiki;
}
public void Add(byte[] domain) {hash.Add_if_dupe_use_nth(domain, domain);} // NOTE: must be Add_if_dupe_use_nth to replace existing wikis
public void Init() {
int len = this.Len();
for (int i = 0; i < len; i++) {
byte[] domain = (byte[])hash.Get_at(i);
Xowe_wiki wiki = app.Wiki_mgr().Get_by_key_or_make(domain);
wiki.Init_assert();
}
init = false;
}
private boolean init = true;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_len)) return this.Len();
else if (ctx.Match(k, Invk_get_at)) return this.Get_at(m.ReadInt("v"));
else return GfoInvkAble_.Rv_unhandled;
// return this;
} private static final String Invk_len = "len", Invk_get_at = "get_at";
}

57
400_xowa/src/gplx/xowa/bldrs/setups/maints/Xow_maint_mgr.java Normal file
View File

@@ -0,0 +1,57 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
public class Xow_maint_mgr implements GfoInvkAble {
public Xow_maint_mgr(Xowe_wiki wiki) {
this.wiki = wiki;
maint_mgr = wiki.Appe().Setup_mgr().Maint_mgr();
} private Xowe_wiki wiki; private Xoa_maint_mgr maint_mgr;
public DateAdp Wmf_dump_date() {
maint_mgr.Wmf_dump_status_loaded_assert();
return wmf_dump_date;
} public Xow_maint_mgr Wmf_dump_date_(DateAdp v) {this.wmf_dump_date = v; return this;} private DateAdp wmf_dump_date;
public boolean Wmf_dump_done() {return wmf_dump_done;} public Xow_maint_mgr Wmf_dump_done_(boolean v) {this.wmf_dump_done = v; return this;} private boolean wmf_dump_done;
public byte[] Wmf_dump_status() {return wmf_dump_status;} public Xow_maint_mgr Wmf_dump_status_(byte[] v) {this.wmf_dump_status = v; return this;} private byte[] wmf_dump_status;
public DateAdp Wiki_dump_date() {
if (wiki_dump_date == null)
wiki_dump_date = wiki.Db_mgr().Dump_date_query();
return wiki_dump_date;
} private DateAdp wiki_dump_date;
public boolean Wiki_update_needed() {
if (this.Wiki_dump_date() == null) return false; // will be null if a custom wiki (i.e.: not on http://dumps.wikimedia.org/backup-index.html)
return this.Wmf_dump_date().Diff(this.Wiki_dump_date()).Total_days().To_double() > 1;
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_wmf_dump_date)) return DateAdp_.Xto_str_fmt_or(Wmf_dump_date(), "yyyy-MM-dd", "");
else if (ctx.Match(k, Invk_wmf_dump_date_)) Wmf_dump_date_(m.ReadDate("v"));
else if (ctx.Match(k, Invk_wmf_dump_done)) return Yn.To_str(wmf_dump_done);
else if (ctx.Match(k, Invk_wmf_dump_done_)) wmf_dump_done = m.ReadYn("v");
else if (ctx.Match(k, Invk_wmf_dump_status)) return String_.new_u8(wmf_dump_status);
else if (ctx.Match(k, Invk_wmf_dump_status_)) wmf_dump_status = m.ReadBry("v");
else if (ctx.Match(k, Invk_wiki_dump_date)) return DateAdp_.Xto_str_fmt_or(Wiki_dump_date(), "yyyy-MM-dd", "");
else if (ctx.Match(k, Invk_wiki_dump_date_)) wiki_dump_date = m.ReadDate("v");
else if (ctx.Match(k, Invk_wiki_update_needed)) return Yn.To_str(Wiki_update_needed());
else if (ctx.Match(k, Invk_wiki_dump_date_)) wiki_dump_date = m.ReadDate("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
}
private static final String Invk_wmf_dump_date = "wmf_dump_date", Invk_wmf_dump_date_ = "wmf_dump_date_", Invk_wmf_dump_done = "wmf_dump_done", Invk_wmf_dump_done_ = "wmf_dump_done_"
, Invk_wmf_dump_status = "wmf_dump_status", Invk_wmf_dump_status_ = "wmf_dump_status_", Invk_wiki_dump_date = "wiki_dump_date", Invk_wiki_dump_date_ = "wiki_dump_date_"
, Invk_wiki_update_needed = "wiki_update_needed"
;
}