Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2024-10-27 20:34:16 +00:00
Code Issues Projects Releases Wiki Activity

DownloadCentral: Cleanup filter_by_lang

Browse Source
This commit is contained in:
gnosygnu 2016-10-21 10:00:14 -04:00
parent 55ea1fa39a
commit bd6576b716
10 changed files with 133 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
400_xowa/src/gplx/xowa/addons/bldrs/centrals/Xobc_filter_mgr.java Normal file
View File

@ -0,0 +1,37 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.centrals; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
import gplx.xowa.addons.bldrs.centrals.tasks.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.langs.*;
public class Xobc_filter_mgr {
public static Xobc_task_itm[] Filter_by_lang(Xobc_task_regy__base task_list, String lang_key_str) {
List_adp tmp = List_adp_.New();
// loop tasks and find matches
int len = task_list.Len();
for (int i = 0; i < len; ++i) {
Xobc_task_itm task = (Xobc_task_itm)task_list.Get_at(i);
Xobc_task_key task_key_itm = Xobc_task_key.To_itm(task.Task_key());
if (Xow_domain_itm_.Match_lang(task_key_itm.Wiki_domain_itm(), lang_key_str))
tmp.Add(task);
}
return (Xobc_task_itm[])tmp.To_ary_and_clear(Xobc_task_itm.class);
}
}

11
400_xowa/src/gplx/xowa/addons/bldrs/centrals/Xobc_task_mgr.java
View File

@ -51,21 +51,20 @@ public class Xobc_task_mgr implements Xog_json_wkr {
user_db.Done_task_tbl().Select_all(todo_mgr, done_mgr);
return this;
}
public static final String Lang_key__all = "all";
public void Reload() {
Gfo_log_.Instance.Info("task_mgr.reload.bgn");
Gfobj_nde root = Gfobj_nde.New();
Gfobj_nde lists_nde = root.New_nde("lists");
work_mgr.Save_to(lists_nde.New_ary("work"), Lang_key__all);
todo_mgr.Save_to(lists_nde.New_ary("todo"), Lang_key__all);
done_mgr.Save_to(lists_nde.New_ary("done"), Lang_key__all);
work_mgr.Save_to(lists_nde.New_ary("work"));
todo_mgr.Save_to(lists_nde.New_ary("todo"));
done_mgr.Save_to(lists_nde.New_ary("done"));
cbk_mgr.Send_json(cbk_trg, "xo.bldr.core.reload__recv", root);
}
public void Filter_by_lang(String lang_key) {
Gfo_log_.Instance.Info("task_mgr.reload.bgn");
Gfo_log_.Instance.Info("task_mgr.filter_by_lang.bgn");
Gfobj_nde root = Gfobj_nde.New();
Gfobj_nde lists_nde = root.New_nde("lists").Add_str("list_name", "todo");
todo_mgr.Save_to(lists_nde.New_ary("todo"), lang_key);
todo_mgr.Save_to(lists_nde.New_ary("todo"), Xobc_filter_mgr.Filter_by_lang(todo_mgr, lang_key));
cbk_mgr.Send_json(cbk_trg, "xo.bldr.core.reload_list__recv", root);
}
public void Transfer(Xobc_task_regy__base src, Xobc_task_regy__base trg, Xobc_task_itm task) {

2
400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/Xobc_task_regy_tbl.java
View File

@ -73,7 +73,7 @@ public class Xobc_task_regy_tbl implements Db_tbl {
String sql = Db_sql_.Make_by_fmt(String_.Ary
( "SELECT *"
, "FROM task_regy"
, "WHERE task_key LIKE '{0}%'"
, "WHERE task_key LIKE '{0}%'" // DEPENDENCY:Xobc_task_key
, "AND task_seqn != 999999"
), wiki_domain);
List_adp list = List_adp_.New();

46
400_xowa/src/gplx/xowa/addons/bldrs/centrals/tasks/Xobc_task_key.java Normal file
View File

@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.centrals.tasks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
import gplx.xowa.wikis.domains.*;
public class Xobc_task_key {
public Xobc_task_key(String wiki_domain, String wiki_date, String task_type) {
this.wiki_domain = wiki_domain;
this.wiki_date = wiki_date;
this.task_type = task_type;
}
public String Wiki_domain() {return wiki_domain;} private final String wiki_domain;
public String Wiki_date() {return wiki_date;} private final String wiki_date;
public String Wiki_date_ui() {return String_.Replace(wiki_date, ".", "-");}
public String Task_type() {return task_type;} private final String task_type;
public String Task_type_ui() {
if (String_.Eq(task_type, "html")) return "Articles";
else if (String_.Eq(task_type, "file")) return "Images";
else if (String_.Eq(task_type, "text")) return "Source";
else if (String_.Eq(task_type, "patch")) return "Patch";
else return task_type;
}
public Xow_domain_itm Wiki_domain_itm() {return Xow_domain_itm_.parse(Bry_.new_u8(wiki_domain));}
public static Xobc_task_key To_itm(String task_key) {
String[] ary = String_.Split(task_key, "|");
return new Xobc_task_key(ary[0], ary[1], ary[2]);
}
public static String To_str(String wiki_domain, String wiki_date, String task_type) {
return String_.Concat(wiki_domain, "|", wiki_date, "|", task_type);
}
}

19
400_xowa/src/gplx/xowa/addons/bldrs/centrals/tasks/Xobc_task_regy__base.java
View File

@ -30,24 +30,13 @@ public abstract class Xobc_task_regy__base {
public void Del_by(int i) {hash.Del(i);}
public void Sort() {hash.Sort();}
public void Save_to(Gfobj_ary ary, String lang_key) {
List_adp list = List_adp_.New();
int len = hash.Len();
for (int i = 0; i < len; ++i) {
Xobc_task_itm sub_task = (Xobc_task_itm)hash.Get_at(i);
if (!String_.Eq(lang_key, Xobc_task_mgr.Lang_key__all)) {
String task_key = sub_task.Task_key();
if (!String_.Has_at_bgn(task_key, lang_key + ".")) continue;
}
list.Add(sub_task);
}
len = list.Len();
public void Save_to(Gfobj_ary ary) {Save_to(ary, (Xobc_task_itm[])hash.To_ary(Xobc_task_itm.class));}
public void Save_to(Gfobj_ary ary, Xobc_task_itm[] itms) {
int len = itms.length;
Gfobj_nde[] sub_ndes = new Gfobj_nde[len];
for (int i = 0; i < len; ++i) {
Gfobj_nde sub_nde = sub_ndes[i] = Gfobj_nde.New();
Xobc_task_itm sub_task = (Xobc_task_itm)list.Get_at(i);
sub_task.Save_to(sub_nde);
itms[i].Save_to(sub_nde);
}
ary.Ary_(sub_ndes);
}

10
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java
View File

@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
import gplx.core.progs.*; import gplx.core.ios.zips.*; import gplx.core.ios.streams.*; import gplx.core.security.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.fsdb.*; import gplx.fsdb.meta.*;
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.hosts.*; import gplx.xowa.addons.bldrs.centrals.tasks.*;
public class Pack_file_mgr {
public void Exec(Xowe_wiki wiki, Pack_file_cfg cfg) {
// init
@ -98,7 +98,7 @@ public class Pack_file_mgr {
int pack_list_len = pack_list.Len();
// create task
String task_key = Task_key__build(wiki.Domain_str(), wiki_date, task_type);
String task_key = Xobc_task_key.To_str(wiki.Domain_str(), wiki_date, task_type);
String task_name = Build_task_name(tmp_bfr, wiki, wiki_date, task_type, raw_len);
Xobc_task_regy_tbl task_regy_tbl = bc_db.Tbl__task_regy();
int task_id = bc_db.Conn().Sys_mgr().Autonum_next("task_regy.task_id");
@ -172,10 +172,4 @@ public class Pack_file_mgr {
bc_db.Tbl__step_regy().Insert(step_id, Xobc_step_itm.Type__wiki_import);
bc_db.Tbl__import_step().Insert(step_id, gplx.xowa.addons.bldrs.centrals.dbs.datas.Xobc_host_regy_tbl.Host_id__archive_org, wiki_abrv, wiki_date, zip_url.NameAndExt(), itm.Tid(), Xobc_zip_type.Type__zip, zip_md5, zip_len, raw_size, 0, 0);
}
private static String Task_key__build(String wiki_domain, String wiki_date, String task_type) {
return String_.Concat(wiki_domain, "|", wiki_date, "|", task_type);
}
public static String[] Task_key__parse(String task_key) {
return String_.Split(task_key, "|");
}
}

4
400_xowa/src/gplx/xowa/addons/bldrs/infos/Xobc_info_html.java
View File

@ -62,10 +62,8 @@ class Xobc_info_html extends Xow_special_wtr__base {
String torrent_fil = null;
String key = data_db.Tbl__task_regy().Select_key_by_id_or_null(task_id);
if (key == null) torrent_fil = "failed to get torrent for " + Int_.To_str(task_id);
else {// convert task_key to name; en.wikipedia.org|2016.06|html
else {
String src_dir = host_eval.Eval_src_dir(data_db, host_id, wiki_domain);
String[] ary = String_.Split(key, "|");
ary[0] = String_.new_u8(wiki_domain.Abrv_wm());
torrent_fil = String_.Format("{0}Xowa_{1}wiki_latest_archive.torrent", src_dir, wiki_domain.Lang_orig_key()); // EX: https://archive.org/download/Xowa_dewiki_latest/Xowa_dewiki_latest_archive.torrent
}

19
400_xowa/src/gplx/xowa/wikis/domains/Xow_domain_itm_.java
View File

@ -73,6 +73,25 @@ public class Xow_domain_itm_ {
return new_other(raw);
}
}
public static boolean Match_lang(Xow_domain_itm domain, String match) {
// exit early if "*"
if (String_.Eq(match, Lang_key__all)) return true;
// get lang
String cur = String_.new_u8(domain.Lang_actl_key());
// return true if direct match; EX: "en" <-> "en"; "de" <-> "de"
if (String_.Eq(cur, match)) return true;
// handle special cases
if (String_.Eq(match, "en"))
return String_.In(domain.Domain_str(), "simple.wikipedia.org", "species.wikimedia.org", "www.wikidata.org", "commons.wikimedia.org");
else if (String_.Eq(match, "zh"))
return String_.Eq(cur, "lzh");
return false;
}
public static final String Lang_key__all = "*";
private static Xow_domain_itm new_other(byte[] raw) {return Xow_domain_itm.new_(raw, Xow_domain_tid_.Tid__other, Xol_lang_stub_.Key__unknown);}
private static byte[] Get_lang_code_for_mw_messages_file(byte[] v) {
Object o = alt_domain__lang_by_subdomain.Get_by_bry(v);

15
400_xowa/src/gplx/xowa/wikis/domains/Xow_domain_itm_tst.java
View File

@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.wikis.domains; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import org.junit.*;
import org.junit.*; import gplx.core.tests.*;
public class Xow_domain_itm_tst {
private final Xow_domain_fxt fxt = new Xow_domain_fxt();
@Test public void Parse_en_wikipedia() {fxt.Test_parse("en.wikipedia.org" , "en" , "en" , Xow_domain_tid_.Tid__wikipedia);}
@ -31,6 +31,11 @@ public class Xow_domain_itm_tst {
@Test public void Parse_ua_wikimedia_org() {fxt.Test_parse("ua.wikimedia.org" , "ua" , "uk" , Xow_domain_tid_.Tid__wikimedia);}
@Test public void Parse_ar_wikimedia_org() {fxt.Test_parse("ar.wikimedia.org" , "ar" , "es" , Xow_domain_tid_.Tid__wikimedia);}
@Test public void Parse_blank() {fxt.Test_parse("" , "" , "" , Xow_domain_tid_.Tid__other);}
@Test public void Match_lang() {
fxt.Test__match_lang_y("en", "en.wikipedia.org", "en.wiktionary.org", "simple.wikipedia.org", "species.wikimedia.org", "www.wikidata.org", "commons.wikimedia.org");
fxt.Test__match_lang_y("fr", "fr.wikipedia.org", "fr.wiktionary.org");
fxt.Test__match_lang_y("zh", "zh-classical.wikipedia.org");
}
}
class Xow_domain_fxt {
public void Test_parse(String domain, String expd_orig_lang, String expd_actl_lang, int expd_tid) {
@ -39,4 +44,12 @@ class Xow_domain_fxt {
Tfds.Eq_str(expd_actl_lang, String_.new_a7((actl.Lang_actl_key())));
Tfds.Eq_int(expd_tid, actl.Domain_type_id());
}
public void Test__match_lang_y(String lang_code, String... domains) {Test__match_lang(Bool_.Y, lang_code, domains);}
public void Test__match_lang(boolean expd, String lang_key_str, String[] domains) {
int len = domains.length;
for (int i = 0; i < len; ++i) {
Xow_domain_itm domain = Xow_domain_itm_.parse(Bry_.new_u8(domains[i]));
Gftest.Eq__bool(expd, Xow_domain_itm_.Match_lang(domain, lang_key_str), lang_key_str + "|" + domains[i]);
}
}
}

11
400_xowa/src/gplx/xowa/xtns/xowa_cmds/wiki_setups/Xop_wiki_setup_mgr.java
View File

@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.xtns.xowa_cmds.wiki_setups; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.xowa_cmds.*;
import gplx.langs.mustaches.*;
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
import gplx.xowa.addons.bldrs.exports.packs.files.*;
import gplx.xowa.addons.bldrs.exports.packs.files.*; import gplx.xowa.addons.bldrs.centrals.tasks.*;
import gplx.xowa.parsers.*;
class Xop_wiki_setup_mgr {
private final Mustache_tkn_parser parser = new Mustache_tkn_parser();
@ -51,13 +51,10 @@ class Xop_wiki_setup_mgr {
Xows_task_itm[] rv = new Xows_task_itm[len];
for (int i = 0; i < len; ++i) {
Xobc_task_regy_itm task_row = task_rows[i];
String[] task_key_parts = Pack_file_mgr.Task_key__parse(String_.new_u8(task_row.Key()));
String task_key_type = task_key_parts[2];
if (String_.Eq(task_key_type, "html")) task_key_type = "Articles";
else if (String_.Eq(task_key_type, "file")) task_key_type = "Images";
Xobc_task_key task_key = Xobc_task_key.To_itm(String_.new_u8(task_row.Key()));
String task_key_type = task_key.Task_type_ui();
Xows_file_itm[] files = Make_file_ary(url_list_bfr, wiki_domain, task_row.Id());
rv[i] = new Xows_task_itm(task_row.Seqn(), wiki_domain, task_row.Name(), Bry_.new_u8(task_key_type), Bry_.Replace(Bry_.new_u8(task_key_parts[1]), Byte_ascii.Dot, Byte_ascii.Dash), files);
rv[i] = new Xows_task_itm(task_row.Seqn(), wiki_domain, task_row.Name(), Bry_.new_u8(task_key_type), Bry_.new_u8(task_key.Wiki_date_ui()), files);
}
Array_.Sort(rv, new Xows_task_itm_sorter());
return rv;