Archives
/
gnosygnu_xowa
mirror of https://github.com/gnosygnu/xowa
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

General: Remove text database support

Browse Source
pull/620/head
gnosygnu 6 years ago
parent fe0d1abde5
commit be3979c5af
44 changed files with 663 additions and 1954 deletions
Show Stats Download Patch File Download Diff File

8
100_core/src/gplx/core/tests/Gftest.java
Unescape View File

@ -152,9 +152,9 @@ public class Gftest {
if (idx < len) {
switch (type_id) {
case Type_ids_.Id__bool: bfr.Add_yn(Bool_.Cast(Array_.Get_at(ary, idx))); break;
case Type_ids_.Id__bry: bfr.Add_safe((byte[])Array_.Get_at(ary, idx)); break;
case Type_ids_.Id__bry: bfr.Add_safe((byte[])Array_.Get_at(ary, idx)); break;
case Type_ids_.Id__long: bfr.Add_long_variable(Long_.cast(Array_.Get_at(ary, idx))); break;
case Type_ids_.Id__int: bfr.Add_int_variable(Int_.Cast(Array_.Get_at(ary, idx))); break;
case Type_ids_.Id__int: bfr.Add_int_variable(Int_.Cast(Array_.Get_at(ary, idx))); break;
case Type_ids_.Id__byte: bfr.Add_int_variable((int)(Byte_.Cast(Array_.Get_at(ary, idx)))); break;
default: throw Err_.new_unhandled_default(type_id);
}
@ -177,9 +177,9 @@ public class Gftest {
else {
switch (tid) {
case Type_ids_.Id__bool: eq = Bool_.Cast(expd_obj) == Bool_.Cast(actl_obj); break;
case Type_ids_.Id__bry: eq = Bry_.Eq((byte[])expd_obj, (byte[])actl_obj); break;
case Type_ids_.Id__bry: eq = Bry_.Eq((byte[])expd_obj, (byte[])actl_obj); break;
case Type_ids_.Id__long: eq = Long_.cast(expd_obj) == Long_.cast(actl_obj); break;
case Type_ids_.Id__int: eq = Int_.Cast(expd_obj) == Int_.Cast(actl_obj); break;
case Type_ids_.Id__int: eq = Int_.Cast(expd_obj) == Int_.Cast(actl_obj); break;
case Type_ids_.Id__byte: eq = Byte_.Cast(expd_obj) == Byte_.Cast(actl_obj); break;
}
}

1
140_dbs/src/gplx/dbs/Dbmeta_fld_list.java
Unescape View File

@ -40,6 +40,7 @@ public class Dbmeta_fld_list {
public String Add_str_null(String name, int len) {return Add(Dbmeta_fld_itm.new_str(name, len).Nullable_y_());}
public String Add_str_dflt(String name, int len, String dflt)
{return Add(Dbmeta_fld_itm.new_str(name, len).Default_(dflt));}
public Dbmeta_fld_list Bld_int(String name) {Add_int(name); return this;}
public Dbmeta_fld_list Bld_str(String name) {return Bld_str(name, 255);}
public Dbmeta_fld_list Bld_str(String name, int len){Add_str(name, len); return this;}

18
140_dbs/src/gplx/dbs/Dbmeta_fld_tid.java
Unescape View File

@ -65,7 +65,21 @@ public class Dbmeta_fld_tid {
default: throw Err_.new_unhandled(tid);
}
}
public static Dbmeta_fld_itm To_itm(String raw) {
return null;
public static int Get_by_obj(Object o) {
int type_id = Type_ids_.To_id_by_obj(o);
switch (type_id) {
case Type_ids_.Id__bool: return Dbmeta_fld_tid.Tid__bool;
case Type_ids_.Id__byte: return Dbmeta_fld_tid.Tid__byte;
case Type_ids_.Id__short: return Dbmeta_fld_tid.Tid__short;
case Type_ids_.Id__int: return Dbmeta_fld_tid.Tid__int;
case Type_ids_.Id__long: return Dbmeta_fld_tid.Tid__long;
case Type_ids_.Id__float: return Dbmeta_fld_tid.Tid__float;
case Type_ids_.Id__double: return Dbmeta_fld_tid.Tid__double;
case Type_ids_.Id__str: return Dbmeta_fld_tid.Tid__str;
case Type_ids_.Id__bry: return Dbmeta_fld_tid.Tid__bry;
case Type_ids_.Id__date: return Dbmeta_fld_tid.Tid__date;
case Type_ids_.Id__decimal: return Dbmeta_fld_tid.Tid__decimal;
default: throw Err_.new_unhandled_default(type_id);
}
}
}

4
400_xowa/src/gplx/core/ios/Io_stream_rdr_process.java
Unescape View File

@ -30,7 +30,9 @@ public class Io_stream_rdr_process implements Io_stream_rdr {
ProcessBuilder pb = new ProcessBuilder(process_args);
pb.redirectErrorStream(false);
try {process = pb.start();}
catch (Exception e) {throw Err_.new_exc(e, "core", "process init failed", "args", String_.AryXtoStr(process_args));}
catch (Exception e) {
throw Err_.new_exc(e, "core", "process init failed", "args", String_.AryXtoStr(process_args));
}
stream_read = process.getInputStream();
return this;
}

221
400_xowa/src/gplx/xowa/addons/wikis/searchs/specials/Srch_special_searcher_tst.java
Unescape View File

@ -1,221 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
//namespace gplx.xowa.addons.wikis.searchs.v1s {
// import org.junit.*; using gplx.xowa.wikis.tdbs; using gplx.xowa.wikis.data.tbls;
// public class Xosrh_core_tst {
// @Before public void Init() {fxt.Clear();} private Xos_search_mgr_fxt fxt = new Xos_search_mgr_fxt();
// @Test public void Basic() {
// fxt.Init_basic();
// fxt.Test_search_exact("b2", "B2_22", "B2_12", "B2__2");
// fxt.Test_search_exact("a" , "A___0");
// fxt.Test_search_exact("b1a"); // missing: mid
// fxt.Test_search_exact("d"); // missing: end
// fxt.Test_search_exact("$"); // missing: bgn
// fxt.Test_search_match_bgn("b*", "B3_23", "B2_22", "B1_21", "B3_13", "B2_12", "B1_11", "B3__3", "B2__2", "B1__1");
// }
// @Test public void Page_size() {
// fxt.Init_basic();
// fxt.Search_mgr().Page_mgr().Itms_per_page_(1);
// fxt.Test_search("b*", 0, "B3_23");
// fxt.Test_search("b*", 1, "B2_22");
// fxt.Test_search("b*", 2, "B1_21");
// fxt.Test_search("b*", 3, "B3_13");
// }
// @Test public void Url() {
// Xoa_url url = Xow_url_parser_old.Parse_url(fxt.App(), fxt.Wiki(), "Special:Search/Abc?fulltext=y&xowa_sort=len_desc");
// fxt.Search_mgr().Args_mgr().Clear().Parse(url.Args());
// Tfds.Eq(Srch_rslt_row_sorter.Tid_len_dsc, fxt.Search_mgr().Args_mgr().Sort_tid());
// }
// @Test public void Url_arg_title() {// http://en.wikipedia.org/wiki/Special:Search/Earth?fulltext=yes&title=Mars
// fxt.Test_url_search_bry("Special:Search?fulltext=y&search=Abc" , "Abc"); // query arg
//// fxt.Test_url_search_bry("Special:Search/Abc?fulltext=y" , "Abc"); // leaf
// fxt.Test_url_search_bry("Special:Search/Abc?fulltext=y&search=Def" , "Def"); // leaf overrides query arg
// }
// @Test public void Url_ns() {
// fxt.Test_url__ns("Special:Search?search=Abc&ns0=1&ns1=1", "0|1");
// fxt.Test_url__ns("Special:Search?search=Abc&ns*=1", "*");
// fxt.Test_url__ns("Special:Search?search=Abc", "0");
// }
// @Test public void Html() {
// fxt.Init_basic();
// fxt.Test_html_by_url("B1", "", String_.Concat_lines_nl
// ( "Result '''1''' of '''3''' for '''B1'''<br/>"
// , "{|"
// , "|-"
// , "| [[Special:Search/B1?fulltext=y&xowa_page_index=0|&lt;]]"
// , "| [[Special:Search/B1?fulltext=y&xowa_page_index=1|&gt;]]"
// , "|-"
// , "| [[Special:Search/B1?fulltext=y&xowa_sort=len_desc|length]]"
// , "| [[Special:Search/B1?fulltext=y&xowa_sort=title_asc|title]]"
// , "|-"
// , "| 42 || [[B1 21]]"
// , "|-"
// , "| 22 || [[B1 11]]"
// , "|-"
// , "| 2 || [[B1 1]]"
// , "|-"
// , "| [[Special:Search/B1?fulltext=y&xowa_page_index=0|&lt;]]"
// , "| [[Special:Search/B1?fulltext=y&xowa_page_index=1|&gt;]]"
// , "|}"
// ));
// }
//// @Test public void Page_next() {
//// fxt.Init_basic();
//// fxt.Search_mgr().Page_size_(1);
//// fxt.Test_search(Srch_special_page.Match_tid_all, "B1", 0, "B1 1");
//// fxt.Test_search(Srch_special_page.Match_tid_all, "B1", 1, "B1 11");
//// }
//// @Test public void Misc_url() {
//// fxt.Init_basic();
//// fxt.Search_mgr().Page_size_(1);
//// fxt.Expd_address_page_("Special:Search/B1");
//// fxt.Test_search(Srch_special_page.Match_tid_all, "B1", 0, "B1 1");
//// }
// @Test public void Sort_defaults_to_len_desc() {
// fxt.Init_basic();
// fxt.Search_mgr().Page_mgr().Itms_per_page_(3);
// fxt.Test_search2(Srch_special_page.Match_tid_bgn, "b" , 0, Srch_rslt_row_sorter.Tid_ttl_asc , "B1_11", "B1_21", "B1__1"); // sort by name; note that _ sorts after alphabet
// fxt.Test_search2(Srch_special_page.Match_tid_bgn, "b" , 1, Srch_rslt_row_sorter.Tid_none , "B2_12", "B2_22", "B2__2"); // sort by name still; next page should not reset
// fxt.Test_search2(Srch_special_page.Match_tid_bgn, "b2" , 0, Srch_rslt_row_sorter.Tid_none , "B2_22", "B2_12", "B2__2"); // sort by len desc; new search should reset
// }
// }
// class Xos_search_mgr_fxt {
// Xoae_app app; Xowe_wiki wiki; Bry_bfr bfr = Bry_bfr_.Reset(500); Srch_special_page search_mgr;
// public Xoae_app App() {return app;}
// public Xowe_wiki Wiki() {return wiki;}
// public Xobl_regy_itm regy_itm_(int id, String bgn, String end, int count) {return new Xobl_regy_itm(id, Bry_.new_u8(bgn), Bry_.new_u8(end), count);}
// public Xowd_page_itm data_ttl_(int id, String ttl) {return data_ttl_(id, 0, 0, false, 0, ttl);}
// public Xowd_page_itm data_ttl_(int id, int fil, int row, boolean redirect, int len, String ttl) {return new Xowd_page_itm().Init(id, Bry_.new_u8(ttl), redirect, len, fil, row);}
// public Xowd_page_itm data_id_(int id, String ttl) {return data_id_(id, Xow_ns_.Tid__main, ttl);}
// public Xowd_page_itm data_id_(int id, int ns, String ttl) {return new Xowd_page_itm().Id_(id).Ns_id_(ns).Ttl_page_db_(Bry_.new_u8(ttl)).Text_db_id_(0).Text_len_(0);}
// public Xobl_search_ttl data_sttl_(String word, params int[] ids) {return new Xobl_search_ttl(Bry_.new_u8(word), data_ttl_word_page_ary_(ids));}
// public Xobl_search_ttl_page[] data_ttl_word_page_ary_(params int[] ids) {
// int ids_len = ids.length;
// Xobl_search_ttl_page[] rv = new Xobl_search_ttl_page[ids_len];
// for (int i = 0; i < ids_len; i++) {
// int id = ids[i];
// rv[i] = new Xobl_search_ttl_page(id, id * 2);
// }
// return rv;
// }
// public void Init_regy_site(byte dir_info, params Xobl_regy_itm[] ary) {Init_regy(wiki.Tdb_fsys_mgr().Url_site_reg(dir_info), ary);}
// public void Init_regy_ns (String ns_num, byte tid, params Xobl_regy_itm[] ary) {Init_regy(wiki.Tdb_fsys_mgr().Url_ns_reg(ns_num, tid), ary);}
// public void Init_regy(Io_url url, Xobl_regy_itm[] ary) {
// int ary_len = ary.length;
// for (int i = 0; i < ary_len; i++) {
// Xobl_regy_itm itm = ary[i];
// itm.Srl_save(tmp_bfr);
// }
// Io_mgr.Instance.SaveFilBfr(url, tmp_bfr);
// } private Bry_bfr tmp_bfr = Bry_bfr_.Reset(255);
// public void Init_data(Io_url fil, params Xobl_data_itm[] ary) {
// Xob_xdat_file xdat_file = new Xob_xdat_file();
// int ary_len = ary.length;
// for (int i = 0; i < ary_len; i++) {
// Xobl_data_itm itm = ary[i];
// itm.Srl_save(tmp_bfr);
// xdat_file.Insert(bfr, tmp_bfr.To_bry_and_clear());
// }
// xdat_file.Save(fil);
// }
// public void Init_basic() {
// this.Init_regy_ns(wiki.Ns_mgr().Ns_main().Num_str(), Xotdb_dir_info_.Tid_search_ttl, this.regy_itm_(0, "A", "C", 5));
// this.Init_data(wiki.Tdb_fsys_mgr().Url_ns_fil(Xotdb_dir_info_.Tid_search_ttl, Xow_ns_.Tid__main, 0)
// , this.data_sttl_("a" , 0)
// , this.data_sttl_("b1" , 1, 11, 21)
// , this.data_sttl_("b2" , 2, 12, 22)
// , this.data_sttl_("b3" , 3, 13, 23)
// , this.data_sttl_("c" , 4)
// );
// this.Init_regy_site(Xotdb_dir_info_.Tid_id, this.regy_itm_(0, "A", "C", 11));
// this.Init_data(wiki.Tdb_fsys_mgr().Url_site_fil(Xotdb_dir_info_.Tid_id, 0)
// , this.data_id_( 0, "A___0")
// , this.data_id_( 1, "B1__1")
// , this.data_id_( 2, "B2__2")
// , this.data_id_( 3, "B3__3")
// , this.data_id_( 4, "C___4")
// , this.data_id_(11, "B1_11")
// , this.data_id_(12, "B2_12")
// , this.data_id_(13, "B3_13")
// , this.data_id_(21, "B1_21")
// , this.data_id_(22, "B2_22")
// , this.data_id_(23, "B3_23")
// );
// search_mgr.Page_mgr().Ns_mgr().Add_all(); // WORKAROUND: xdat fmt does not store ns with search data; pages will be retrieved with ns_id = null; force ns_all (instead of allowing ns_main default);
// }
// public void Clear() {
// Io_mgr.Instance.InitEngine_mem();
// app = Xoa_app_fxt.Make__app__edit();
// wiki = Xoa_app_fxt.Make__wiki__edit(app);
// search_mgr = wiki.Special_mgr().Page_search();
// wiki.Appe().Gui_mgr().Search_suggest_mgr().Args_default_str_("ns*=1"); // WORKAROUND: xdat fmt does not store ns with search data; pages will be retrieved with ns_id = null; force ns_all (instead of allowing ns_main default);
// }
// public Srch_special_page Search_mgr() {return search_mgr;}
// public void Test_url_search_bry(String url_str, String expd) {
// Xoa_url url = Xow_url_parser_old.Parse_url(app, wiki, url_str);
// search_mgr.Args_mgr().Clear().Parse(url.Args());
// Tfds.Eq(expd, String_.new_u8(search_mgr.Args_mgr().Search_bry()));
// }
// public void Test_url__ns(String url_str, String expd) {
// Xoa_url url = Xow_url_parser_old.Parse_url(app, wiki, url_str);
// search_mgr.Args_mgr().Clear().Parse(url.Args());
// Tfds.Eq(expd, String_.new_a7(search_mgr.Args_mgr().Ns_mgr().Xto_hash_key()));
// }
// public void Test_search_exact(String ttl_str, params String[] expd_ary) {Test_search(ttl_str, 0, expd_ary);}
// public void Test_search_match_bgn(String ttl_str, params String[] expd_ary) {Test_search(ttl_str, 0, expd_ary);}
// public void Test_search(String ttl_str, int page_idx, params String[] expd_ary) {
// byte[] ttl_bry = Bry_.new_a7(ttl_str);
// Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b128();
// Xosrh_rslt_grp page = search_mgr.Page_mgr().Search(bfr, wiki, ttl_bry, page_idx, search_mgr.Page_mgr());
// bfr.Mkr_rls();
// Tfds.Eq_ary(expd_ary, Search_itms_to_int_ary(page));
// }
// public void Test_html_by_url(String ttl_str, String args_str, String expd_html) {
// wiki.Init_needed_(false);
// byte[] ttl_bry = Bry_.new_a7(ttl_str);
// Xoa_ttl ttl = Xoa_ttl.Parse(wiki, ttl_bry);
// Xoae_page page = Xoae_page.New_test(wiki, ttl);
// byte[] url_bry = Bry_.new_a7("http://en.wikipedia.org/wiki/Special:Search/" + ttl_str + args_str);
// Xoa_url url = wiki.Appe().Url_parser().Parse(url_bry);
// search_mgr.Special__gen(url, page, wiki, ttl);
// Tfds.Eq_str_lines(expd_html, String_.new_u8(page.Root().Data_htm()));
// }
// public void Test_search2(byte match_tid, String ttl_str, int page_idx, byte sort_tid, params String[] expd_ary) {
// Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b128();
// Xow_url_parser_old url_parser = new Xow_url_parser_old();
// byte[] url_raw = Bry_.new_a7("Special:Search/" + ttl_str + ((match_tid == Srch_special_page.Match_tid_all) ? "" : "*") + "?fulltext=y" + Srch_rslt_row_sorter.Xto_url_arg(sort_tid) + "&xowa_page_size=1&xowa_page_index=" + page_idx);
// Xoa_url url = url_parser.Parse(url_raw);
// Xoa_ttl ttl = Xoa_ttl.Parse(wiki, url_raw);
// Xoae_page page = wiki.Ctx().Page();
// search_mgr.Special__gen(url, page, wiki, ttl);
// Xosrh_rslt_grp cur_grp = search_mgr.Cur_grp();
// bfr.Mkr_rls();
// Tfds.Eq_ary(expd_ary, Search_itms_to_int_ary(cur_grp));
// }
// String[] Search_itms_to_int_ary(Xosrh_rslt_grp page) {
// int itms_len = page.Itms_len();
// String[] rv = new String[itms_len];
// for (int i = 0; i < itms_len; i++) {
// Xowd_page_itm itm = page.Itms_get_at(i);
// rv[i] = String_.new_u8(itm.Ttl_page_db());
// }
// return rv;
// }
// }
// interface Xobl_data_itm {
// void Srl_save(Bry_bfr bfr);
// }
//}

92
400_xowa/src/gplx/xowa/bldrs/Db_mgr_fxt.java
Unescape View File

@ -1,92 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.primitives.*; import gplx.core.strings.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.addons.wikis.ctgs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.infos.*;
import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*;
public class Db_mgr_fxt {
public Db_mgr_fxt Ctor_fsys() {bldr_fxt = new Xob_fxt().Ctor(Xoa_test_.Url_root().GenSubDir("root")); return this;}
public Db_mgr_fxt Ctor_mem() {bldr_fxt = new Xob_fxt().Ctor_mem(); return this;} private Xob_fxt bldr_fxt;
public Xowd_page_itm page_(int id, String modified_on, boolean type_redirect, int text_len) {return new Xowd_page_itm().Id_(id).Modified_on_(DateAdp_.parse_gplx(modified_on)).Redirected_(type_redirect).Text_len_(text_len);}
public Xowe_wiki Wiki() {return bldr_fxt.Wiki();}
public Xob_bldr Bldr() {return bldr_fxt.Bldr();}
public Db_mgr_fxt doc_ary_(Xowd_page_itm... v) {bldr_fxt.doc_ary_(v); return this;}
public Xowd_page_itm doc_(int id, String date, String title, String text) {return bldr_fxt.doc_(id, date, title, text);}
public Xowd_page_itm doc_wo_date_(int id, String title, String text) {return bldr_fxt.doc_(id, "2012-01-02 03:04", title, text);}
public Xowd_page_itm doc_ttl_(int id, String title) {return bldr_fxt.doc_(id, "2012-01-02 03:04", title, "IGNORE");}
public Db_mgr_fxt Init_fil(String url, String raw) {return Init_fil(Io_url_.new_fil_(url), raw);}
public Db_mgr_fxt Init_fil(Io_url url, String raw) {Io_mgr.Instance.SaveFilStr(url, raw); return this;}
public Db_mgr_fxt Exec_run(Xob_page_wkr wkr) {bldr_fxt.Run_page_wkrs(wkr); return this;}
public Db_mgr_fxt Exec_run(Xob_cmd cmd) {bldr_fxt.Run_cmds(cmd); return this;}
public Db_mgr_fxt Exec_run(Xobd_parser_wkr wkr) {bldr_fxt.Run(wkr); return this;}
public void Init_page_insert(Int_obj_ref page_id_next, int ns_id, String[] ttls) {
Xowe_wiki wiki = this.Wiki();
int len = ttls.length;
DateAdp modified_on = Datetime_now.Dflt_add_min_(0);
Xowd_page_tbl tbl_page = wiki.Db_mgr_as_sql().Core_data_mgr().Tbl__page();
tbl_page.Insert_bgn();
for (int i = 0; i < len; i++) {
String ttl = ttls[i];
int page_id = page_id_next.Val();
tbl_page.Insert_cmd_by_batch(page_id, ns_id, Bry_.new_u8(ttl), false, modified_on, 0, page_id, 0, 0, -1);
page_id_next.Val_add(1);
}
tbl_page.Insert_end();
}
public void Test_load_ttl(int ns_id, String ttl_str, Xowd_page_itm expd) {
Xowe_wiki wiki = bldr_fxt.Wiki();
Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
byte[] ttl_bry = Bry_.new_a7(ttl_str);
wiki.Db_mgr_as_sql().Load_mgr().Load_by_ttl(actl, ns, ttl_bry);
Tfds.Eq(expd.Id(), actl.Id());
Tfds.Eq_date(expd.Modified_on(), actl.Modified_on());
Tfds.Eq(expd.Redirected(), actl.Redirected());
Tfds.Eq(expd.Text_len(), actl.Text_len());
} private Xowd_page_itm actl = new Xowd_page_itm();
public void Test_load_page(int ns_id, int page_id, String expd) {
Xowe_wiki wiki = bldr_fxt.Wiki();
Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
wiki.Db_mgr_as_sql().Load_mgr().Load_page(actl.Id_(page_id), ns);
Tfds.Eq(expd, String_.new_a7(actl.Text()));
}
int[] Xto_int_ary(List_adp rslts) {
int len = rslts.Count();
int[] rv = new int[len];
for (int i = 0; i < len; i++) {
Xowd_page_itm page = (Xowd_page_itm)rslts.Get_at(i);
rv[i] = page.Id();
}
return rv;
}
public void Test_file(String url, String expd) {
String actl = Io_mgr.Instance.LoadFilStr(url);
Tfds.Eq_str_lines(expd, actl);
}
public void Init_db_sqlite() {
Xowe_wiki wiki = this.Wiki();
Db_conn_pool.Instance.Rls_all();
Db_conn_bldr.Instance.Reg_default_sqlite();
Io_mgr.Instance.DeleteDir_cmd(wiki.Fsys_mgr().Root_dir()).MissingIgnored_().Exec();
wiki.Db_mgr_create_as_sql().Core_data_mgr().Init_by_make(Xowd_core_db_props.Test, Xob_info_session.Test);
Io_mgr.Instance.SaveFilStr(wiki.Import_cfg().Src_dir().GenSubFil("a.xml"), "<test/>");
}
public void Rls() {
this.Wiki().Db_mgr_as_sql().Core_data_mgr().Rls();
}
}

6
400_xowa/src/gplx/xowa/bldrs/Xob_cmd_keys.java
Unescape View File

@ -36,12 +36,6 @@ public class Xob_cmd_keys {
, Key_diff_merge = "diff.merge"
, Key_text_delete_page = "text.delete_page"
, Key_tdb_text_init = "tdb.text.init" // "core.init"
, Key_tdb_make_page = "tdb.text.page" // "core.make_page"
, Key_tdb_make_id = "core.make_id"
, Key_tdb_calc_stats = "core.calc_stats"
, Key_tdb_text_wdata_qid = "tdb.text.wdata.qid"
, Key_tdb_text_wdata_pid = "tdb.text.wdata.pid"
, Key_exec_sql = "import.sql.exec_sql"
, Key_decompress_bz2 = "core.decompress_bz2"
;

9
400_xowa/src/gplx/xowa/bldrs/Xob_cmd_mgr.java
Unescape View File

@ -49,16 +49,11 @@ public class Xob_cmd_mgr implements Gfo_invk {
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_util_delete)) return Add(new Xob_delete_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_util_download)) return Add(new Xob_download_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_util_xml_dump)) return Add(new Xob_xml_dumper_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_qid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_qid_sql().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_sql(null, wiki).Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_qid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_qid(null).Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid(null).Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_db)) return Add(new Xob_wdata_db_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_site_meta)) return Add(new Xob_site_meta_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_init)) return Add(new Xob_init_tdb(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_id)) return Xml_rdr_direct_add(wiki, new Xob_make_id_wkr(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_calc_stats)) return Add(new Xob_calc_stats_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_qid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_qid_txt().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_txt().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_diff_build)) return Add(new Xob_diff_build_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_exec_sql)) return Add(new Xob_exec_sql_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_decompress_bz2)) return Add(new Xob_decompress_bz2_cmd(bldr, wiki));

154
400_xowa/src/gplx/xowa/bldrs/Xob_fxt.java
Unescape View File

@ -1,154 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.tests.*; import gplx.core.ios.*; import gplx.core.times.*;
import gplx.dbs.*; import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_fxt {
private final Tst_mgr tst_mgr = new Tst_mgr();
private final DateAdp_parser dateParser = DateAdp_parser.new_();
public Xob_fxt Ctor_mem() {
Io_mgr.Instance.InitEngine_mem();
return Ctor(Io_url_.mem_dir_("mem/xowa/"));
}
public Xob_fxt Ctor(Io_url root_dir) {
Db_conn_bldr.Instance.Reg_default_sqlite();
app = Xoa_app_fxt.Make__app__edit("linux", root_dir);
wiki = Xoa_app_fxt.Make__wiki__edit(app);
bldr = Xoa_app_fxt.bldr_(app);
return this;
}
public Xoae_app App() {return app;} private Xoae_app app;
public Xob_bldr Bldr() {return bldr;} private Xob_bldr bldr;
public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki;
public Xob_fxt Fil_expd(Io_url url, String... expd) {
String text = String_.Concat_lines_nl_skip_last(expd); // skipLast b/c if trailing line wanted, easier to pass in extra argument for ""
expd_list.Add(new Io_fil_chkr(url, text));
return this;
} List_adp expd_list = List_adp_.New();
public Xob_fxt Fil_skip(Io_url... urls) {
for (int i = 0; i < urls.length; i++)
skip_list.Add(urls[i]);
return this;
} List_adp skip_list = List_adp_.New();
public Xob_fxt doc_ary_(Xowd_page_itm... v) {doc_ary = v; return this;} private Xowd_page_itm[] doc_ary;
public Xowd_page_itm doc_wo_date_(int id, String title, String text) {return doc_(id, "2012-01-02 13:14", title, text);}
public Xowd_page_itm doc_(int id, String date, String title, String text) {
Xowd_page_itm rv = new Xowd_page_itm().Id_(id).Ttl_(Bry_.new_u8(title), wiki.Ns_mgr()).Text_(Bry_.new_u8(text));
int[] modified_on = new int[7];
dateParser.Parse_iso8651_like(modified_on, date);
rv.Modified_on_(DateAdp_.seg_(modified_on));
return rv;
}
public Xob_fxt Run_id() {
Xob_make_id_wkr wkr = new Xob_make_id_wkr(bldr, wiki);
Run_page_wkrs(wkr);
return this;
}
private void Run_wkr(Xob_page_wkr wkr) {
wkr.Page_wkr__bgn();
for (int i = 0; i < doc_ary.length; i++) {
Xowd_page_itm page = doc_ary[i];
wkr.Page_wkr__run(page);
}
wkr.Page_wkr__end();
}
private void tst_fils(Io_url[] ary) {
Io_fil[] actls = Get_actl(ary);
Io_fil_chkr[] expds = (Io_fil_chkr[])expd_list.To_ary(Io_fil_chkr.class);
tst_mgr.Tst_ary("all", expds, actls);
}
Io_fil[] Get_actl(Io_url[] ary) {
int len = ary.length;
Io_fil[] rv = new Io_fil[len];
for (int i = 0; i < len; i++) {
Io_url url = ary[i];
String data = Io_mgr.Instance.LoadFilStr(url);
rv[i] = new Io_fil(url, data);
}
return rv;
}
public Xob_fxt Run_tmpl_dump() {
Xob_parse_dump_templates_cmd wkr = new Xob_parse_dump_templates_cmd(bldr, wiki);
Run_wkr(wkr);
tst_fils(wkr.Dump_url_gen().Prv_urls());
return this;
}
public Xob_fxt Run(Xobd_parser_wkr... wkrs) {
Xobd_parser parser_wkr = new Xobd_parser(bldr);
int len = wkrs.length;
for (int i = 0; i < len; i++)
parser_wkr.Wkr_add(wkrs[i]);
Run_page_wkrs(parser_wkr);
return this;
}
public Xob_fxt Run_page_wkrs(Xob_page_wkr... wkrs) {
int doc_ary_len = doc_ary.length;
for (int j = 0; j < wkrs.length; j++) {
Xob_page_wkr wkr = wkrs[j];
wkr.Page_wkr__bgn();
for (int i = 0; i < doc_ary_len; i++) {
Xowd_page_itm page = doc_ary[i];
wkr.Page_wkr__run(page);
}
wkr.Page_wkr__end();
}
Test_expd_files();
return this;
}
public Xob_fxt Run_cmds(Xob_cmd... cmds) {
for (int j = 0; j < cmds.length; j++) {
Xob_cmd cmd = cmds[j];
cmd.Cmd_bgn(bldr);
cmd.Cmd_run();
cmd.Cmd_end();
}
Test_expd_files();
return this;
}
private void Test_expd_files() {
if (expd_list.Count() > 0) {
Io_fil_chkr[] expd = (Io_fil_chkr[])expd_list.To_ary(Io_fil_chkr.class);
Io_fil[] actl = wiki_();
tst_mgr.Tst_ary("all", expd, actl);
}
}
Io_fil[] wiki_() {
List_adp rv = List_adp_.New();
wiki_fil_add(rv, wiki.Tdb_fsys_mgr().Ns_dir());
wiki_fil_add(rv, wiki.Tdb_fsys_mgr().Site_dir());
rv.Sort();
return (Io_fil[])rv.To_ary(Io_fil.class);
}
private void wiki_fil_add(List_adp list, Io_url root_dir) {
Io_url[] ary = Io_mgr.Instance.QueryDir_args(root_dir).Recur_().ExecAsUrlAry();
for (int i = 0; i < ary.length; i++) {
Io_url url = ary[i];
Io_fil fil = new Io_fil(url, Io_mgr.Instance.LoadFilStr_args(url).MissingIgnored_().Exec());
list.Add(fil);
}
}
public void Run_page_wkr(Xob_page_wkr wkr, Xowd_page_itm... pages) {
int len = pages.length;
wkr.Page_wkr__bgn();
for (int i = 0; i < len; i++) {
Xowd_page_itm page = pages[i];
wkr.Page_wkr__run(page);
}
wkr.Page_wkr__end();
}
}

53
400_xowa/src/gplx/xowa/bldrs/Xobldr_fxt.java
Unescape View File

@ -0,0 +1,53 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.tests.*; import gplx.core.times.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xobldr_fxt {
private final DateAdp_parser dateParser = DateAdp_parser.new_();
public Xoae_app App() {return app;} private Xoae_app app;
public Xob_bldr Bldr() {return bldr;} private Xob_bldr bldr;
public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki;
public Xobldr_fxt Ctor_mem() {
Io_mgr.Instance.InitEngine_mem();
return Ctor(Io_url_.mem_dir_("mem/xowa/"));
}
private Xobldr_fxt Ctor(Io_url root_dir) {
Db_conn_bldr.Instance.Reg_default_sqlite();
app = Xoa_app_fxt.Make__app__edit("linux", root_dir);
wiki = Xoa_app_fxt.Make__wiki__edit(app);
bldr = Xoa_app_fxt.bldr_(app);
return this;
}
public Xowd_page_itm New_page_wo_date(int id, String title, String text) {return New_page(id, "2012-01-02 13:14", title, text);}
public Xowd_page_itm New_page(int id, String date, String title, String text) {
Xowd_page_itm rv = new Xowd_page_itm().Id_(id).Ttl_(Bry_.new_u8(title), wiki.Ns_mgr()).Text_(Bry_.new_u8(text));
int[] modified_on = new int[7];
dateParser.Parse_iso8651_like(modified_on, date);
rv.Modified_on_(DateAdp_.seg_(modified_on));
return rv;
}
public void Run_page_wkr(Xob_page_wkr wkr, Xowd_page_itm... pages) {
int len = pages.length;
wkr.Page_wkr__bgn();
for (int i = 0; i < len; i++) {
Xowd_page_itm page = pages[i];
wkr.Page_wkr__run(page);
}
wkr.Page_wkr__end();
}
}

105
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Srch_bldr_wkr_base.java
Unescape View File

@ -1,105 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.core.primitives.*; import gplx.core.ios.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.wtrs.*;
import gplx.xowa.langs.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
public abstract class Srch_bldr_wkr_base extends Xob_itm_dump_base implements Xob_page_wkr {
private final Ordered_hash list = Ordered_hash_.New(); private Xol_lang_itm lang;
public abstract String Page_wkr__key();
public void Page_wkr__bgn() {
make_dir = wiki.Tdb_fsys_mgr().Ns_dir();
this.Init_dump(this.Page_wkr__key(), make_dir);
lang = wiki.Lang(); // wiki.Appe().Lang_mgr().Lang_en(); // NOTE: was .Lang_en which is wrong (should match lang of wiki); DATE:2013-05-11
tmp_wtr_mgr = new Xob_tmp_wtr_mgr(new Xob_tmp_wtr_wkr__ttl(temp_dir, dump_fil_len));
if (wiki.Db_mgr().Tid() == Xodb_mgr_sql.Tid_sql) // if sqlite, hard-code to ns_main; aggregates all ns into one
ns_main = wiki.Ns_mgr().Ns_main();
} private Xob_tmp_wtr_mgr tmp_wtr_mgr; private Xow_ns ns_main;
public void Page_wkr__run(Xowd_page_itm page) {
// if (page.Ns_id() != Xow_ns_.Tid__main) return; // limit to main ns for now
try {
byte[] ttl = page.Ttl_page_db();
byte[][] words = Split_ttl_into_words(lang, list, dump_bfr, ttl);
Xob_tmp_wtr wtr = tmp_wtr_mgr.Get_or_new(ns_main == null ? page.Ns() : ns_main);
int words_len = words.length;
int row_len = 0;
for (int i = 0; i < words_len; i++) {
byte[] word = words[i];
row_len += word.length + 13; // 13=5(id) + 5(page_len) + 3(dlms)
}
if (wtr.FlushNeeded(row_len)) wtr.Flush(bldr.Usr_dlg());
for (int i = 0; i < words_len; i++) {
byte[] word = words[i];
wtr.Bfr() .Add(word) .Add_byte(Byte_ascii.Pipe)
.Add_base85_len_5(page.Id()) .Add_byte(Byte_ascii.Semic)
.Add_base85_len_5(page.Text().length) .Add_byte(Byte_ascii.Nl);
}
} catch (Exception e) {bldr.Usr_dlg().Warn_many("", "", "search_index:fatal error: err=~{0}", Err_.Message_gplx_full(e));} // never let single page crash entire import
}
public void Page_wkr__run_cleanup() {}
public void Page_wkr__end() {
tmp_wtr_mgr.Flush_all(bldr.Usr_dlg());
dump_bfr.ClearAndReset();
Xobdc_merger.Ns(bldr.Usr_dlg(), tmp_wtr_mgr.Regy(), Xotdb_dir_info_.Name_search_ttl, temp_dir, make_dir, sort_mem_len, Io_line_rdr_key_gen_.first_pipe, this.Make_cmd_site());
tmp_wtr_mgr.Rls_all();
if (delete_temp) Io_mgr.Instance.DeleteDirDeep(temp_dir);
}
public abstract Io_make_cmd Make_cmd_site();
public static byte[][] Split_ttl_into_words(Xol_lang_itm lang, Ordered_hash list, Bry_bfr bfr, byte[] ttl) {
if (lang != null) // null lang passed in by searcher
ttl = lang.Case_mgr().Case_build_lower(ttl);
int ttl_len = ttl.length; Bry_obj_ref word_ref = Bry_obj_ref.New(Bry_.Empty);
int i = 0; boolean word_done = false;
while (true) {
if (word_done || i == ttl_len) {
if (bfr.Len() > 0) {
byte[] word = bfr.To_bry_and_clear();
word_ref.Val_(word);
if (!list.Has(word_ref)) list.Add(word_ref, word); // don't add same word twice; EX: Title of "Can Can" should only have "Can" in index
}
if (i == ttl_len) break;
word_done = false;
}
byte b = ttl[i];
switch (b) {
case Byte_ascii.Underline: // underline is word-breaking; EX: A_B -> A, B
case Byte_ascii.Space: // should not occur, but just in case (only underscores)
case Byte_ascii.Tab: case Byte_ascii.Nl: case Byte_ascii.Cr: // should not occur in titles, but just in case
case Byte_ascii.Dash: // treat hypenated words separately
case Byte_ascii.Dot: // treat abbreviations as separate words; EX: A.B.C.
case Byte_ascii.Bang: case Byte_ascii.Hash: case Byte_ascii.Dollar: case Byte_ascii.Percent:
case Byte_ascii.Amp: case Byte_ascii.Paren_bgn: case Byte_ascii.Paren_end: case Byte_ascii.Star:
case Byte_ascii.Comma: case Byte_ascii.Slash:
case Byte_ascii.Colon: case Byte_ascii.Semic: case Byte_ascii.Gt:
case Byte_ascii.Question: case Byte_ascii.At: case Byte_ascii.Brack_bgn: case Byte_ascii.Brack_end:
case Byte_ascii.Pow: case Byte_ascii.Tick:
case Byte_ascii.Curly_bgn: case Byte_ascii.Pipe: case Byte_ascii.Curly_end: case Byte_ascii.Tilde:
case Byte_ascii.Quote: case Byte_ascii.Apos: // FUTURE: apos will split "Earth's" to Earth and s; should remove latter
++i;
word_done = true;
break;
default:
bfr.Add_byte(b);
++i;
break;
}
}
byte[][] rv = (byte[][])list.To_ary(byte[].class);
list.Clear(); list.Resize_bounds(16);
return rv;
}
}

113
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_calc_stats_cmd.java
Unescape View File

@ -1,113 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.data.site_stats.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.tdbs.hives.*; import gplx.xowa.wikis.tdbs.xdats.*;
public class Xob_calc_stats_cmd extends Xob_itm_basic_base implements Xob_cmd {
public Xob_calc_stats_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_tdb_calc_stats;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_run() {Exec();}
public void Cmd_end() {}
public void Cmd_term() {}
private void Exec() {
int ns_len = wiki.Ns_mgr().Ords_len();
int total = 0;
for (int i = 0; i < ns_len; i++) {
Xow_ns ns = wiki.Ns_mgr().Ords_ary()[i];
int ns_count = Calc_counts(ns);
ns.Count_(ns_count);
total += ns_count;
}
int count_main = Calc_count_articles(wiki.Ns_mgr().Ns_main());
int count_file = Calc_count_articles(wiki.Ns_mgr().Ns_file());
Bry_bfr bfr = Bry_bfr_.New();
Gen_call(Bool_.Y, bfr, Xowe_wiki.Invk_stats);
Gen_call(Bool_.N, bfr, Xowd_site_stats_mgr.Invk_number_of_articles_, count_main);
Gen_call(Bool_.N, bfr, Xowd_site_stats_mgr.Invk_number_of_files_, count_file);
Gen_call(Bool_.N, bfr, Xowd_site_stats_mgr.Invk_number_of_pages_, total);
for (int i = 0; i < ns_len; i++) {
Xow_ns ns = wiki.Ns_mgr().Ords_ary()[i];
if (ns.Id() < 0) continue;
bfr.Add_byte_nl();
Gen_call(Bool_.N, bfr, Xowd_site_stats_mgr.Invk_number_of_articles_in_ns_, ns.Num_str(), Int_.To_str_pad_bgn_zero(ns.Count(), 10));
}
bfr.Add_byte_nl().Add_byte(Byte_ascii.Semic).Add_byte_nl();
Io_url wiki_gfs = Wiki_gfs_url(wiki);
Io_mgr.Instance.SaveFilBfr(wiki_gfs, bfr);
}
private void Gen_call(boolean first, Bry_bfr bfr, String key, Object... vals) {
if (!first) bfr.Add_byte(Byte_ascii.Dot);
bfr.Add_str_u8(key);
int len = vals.length;
if (len > 0) {
bfr.Add_byte(Byte_ascii.Paren_bgn);
for (int i = 0; i < len; i++) {
if (i != 0) bfr.Add_byte(Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
Object val = vals[i];
bfr.Add_str_u8(Object_.Xto_str_strict_or_null_mark(val));
}
bfr.Add_byte(Byte_ascii.Paren_end);
}
}
int Calc_counts(Xow_ns ns) {
Io_url reg_url = wiki.Tdb_fsys_mgr().Url_ns_reg(ns.Num_str(), Xotdb_dir_info_.Tid_ttl);
Xowd_regy_mgr reg_mgr = new Xowd_regy_mgr(reg_url);
int files_ary_len = reg_mgr.Files_ary().length;
int count = 0;
for (int i = 0; i < files_ary_len; i++) {
count += reg_mgr.Files_ary()[i].Count();
}
return count;
}
int Calc_count_articles(Xow_ns ns) {
Io_url hive_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest(Xotdb_dir_info_.Name_ns, ns.Num_str(), Xotdb_dir_info_.Name_title);
return Calc_count_articles_dir(ns, hive_dir);
}
int Calc_count_articles_dir(Xow_ns ns, Io_url dir) {
Io_url[] subs = Io_mgr.Instance.QueryDir_args(dir).DirInclude_().ExecAsUrlAry();
int count = 0;
int subs_len = subs.length;
bldr.Usr_dlg().Prog_one(GRP_KEY, "count", "calculating: ~{0}", dir.Raw());
for (int i = 0; i < subs_len; i++) {
Io_url sub = subs[i];
if (sub.Type_dir())
count += Calc_count_articles_dir(ns, sub);
else
count += Calc_count_articles_fil(ns, sub);
}
return count;
}
int Calc_count_articles_fil(Xow_ns ns, Io_url fil) {
if (String_.Eq(fil.NameAndExt(), Xotdb_dir_info_.Name_reg_fil)) return 0;
int rv = 0;
byte[] bry = Io_mgr.Instance.LoadFilBry(fil);
Xob_xdat_file xdat_file = new Xob_xdat_file().Parse(bry, bry.length, fil);
Xowd_page_itm page = Xowd_page_itm.new_tmp();
int count = xdat_file.Count();
for (int i = 0; i < count; i++) {
byte[] ttl_bry = xdat_file.Get_bry(i);
Xotdb_page_itm_.Txt_ttl_load(page, ttl_bry);
rv += page.Redirected() ? 0 : 1;
}
return rv;
}
static final String GRP_KEY = "xowa.bldr.calc_stats";
public static Io_url Wiki_gfs_url(Xowe_wiki wiki) {return wiki.Fsys_mgr().Root_dir().GenSubFil_nest("cfg", "wiki_stats.gfs");}
}

3
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_init_base_tst.java
Unescape View File

@ -15,6 +15,7 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import org.junit.*; import gplx.xowa.htmls.portal.*; import gplx.xowa.wikis.xwikis.*;
import gplx.xowa.bldrs.cmds.texts.sqls.*;
public class Xob_init_base_tst {
@Before public void init() {fxt.Clear();} private Xob_init_base_fxt fxt = new Xob_init_base_fxt();
@Test public void Dirty_wiki_itms() {
@ -24,7 +25,7 @@ public class Xob_init_base_tst {
Xow_xwiki_itm xwiki_itm = app.Usere().Wiki().Xwiki_mgr().Add_by_atrs("en.wikipedia.org", "en.wikipedia.org");
xwiki_itm.Offline_(Bool_.Y); // simulate add via Available_from_fsys; DATE:2014-09-21
Tfds.Eq("", wikis_list.Itms_as_html()); // still empty
new Xob_init_tdb(app.Bldr(), wiki).Cmd_end(); // mock "init" task
new Xob_init_cmd(app.Bldr(), wiki).Cmd_end(); // mock "init" task
Tfds.Eq("\n <li><a href=\"/site/en.wikipedia.org/\" class='xowa-hover-off'>en.wikipedia.org</a></li>", wikis_list.Itms_as_html()); // no longer empty
}
}

26
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_init_tdb.java
Unescape View File

@ -1,26 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.xowa.xtns.wbases.imports.*;
public class Xob_init_tdb extends Xob_init_base {
public Xob_init_tdb(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki);}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_text_init;}
@Override public void Cmd_ini_wdata(Xob_bldr bldr, Xowe_wiki wiki) {
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_wdata_qid);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_wdata_pid);
}
@Override public void Cmd_run_end(Xowe_wiki wiki) {}
}

36
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_make_id_wkr.java
Unescape View File

@ -1,36 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.core.ios.*; import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_make_id_wkr extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
public Xob_make_id_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Page_wkr__key() {return KEY;} public static final String KEY = "core.make_id";
public void Page_wkr__bgn() {
this.Init_dump(KEY, wiki.Tdb_fsys_mgr().Site_dir().GenSubDir(Xotdb_dir_info_.Name_id));
}
public void Page_wkr__run(Xowd_page_itm page) {
byte[] ttl = page.Ttl_page_db();
if (dump_bfr.Len() + row_fixed_len + ttl.length > dump_fil_len) Io_mgr.Instance.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
Xotdb_page_itm_.Txt_id_save(dump_bfr, page);
}
public void Page_wkr__run_cleanup() {}
public void Page_wkr__end() {
this.Term_dump(new Xob_make_cmd_site(bldr.Usr_dlg(), make_dir, make_fil_len));
if (delete_temp) Io_mgr.Instance.DeleteDirDeep(temp_dir);
}
static final int row_fixed_len = 25 + 1 + 7; // 25=5 base_85 flds; 1=Redirect; 7=dlm
}

162
400_xowa/src/gplx/xowa/wikis/data/Xow_data_mgr_tst.java
Unescape View File

@ -1,162 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis.data; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import org.junit.*; import gplx.xowa.wikis.tdbs.*;
import gplx.xowa.wikis.nss.*;
public class Xow_data_mgr_tst {
Xow_data_mgr_fxt fxt = new Xow_data_mgr_fxt();
@Before public void init() {fxt.Clear(); Datetime_now.Manual_y_();}
@After public void term() {Datetime_now.Manual_n_();}
@Test public void Create() {
fxt .Create("A1", "A1 data")
.Create("B12", "B12 data")
.Create("C123", "C123 data")
.Tst_regy_title("0|A1|C123|3\n")
.Tst_data_title(String_.Concat_lines_nl
( "!!!!>|!!!!?|!!!!@|"
, "!!!!!|!!!!!|!!!!!|0|!!!!(|A1"
, "!!!!\"|!!!!!|!!!!\"|0|!!!!)|B12"
, "!!!!#|!!!!!|!!!!#|0|!!!!*|C123"
))
.Tst_data_page(String_.Concat_lines_nl
( "!!!!9|!!!!;|!!!!=|"
, "!!!!!\t##PX+\tA1\tA1 data\t"
, "!!!!\"\t##PX/\tB12\tB12 data\t"
, "!!!!#\t##PX0\tC123\tC123 data\t"
))
;
}
@Test public void Update() {
fxt .Create("A1", "A1 data")
.Create("B12", "B12 data")
.Create("C123", "C123 data")
.Update("B12", "B12 changed")
.Tst_regy_title("0|A1|C123|3\n")
.Tst_data_title(String_.Concat_lines_nl
( "!!!!>|!!!!?|!!!!@|"
, "!!!!!|!!!!!|!!!!!|0|!!!!(|A1"
, "!!!!\"|!!!!!|!!!!\"|0|!!!!,|B12"
, "!!!!#|!!!!!|!!!!#|0|!!!!*|C123"
))
.Tst_data_page(String_.Concat_lines_nl
( "!!!!9|!!!!>|!!!!=|"
, "!!!!!\t##PX+\tA1\tA1 data\t"
, "!!!!\"\t##PX/\tB12\tB12 changed\t"
, "!!!!#\t##PX0\tC123\tC123 data\t"
))
;
}
@Test public void Update_zip() {
// fxt.Wiki().Fsys_mgr().Dir_regy()[Xow_ns_.Tid__main].Ext_tid_(gplx.core.ios.streams.Io_stream_tid_.Tid__zip);
// fxt.Wiki().Data_mgr().Zip_mgr_(new Io_zip_mgr_mok());
// fxt .Create("A1", "A1 data")
// .Create("B12", "B12 data")
// .Create("C123", "C123 data")
// .Update("B12", "B12 changed")
// .Tst_regy_title("0|A1|C123|3\n")
// .Tst_data_title(String_.Concat_lines_nl
// ( "!!!!>|!!!!?|!!!!@|"
// , "!!!!!|!!!!!|!!!!!|0|!!!!(|A1"
// , "!!!!\"|!!!!!|!!!!\"|0|!!!!,|B12"
// , "!!!!#|!!!!!|!!!!#|0|!!!!*|C123"
// ))
// .Tst_data_page(String_.Concat_lines_nl
// ( "zipped:!!!!9|!!!!>|!!!!=|"
// , "!!!!!\t##PX+\tA1\tA1 data\t"
// , "!!!!\"\t##PX/\tB12\tB12 changed\t"
// , "!!!!#\t##PX0\tC123\tC123 data\t"
// ))
// ;
}
@Test public void Create_out_of_order() {
fxt .Create("C123", "C123 data")
.Create("B12", "B12 data")
.Create("A1", "A1 data")
.Tst_regy_title("0|A1|C123|3\n")
.Tst_data_title(String_.Concat_lines_nl
( "!!!!>|!!!!?|!!!!@|"
, "!!!!#|!!!!!|!!!!#|0|!!!!(|A1"
, "!!!!\"|!!!!!|!!!!\"|0|!!!!)|B12"
, "!!!!!|!!!!!|!!!!!|0|!!!!*|C123"
))
.Tst_data_page(String_.Concat_lines_nl
( "!!!!=|!!!!;|!!!!9|"
, "!!!!!\t##PX+\tC123\tC123 data\t"
, "!!!!\"\t##PX/\tB12\tB12 data\t"
, "!!!!#\t##PX0\tA1\tA1 data\t"
))
;
}
@Test public void Rename() {
fxt .Create("A1", "A1 data")
.Create("B12", "B12 data")
.Create("C123", "C123 data")
.Rename("C123", "C1234")
.Tst_regy_title("0|A1|C123|3\n")
.Tst_data_title(String_.Concat_lines_nl
( "!!!!>|!!!!?|!!!!@|"
, "!!!!!|!!!!!|!!!!!|0|!!!!(|A1"
, "!!!!\"|!!!!!|!!!!\"|0|!!!!)|B12"
, "!!!!#|!!!!!|!!!!#|0|!!!!*|C123"
))
.Tst_data_page(String_.Concat_lines_nl
( "!!!!9|!!!!;|!!!!=|"
, "!!!!!\t##PX+\tA1\tA1 data\t"
, "!!!!\"\t##PX/\tB12\tB12 data\t"
, "!!!!#\t##PX0\tC123\tC123 data\t"
))
;
}
}
class Xow_data_mgr_fxt {
Xoae_app app;
public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki;
public void Clear() {
app = Xoa_app_fxt.Make__app__edit();
wiki = Xoa_app_fxt.Make__wiki__edit(app);
wiki.Db_mgr().Save_mgr().Page_id_next_(0);
}
public Xow_data_mgr_fxt Create(String ttl_str, String data) {
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl_str));
wiki.Db_mgr().Save_mgr().Data_create(wiki, ttl, Bry_.new_u8(data));
return this;
}
public Xow_data_mgr_fxt Update(String ttl_str, String data) {
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl_str));
Xoae_page page = Xoae_page.New_test(wiki, ttl);
wiki.Db_mgr().Save_mgr().Data_update(page, Bry_.new_u8(data));
return this;
}
public Xow_data_mgr_fxt Rename(String old_ttl, String new_ttl) {
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(old_ttl));
Xoae_page page = Xoae_page.New_test(wiki, ttl);
wiki.Db_mgr().Save_mgr().Data_rename(page, ttl.Ns().Id(), Bry_.new_u8(new_ttl));
return this;
}
public Xow_data_mgr_fxt Tst_regy_title(String expd) {return Tst_regy(Xotdb_dir_info_.Name_title, expd);}
Xow_data_mgr_fxt Tst_regy(String name, String expd) {
Io_url file_orig = Io_url_.mem_fil_("mem/xowa/wiki/en.wikipedia.org/ns/000/" + name + "/reg.csv");
Tfds.Eq_str_lines(expd, Io_mgr.Instance.LoadFilStr(file_orig));
return this;
}
public Xow_data_mgr_fxt Tst_data_page(String expd) {return Tst_data(Xotdb_dir_info_.Tid_page , Xow_ns_.Tid__main, 0, expd);}
public Xow_data_mgr_fxt Tst_data_title(String expd) {return Tst_data(Xotdb_dir_info_.Tid_ttl, Xow_ns_.Tid__main, 0, expd);}
public Xow_data_mgr_fxt Tst_data(byte dir_tid, int ns_id, int fil, String expd) {
Io_url url = wiki.Tdb_fsys_mgr().Url_ns_fil(dir_tid, ns_id, fil);
Tfds.Eq_str_lines(expd, Io_mgr.Instance.LoadFilStr(url));
return this;
}
}

58
400_xowa/src/gplx/xowa/wikis/tdbs/Xotdb_page_raw_parser.java
Unescape View File

@ -1,58 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import gplx.core.brys.*; import gplx.core.ios.*; import gplx.core.encoders.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.tdbs.bldrs.*;
public class Xotdb_page_raw_parser {
public void Init(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, int load_len) {
this.wiki = wiki; ns_mgr = wiki.Ns_mgr();
rdr = new Io_line_rdr(usr_dlg, new Io_url[1]);
rdr.Line_dlm_(Byte_ascii.Tab).Load_len_(load_len).Key_gen_(Io_line_rdr_key_gen_all.Instance);
}
public void Init_ns(Xow_ns ns_itm) {this.ns_itm = ns_itm;}
public void Reset_one(Io_url url) {
rdr.Reset_one(url);
}
public void Load(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xow_ns ns_itm, Io_url[] urls, int load_len) {
this.wiki = wiki; ns_mgr = wiki.Ns_mgr(); this.ns_itm = ns_itm;
rdr = new Io_line_rdr(usr_dlg, urls);
rdr.Line_dlm_(Byte_ascii.Tab).Load_len_(load_len).Key_gen_(Io_line_rdr_key_gen_all.Instance);
} Io_line_rdr rdr; Xowe_wiki wiki; Xow_ns_mgr ns_mgr; Xow_ns ns_itm;
public void Skip_first_line() {
rdr.Read_next();
int pos = Bry_find_.Find_fwd(rdr.Bfr(), Byte_ascii.Nl);
// rdr.Move(pos + 1);
rdr.Truncate(pos + 1);
}
public boolean Read(Xowd_page_itm page) {
boolean read = false;
read = rdr.Read_next(); if (!read) return false;
int id = Base85_.To_int_by_bry(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 2);
page.Id_(id);
read = rdr.Read_next(); if (!read) throw Err_.new_wo_type("could not read timestamp");
int timestamp = Base85_.To_int_by_bry(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 1);
page.Modified_on_(Int_flag_bldr_.To_date_short(timestamp));
read = rdr.Read_next(); if (!read) throw Err_.new_wo_type("could not read ttl");
byte[] ttl = Bry_.Mid(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 1);
page.Ttl_(ttl, ns_mgr);
read = rdr.Read_next(); if (!read) throw Err_.new_wo_type("could not read text");
byte[] text = Bry_.Mid(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 1);
page.Text_(text);
rdr.Bfr_last_read_add(1);
return true;
}
}

41
400_xowa/src/gplx/xowa/wikis/tdbs/stats/Xob_stat_itm.java
Unescape View File

@ -1,41 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis.tdbs.stats; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*;
import gplx.core.strings.*;
public class Xob_stat_itm {
public String Ns() {return ns;} private String ns;
public int Fils;
public long Size, SizeMax = Int_.Min_value, SizeMin = Int_.Max_value;
public int SizeMaxIdx, SizeMinIdx;
public void Tally(long size, int idx) {
Fils++;
Size += size;
if (size > SizeMax) {SizeMax = size; SizeMaxIdx = idx;}
if (size < SizeMin) {SizeMin = size; SizeMinIdx = idx;}
}
public void To_str(String_bldr sb) {
XtoStr_fld(sb, ns).XtoStr_fld(sb, Fils).XtoStr_fld(sb, Size).XtoStr_fld(sb, SizeMax).XtoStr_fld(sb, SizeMaxIdx).XtoStr_fld(sb, SizeMin);
sb.Add(Int_.To_str(SizeMinIdx));
}
Xob_stat_itm XtoStr_fld(String_bldr sb, long v) {sb.Add(Long_.To_str(v)).Add(Xob_stat_itm.Dlm); return this;}
Xob_stat_itm XtoStr_fld(String_bldr sb, int v) {sb.Add(Int_.To_str(v)).Add(Xob_stat_itm.Dlm); return this;}
Xob_stat_itm XtoStr_fld(String_bldr sb, String v) {sb.Add(v).Add(Xob_stat_itm.Dlm); return this;}
public Xob_stat_itm(String ns) {
this.ns = ns;
}
public Object NewByKey(Object o) {return new Xob_stat_itm((String)o);} public static final Xob_stat_itm Instance = new Xob_stat_itm(); Xob_stat_itm() {}
public static final char Dlm = '|';
}

67
400_xowa/src/gplx/xowa/wikis/tdbs/stats/Xob_stat_mgr.java
Unescape View File

@ -1,67 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis.tdbs.stats; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*;
import gplx.core.strings.*;
import gplx.xowa.wikis.nss.*;
public class Xob_stat_mgr {
public Xob_stat_type GetOrNew(byte tid) {
Xob_stat_type rv = (Xob_stat_type)regy.Get_by(tid);
if (rv == null) {
rv = new Xob_stat_type(tid);
regy.Add(tid, rv);
}
return rv;
}
public String Print(Xow_ns_mgr nsMgr) {
String_bldr sb = String_bldr_.new_();
for (int i = 0; i < regy.Count(); i++) {
Xob_stat_type typ = (Xob_stat_type)regy.Get_at(i);
sb.Add(String_.PadEnd(Xotdb_dir_info_.Tid_name(typ.Tid()), 6, " "));
}
sb.Add_str_w_crlf("ns");
String[] nsAry = GetNmsAry(nsMgr);
for (String ns : nsAry) {
for (int i = 0; i < regy.Count(); i++) {
Xob_stat_type typ = (Xob_stat_type)regy.Get_at(i);
Xob_stat_itm itm = (Xob_stat_itm)typ.GetOrNew(ns);
sb.Add(Int_.To_str_pad_bgn_zero(itm.Fils, 5)).Add(" ");
}
sb.Add_str_w_crlf(ns);
}
return sb.To_str();
}
public String To_str() {
String_bldr sb = String_bldr_.new_();
for (int i = 0; i < regy.Count(); i++) {
Xob_stat_type typ = (Xob_stat_type)regy.Get_at(i);
typ.To_str(sb);
}
return sb.To_str();
}
String[] GetNmsAry(Xow_ns_mgr nsMgr) {
Ordered_hash nsRegy = Ordered_hash_.New();
for (int i = 0; i < regy.Count(); i++) {
Xob_stat_type typ = (Xob_stat_type)regy.Get_at(i);
for (int j = 0; j < typ.Count(); j++) {
Xob_stat_itm itm = (Xob_stat_itm)typ.GetAt(j);
if (!nsRegy.Has(itm.Ns()))
nsRegy.Add_as_key_and_val(itm.Ns());
}
}
return (String[])nsRegy.To_ary(String.class);
}
Ordered_hash regy = Ordered_hash_.New();
}

41
400_xowa/src/gplx/xowa/wikis/tdbs/stats/Xob_stat_type.java
Unescape View File

@ -1,41 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis.tdbs.stats; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*;
import gplx.core.strings.*; import gplx.xowa.wikis.tdbs.*;
public class Xob_stat_type {
public byte Tid() {return tid;} private byte tid;
public Xob_stat_type(byte tid) {this.tid = tid;}
public Xob_stat_itm GetOrNew(String ns) {
Xob_stat_itm rv = (Xob_stat_itm)regy.Get_by(ns);
if (rv == null) {
rv = new Xob_stat_itm(ns);
regy.Add(ns, rv);
}
return rv;
}
public Xob_stat_itm GetAt(int i) {return (Xob_stat_itm)regy.Get_at(i);}
public int Count() {return regy.Count();}
public void To_str(String_bldr sb) {
for (int i = 0; i < regy.Count(); i++) {
Xob_stat_itm itm = (Xob_stat_itm)regy.Get_at(i);
sb.Add(Xotdb_dir_info_.Tid_name(tid)).Add(Xob_stat_itm.Dlm);
itm.To_str(sb);
sb.Add(Byte_ascii.Nl);
}
}
Ordered_hash regy = Ordered_hash_.New();
public static final Xob_stat_type Instance = new Xob_stat_type(); Xob_stat_type() {}
}

1
400_xowa/src/gplx/xowa/xtns/pfuncs/ttls/Pfunc_filepath_tst.java
Unescape View File

@ -47,7 +47,6 @@ public class Pfunc_filepath_tst {
}
@Test public void Wiki_is_commons() {
fxt.Init_page_create(commons_wiki, "File:A.png", "");
commons_wiki.Tdb_fsys_mgr().Tdb_dir_regy()[Xotdb_dir_info_.Tid_page].Ext_tid_(gplx.core.ios.streams.Io_stream_tid_.Tid__raw);
mock_wkr.Redirect_("A.png", "A.png").Repo_idx_(1);
fxt.Test_parse_tmpl_str_test("{{filepath:A.png}}", "{{test}}", "file:///mem/xowa/file/commons/trg/orig/7/0/1/c/A.png");
}

7
400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_wikibase_tst.java
Unescape View File

@ -50,8 +50,11 @@ public class Scrib_lib_wikibase_tst {
fxt.Test_scrib_proc_bool(lib, Scrib_lib_wikibase.Invk_entityExists, Object_.Ary("Q1" ), false);
}
@Test public void GetEntityId() {
wdata_fxt.Init_links_add("enwiki", "Earth", "q2");
fxt.Test_scrib_proc_str(lib, Scrib_lib_wikibase.Invk_getEntityId, Object_.Ary("Earth" ), "q2");
wdata_fxt.Init__docs__add(wdata_fxt.Wdoc("Q2")
.Add_sitelink("enwiki", "Earth")
);
fxt.Test_scrib_proc_str(lib, Scrib_lib_wikibase.Invk_getEntityId, Object_.Ary("Earth" ), "Q2");
fxt.Test_scrib_proc_str(lib, Scrib_lib_wikibase.Invk_getEntityId, Object_.Ary("missing_page" ), "");
fxt.Test_scrib_proc_str(lib, Scrib_lib_wikibase.Invk_getEntityId, Object_.Ary("" ), ""); // PAGE:en.w:Water_treader DATE:2018-07-01
}

0
400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_regx_group_mgr.java → 400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_regx_grp_mgr.java
Unescape View File

3
400_xowa/src/gplx/xowa/xtns/wbases/Wdata_doc_bldr.java
Unescape View File

@ -32,7 +32,8 @@ public class Wdata_doc_bldr {
public Wdata_doc_bldr Add_description(String lang, String text) {byte[] key = Bry_.new_u8(lang); descr_list.Add(key, new Wdata_langtext_itm(key, Bry_.new_u8(text))); return this;}
public Wdata_doc_bldr Add_label(String lang, String text) {byte[] key = Bry_.new_u8(lang); label_list.Add(key, new Wdata_langtext_itm(key, Bry_.new_u8(text))); return this;}
public Wdata_doc_bldr Add_sitelink(String site, String link, String... bdgs) {
byte[] key = Bry_.new_u8(site); slink_list.Add(key, new Wdata_sitelink_itm(key, Bry_.new_u8(link), Bry_.Ary(bdgs)));
byte[] key = Bry_.new_u8(site);
slink_list.Add(key, new Wdata_sitelink_itm(key, Bry_.new_u8(link), Bry_.Ary(bdgs)));
return this;
}
public Wdata_doc_bldr Add_alias(String lang, String... ary){byte[] key = Bry_.new_u8(lang); alias_list.Add(key, new Wdata_alias_itm (key, Bry_.Ary(ary))); return this;}

79
400_xowa/src/gplx/xowa/xtns/wbases/Wdata_wiki_mgr_fxt.java
Unescape View File

@ -15,12 +15,10 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
import gplx.core.primitives.*; import gplx.langs.jsons.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.xwikis.*; import gplx.xowa.guis.*; import gplx.xowa.xtns.wbases.imports.*; import gplx.xowa.wikis.pages.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.langs.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.domains.*;
import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.pfuncs.*; import gplx.xowa.xtns.wbases.claims.*; import gplx.xowa.xtns.wbases.claims.enums.*; import gplx.xowa.xtns.wbases.claims.itms.*; import gplx.xowa.xtns.wbases.stores.*;
import gplx.xowa.wikis.tdbs.hives.*; import gplx.xowa.wikis.tdbs.xdats.*;
public class Wdata_wiki_mgr_fxt {
private Xoae_app app; private Xowe_wiki wiki; private Wdata_doc_bldr wdoc_bldr;
private final Wdata_xwiki_link_wtr wdata_lang_wtr = new Wdata_xwiki_link_wtr();
private final Bry_bfr tmp_time_bfr = Bry_bfr_.New();
public Xowe_wiki Wiki() {return parser_fxt.Wiki();}
@ -39,15 +37,13 @@ public class Wdata_wiki_mgr_fxt {
parser_fxt.Reset();
}
return this;
} private Xoae_app app; private Xowe_wiki wiki; private Wdata_doc_bldr wdoc_bldr;
}
public Xoae_app App() {return app;}
public Wdata_wiki_mgr Wdata_mgr() {return wdata_mgr;} private Wdata_wiki_mgr wdata_mgr;
public Xop_fxt Parser_fxt() {return parser_fxt;} private Xop_fxt parser_fxt;
public void Init_lang_fallbacks(String... fallbacks) {
wiki.Lang().Fallback_bry_(Bry_.new_a7(String_.Concat_with_str(",", fallbacks)));
}
public Wdata_doc_bldr Wdoc_bldr(String qid) {return wdoc_bldr.Qid_(qid);}
public Json_doc Make_json(String src) {return app.Utl__json_parser().Parse_by_apos(src);}
public Wbase_claim_base Make_claim_novalue(int pid) {return Wbase_claim_value.New_novalue(pid);}
public Wbase_claim_base Make_claim_somevalue(int pid) {return Wbase_claim_value.New_somevalue(pid);}
public Wbase_claim_base Make_claim_string(int pid, String val) {return Make_claim_string(pid, Bry_.new_u8(val));}
@ -74,7 +70,16 @@ public class Wdata_wiki_mgr_fxt {
return rv;
}
public Wdata_doc_bldr Wdoc(String qid) {return wdoc_bldr.Qid_(qid);}
public Wdata_doc doc_(String qid, Wbase_claim_base... props) {return wdoc_bldr.Qid_(qid).Add_claims(props).Xto_wdoc();}
public void Init_pids_add(String lang_key, String pid_name, int pid) {wdata_mgr.Pid_mgr.Add(Bry_.new_u8(lang_key + "|" + pid_name), pid);}
public void Init_qids_add(String lang_key, int wiki_tid, String ttl, String qid) {
Bry_bfr tmp_bfr = wiki.Utl__bfr_mkr().Get_b512();
wdata_mgr.Qid_mgr.Add(tmp_bfr, Bry_.new_a7(lang_key), wiki_tid, Bry_.new_a7("000"), Bry_.new_a7(ttl), Bry_.new_a7(qid));
tmp_bfr.Mkr_rls();
}
public void Init_lang_fallbacks(String... fallbacks) {wiki.Lang().Fallback_bry_(Bry_.new_a7(String_.Concat_with_str(",", fallbacks)));}
public void Init_xwikis_add(String... prefixes) {
int len = prefixes.length;
for (int i = 0; i < len; i++) {
@ -82,28 +87,6 @@ public class Wdata_wiki_mgr_fxt {
wiki.Xwiki_mgr().Add_by_atrs(prefix, prefix + ".wikipedia.org");
}
}
public void Init_qids_add(String lang_key, int wiki_tid, String ttl, String qid) {
Bry_bfr tmp_bfr = wiki.Utl__bfr_mkr().Get_b512();
wdata_mgr.Qid_mgr.Add(tmp_bfr, Bry_.new_a7(lang_key), wiki_tid, Bry_.new_a7("000"), Bry_.new_a7(ttl), Bry_.new_a7(qid));
tmp_bfr.Mkr_rls();
}
public void Init_pids_add(String lang_key, String pid_name, int pid) {wdata_mgr.Pid_mgr.Add(Bry_.new_u8(lang_key + "|" + pid_name), pid);}
public void Init_links_add(String wiki, String ttl, String qid) {Init_links_add(wiki, "000", ttl, qid);}
public void Init_links_add(String wiki, String ns_num, String ttl, String qid) {
byte[] ttl_bry = Bry_.new_u8(ttl);
Xowd_regy_mgr regy_mgr = app.Hive_mgr().Regy_mgr();
Io_url regy_fil = wdata_mgr.Wdata_wiki().Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "qid").GenSubFil_nest(wiki, ns_num, "reg.csv");
regy_mgr.Init(regy_fil);
regy_mgr.Create(ttl_bry);
regy_mgr.Save();
Bry_bfr bfr = Bry_bfr_.New();
byte[] itm = bfr.Add(ttl_bry).Add_byte(Byte_ascii.Pipe).Add(Bry_.new_a7(qid)).Add_byte_nl().To_bry_and_clear();
Xob_xdat_file xdat_file = new Xob_xdat_file();
xdat_file.Insert(bfr, itm);
Io_url file_orig = Xob_wdata_qid_base_tst.ttl_(app.Wiki_mgr().Wdata_mgr().Wdata_wiki(), wiki, ns_num, 0);
xdat_file.Save(file_orig);
}
public void Init_external_links_mgr_clear() {wiki.Parser_mgr().Ctx().Page().Wdata_external_lang_links().Reset();}
public void Init_external_links_mgr_add(String... langs) {
Wdata_external_lang_links_data external_lang_links = wiki.Parser_mgr().Ctx().Page().Wdata_external_lang_links();
@ -117,6 +100,23 @@ public class Wdata_wiki_mgr_fxt {
external_lang_links.Langs_add(Bry_.new_a7(lang));
}
}
public void Init__docs__add(Wdata_doc_bldr bldr) {Init__docs__add(bldr.Xto_wdoc());}
public void Init__docs__add(Wdata_doc wdoc) {
wdata_mgr.Doc_mgr.Add(wdoc.Qid(), wdoc);
Bry_bfr tmp_bfr = Bry_bfr_.New();
Wbase_qid_mgr qid_mgr = wdata_mgr.Qid_mgr;
Ordered_hash slinks = wdoc.Slink_list();
int slinks_len = slinks.Len();
for (int i = 0; i < slinks_len; i++) {
Wdata_sitelink_itm slink = (Wdata_sitelink_itm)slinks.Get_at(i);
Xow_domain_itm domain = Xow_abrv_wm_.Parse_to_domain_itm(slink.Site());
Xoa_ttl page_ttl = wiki.Ttl_parse(slink.Name());
qid_mgr.Add(tmp_bfr, domain.Lang_actl_key(), domain.Domain_type_id(), page_ttl.Ns().Num_bry(), page_ttl.Page_db(), wdoc.Qid());
}
}
public void Test_link(String ttl_str, String expd) {Test_link(Xoa_ttl.Parse(wiki, Xow_ns_.Tid__main, Bry_.new_u8(ttl_str)), expd);}
public void Test_link(Xoa_ttl ttl, String expd) {
byte[] qid_ttl = wdata_mgr.Qid_mgr.Get_qid_or_null(wiki, ttl);
@ -124,7 +124,6 @@ public class Wdata_wiki_mgr_fxt {
}
public void Test_parse_pid_null(String val) {Test_parse_pid(val, Wbase_pid.Id_null);}
public void Test_parse_pid(String val, int expd) {Tfds.Eq(expd, Wbase_statement_mgr_.Parse_pid(num_parser, Bry_.new_a7(val)));} private Gfo_number_parser num_parser = new Gfo_number_parser();
public void Init__docs__add(Wdata_doc page) {wdata_mgr.Doc_mgr.Add(page.Qid(), page);}
public void Test_parse(String raw, String expd) {
parser_fxt.Test_parse_page_tmpl_str(raw, expd);
}
@ -157,7 +156,7 @@ public class Wdata_wiki_mgr_fxt {
tmp_langs.Clear();
Wdata_xwiki_link_wtr.Write_wdata_links(tmp_langs, wiki, Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl)), wiki.Parser_mgr().Ctx().Page().Wdata_external_lang_links());
Tfds.Eq_ary_str(expd, Test_xwiki_links_xto_str_ary(tmp_langs));
} List_adp tmp_langs = List_adp_.New();
} private List_adp tmp_langs = List_adp_.New();
String[] Test_xwiki_links_xto_str_ary(List_adp list) {
int len = list.Count();
String[] rv = new String[len];
@ -175,4 +174,20 @@ public class Wdata_wiki_mgr_fxt {
Wdata_wiki_mgr.Write_json_as_html(wdata_mgr.Jdoc_parser(), bfr, raw_bry);
Tfds.Eq(expd, bfr.To_str_and_rls());
}
public static String New_json(String entity_id, String grp_key, String[] grp_vals) {
Bry_bfr bfr = Bry_bfr_.New();
bfr.Add_str_a7("{ 'entity':'").Add_str_u8(entity_id).Add_byte(Byte_ascii.Apos).Add_byte_nl();
bfr.Add_str_a7(", 'datatype':'commonsMedia'\n");
bfr.Add_str_a7(", '").Add_str_u8(grp_key).Add_str_a7("':").Add_byte_nl();
int len = grp_vals.length;
for (int i = 0; i < len; i += 2) {
bfr.Add_byte_repeat(Byte_ascii.Space, 2);
bfr.Add_byte(i == 0 ? Byte_ascii.Curly_bgn : Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i ]).Add_byte(Byte_ascii.Apos).Add_byte(Byte_ascii.Colon);
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i + 1]).Add_byte(Byte_ascii.Apos).Add_byte_nl();
}
bfr.Add_str_a7(" }").Add_byte_nl();
bfr.Add_str_a7("}").Add_byte_nl();
return String_.Replace(bfr.To_str_and_clear(), "'", "\"");
}
}

26
400_xowa/src/gplx/xowa/xtns/wbases/Wdata_wiki_mgr_tst.java
Unescape View File

@ -19,23 +19,31 @@ public class Wdata_wiki_mgr_tst {
private final Wdata_prop_val_visitor__fxt fxt = new Wdata_prop_val_visitor__fxt();
@Test public void Basic() {
Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt().Init();
fxt.Init_links_add("enwiki", "Q1", "Q1_en");
fxt.Test_link("Q1", "Q1_en");
fxt.Test_link("Q2", null);
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_sitelink("enwiki", "Q1_en")
);
fxt.Test_link("Q1_en" , "Q1");
fxt.Test_link("Q1_nil", null);
}
@Test public void Case_sensitive() { // PURPOSE: wikidata lkp should be case_sensitive; a vs A DATE:2013-09-03
Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt().Init();
fxt.Init_links_add("enwiki", "Page", "Page_data");
fxt.Test_link("Page", "Page_data");
fxt.Test_link("PAGE", null);
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_sitelink("enwiki", "Q1_EN")
);
fxt.Test_link("Q1_EN", "Q1");
fxt.Test_link("q1_en", null);
}
@Test public void Non_canonical_ns() { // PURPOSE: handle wikidata entries in non-canonical ns; EX:ukwikisource and Author; PAGE:uk.s:Автор:Богдан_Гаврилишин DATE:2014-07-23
Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt().Init();
Xowe_wiki wiki = fxt.Wiki();
wiki.Ns_mgr().Add_new(124, "Test_ns");
fxt.Init_links_add("enwiki", "000", "Test_ns:Test_page", "pass"); // NOTE: wdata will save to "000" ns, b/c "124" ns is not canonical
Xoa_ttl ttl = Xoa_ttl.Parse(fxt.Wiki(), 124, Bry_.new_a7("Test_page"));
fxt.Test_link(ttl, "pass");
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_sitelink("enwiki", "Test_ns:Test_page")
);
fxt.Test_link(Xoa_ttl.Parse(fxt.Wiki(), 124, Bry_.new_a7("Test_page")), "Q1"); // NOTE: wdata will save to "000" ns, b/c "124" ns is not canonical
}
@Test public void Write_json_as_html() {
Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt().Init();

2
400_xowa/src/gplx/xowa/xtns/wbases/claims/enums/Wbase_claim_type_.java
Unescape View File

@ -34,7 +34,7 @@ public class Wbase_claim_type_ {
, Tid__tabular_data = 14
, Tid__lexeme = 15
;
public static final Wbase_enum_hash Reg = new Wbase_enum_hash("claim.data_type", 15);
public static final Wbase_enum_hash Reg = new Wbase_enum_hash("claim.data_type", 16);
public static final Wbase_enum_itm
Itm__unknown = New(Tid__unknown , "unknown")
, Itm__bad = New(Tid__bad , "bad") // NOTE: wikidata identifies several entries as "bad"; Q1615351|'s-Graveland, Q107538|Baco; DATE:2013-10-20

40
400_xowa/src/gplx/xowa/xtns/wbases/imports/Wdata_idx_mgr_base.java
Unescape View File

@ -1,40 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.langs.jsons.*; import gplx.core.ios.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.parsers.*;
import gplx.xowa.xtns.wbases.imports.*;
abstract class Wdata_idx_mgr_base {
public void Ctor(Xob_itm_dump_base wkr, Xob_bldr bldr, Xowe_wiki wiki, int dump_fil_len) {
this.wkr = wkr; this.wiki = wiki; this.bldr = bldr; this.dump_fil_len = dump_fil_len;
} Xob_itm_dump_base wkr; protected Xowe_wiki wiki; Xob_bldr bldr; Xol_csv_parser csv_parser = Xol_csv_parser.Instance; protected Ordered_hash hash = Ordered_hash_.New(); protected int dump_fil_len;
public void Flush() {
int len = hash.Count();
for (int i = 0; i < len; i++) {
Wdata_idx_wtr wtr = (Wdata_idx_wtr)hash.Get_at(i);
wtr.Flush();
}
}
public void Make() {
int len = hash.Count();
for (int i = 0; i < len; i++) {
Wdata_idx_wtr wtr = (Wdata_idx_wtr)hash.Get_at(i);
wtr.Make(bldr.Usr_dlg(), wkr.Make_fil_len());
}
if (wkr.Delete_temp()) Io_mgr.Instance.DeleteDirDeep(wkr.Temp_dir());
}
}

52
400_xowa/src/gplx/xowa/xtns/wbases/imports/Wdata_idx_wtr.java
Unescape View File

@ -1,52 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.core.ios.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*;
class Wdata_idx_wtr {
public Wdata_idx_wtr(Io_url dump_dir, int dump_fil_max, Io_url make_dir) {
this.dump_dir = dump_dir;
this.dump_fil_max = dump_fil_max;
dump_url_gen = Io_url_gen_.dir_(dump_dir);
this.make_dir = make_dir;
} private Bry_bfr dump_bfr = Bry_bfr_.Reset(2 * Io_mgr.Len_kb); int dump_fil_max; Io_url dump_dir, make_dir; Io_url_gen dump_url_gen;
public void Write(byte[] ttl, byte[] qid) {
if (dump_bfr.Len() + ttl.length + qid.length + 2 > dump_fil_max) Flush(); // 2 = "|" + "\n"; NOTE: all items have format of "data|qid\n"
dump_bfr.Add(ttl).Add_byte_pipe().Add(qid).Add_byte_nl();
}
public void Flush() {
Io_mgr.Instance.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
}
public void Make(Gfo_usr_dlg usr_dlg, int make_fil_len) {
Xobdc_merger.Basic(usr_dlg, dump_url_gen, dump_dir.OwnerDir().GenSubDir("sort"), dump_fil_max, Io_line_rdr_key_gen_.first_pipe, new Xob_make_cmd_site(usr_dlg, make_dir, make_fil_len));
}
public static Wdata_idx_wtr new_qid_(Xowe_wiki wdata_wiki, String wiki_str, String ns_num, int dump_fil_max) {
Io_url dump_dir = wdata_wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest("wdata.qid", "qid", wiki_str, ns_num, "dump"); // /xowa/wiki/www.wikidata.org/tmp/wdata_qid/ + enwiki/000/dump/
Io_url make_dir = dir_qid_(wdata_wiki, wiki_str, ns_num); // /xowa/wiki/www.wikidata.org/site/data/qid/ + enwiki/000/
return new Wdata_idx_wtr(dump_dir, dump_fil_max, make_dir);
}
public static Wdata_idx_wtr new_pid_(Xowe_wiki wdata_wiki, String lang_key, int dump_fil_max) {
Io_url dump_dir = wdata_wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest("wdata.pid", "pid", lang_key, "dump"); // /xowa/wiki/www.wikidata.org/tmp/wdata_pid/ + en/
Io_url make_dir = dir_pid_(wdata_wiki, lang_key); // /xowa/wiki/www.wikidata.org/site/data/pid/ + en/
return new Wdata_idx_wtr(dump_dir, dump_fil_max, make_dir);
}
public static Io_url dir_qid_(Xowe_wiki wiki, String wiki_str, String ns_num) {
return wiki.Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "qid", wiki_str, ns_num); // /xowa/wiki/www.wikidata.org/site/data/ + qid/enwiki/000/
}
public static Io_url dir_pid_(Xowe_wiki wiki, String lang_key) {
return wiki.Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "pid", lang_key); // /xowa/wiki/www.wikidata.org/site/data/ + pid/en/
}
}

94
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_pid.java
Unescape View File

@ -0,0 +1,94 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.claims.enums.*; import gplx.xowa.xtns.wbases.parsers.*; import gplx.xowa.xtns.wbases.dbs.*;
import gplx.langs.jsons.*;
public class Xob_wdata_pid extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
private Db_conn conn;
private Wbase_pid_tbl tbl__pid;
private Xowb_prop_tbl tbl__prop;
private Json_parser jdoc_parser;
private final Ordered_hash datatype_hash = Ordered_hash_.New_bry();
public Xob_wdata_pid(Db_conn conn) {
this.conn = conn;
}
public Xob_wdata_pid Ctor(Xob_bldr bldr, Xowe_wiki wiki) {
this.Cmd_ctor(bldr, wiki);
this.jdoc_parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
return this;
}
public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_pid;}
public void Page_wkr__bgn() {this.Pid__bgn();}
public void Page_wkr__run(Xowd_page_itm page) {
if (page.Ns_id() != Wdata_wiki_mgr.Ns_property) return;
Json_doc jdoc = jdoc_parser.Parse(page.Text());
if (jdoc == null) {
bldr.Usr_dlg().Warn_many(GRP_KEY, "json.invalid", "json is invalid: ns=~{0} id=~{1}", page.Ns_id(), String_.new_u8(page.Ttl_page_db()));
return;
}
Pid__run(jdoc);
}
public void Page_wkr__run_cleanup() {}
public void Page_wkr__end() {this.Pid__end();}
public void Pid__bgn() {
if (conn == null) // conn will be null unless test
conn = wiki.Data__core_mgr().Db__wbase().Conn();
// init datatype_hash
Wbase_enum_hash enum_hash = Wbase_claim_type_.Reg;
byte len = (byte)enum_hash.Len();
for (byte i = 0; i < len; i++) {
Wbase_claim_type claim_type = (Wbase_claim_type)enum_hash.Get_itm_or(i, null);
datatype_hash.Add(Bry_.new_u8(claim_type.Key_for_scrib()), claim_type);
}
// init wbase_pid
tbl__pid = Wbase_pid_tbl.New_make(conn);
tbl__pid.Create_tbl();
tbl__pid.Insert_bgn();
// init wbase_prop
tbl__prop = new Xowb_prop_tbl(conn);
tbl__prop.Create_tbl();
tbl__prop.Insert_bgn();
}
public void Pid__run(Json_doc jdoc) {
Wdata_doc_parser wdoc_parser = app.Wiki_mgr().Wdata_mgr().Wdoc_parser(jdoc);
byte[] pid = wdoc_parser.Parse_qid(jdoc);
// add datatype
byte[] datatype = jdoc.Root_nde().Get_as_bry(Wdata_dict_mainsnak.Itm__datatype.Key_str());
Wbase_claim_type claim_type = (Wbase_claim_type)datatype_hash.Get_by_or_fail(datatype);
tbl__prop.Insert_cmd_by_batch(pid, claim_type.Tid());
// add langs
Ordered_hash list = wdoc_parser.Parse_langvals(pid, jdoc, Bool_.Y);
int len = list.Count();
for (int i = 0; i < len; ++i) {
Wdata_langtext_itm label = (Wdata_langtext_itm)list.Get_at(i);
tbl__pid.Insert_cmd_by_batch(label.Lang(), label.Text(), pid);
}
}
public void Pid__end() {
tbl__pid.Insert_end();
tbl__pid.Create_idx();
tbl__prop.Insert_end();
}
private static final String GRP_KEY = "xowa.wdata.pid_wkr";
}

62
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_pid_base.java
Unescape View File

@ -1,62 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.langs.jsons.*; import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.parsers.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public abstract class Xob_wdata_pid_base extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
private Json_parser parser;
public Xob_wdata_pid_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); return this;}
public abstract void Pid_bgn();
public abstract void Pid_add(byte[] src_lang, byte[] src_ttl, byte[] trg_ttl);
public abstract void Pid_datatype(byte[] pid, byte[] datatype_bry);
public abstract void Pid_end();
public abstract String Page_wkr__key();
public void Page_wkr__bgn() {
this.Init_dump(this.Page_wkr__key(), wiki.Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "pid")); // NOTE: must pass in correct make_dir in order to delete earlier version (else make_dirs will append)
this.parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
this.Pid_bgn();
}
public void Page_wkr__run(Xowd_page_itm page) {
if (page.Ns_id() != Wdata_wiki_mgr.Ns_property) return;
Json_doc jdoc = parser.Parse(page.Text());
if (jdoc == null) {
bldr.Usr_dlg().Warn_many(GRP_KEY, "json.invalid", "json is invalid: ns=~{0} id=~{1}", page.Ns_id(), String_.new_u8(page.Ttl_page_db()));
return;
}
Parse_jdoc(jdoc);
}
public void Page_wkr__run_cleanup() {}
public void Parse_jdoc(Json_doc jdoc) {
Wdata_doc_parser wdoc_parser = app.Wiki_mgr().Wdata_mgr().Wdoc_parser(jdoc);
byte[] qid = wdoc_parser.Parse_qid(jdoc);
// add datatype
byte[] datatype = jdoc.Root_nde().Get_as_bry(Wdata_dict_mainsnak.Itm__datatype.Key_str());
this.Pid_datatype(qid, datatype);
// add langs
Ordered_hash list = wdoc_parser.Parse_langvals(qid, jdoc, Bool_.Y);
int len = list.Count();
for (int i = 0; i < len; ++i) {
Wdata_langtext_itm label = (Wdata_langtext_itm)list.Get_at(i);
this.Pid_add(label.Lang(), label.Text(), qid);
}
}
public void Page_wkr__end() {this.Pid_end();}
private static final String GRP_KEY = "xowa.wdata.pid_wkr";
}

74
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_pid_base_tst.java
Unescape View File

@ -1,74 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import org.junit.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*;
import gplx.xowa.bldrs.*;
public class Xob_wdata_pid_base_tst {
private final Xob_fxt fxt = new Xob_fxt().Ctor_mem();
Io_url reg_(Xowe_wiki wdata, String wiki) {return Wdata_idx_wtr.dir_pid_(wdata, wiki).GenSubFil(Xotdb_dir_info_.Name_reg_fil);}
Io_url ttl_(Xowe_wiki wdata, String wiki, int fil_id) {
Io_url root = Wdata_idx_wtr.dir_pid_(wdata, wiki);
return Xotdb_fsys_mgr.Url_fil(root, fil_id, Xotdb_dir_info_.Bry_xdat);
}
@Test public void Basic() {
fxt.Wiki().Ns_mgr().Add_new(Wdata_wiki_mgr.Ns_property, "Property");
fxt.doc_ary_
( fxt.doc_wo_date_(2, "Property:P2", json_("p2", "label", String_.Ary("en", "p2_en", "fr", "p2_fr")))
, fxt.doc_wo_date_(1, "Property:P1", json_("p1", "label", String_.Ary("en", "p1_en", "fr", "p1_fr")))
)
.Fil_expd(ttl_(fxt.Wiki(), "en", 0)
, "!!!!*|!!!!*|"
, "p1_en|p1"
, "p2_en|p2"
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "en")
, "0|p1_en|p2_en|2"
, ""
)
.Fil_expd(ttl_(fxt.Wiki(), "fr", 0)
, "!!!!*|!!!!*|"
, "p1_fr|p1"
, "p2_fr|p2"
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "fr")
, "0|p1_fr|p2_fr|2"
, ""
)
.Run_page_wkrs(new Xob_wdata_pid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
;
}
public static String json_(String entity_id, String grp_key, String[] grp_vals) {
Bry_bfr bfr = Bry_bfr_.New();
bfr.Add_str_a7("{ 'entity':'").Add_str_u8(entity_id).Add_byte(Byte_ascii.Apos).Add_byte_nl();
bfr.Add_str_a7(", 'datatype':'commonsMedia'\n");
bfr.Add_str_a7(", '").Add_str_u8(grp_key).Add_str_a7("':").Add_byte_nl();
int len = grp_vals.length;
for (int i = 0; i < len; i += 2) {
bfr.Add_byte_repeat(Byte_ascii.Space, 2);
bfr.Add_byte(i == 0 ? Byte_ascii.Curly_bgn : Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i ]).Add_byte(Byte_ascii.Apos).Add_byte(Byte_ascii.Colon);
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i + 1]).Add_byte(Byte_ascii.Apos).Add_byte_nl();
}
bfr.Add_str_a7(" }").Add_byte_nl();
bfr.Add_str_a7("}").Add_byte_nl();
return String_.Replace(bfr.To_str_and_clear(), "'", "\"");
}
}

62
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_pid_sql.java
Unescape View File

@ -1,62 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.xtns.wbases.dbs.*; import gplx.xowa.xtns.wbases.claims.enums.*;
public class Xob_wdata_pid_sql extends Xob_wdata_pid_base {
private Db_conn conn;
private Wbase_pid_tbl tbl__pid;
private Xowb_prop_tbl tbl__prop;
private final Ordered_hash datatype_hash = Ordered_hash_.New_bry();
public Xob_wdata_pid_sql(Db_conn conn, Xowe_wiki wiki) {
this.conn = conn;
this.wiki = wiki;
}
@Override public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_pid;}
@Override public void Pid_bgn() {
if (conn == null)
conn = wiki.Data__core_mgr().Db__wbase().Conn();
// init datatype_hash
Wbase_enum_hash enum_hash = Wbase_claim_type_.Reg;
byte len = (byte)enum_hash.Len();
for (byte i = 0; i < len; i++) {
Wbase_claim_type claim_type = (Wbase_claim_type)enum_hash.Get_itm_or(i, null);
datatype_hash.Add(Bry_.new_u8(claim_type.Key_for_scrib()), claim_type);
}
// init wbase_pid
tbl__pid = Wbase_pid_tbl.New_make(conn);
tbl__pid.Create_tbl();
tbl__pid.Insert_bgn();
// init wbase_prop
tbl__prop = new Xowb_prop_tbl(conn);
tbl__prop.Create_tbl();
tbl__prop.Insert_bgn();
}
@Override public void Pid_add(byte[] lang_key, byte[] ttl, byte[] pid) {
tbl__pid.Insert_cmd_by_batch(lang_key, ttl, pid);
}
@Override public void Pid_datatype(byte[] pid, byte[] datatype_bry) {
Wbase_claim_type claim_type = (Wbase_claim_type)datatype_hash.Get_by_or_fail(datatype_bry);
tbl__prop.Insert_cmd_by_batch(pid, claim_type.Tid());
}
@Override public void Pid_end() {
tbl__pid.Insert_end();
tbl__pid.Create_idx();
tbl__prop.Insert_end();
}
}

91
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_pid_sql_tst.java
Unescape View File

@ -1,91 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import org.junit.*; import gplx.core.tests.*;
import gplx.*;
import gplx.dbs.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xob_wdata_pid_sql_tst {
private Db_conn conn;
private final Xob_fxt fxt = new Xob_fxt().Ctor_mem();
private final Gfo_db_tester db_tester = new Gfo_db_tester();
@Before public void init() {
Io_mgr.Instance.InitEngine_mem();
Db_conn_bldr.Instance.Reg_default_mem();
this.conn = Db_conn_bldr.Instance.New(Io_url_.mem_fil_("mem/db/wbase.xowa"));
}
@Test public void Basic() {
fxt.Wiki().Ns_mgr().Add_new(Wdata_wiki_mgr.Ns_property, "Property");
Xob_wdata_pid_sql wkr = new Xob_wdata_pid_sql(conn, null);
wkr.Ctor(fxt.Bldr(), fxt.Wiki());
fxt.Run_page_wkr(wkr
, fxt.doc_wo_date_(2, "Property:P2", Xob_wdata_tst_utl.Json("p2", "label", String_.Ary("en", "p2_en", "fr", "p2_fr")))
, fxt.doc_wo_date_(1, "Property:P1", Xob_wdata_tst_utl.Json("p1", "label", String_.Ary("en", "p1_en", "fr", "p1_fr")))
);
db_tester.Test__select_tbl(conn
, "wbase_pid", new Dbmeta_fld_list().Bld_str("src_lang").Bld_str("src_ttl").Bld_str("trg_ttl")
, String_.Ary("en", "p2_en", "p2")
, String_.Ary("fr", "p2_fr", "p2")
, String_.Ary("en", "p1_en", "p1")
, String_.Ary("fr", "p1_fr", "p1")
);
}
}
class Gfo_db_tester {
public void Test__select_tbl(Db_conn conn, String tbl, Dbmeta_fld_list flds, String[]... expd_rows) {
Db_rdr rdr = Db_rdr_.Empty;
try {
rdr = conn.Stmt_select_all(tbl, flds).Exec_select__rls_auto();
int row_idx = 0;
while (rdr.Move_next()) {
String[] actl_row = new String[flds.Len()];
for (int i = 0; i < flds.Len(); i++)
actl_row[i] = Object_.Xto_str_strict_or_null(rdr.Read_at(i));
String[] expd_row = expd_rows[row_idx++];
Gftest.Eq__ary(expd_row, actl_row, "row compare failed", "row", row_idx);
}
}
finally {
rdr.Rls();
}
}
public void Test__objary(Object[][] expd, Object[][] actl) {
// gplx.core.strings.String_bldr_.new_()
}
}
class Xob_wdata_tst_utl {
public static String Json(String entity_id, String grp_key, String[] grp_vals) {
Bry_bfr bfr = Bry_bfr_.New();
bfr.Add_str_a7("{ 'entity':'").Add_str_u8(entity_id).Add_byte(Byte_ascii.Apos).Add_byte_nl();
bfr.Add_str_a7(", 'datatype':'commonsMedia'\n");
bfr.Add_str_a7(", '").Add_str_u8(grp_key).Add_str_a7("':").Add_byte_nl();
int len = grp_vals.length;
for (int i = 0; i < len; i += 2) {
bfr.Add_byte_repeat(Byte_ascii.Space, 2);
bfr.Add_byte(i == 0 ? Byte_ascii.Curly_bgn : Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i ]).Add_byte(Byte_ascii.Apos).Add_byte(Byte_ascii.Colon);
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i + 1]).Add_byte(Byte_ascii.Apos).Add_byte_nl();
}
bfr.Add_str_a7(" }").Add_byte_nl();
bfr.Add_str_a7("}").Add_byte_nl();
return String_.Replace(bfr.To_str_and_clear(), "'", "\"");
}
}

127
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_pid_tst.java
Unescape View File

@ -0,0 +1,127 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import org.junit.*; import gplx.core.tests.*;
import gplx.*; import gplx.dbs.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xob_wdata_pid_tst {
private Db_conn conn;
private final Xobldr_fxt fxt = new Xobldr_fxt().Ctor_mem();
private final Gfo_db_tester db_tester = new Gfo_db_tester();
@Before public void init() {
Io_mgr.Instance.InitEngine_mem();
Db_conn_bldr.Instance.Reg_default_mem();
this.conn = Db_conn_bldr.Instance.New(Io_url_.mem_fil_("mem/db/wbase.xowa"));
}
@Test public void Basic() {
fxt.Wiki().Ns_mgr().Add_new(Wdata_wiki_mgr.Ns_property, "Property");
Xob_wdata_pid wkr = new Xob_wdata_pid(conn);
wkr.Ctor(fxt.Bldr(), fxt.Wiki());
fxt.Run_page_wkr(wkr
, fxt.New_page_wo_date(2, "Property:P2", Xob_wdata_tst_utl.Json("p2", "label", String_.Ary("en", "p2_en", "fr", "p2_fr")))
, fxt.New_page_wo_date(1, "Property:P1", Xob_wdata_tst_utl.Json("p1", "label", String_.Ary("en", "p1_en", "fr", "p1_fr")))
);
db_tester.Test__select_tbl(conn, "wbase_pid", new Dbmeta_fld_list().Bld_str("src_lang").Bld_str("src_ttl").Bld_str("trg_ttl")
, Object_.Ary("en", "p2_en", "p2")
, Object_.Ary("fr", "p2_fr", "p2")
, Object_.Ary("en", "p1_en", "p1")
, Object_.Ary("fr", "p1_fr", "p1")
);
db_tester.Test__select_tbl(conn, "wbase_prop", new Dbmeta_fld_list().Bld_str("wbp_pid").Bld_int("wbp_datatype")
, Object_.Ary("p2", 12)
, Object_.Ary("p1", 12)
);
}
}
class Gfo_db_tester {
public void Test__select_tbl(Db_conn conn, String tbl, Dbmeta_fld_list flds, Object[]... expd_rows) {
// get actl
List_adp actl_list = List_adp_.New();
Db_rdr rdr = Db_rdr_.Empty;
try {
rdr = conn.Stmt_select_all(tbl, flds).Exec_select__rls_auto();
while (rdr.Move_next()) {
Object[] actl_row = new Object[flds.Len()];
actl_list.Add(actl_row);
for (int i = 0; i < flds.Len(); i++) {
Object val = rdr.Read_at(i);
int val_tid = Dbmeta_fld_tid.Get_by_obj(val);
Dbmeta_fld_itm fld = flds.Get_at(i);
if (val_tid != fld.Type().Tid_ansi())
actl_row[i] = Object_.Xto_str_strict_or_null_mark(val) + "|shouldBe=" + String_.new_u8(fld.Type().Name()) + "|was=" + Type_.Name_by_obj(val);
else
actl_row[i] = val;
}
}
}
finally {
rdr.Rls();
}
Object[][] actl_rows = (Object[][])actl_list.To_ary_and_clear(Object[].class);
// validate expd datatypes
for (Object[] expd_row : expd_rows) {
int len = expd_row.length;
for (int i = 0; i < len; i++) {
Object val = expd_row[i];
int val_tid = Dbmeta_fld_tid.Get_by_obj(val);
Dbmeta_fld_itm fld = flds.Get_at(i);
if (val_tid != fld.Type().Tid_ansi())
expd_row[i] = Object_.Xto_str_strict_or_null_mark(val) + "|shouldBe=" + String_.new_u8(fld.Type().Name()) + "|was=" + Type_.Name_by_obj(val);
}
}
Gftest.Eq__ary(To_str_ary(expd_rows), To_str_ary(actl_rows), "rows mismatch");
}
public byte[][] To_str_ary(Object[][] rows) {
Bry_bfr bfr = Bry_bfr_.New();
byte[][] rv = new byte[rows.length][];
int rows_len = rows.length;
for (int i = 0; i < rows_len; i++) {
Object[] row = rows[i];
for (int j = 0; j < row.length; j++) {
if (j != 0) bfr.Add_byte_pipe();
bfr.Add_obj(row[j]);
}
bfr.Add_byte_nl();
rv[i] = bfr.To_bry_and_clear();
}
return rv;
}
}
class Xob_wdata_tst_utl {
public static String Json(String entity_id, String grp_key, String[] grp_vals) {
Bry_bfr bfr = Bry_bfr_.New();
bfr.Add_str_a7("{ 'entity':'").Add_str_u8(entity_id).Add_byte(Byte_ascii.Apos).Add_byte_nl();
bfr.Add_str_a7(", 'datatype':'commonsMedia'\n");
bfr.Add_str_a7(", '").Add_str_u8(grp_key).Add_str_a7("':").Add_byte_nl();
int len = grp_vals.length;
for (int i = 0; i < len; i += 2) {
bfr.Add_byte_repeat(Byte_ascii.Space, 2);
bfr.Add_byte(i == 0 ? Byte_ascii.Curly_bgn : Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i ]).Add_byte(Byte_ascii.Apos).Add_byte(Byte_ascii.Colon);
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i + 1]).Add_byte(Byte_ascii.Apos).Add_byte_nl();
}
bfr.Add_str_a7(" }").Add_byte_nl();
bfr.Add_str_a7("}").Add_byte_nl();
return String_.Replace(bfr.To_str_and_clear(), "'", "\"");
}
}

48
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_pid_txt.java
Unescape View File

@ -1,48 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xob_wdata_pid_txt extends Xob_wdata_pid_base {
@Override public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_tdb_text_wdata_pid;}
@Override public void Pid_bgn() {
pid_bldr = new Wdata_idx_bldr_pid(this, bldr, wiki, dump_fil_len);
} Wdata_idx_bldr_pid pid_bldr;
@Override public void Pid_add(byte[] lang_key, byte[] prop_key, byte[] qid) {
pid_bldr.Add(lang_key, prop_key, qid);
}
@Override public void Pid_datatype(byte[] pid, byte[] datatype_bry) {}
@Override public void Pid_end() {
pid_bldr.Flush();
pid_bldr.Make();
}
}
class Wdata_idx_bldr_pid extends Wdata_idx_mgr_base {
public Wdata_idx_bldr_pid(Xob_itm_dump_base wkr, Xob_bldr bldr, Xowe_wiki wiki, int dump_fil_len) {this.Ctor(wkr, bldr, wiki, dump_fil_len);}
public void Add(byte[] lang, byte[] prop_key, byte[] pid) {
Wdata_idx_wtr wtr = Get_or_new(lang);
wtr.Write(prop_key, pid);
}
public Wdata_idx_wtr Get_or_new(byte[] lang_bry) {
String lang = String_.Lower(String_.new_u8(lang_bry)); // NOTE: for some reason, both "en" and "En" can be added; normalize case
Object rv = hash.Get_by(lang);
if (rv == null) {
Wdata_idx_wtr wtr = Wdata_idx_wtr.new_pid_(wiki, lang, dump_fil_len);
hash.Add(lang, wtr);
return wtr;
}
return (Wdata_idx_wtr)rv;
}
}

55
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_qid_base.java → 400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_qid.java
Unescape View File

@ -14,32 +14,44 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.langs.jsons.*; import gplx.core.ios.*; import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.parsers.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.wikis.nss.*;
import gplx.langs.jsons.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.bldrs.wms.sites.*;
public abstract class Xob_wdata_qid_base extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.dbs.*; import gplx.xowa.xtns.wbases.parsers.*;
public class Xob_wdata_qid extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
private Db_conn conn;
private Wbase_qid_tbl tbl;
private final Object thread_lock = new Object();
private Json_parser parser; private Xob_wbase_ns_parser ns_parser; private final Xob_wbase_ns_parser_rslt ns_parser_rslt = new Xob_wbase_ns_parser_rslt();
public Xob_wdata_qid_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); return this;}
public abstract String Page_wkr__key();
public abstract void Qid_bgn();
public abstract void Qid_add(byte[] wiki_key, int ns_id, byte[] ttl, byte[] qid);
public abstract void Qid_end();
public Xob_wdata_qid(Db_conn conn) {
this.conn = conn;
}
public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_qid;}
public Xob_wdata_qid Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); return this;}
public void Page_wkr__bgn() {
this.Init_dump(this.Page_wkr__key(), wiki.Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "qid")); // NOTE: must pass in correct make_dir in order to delete earlier version (else make_dirs will append)
this.parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
this.ns_parser = new Xob_wbase_ns_parser(bldr.App().Fsys_mgr().Cfg_site_meta_fil());
this.Qid_bgn();
this.Qid__bgn();
}
public void Page_wkr__run(Xowd_page_itm page) {
if (page.Ns_id() != Xow_ns_.Tid__main) return; // qid pages are only in the Main Srch_rslt_cbk
Json_doc jdoc = parser.Parse(page.Text());
if (jdoc == null) {bldr.Usr_dlg().Warn_many("", "", "json is invalid: ns=~{0} id=~{1}", page.Ns_id(), String_.new_u8(page.Ttl_page_db())); return;}
this.Parse_jdoc(jdoc);
this.Qid__run(jdoc);
}
public void Page_wkr__run_cleanup() {}
public void Parse_jdoc(Json_doc jdoc) {
public void Page_wkr__end() {this.Qid__end();}
public void Qid__bgn() {
if (conn == null) {
Xow_db_file wbase_db = Make_wbase_db(wiki.Db_mgr_as_sql().Core_data_mgr());
conn = wbase_db.Conn();
}
tbl = Wbase_qid_tbl.New_make(conn, false);
tbl.Create_tbl();
tbl.Insert_bgn();
}
public void Qid__run(Json_doc jdoc) {
synchronized (thread_lock) {
Wdata_doc_parser wdoc_parser = app.Wiki_mgr().Wdata_mgr().Wdoc_parser(jdoc);
byte[] qid = wdoc_parser.Parse_qid(jdoc);
@ -54,12 +66,21 @@ public abstract class Xob_wdata_qid_base extends Xob_itm_dump_base implements Xo
if (sitelink_ns != Xow_ns_.Tid__main) // ttl not in main; chop off ns portion; EX:Aide:French_title -> French_title
sitelink_ttl = Bry_.Mid(sitelink_ttl, ns_parser_rslt.Ttl_bgn(), sitelink_ttl.length);
sitelink_ttl = wiki.Lang().Case_mgr().Case_build_1st_upper(tmp_bfr, sitelink_ttl, 0, sitelink_ttl.length);
this.Qid_add(sitelink.Site(), sitelink_ns, Xoa_ttl.Replace_spaces(sitelink_ttl), qid); // NOTE: always convert spaces to underscores; EX: "A B" -> "A_B" DATE:2015-04-21
tbl.Insert_cmd_by_batch(sitelink.Site(), sitelink_ns, Xoa_ttl.Replace_spaces(sitelink_ttl), qid); // NOTE: always convert spaces to underscores; EX: "A B" -> "A_B" DATE:2015-04-21
}
}
}
public void Page_wkr__end() {
this.Qid_end();
// wiki.Data__core_mgr().Db__wbase().Tbl__cfg().Insert_int("", "", 1);
public void Qid__end() {
tbl.Insert_end();
tbl.Create_idx();
}
public static Xow_db_file Make_wbase_db(Xow_db_mgr db_mgr) {
boolean db_is_all_or_few = db_mgr.Props().Layout_text().Tid_is_all_or_few();
Xow_db_file wbase_db = db_is_all_or_few
? db_mgr.Db__core()
: db_mgr.Dbs__make_by_tid(Xow_db_file_.Tid__wbase);
if (db_is_all_or_few)
db_mgr.Db__wbase_(wbase_db);
return wbase_db;
}
}

173
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_qid_base_tst.java
Unescape View File

@ -1,173 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import org.junit.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*; import gplx.dbs.*;
import gplx.xowa.bldrs.wms.sites.*;
public class Xob_wdata_qid_base_tst {
private gplx.xowa.bldrs.Xob_fxt fxt; // NOTE: reset memory instance (don't just call clear)
@Before public void init() {
this.fxt = new gplx.xowa.bldrs.Xob_fxt().Ctor_mem();
gplx.dbs.Db_conn_bldr.Instance.Reg_default_mem();
}
@Test public void Basic() {
fxt.doc_ary_
( fxt.doc_wo_date_(2, "q2", Xob_wdata_pid_base_tst.json_("q2", "links", String_.Ary("enwiki", "q2_en", "frwiki", "q2_fr")))
, fxt.doc_wo_date_(1, "q1", Xob_wdata_pid_base_tst.json_("q1", "links", String_.Ary("enwiki", "q1_en", "frwiki", "q1_fr")))
)
.Fil_expd(ttl_(fxt.Wiki(), "enwiki", "000", 0)
, "!!!!*|!!!!*|"
, "Q1_en|q1"
, "Q2_en|q2"
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "enwiki", "000")
, "0|Q1_en|Q2_en|2"
, ""
)
.Fil_expd(ttl_(fxt.Wiki(), "frwiki", "000", 0)
, "!!!!*|!!!!*|"
, "Q1_fr|q1"
, "Q2_fr|q2"
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "frwiki", "000")
, "0|Q1_fr|Q2_fr|2"
, ""
)
.Run_page_wkrs(new Xob_wdata_qid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
;
}
@Test public void Ns() {
// setup db
Site_core_db json_db = new Site_core_db(fxt.App().Fsys_mgr().Cfg_site_meta_fil());
Site_namespace_tbl ns_tbl = json_db.Tbl__namespace();
ns_tbl.Insert(Bry_.new_a7("en.w"), Xow_ns_.Tid__help, Xow_ns_case_.Bry__1st, Bry_.Empty, Bry_.new_a7("Help"), Bool_.N, Bool_.N, Bry_.Empty);
ns_tbl.Insert(Bry_.new_a7("fr.w"), Xow_ns_.Tid__help, Xow_ns_case_.Bry__1st, Bry_.Empty, Bry_.new_a7("Aide"), Bool_.N, Bool_.N, Bry_.Empty);
// run test
fxt.doc_ary_
( fxt.doc_wo_date_(1, "11", Xob_wdata_pid_base_tst.json_("q1", "links", String_.Ary("enwiki", "Help:Q1_en", "frwiki", "Aide:Q1_fr")))
)
.Fil_expd(ttl_(fxt.Wiki(), "enwiki", "012", 0)
, "!!!!*|"
, "Q1_en|q1"
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "enwiki", "012")
, "0|Q1_en|Q1_en|1"
, ""
)
.Fil_expd(ttl_(fxt.Wiki(), "frwiki", "012", 0)
, "!!!!*|"
, "Q1_fr|q1"
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "frwiki", "012")
, "0|Q1_fr|Q1_fr|1"
, ""
)
.Run_page_wkrs(new Xob_wdata_qid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
;
}
@Test public void Links_w_name() { // PURPOSE: wikidata changed links node from "enwiki:A" to "enwiki:{name:A,badges:[]}"; DATE:2013-09-14
String q1_str = String_.Concat_lines_nl
( "{ \"entity\":\"q1\""
, ", \"links\":"
, " { \"enwiki\":\"q1_en\""
, " , \"frwiki\":\"q1_fr\""
, " }"
, "}"
);
String q2_str = String_.Concat_lines_nl
( "{ \"entity\":[\"item\",2]"
, ", \"links\":"
, " { \"enwiki\":{\"name\":\"q2_en\",\"badges\":[]}"
, " , \"frwiki\":{\"name\":\"q2_fr\",\"badges\":[]}"
, " }"
, "}"
);
fxt.doc_ary_
( fxt.doc_wo_date_(1, "q1", q1_str)
, fxt.doc_wo_date_(2, "q2", q2_str)
)
.Fil_expd(ttl_(fxt.Wiki(), "enwiki", "000", 0)
, "!!!!*|!!!!*|"
, "Q1_en|q1"
, "Q2_en|q2"
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "enwiki", "000")
, "0|Q1_en|Q2_en|2"
, ""
)
.Fil_expd(ttl_(fxt.Wiki(), "frwiki", "000", 0)
, "!!!!*|!!!!*|"
, "Q1_fr|q1"
, "Q2_fr|q2"
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "frwiki", "000")
, "0|Q1_fr|Q2_fr|2"
, ""
)
.Run_page_wkrs(new Xob_wdata_qid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
;
}
@Test public void Spaces() { // PURPOSE: assert that ttls with spaces are converted to unders DATE:2015-04-21
fxt.doc_ary_
( fxt.doc_wo_date_(2, "q2", Xob_wdata_pid_base_tst.json_("q2", "links", String_.Ary("enwiki", "q2 en", "frwiki", "q2 fr"))) // note "q2 en" not "q2_en"
, fxt.doc_wo_date_(1, "q1", Xob_wdata_pid_base_tst.json_("q1", "links", String_.Ary("enwiki", "q1 en", "frwiki", "q1 fr")))
)
.Fil_expd(ttl_(fxt.Wiki(), "enwiki", "000", 0)
, "!!!!*|!!!!*|"
, "Q1_en|q1"
, "Q2_en|q2" // NOTE: q2_en
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "enwiki", "000")
, "0|Q1_en|Q2_en|2"
, ""
)
.Fil_expd(ttl_(fxt.Wiki(), "frwiki", "000", 0)
, "!!!!*|!!!!*|"
, "Q1_fr|q1"
, "Q2_fr|q2"
, ""
)
.Fil_expd
( reg_(fxt.Wiki(), "frwiki", "000")
, "0|Q1_fr|Q2_fr|2"
, ""
)
.Run_page_wkrs(new Xob_wdata_qid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
;
}
public static Io_url reg_(Xowe_wiki wdata, String wiki, String ns_id) {
return Wdata_idx_wtr.dir_qid_(wdata, wiki, ns_id).GenSubFil(Xotdb_dir_info_.Name_reg_fil);
}
public static Io_url ttl_(Xowe_wiki wdata, String wiki, String ns_id, int fil_id) {
Io_url root = Wdata_idx_wtr.dir_qid_(wdata, wiki, ns_id);
return Xotdb_fsys_mgr.Url_fil(root, fil_id, Xotdb_dir_info_.Bry_xdat);
}
}

44
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_qid_sql.java
Unescape View File

@ -1,44 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.xtns.wbases.dbs.*;
public class Xob_wdata_qid_sql extends Xob_wdata_qid_base {
private Wbase_qid_tbl tbl;
@Override public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_qid;}
@Override public void Qid_bgn() {
Xow_db_file wbase_db = Make_wbase_db(wiki.Db_mgr_as_sql().Core_data_mgr());
tbl = Wbase_qid_tbl.New_make(wbase_db.Conn(), false);
tbl.Create_tbl();
tbl.Insert_bgn();
}
@Override public void Qid_add(byte[] wiki_key, int ns_id, byte[] ttl, byte[] qid) {
tbl.Insert_cmd_by_batch(wiki_key, ns_id, ttl, qid);
}
@Override public void Qid_end() {
tbl.Insert_end();
tbl.Create_idx();
}
public static Xow_db_file Make_wbase_db(Xow_db_mgr db_mgr) {
boolean db_is_all_or_few = db_mgr.Props().Layout_text().Tid_is_all_or_few();
Xow_db_file wbase_db = db_is_all_or_few
? db_mgr.Db__core()
: db_mgr.Dbs__make_by_tid(Xow_db_file_.Tid__wbase);
if (db_is_all_or_few)
db_mgr.Db__wbase_(wbase_db);
return wbase_db;
}
}

107
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_qid_tst.java
Unescape View File

@ -0,0 +1,107 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import org.junit.*; import gplx.core.tests.*;
import gplx.*; import gplx.dbs.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.nss.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.bldrs.wms.sites.*;
public class Xob_wdata_qid_tst {
private Db_conn conn;
private final Xobldr_fxt fxt = new Xobldr_fxt().Ctor_mem();
private Xob_wdata_qid wkr;
private final Gfo_db_tester db_tester = new Gfo_db_tester();
private final Dbmeta_fld_list flds__wbase_qid = new Dbmeta_fld_list().Bld_str("src_wiki").Bld_int("src_ns").Bld_str("src_ttl").Bld_str("trg_ttl");
@Before public void init() {
Io_mgr.Instance.InitEngine_mem();
Db_conn_bldr.Instance.Reg_default_mem();
this.conn = Db_conn_bldr.Instance.New(Io_url_.mem_fil_("mem/db/wbase.xowa"));
this.wkr = new Xob_wdata_qid(conn);
wkr.Ctor(fxt.Bldr(), fxt.Wiki());
}
@Test public void Basic() {
fxt.Run_page_wkr(wkr
, fxt.New_page_wo_date(2, "q2", Wdata_wiki_mgr_fxt.New_json("q2", "links", String_.Ary("enwiki", "q2_en", "frwiki", "q2_fr")))
, fxt.New_page_wo_date(1, "q1", Wdata_wiki_mgr_fxt.New_json("q1", "links", String_.Ary("enwiki", "q1_en", "frwiki", "q1_fr")))
);
db_tester.Test__select_tbl(conn, "wbase_qid", flds__wbase_qid
, Object_.Ary("enwiki", 0, "Q2_en", "q2")
, Object_.Ary("frwiki", 0, "Q2_fr", "q2")
, Object_.Ary("enwiki", 0, "Q1_en", "q1")
, Object_.Ary("frwiki", 0, "Q1_fr", "q1")
);
}
@Test public void Ns() {
Site_core_db json_db = new Site_core_db(fxt.App().Fsys_mgr().Cfg_site_meta_fil());
Site_namespace_tbl ns_tbl = json_db.Tbl__namespace();
ns_tbl.Insert(Bry_.new_a7("en.w"), Xow_ns_.Tid__help, Xow_ns_case_.Bry__1st, Bry_.Empty, Bry_.new_a7("Help"), Bool_.N, Bool_.N, Bry_.Empty);
ns_tbl.Insert(Bry_.new_a7("fr.w"), Xow_ns_.Tid__help, Xow_ns_case_.Bry__1st, Bry_.Empty, Bry_.new_a7("Aide"), Bool_.N, Bool_.N, Bry_.Empty);
fxt.Run_page_wkr(wkr
, fxt.New_page_wo_date(1, "11", Wdata_wiki_mgr_fxt.New_json("q1", "links", String_.Ary("enwiki", "Help:Q1_en", "frwiki", "Aide:Q1_fr")))
);
db_tester.Test__select_tbl(conn, "wbase_qid", flds__wbase_qid
, Object_.Ary("enwiki", 12, "Q1_en", "q1")
, Object_.Ary("frwiki", 12, "Q1_fr", "q1")
);
}
@Test public void Links_w_name() { // PURPOSE: wikidata changed links node from "enwiki:A" to "enwiki:{name:A,badges:[]}"; DATE:2013-09-14
String q1_str = String_.Concat_lines_nl
( "{ \"entity\":\"q1\""
, ", \"links\":"
, " { \"enwiki\":\"q1_en\""
, " , \"frwiki\":\"q1_fr\""
, " }"
, "}"
);
String q2_str = String_.Concat_lines_nl
( "{ \"entity\":[\"item\",2]"
, ", \"links\":"
, " { \"enwiki\":{\"name\":\"q2_en\",\"badges\":[]}"
, " , \"frwiki\":{\"name\":\"q2_fr\",\"badges\":[]}"
, " }"
, "}"
);
fxt.Run_page_wkr(wkr
, fxt.New_page_wo_date(1, "q1", q1_str)
, fxt.New_page_wo_date(2, "q2", q2_str)
);
db_tester.Test__select_tbl(conn, "wbase_qid", flds__wbase_qid
, Object_.Ary("enwiki", 0, "Q1_en", "q1")
, Object_.Ary("frwiki", 0, "Q1_fr", "q1")
, Object_.Ary("enwiki", 0, "Q2_en", "q2")
, Object_.Ary("frwiki", 0, "Q2_fr", "q2")
);
}
@Test public void Spaces() { // PURPOSE: assert that ttls with spaces are converted to unders DATE:2015-04-21
fxt.Run_page_wkr(wkr
, fxt.New_page_wo_date(2, "q2", Wdata_wiki_mgr_fxt.New_json("q2", "links", String_.Ary("enwiki", "q2 en", "frwiki", "q2 fr"))) // note "q2 en" not "q2_en"
, fxt.New_page_wo_date(1, "q1", Wdata_wiki_mgr_fxt.New_json("q1", "links", String_.Ary("enwiki", "q1 en", "frwiki", "q1 fr")))
);
db_tester.Test__select_tbl(conn, "wbase_qid", flds__wbase_qid
, Object_.Ary("enwiki", 0, "Q2_en", "q2") // NOTE: q2_en
, Object_.Ary("frwiki", 0, "Q2_fr", "q2")
, Object_.Ary("enwiki", 0, "Q1_en", "q1")
, Object_.Ary("frwiki", 0, "Q1_fr", "q1")
);
}
}

46
400_xowa/src/gplx/xowa/xtns/wbases/imports/Xob_wdata_qid_txt.java
Unescape View File

@ -1,46 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.xowa.bldrs.*;
public class Xob_wdata_qid_txt extends Xob_wdata_qid_base {
private Wdata_idx_bldr_qid qid_bldr;
@Override public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_tdb_text_wdata_qid;}
@Override public void Qid_bgn() {qid_bldr = new Wdata_idx_bldr_qid().Ctor(this, bldr, wiki, dump_fil_len);}
@Override public void Qid_add(byte[] wiki_key, int ns_id, byte[] ttl, byte[] qid) {
qid_bldr.Add(String_.new_u8(wiki_key), Int_.To_str_pad_bgn_zero(ns_id, 3), ttl, qid);
}
@Override public void Qid_end() {
qid_bldr.Flush();
qid_bldr.Make();
}
}
class Wdata_idx_bldr_qid extends Wdata_idx_mgr_base {
public Wdata_idx_bldr_qid Ctor(Xob_wdata_qid_base wkr, Xob_bldr bldr, Xowe_wiki wiki, int dump_fil_len) {super.Ctor(wkr, bldr, wiki, dump_fil_len); return this;}
public void Add(String wiki_key, String ns_num_str, byte[] ttl, byte[] qid) {
Wdata_idx_wtr wtr = Get_or_new(wiki_key, ns_num_str);
wtr.Write(ttl, qid);
}
private Wdata_idx_wtr Get_or_new(String wiki_key, String ns_num_str) {
String wtr_key = wiki_key + "|" + ns_num_str;
Object rv = hash.Get_by(wtr_key);
if (rv == null) {
Wdata_idx_wtr wtr = Wdata_idx_wtr.new_qid_(wiki, wiki_key, ns_num_str, dump_fil_len);
hash.Add(wtr_key, wtr);
return wtr;
}
return (Wdata_idx_wtr)rv;
}
}

15
400_xowa/src/gplx/xowa/xtns/wbases/imports/json/Xowb_json_dump_db.java
Unescape View File

@ -22,7 +22,7 @@ import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.parsers.*;
class Xowb_json_dump_db {
private final Xoae_app app; private final Gfo_usr_dlg usr_dlg; private final Xowe_wiki wiki; private final Xob_bldr bldr;
private final Json_parser json_parser;
private final Xob_wdata_pid_sql pid_cmd; private final Xob_wdata_qid_sql qid_cmd = new Xob_wdata_qid_sql();
private final Xob_wdata_pid pid_cmd; private final Xob_wdata_qid qid_cmd;
private Xow_ns_mgr ns_mgr; private Xow_db_mgr db_mgr;
private Xowd_page_tbl page_tbl; private Xob_ns_to_db_mgr ns_to_db_mgr;
private Io_stream_zip_mgr text_zip_mgr; private byte text_zip_tid;
@ -32,7 +32,8 @@ class Xowb_json_dump_db {
this.app = bldr.App(); this.usr_dlg = app.Usr_dlg(); this.wiki = wiki; this.bldr = bldr;
this.json_parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
this.ns_mgr = wiki.Ns_mgr();
this.pid_cmd = new Xob_wdata_pid_sql(wiki.Data__core_mgr().Db__wbase().Conn(), null);
this.pid_cmd = new Xob_wdata_pid(wiki.Data__core_mgr().Db__wbase().Conn());
this.qid_cmd = new Xob_wdata_qid(wiki.Data__core_mgr().Db__wbase().Conn());
}
public void Parse_all_bgn(long src_fil_len, String src_fil_name) {
// load wiki
@ -57,7 +58,7 @@ class Xowb_json_dump_db {
this.page_modified_on = Datetime_now.Get();
page_tbl.Insert_bgn();
qid_cmd.Page_wkr__bgn();
pid_cmd.Pid_bgn();
pid_cmd.Pid__bgn();
}
public void Parse_doc(byte[] json_bry) {
// parse to jdoc
@ -78,17 +79,17 @@ class Xowb_json_dump_db {
// insert text
if (jdoc_is_qid) {
qid_cmd.Parse_jdoc(jdoc);
qid_cmd.Qid__run(jdoc);
++page_count_main;
}
else
pid_cmd.Parse_jdoc(jdoc);
pid_cmd.Pid__run(jdoc);
}
public void Parse_all_end() {
page_tbl.Insert_end();
page_tbl.Create_idx();
qid_cmd.Qid_end();
pid_cmd.Pid_end();
qid_cmd.Qid__end();
pid_cmd.Pid__end();
ns_to_db_mgr.Rls_all();
// cleanup core

172
400_xowa/src/gplx/xowa/xtns/wbases/pfuncs/Wdata_pf_property__basic__tst.java
Unescape View File

@ -18,117 +18,189 @@ import org.junit.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*
public class Wdata_pf_property__basic__tst {
@Before public void init() {fxt.Init();} private final Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt();
@Test public void String() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_string(1, "a")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_string(1, "a"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "a");
fxt.Test_parse("{{#property:p2}}", "");
}
@Test public void Entity() {
fxt.Init_links_add("enwiki", "Test_page", "Q1");
fxt.Init__docs__add(fxt.Wdoc_bldr("Q2").Add_label("en", "b").Xto_wdoc());
fxt.Init__docs__add(fxt.doc_("Q1", fxt.Make_claim_entity_qid(1, 2)));
fxt.Init__docs__add(fxt.Wdoc("Q2")
.Add_label("en", "b")
);
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_entity_qid(1, 2))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "b");
}
@Test public void Entity_fr() { // PURPOSE: non-English wiki should default to English label if non-English label not available; DATE:2013-12-19
fxt.Wiki().Wdata_wiki_lang_(Bry_.new_a7("fr")); // set wiki to French
fxt.Init_links_add("frwiki", "Test_page", "Q1"); // create link for en:Test_page in wikidata
fxt.Init__docs__add(fxt.doc_("Q1", fxt.Make_claim_entity_qid(1, 2))); // create wdata page Q1 with prop entity reference to Q2
fxt.Init__docs__add(fxt.Wdoc_bldr("Q2").Add_label("en", "b").Xto_wdoc()); // create wdata page Q2 with label in en (not fr)
fxt.Test_parse("{{#property:p1}}", "b"); // parse; should get en label
// set wiki to French
fxt.Wiki().Wdata_wiki_lang_(Bry_.new_a7("fr"));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_entity_qid(1, 2))
.Add_sitelink("frwiki", "Test_page")
);
// create wdata page Q2 with label in en (not fr)
fxt.Init__docs__add(fxt.Wdoc("Q2")
.Add_label("en", "b")
);
// parse; should get en label
fxt.Test_parse("{{#property:p1}}", "b");
}
@Test public void Entity_missing() { // PURPOSE: wiki may refer to entity that no longer exists; EX: {{#property:p1}} which links to Q1, but p1 links to Q2 and Q2 was deleted; DATE:2014-02-01
fxt.Init_links_add("enwiki", "Test_page", "q1"); // create link for en:Test_page in wikidata
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_entity_qid(1, 2))); // create wdata page Q1 with prop entity reference to Q2; note that Q2 is not created
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_entity_qid(1, 2)) // create wdata page Q1 with prop entity reference to Q2; note that Q2 is not created
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", ""); // parse; get ""
}
@Test public void Time() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_time(1, "2012-01-02 03:04:05")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_time(1, "2012-01-02 03:04:05"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "30405 2 Jan 2012"); // NOTE: format is missing ":" b/c test does not init messages for html_wtr; DATE:2015-08-03
}
@Test public void Geodata() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_geo(1, "6.789", "1.2345")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_geo(1, "6.789", "1.2345"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "1°14&#39;4.2&#34;N, 6°47&#39;20.4&#34;E");
}
@Test public void Quantity__plus_minus__y() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234", "1", "+1236", "+1232")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_quantity(1, "+1234", "1", "+1236", "+1232"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "1,234±2");
}
@Test public void Quantity__plus_minus__n() { // PURPOSE:do not output ± if lbound == val == ubound; PAGE:en.w:Tintinan DATE:2015-08-02
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234", "1", "+1234", "+1234")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_quantity(1, "+1234", "1", "+1234", "+1234"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "1,234");
}
@Test public void Quantity__range() { // PURPOSE:do not output ± if lbound == val == ubound; PAGE:en.w:Tintinan DATE:2015-08-02
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234", "1", "+1236", "+1233")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_quantity(1, "+1234", "1", "+1236", "+1233"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "1,233-1,236");
}
@Test public void Quantity__long() { // PURPOSE: must cast to long for large numbers; EX:{{#property:P1082}} PAGE:en.w:Earth; DATE:2015-08-02
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+4321000000", "1", "4321000000", "4321000000")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_quantity(1, "+4321000000", "1", "4321000000", "4321000000"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "4,321,000,000");
}
@Test public void Quantity__unit__entity() {// PURPOSE: get entity name; EX:{{#invoke:Wikidata|getUnits|P2386|FETCH_WIKIDATA}} PAGE:en.w:Arecibo_Observatory; DATE:2016-10-11
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234", "http://www.wikidata.org/entity/q2", "+1236", "+1232")));
Wdata_doc wdoc = fxt.doc_("q2", fxt.Make_claim_string(2, "a"));
wdoc.Label_list().Add(Bry_.new_a7("en"), new gplx.xowa.xtns.wbases.core.Wdata_langtext_itm(Bry_.new_a7("en"), Bry_.new_a7("meter")));
fxt.Init__docs__add(wdoc);
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_quantity(1, "+1234", "http://www.wikidata.org/entity/Q2", "+1236", "+1232"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Init__docs__add(fxt.Wdoc("Q2")
.Add_claims(fxt.Make_claim_string(2, "a"))
.Add_label("en", "meter")
);
fxt.Test_parse("{{#property:p1}}", "1,234±2 meter");
}
@Test public void Quantity__decimal() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234.50", "1", "+1236.75", "+1232.25")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_quantity(1, "+1234.50", "1", "+1236.75", "+1232.25"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "1,234.5±2.25");
}
@Test public void Monolingualtext() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_monolingual(1, "la", "Lorem ipsum dolor sit amet")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_monolingual(1, "la", "Lorem ipsum dolor sit amet"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "Lorem ipsum dolor sit amet");
}
@Test public void Novalue() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_novalue(1)));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_novalue(1))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "novalue");
}
@Test public void Somevalue() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_somevalue(1)));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_somevalue(1))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "somevalue");
}
@Test public void Multiple() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_string(1, "a"), fxt.Make_claim_string(1, "b")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_string(1, "a"), fxt.Make_claim_string(1, "b"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1}}", "a"); // only take first; DATE:2015-08-02
}
@Test public void Q() {
fxt.Init_links_add("enwiki", "Test_page", "q2");
fxt.Init__docs__add(fxt.doc_("q2", fxt.Make_claim_string(1, "a")));
fxt.Test_parse("{{#property:p1|q=q2}}", "a");
fxt.Init__docs__add(fxt.Wdoc("Q2")
.Add_claims(fxt.Make_claim_string(1, "a"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#property:p1|q=Q2}}", "a");
}
@Test public void Of() {
fxt.Init_links_add("enwiki", "Of_page", "q2");
fxt.Init__docs__add(fxt.doc_("q2", fxt.Make_claim_string(1, "a")));
fxt.Init__docs__add(fxt.Wdoc("Q2")
.Add_claims(fxt.Make_claim_string(1, "a"))
.Add_sitelink("enwiki", "Of_page")
);
fxt.Test_parse("{{#property:p1|of=Of_page}}", "a");
}
@Test public void From() {
fxt.Init__docs__add(fxt.doc_("Property:p2", fxt.Make_claim_string(1, "a")));
fxt.Test_parse("{{#property:p1|from=p2}}", "a");
fxt.Init__docs__add(fxt.Wdoc("Property:P2")
.Add_claims(fxt.Make_claim_string(1, "a"))
);
fxt.Test_parse("{{#property:p1|from=P2}}", "a");
fxt.Test_parse("{{#property:p1|from=}}", "");
}
@Test public void Pid_as_name() {
fxt.Init_links_add("enwiki", "Test_page", "q2");
fxt.Init__docs__add(fxt.Wdoc("Q2")
.Add_claims(fxt.Make_claim_string(1, "a"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Init_pids_add("en", "astronomic symbol", 1);
fxt.Init__docs__add(fxt.doc_("q2", fxt.Make_claim_string(1, "a")));
fxt.Test_parse("{{#property:astronomic symbol}}", "a");
}
@Test public void Empty_arg() { // PURPOSE: {{#property:p1|}} should not fail / warn; DATE:2013-11-15
fxt.Init_links_add("enwiki", "Test_page", "q2");
fxt.Init__docs__add(fxt.Wdoc("Q2")
.Add_claims(fxt.Make_claim_string(1, "a"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Init_pids_add("en", "astronomic symbol", 1);
fxt.Init__docs__add(fxt.doc_("q2", fxt.Make_claim_string(1, "a")));
fxt.Test_parse("{{#property:p1|}}", "a");
}
@Test public void Parse_pid() {

19
400_xowa/src/gplx/xowa/xtns/wbases/pfuncs/Wdata_pf_statements__basic__tst.java
Unescape View File

@ -18,14 +18,23 @@ import org.junit.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*
public class Wdata_pf_statements__basic__tst {
@Before public void init() {fxt.Init();} private final Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt();
@Test public void String() {
fxt.Init_links_add("enwiki", "Test_page", "q1");
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_string(1, "a")));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_string(1, "a"))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Test_parse("{{#statements:p1}}", "a");
}
@Test public void Entity() {
fxt.Init_links_add("enwiki", "Test_page", "Q1");
fxt.Init__docs__add(fxt.Wdoc_bldr("Q2").Add_label("en", "b").Xto_wdoc());
fxt.Init__docs__add(fxt.doc_("Q1", fxt.Make_claim_entity_qid(1, 2)));
fxt.Init__docs__add(fxt.Wdoc("Q1")
.Add_claims(fxt.Make_claim_entity_qid(1, 2))
.Add_sitelink("enwiki", "Test_page")
);
fxt.Init__docs__add(fxt.Wdoc("Q2")
.Add_label("en", "b")
);
fxt.Test_parse("{{#statements:p1}}", "[[b]]");
}
}

Write Preview
Loading…
Cancel
Save