mirror of
https://github.com/gnosygnu/xowa.git
synced 2024-10-27 20:34:16 +00:00
General: Remove text database support
This commit is contained in:
gnosygnu
parent
fe0d1abde5
commit
be3979c5af
@ -152,9 +152,9 @@ public class Gftest {
|
||||
if (idx < len) {
|
||||
switch (type_id) {
|
||||
case Type_ids_.Id__bool: bfr.Add_yn(Bool_.Cast(Array_.Get_at(ary, idx))); break;
|
||||
case Type_ids_.Id__bry: bfr.Add_safe((byte[])Array_.Get_at(ary, idx)); break;
|
||||
case Type_ids_.Id__bry: bfr.Add_safe((byte[])Array_.Get_at(ary, idx)); break;
|
||||
case Type_ids_.Id__long: bfr.Add_long_variable(Long_.cast(Array_.Get_at(ary, idx))); break;
|
||||
case Type_ids_.Id__int: bfr.Add_int_variable(Int_.Cast(Array_.Get_at(ary, idx))); break;
|
||||
case Type_ids_.Id__int: bfr.Add_int_variable(Int_.Cast(Array_.Get_at(ary, idx))); break;
|
||||
case Type_ids_.Id__byte: bfr.Add_int_variable((int)(Byte_.Cast(Array_.Get_at(ary, idx)))); break;
|
||||
default: throw Err_.new_unhandled_default(type_id);
|
||||
}
|
||||
@ -177,9 +177,9 @@ public class Gftest {
|
||||
else {
|
||||
switch (tid) {
|
||||
case Type_ids_.Id__bool: eq = Bool_.Cast(expd_obj) == Bool_.Cast(actl_obj); break;
|
||||
case Type_ids_.Id__bry: eq = Bry_.Eq((byte[])expd_obj, (byte[])actl_obj); break;
|
||||
case Type_ids_.Id__bry: eq = Bry_.Eq((byte[])expd_obj, (byte[])actl_obj); break;
|
||||
case Type_ids_.Id__long: eq = Long_.cast(expd_obj) == Long_.cast(actl_obj); break;
|
||||
case Type_ids_.Id__int: eq = Int_.Cast(expd_obj) == Int_.Cast(actl_obj); break;
|
||||
case Type_ids_.Id__int: eq = Int_.Cast(expd_obj) == Int_.Cast(actl_obj); break;
|
||||
case Type_ids_.Id__byte: eq = Byte_.Cast(expd_obj) == Byte_.Cast(actl_obj); break;
|
||||
}
|
||||
}
|
||||
|
||||
@ -40,6 +40,7 @@ public class Dbmeta_fld_list {
|
||||
public String Add_str_null(String name, int len) {return Add(Dbmeta_fld_itm.new_str(name, len).Nullable_y_());}
|
||||
public String Add_str_dflt(String name, int len, String dflt)
|
||||
{return Add(Dbmeta_fld_itm.new_str(name, len).Default_(dflt));}
|
||||
public Dbmeta_fld_list Bld_int(String name) {Add_int(name); return this;}
|
||||
public Dbmeta_fld_list Bld_str(String name) {return Bld_str(name, 255);}
|
||||
public Dbmeta_fld_list Bld_str(String name, int len){Add_str(name, len); return this;}
|
||||
|
||||
|
||||
@ -65,7 +65,21 @@ public class Dbmeta_fld_tid {
|
||||
default: throw Err_.new_unhandled(tid);
|
||||
}
|
||||
}
|
||||
public static Dbmeta_fld_itm To_itm(String raw) {
|
||||
return null;
|
||||
public static int Get_by_obj(Object o) {
|
||||
int type_id = Type_ids_.To_id_by_obj(o);
|
||||
switch (type_id) {
|
||||
case Type_ids_.Id__bool: return Dbmeta_fld_tid.Tid__bool;
|
||||
case Type_ids_.Id__byte: return Dbmeta_fld_tid.Tid__byte;
|
||||
case Type_ids_.Id__short: return Dbmeta_fld_tid.Tid__short;
|
||||
case Type_ids_.Id__int: return Dbmeta_fld_tid.Tid__int;
|
||||
case Type_ids_.Id__long: return Dbmeta_fld_tid.Tid__long;
|
||||
case Type_ids_.Id__float: return Dbmeta_fld_tid.Tid__float;
|
||||
case Type_ids_.Id__double: return Dbmeta_fld_tid.Tid__double;
|
||||
case Type_ids_.Id__str: return Dbmeta_fld_tid.Tid__str;
|
||||
case Type_ids_.Id__bry: return Dbmeta_fld_tid.Tid__bry;
|
||||
case Type_ids_.Id__date: return Dbmeta_fld_tid.Tid__date;
|
||||
case Type_ids_.Id__decimal: return Dbmeta_fld_tid.Tid__decimal;
|
||||
default: throw Err_.new_unhandled_default(type_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -30,7 +30,9 @@ public class Io_stream_rdr_process implements Io_stream_rdr {
|
||||
ProcessBuilder pb = new ProcessBuilder(process_args);
|
||||
pb.redirectErrorStream(false);
|
||||
try {process = pb.start();}
|
||||
catch (Exception e) {throw Err_.new_exc(e, "core", "process init failed", "args", String_.AryXtoStr(process_args));}
|
||||
catch (Exception e) {
|
||||
throw Err_.new_exc(e, "core", "process init failed", "args", String_.AryXtoStr(process_args));
|
||||
}
|
||||
stream_read = process.getInputStream();
|
||||
return this;
|
||||
}
|
||||
|
||||
@ -1,221 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
//namespace gplx.xowa.addons.wikis.searchs.v1s {
|
||||
// import org.junit.*; using gplx.xowa.wikis.tdbs; using gplx.xowa.wikis.data.tbls;
|
||||
// public class Xosrh_core_tst {
|
||||
// @Before public void Init() {fxt.Clear();} private Xos_search_mgr_fxt fxt = new Xos_search_mgr_fxt();
|
||||
// @Test public void Basic() {
|
||||
// fxt.Init_basic();
|
||||
// fxt.Test_search_exact("b2", "B2_22", "B2_12", "B2__2");
|
||||
// fxt.Test_search_exact("a" , "A___0");
|
||||
// fxt.Test_search_exact("b1a"); // missing: mid
|
||||
// fxt.Test_search_exact("d"); // missing: end
|
||||
// fxt.Test_search_exact("$"); // missing: bgn
|
||||
// fxt.Test_search_match_bgn("b*", "B3_23", "B2_22", "B1_21", "B3_13", "B2_12", "B1_11", "B3__3", "B2__2", "B1__1");
|
||||
// }
|
||||
// @Test public void Page_size() {
|
||||
// fxt.Init_basic();
|
||||
// fxt.Search_mgr().Page_mgr().Itms_per_page_(1);
|
||||
// fxt.Test_search("b*", 0, "B3_23");
|
||||
// fxt.Test_search("b*", 1, "B2_22");
|
||||
// fxt.Test_search("b*", 2, "B1_21");
|
||||
// fxt.Test_search("b*", 3, "B3_13");
|
||||
// }
|
||||
// @Test public void Url() {
|
||||
// Xoa_url url = Xow_url_parser_old.Parse_url(fxt.App(), fxt.Wiki(), "Special:Search/Abc?fulltext=y&xowa_sort=len_desc");
|
||||
// fxt.Search_mgr().Args_mgr().Clear().Parse(url.Args());
|
||||
// Tfds.Eq(Srch_rslt_row_sorter.Tid_len_dsc, fxt.Search_mgr().Args_mgr().Sort_tid());
|
||||
// }
|
||||
// @Test public void Url_arg_title() {// http://en.wikipedia.org/wiki/Special:Search/Earth?fulltext=yes&title=Mars
|
||||
// fxt.Test_url_search_bry("Special:Search?fulltext=y&search=Abc" , "Abc"); // query arg
|
||||
//// fxt.Test_url_search_bry("Special:Search/Abc?fulltext=y" , "Abc"); // leaf
|
||||
// fxt.Test_url_search_bry("Special:Search/Abc?fulltext=y&search=Def" , "Def"); // leaf overrides query arg
|
||||
// }
|
||||
// @Test public void Url_ns() {
|
||||
// fxt.Test_url__ns("Special:Search?search=Abc&ns0=1&ns1=1", "0|1");
|
||||
// fxt.Test_url__ns("Special:Search?search=Abc&ns*=1", "*");
|
||||
// fxt.Test_url__ns("Special:Search?search=Abc", "0");
|
||||
// }
|
||||
// @Test public void Html() {
|
||||
// fxt.Init_basic();
|
||||
// fxt.Test_html_by_url("B1", "", String_.Concat_lines_nl
|
||||
// ( "Result '''1''' of '''3''' for '''B1'''<br/>"
|
||||
// , "{|"
|
||||
// , "|-"
|
||||
// , "| [[Special:Search/B1?fulltext=y&xowa_page_index=0|<]]"
|
||||
// , "| [[Special:Search/B1?fulltext=y&xowa_page_index=1|>]]"
|
||||
// , "|-"
|
||||
// , "| [[Special:Search/B1?fulltext=y&xowa_sort=len_desc|length]]"
|
||||
// , "| [[Special:Search/B1?fulltext=y&xowa_sort=title_asc|title]]"
|
||||
// , "|-"
|
||||
// , "| 42 || [[B1 21]]"
|
||||
// , "|-"
|
||||
// , "| 22 || [[B1 11]]"
|
||||
// , "|-"
|
||||
// , "| 2 || [[B1 1]]"
|
||||
// , "|-"
|
||||
// , "| [[Special:Search/B1?fulltext=y&xowa_page_index=0|<]]"
|
||||
// , "| [[Special:Search/B1?fulltext=y&xowa_page_index=1|>]]"
|
||||
// , "|}"
|
||||
// ));
|
||||
// }
|
||||
//// @Test public void Page_next() {
|
||||
//// fxt.Init_basic();
|
||||
//// fxt.Search_mgr().Page_size_(1);
|
||||
//// fxt.Test_search(Srch_special_page.Match_tid_all, "B1", 0, "B1 1");
|
||||
//// fxt.Test_search(Srch_special_page.Match_tid_all, "B1", 1, "B1 11");
|
||||
//// }
|
||||
//// @Test public void Misc_url() {
|
||||
//// fxt.Init_basic();
|
||||
//// fxt.Search_mgr().Page_size_(1);
|
||||
//// fxt.Expd_address_page_("Special:Search/B1");
|
||||
//// fxt.Test_search(Srch_special_page.Match_tid_all, "B1", 0, "B1 1");
|
||||
//// }
|
||||
// @Test public void Sort_defaults_to_len_desc() {
|
||||
// fxt.Init_basic();
|
||||
// fxt.Search_mgr().Page_mgr().Itms_per_page_(3);
|
||||
// fxt.Test_search2(Srch_special_page.Match_tid_bgn, "b" , 0, Srch_rslt_row_sorter.Tid_ttl_asc , "B1_11", "B1_21", "B1__1"); // sort by name; note that _ sorts after alphabet
|
||||
// fxt.Test_search2(Srch_special_page.Match_tid_bgn, "b" , 1, Srch_rslt_row_sorter.Tid_none , "B2_12", "B2_22", "B2__2"); // sort by name still; next page should not reset
|
||||
// fxt.Test_search2(Srch_special_page.Match_tid_bgn, "b2" , 0, Srch_rslt_row_sorter.Tid_none , "B2_22", "B2_12", "B2__2"); // sort by len desc; new search should reset
|
||||
// }
|
||||
// }
|
||||
// class Xos_search_mgr_fxt {
|
||||
// Xoae_app app; Xowe_wiki wiki; Bry_bfr bfr = Bry_bfr_.Reset(500); Srch_special_page search_mgr;
|
||||
// public Xoae_app App() {return app;}
|
||||
// public Xowe_wiki Wiki() {return wiki;}
|
||||
// public Xobl_regy_itm regy_itm_(int id, String bgn, String end, int count) {return new Xobl_regy_itm(id, Bry_.new_u8(bgn), Bry_.new_u8(end), count);}
|
||||
// public Xowd_page_itm data_ttl_(int id, String ttl) {return data_ttl_(id, 0, 0, false, 0, ttl);}
|
||||
// public Xowd_page_itm data_ttl_(int id, int fil, int row, boolean redirect, int len, String ttl) {return new Xowd_page_itm().Init(id, Bry_.new_u8(ttl), redirect, len, fil, row);}
|
||||
// public Xowd_page_itm data_id_(int id, String ttl) {return data_id_(id, Xow_ns_.Tid__main, ttl);}
|
||||
// public Xowd_page_itm data_id_(int id, int ns, String ttl) {return new Xowd_page_itm().Id_(id).Ns_id_(ns).Ttl_page_db_(Bry_.new_u8(ttl)).Text_db_id_(0).Text_len_(0);}
|
||||
// public Xobl_search_ttl data_sttl_(String word, params int[] ids) {return new Xobl_search_ttl(Bry_.new_u8(word), data_ttl_word_page_ary_(ids));}
|
||||
// public Xobl_search_ttl_page[] data_ttl_word_page_ary_(params int[] ids) {
|
||||
// int ids_len = ids.length;
|
||||
// Xobl_search_ttl_page[] rv = new Xobl_search_ttl_page[ids_len];
|
||||
// for (int i = 0; i < ids_len; i++) {
|
||||
// int id = ids[i];
|
||||
// rv[i] = new Xobl_search_ttl_page(id, id * 2);
|
||||
// }
|
||||
// return rv;
|
||||
// }
|
||||
// public void Init_regy_site(byte dir_info, params Xobl_regy_itm[] ary) {Init_regy(wiki.Tdb_fsys_mgr().Url_site_reg(dir_info), ary);}
|
||||
// public void Init_regy_ns (String ns_num, byte tid, params Xobl_regy_itm[] ary) {Init_regy(wiki.Tdb_fsys_mgr().Url_ns_reg(ns_num, tid), ary);}
|
||||
// public void Init_regy(Io_url url, Xobl_regy_itm[] ary) {
|
||||
// int ary_len = ary.length;
|
||||
// for (int i = 0; i < ary_len; i++) {
|
||||
// Xobl_regy_itm itm = ary[i];
|
||||
// itm.Srl_save(tmp_bfr);
|
||||
// }
|
||||
// Io_mgr.Instance.SaveFilBfr(url, tmp_bfr);
|
||||
// } private Bry_bfr tmp_bfr = Bry_bfr_.Reset(255);
|
||||
// public void Init_data(Io_url fil, params Xobl_data_itm[] ary) {
|
||||
// Xob_xdat_file xdat_file = new Xob_xdat_file();
|
||||
// int ary_len = ary.length;
|
||||
// for (int i = 0; i < ary_len; i++) {
|
||||
// Xobl_data_itm itm = ary[i];
|
||||
// itm.Srl_save(tmp_bfr);
|
||||
// xdat_file.Insert(bfr, tmp_bfr.To_bry_and_clear());
|
||||
// }
|
||||
// xdat_file.Save(fil);
|
||||
// }
|
||||
// public void Init_basic() {
|
||||
// this.Init_regy_ns(wiki.Ns_mgr().Ns_main().Num_str(), Xotdb_dir_info_.Tid_search_ttl, this.regy_itm_(0, "A", "C", 5));
|
||||
// this.Init_data(wiki.Tdb_fsys_mgr().Url_ns_fil(Xotdb_dir_info_.Tid_search_ttl, Xow_ns_.Tid__main, 0)
|
||||
// , this.data_sttl_("a" , 0)
|
||||
// , this.data_sttl_("b1" , 1, 11, 21)
|
||||
// , this.data_sttl_("b2" , 2, 12, 22)
|
||||
// , this.data_sttl_("b3" , 3, 13, 23)
|
||||
// , this.data_sttl_("c" , 4)
|
||||
// );
|
||||
// this.Init_regy_site(Xotdb_dir_info_.Tid_id, this.regy_itm_(0, "A", "C", 11));
|
||||
// this.Init_data(wiki.Tdb_fsys_mgr().Url_site_fil(Xotdb_dir_info_.Tid_id, 0)
|
||||
// , this.data_id_( 0, "A___0")
|
||||
// , this.data_id_( 1, "B1__1")
|
||||
// , this.data_id_( 2, "B2__2")
|
||||
// , this.data_id_( 3, "B3__3")
|
||||
// , this.data_id_( 4, "C___4")
|
||||
// , this.data_id_(11, "B1_11")
|
||||
// , this.data_id_(12, "B2_12")
|
||||
// , this.data_id_(13, "B3_13")
|
||||
// , this.data_id_(21, "B1_21")
|
||||
// , this.data_id_(22, "B2_22")
|
||||
// , this.data_id_(23, "B3_23")
|
||||
// );
|
||||
// search_mgr.Page_mgr().Ns_mgr().Add_all(); // WORKAROUND: xdat fmt does not store ns with search data; pages will be retrieved with ns_id = null; force ns_all (instead of allowing ns_main default);
|
||||
// }
|
||||
// public void Clear() {
|
||||
// Io_mgr.Instance.InitEngine_mem();
|
||||
// app = Xoa_app_fxt.Make__app__edit();
|
||||
// wiki = Xoa_app_fxt.Make__wiki__edit(app);
|
||||
// search_mgr = wiki.Special_mgr().Page_search();
|
||||
// wiki.Appe().Gui_mgr().Search_suggest_mgr().Args_default_str_("ns*=1"); // WORKAROUND: xdat fmt does not store ns with search data; pages will be retrieved with ns_id = null; force ns_all (instead of allowing ns_main default);
|
||||
// }
|
||||
// public Srch_special_page Search_mgr() {return search_mgr;}
|
||||
// public void Test_url_search_bry(String url_str, String expd) {
|
||||
// Xoa_url url = Xow_url_parser_old.Parse_url(app, wiki, url_str);
|
||||
// search_mgr.Args_mgr().Clear().Parse(url.Args());
|
||||
// Tfds.Eq(expd, String_.new_u8(search_mgr.Args_mgr().Search_bry()));
|
||||
// }
|
||||
// public void Test_url__ns(String url_str, String expd) {
|
||||
// Xoa_url url = Xow_url_parser_old.Parse_url(app, wiki, url_str);
|
||||
// search_mgr.Args_mgr().Clear().Parse(url.Args());
|
||||
// Tfds.Eq(expd, String_.new_a7(search_mgr.Args_mgr().Ns_mgr().Xto_hash_key()));
|
||||
// }
|
||||
// public void Test_search_exact(String ttl_str, params String[] expd_ary) {Test_search(ttl_str, 0, expd_ary);}
|
||||
// public void Test_search_match_bgn(String ttl_str, params String[] expd_ary) {Test_search(ttl_str, 0, expd_ary);}
|
||||
// public void Test_search(String ttl_str, int page_idx, params String[] expd_ary) {
|
||||
// byte[] ttl_bry = Bry_.new_a7(ttl_str);
|
||||
// Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b128();
|
||||
// Xosrh_rslt_grp page = search_mgr.Page_mgr().Search(bfr, wiki, ttl_bry, page_idx, search_mgr.Page_mgr());
|
||||
// bfr.Mkr_rls();
|
||||
// Tfds.Eq_ary(expd_ary, Search_itms_to_int_ary(page));
|
||||
// }
|
||||
// public void Test_html_by_url(String ttl_str, String args_str, String expd_html) {
|
||||
// wiki.Init_needed_(false);
|
||||
// byte[] ttl_bry = Bry_.new_a7(ttl_str);
|
||||
// Xoa_ttl ttl = Xoa_ttl.Parse(wiki, ttl_bry);
|
||||
// Xoae_page page = Xoae_page.New_test(wiki, ttl);
|
||||
// byte[] url_bry = Bry_.new_a7("http://en.wikipedia.org/wiki/Special:Search/" + ttl_str + args_str);
|
||||
// Xoa_url url = wiki.Appe().Url_parser().Parse(url_bry);
|
||||
// search_mgr.Special__gen(url, page, wiki, ttl);
|
||||
// Tfds.Eq_str_lines(expd_html, String_.new_u8(page.Root().Data_htm()));
|
||||
// }
|
||||
// public void Test_search2(byte match_tid, String ttl_str, int page_idx, byte sort_tid, params String[] expd_ary) {
|
||||
// Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b128();
|
||||
// Xow_url_parser_old url_parser = new Xow_url_parser_old();
|
||||
// byte[] url_raw = Bry_.new_a7("Special:Search/" + ttl_str + ((match_tid == Srch_special_page.Match_tid_all) ? "" : "*") + "?fulltext=y" + Srch_rslt_row_sorter.Xto_url_arg(sort_tid) + "&xowa_page_size=1&xowa_page_index=" + page_idx);
|
||||
// Xoa_url url = url_parser.Parse(url_raw);
|
||||
// Xoa_ttl ttl = Xoa_ttl.Parse(wiki, url_raw);
|
||||
// Xoae_page page = wiki.Ctx().Page();
|
||||
// search_mgr.Special__gen(url, page, wiki, ttl);
|
||||
// Xosrh_rslt_grp cur_grp = search_mgr.Cur_grp();
|
||||
// bfr.Mkr_rls();
|
||||
// Tfds.Eq_ary(expd_ary, Search_itms_to_int_ary(cur_grp));
|
||||
// }
|
||||
// String[] Search_itms_to_int_ary(Xosrh_rslt_grp page) {
|
||||
// int itms_len = page.Itms_len();
|
||||
// String[] rv = new String[itms_len];
|
||||
// for (int i = 0; i < itms_len; i++) {
|
||||
// Xowd_page_itm itm = page.Itms_get_at(i);
|
||||
// rv[i] = String_.new_u8(itm.Ttl_page_db());
|
||||
// }
|
||||
// return rv;
|
||||
// }
|
||||
// }
|
||||
// interface Xobl_data_itm {
|
||||
// void Srl_save(Bry_bfr bfr);
|
||||
// }
|
||||
//}
|
||||
@ -1,92 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
|
||||
import gplx.core.primitives.*; import gplx.core.strings.*;
|
||||
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.addons.wikis.ctgs.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.infos.*;
|
||||
import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*;
|
||||
public class Db_mgr_fxt {
|
||||
public Db_mgr_fxt Ctor_fsys() {bldr_fxt = new Xob_fxt().Ctor(Xoa_test_.Url_root().GenSubDir("root")); return this;}
|
||||
public Db_mgr_fxt Ctor_mem() {bldr_fxt = new Xob_fxt().Ctor_mem(); return this;} private Xob_fxt bldr_fxt;
|
||||
public Xowd_page_itm page_(int id, String modified_on, boolean type_redirect, int text_len) {return new Xowd_page_itm().Id_(id).Modified_on_(DateAdp_.parse_gplx(modified_on)).Redirected_(type_redirect).Text_len_(text_len);}
|
||||
public Xowe_wiki Wiki() {return bldr_fxt.Wiki();}
|
||||
public Xob_bldr Bldr() {return bldr_fxt.Bldr();}
|
||||
public Db_mgr_fxt doc_ary_(Xowd_page_itm... v) {bldr_fxt.doc_ary_(v); return this;}
|
||||
public Xowd_page_itm doc_(int id, String date, String title, String text) {return bldr_fxt.doc_(id, date, title, text);}
|
||||
public Xowd_page_itm doc_wo_date_(int id, String title, String text) {return bldr_fxt.doc_(id, "2012-01-02 03:04", title, text);}
|
||||
public Xowd_page_itm doc_ttl_(int id, String title) {return bldr_fxt.doc_(id, "2012-01-02 03:04", title, "IGNORE");}
|
||||
public Db_mgr_fxt Init_fil(String url, String raw) {return Init_fil(Io_url_.new_fil_(url), raw);}
|
||||
public Db_mgr_fxt Init_fil(Io_url url, String raw) {Io_mgr.Instance.SaveFilStr(url, raw); return this;}
|
||||
public Db_mgr_fxt Exec_run(Xob_page_wkr wkr) {bldr_fxt.Run_page_wkrs(wkr); return this;}
|
||||
public Db_mgr_fxt Exec_run(Xob_cmd cmd) {bldr_fxt.Run_cmds(cmd); return this;}
|
||||
public Db_mgr_fxt Exec_run(Xobd_parser_wkr wkr) {bldr_fxt.Run(wkr); return this;}
|
||||
public void Init_page_insert(Int_obj_ref page_id_next, int ns_id, String[] ttls) {
|
||||
Xowe_wiki wiki = this.Wiki();
|
||||
int len = ttls.length;
|
||||
DateAdp modified_on = Datetime_now.Dflt_add_min_(0);
|
||||
Xowd_page_tbl tbl_page = wiki.Db_mgr_as_sql().Core_data_mgr().Tbl__page();
|
||||
tbl_page.Insert_bgn();
|
||||
for (int i = 0; i < len; i++) {
|
||||
String ttl = ttls[i];
|
||||
int page_id = page_id_next.Val();
|
||||
tbl_page.Insert_cmd_by_batch(page_id, ns_id, Bry_.new_u8(ttl), false, modified_on, 0, page_id, 0, 0, -1);
|
||||
page_id_next.Val_add(1);
|
||||
}
|
||||
tbl_page.Insert_end();
|
||||
}
|
||||
public void Test_load_ttl(int ns_id, String ttl_str, Xowd_page_itm expd) {
|
||||
Xowe_wiki wiki = bldr_fxt.Wiki();
|
||||
Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
|
||||
byte[] ttl_bry = Bry_.new_a7(ttl_str);
|
||||
wiki.Db_mgr_as_sql().Load_mgr().Load_by_ttl(actl, ns, ttl_bry);
|
||||
Tfds.Eq(expd.Id(), actl.Id());
|
||||
Tfds.Eq_date(expd.Modified_on(), actl.Modified_on());
|
||||
Tfds.Eq(expd.Redirected(), actl.Redirected());
|
||||
Tfds.Eq(expd.Text_len(), actl.Text_len());
|
||||
} private Xowd_page_itm actl = new Xowd_page_itm();
|
||||
public void Test_load_page(int ns_id, int page_id, String expd) {
|
||||
Xowe_wiki wiki = bldr_fxt.Wiki();
|
||||
Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(ns_id);
|
||||
wiki.Db_mgr_as_sql().Load_mgr().Load_page(actl.Id_(page_id), ns);
|
||||
Tfds.Eq(expd, String_.new_a7(actl.Text()));
|
||||
}
|
||||
int[] Xto_int_ary(List_adp rslts) {
|
||||
int len = rslts.Count();
|
||||
int[] rv = new int[len];
|
||||
for (int i = 0; i < len; i++) {
|
||||
Xowd_page_itm page = (Xowd_page_itm)rslts.Get_at(i);
|
||||
rv[i] = page.Id();
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public void Test_file(String url, String expd) {
|
||||
String actl = Io_mgr.Instance.LoadFilStr(url);
|
||||
Tfds.Eq_str_lines(expd, actl);
|
||||
}
|
||||
public void Init_db_sqlite() {
|
||||
Xowe_wiki wiki = this.Wiki();
|
||||
Db_conn_pool.Instance.Rls_all();
|
||||
Db_conn_bldr.Instance.Reg_default_sqlite();
|
||||
Io_mgr.Instance.DeleteDir_cmd(wiki.Fsys_mgr().Root_dir()).MissingIgnored_().Exec();
|
||||
wiki.Db_mgr_create_as_sql().Core_data_mgr().Init_by_make(Xowd_core_db_props.Test, Xob_info_session.Test);
|
||||
Io_mgr.Instance.SaveFilStr(wiki.Import_cfg().Src_dir().GenSubFil("a.xml"), "<test/>");
|
||||
}
|
||||
public void Rls() {
|
||||
this.Wiki().Db_mgr_as_sql().Core_data_mgr().Rls();
|
||||
}
|
||||
}
|
||||
@ -36,12 +36,6 @@ public class Xob_cmd_keys {
|
||||
, Key_diff_merge = "diff.merge"
|
||||
, Key_text_delete_page = "text.delete_page"
|
||||
|
||||
, Key_tdb_text_init = "tdb.text.init" // "core.init"
|
||||
, Key_tdb_make_page = "tdb.text.page" // "core.make_page"
|
||||
, Key_tdb_make_id = "core.make_id"
|
||||
, Key_tdb_calc_stats = "core.calc_stats"
|
||||
, Key_tdb_text_wdata_qid = "tdb.text.wdata.qid"
|
||||
, Key_tdb_text_wdata_pid = "tdb.text.wdata.pid"
|
||||
, Key_exec_sql = "import.sql.exec_sql"
|
||||
, Key_decompress_bz2 = "core.decompress_bz2"
|
||||
;
|
||||
|
||||
@ -49,16 +49,11 @@ public class Xob_cmd_mgr implements Gfo_invk {
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_util_delete)) return Add(new Xob_delete_cmd(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_util_download)) return Add(new Xob_download_cmd(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_util_xml_dump)) return Add(new Xob_xml_dumper_cmd(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_qid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_qid_sql().Ctor(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_sql(null, wiki).Ctor(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_qid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_qid(null).Ctor(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid(null).Ctor(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_db)) return Add(new Xob_wdata_db_cmd(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_site_meta)) return Add(new Xob_site_meta_cmd(bldr, wiki));
|
||||
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_init)) return Add(new Xob_init_tdb(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_id)) return Xml_rdr_direct_add(wiki, new Xob_make_id_wkr(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_calc_stats)) return Add(new Xob_calc_stats_cmd(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_qid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_qid_txt().Ctor(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_txt().Ctor(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_diff_build)) return Add(new Xob_diff_build_cmd(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_exec_sql)) return Add(new Xob_exec_sql_cmd(bldr, wiki));
|
||||
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_decompress_bz2)) return Add(new Xob_decompress_bz2_cmd(bldr, wiki));
|
||||
|
||||
@ -1,154 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
|
||||
import gplx.core.tests.*; import gplx.core.ios.*; import gplx.core.times.*;
|
||||
import gplx.dbs.*; import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*;
|
||||
import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xob_fxt {
|
||||
private final Tst_mgr tst_mgr = new Tst_mgr();
|
||||
private final DateAdp_parser dateParser = DateAdp_parser.new_();
|
||||
public Xob_fxt Ctor_mem() {
|
||||
Io_mgr.Instance.InitEngine_mem();
|
||||
return Ctor(Io_url_.mem_dir_("mem/xowa/"));
|
||||
}
|
||||
public Xob_fxt Ctor(Io_url root_dir) {
|
||||
Db_conn_bldr.Instance.Reg_default_sqlite();
|
||||
app = Xoa_app_fxt.Make__app__edit("linux", root_dir);
|
||||
wiki = Xoa_app_fxt.Make__wiki__edit(app);
|
||||
bldr = Xoa_app_fxt.bldr_(app);
|
||||
return this;
|
||||
}
|
||||
public Xoae_app App() {return app;} private Xoae_app app;
|
||||
public Xob_bldr Bldr() {return bldr;} private Xob_bldr bldr;
|
||||
public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki;
|
||||
public Xob_fxt Fil_expd(Io_url url, String... expd) {
|
||||
String text = String_.Concat_lines_nl_skip_last(expd); // skipLast b/c if trailing line wanted, easier to pass in extra argument for ""
|
||||
expd_list.Add(new Io_fil_chkr(url, text));
|
||||
return this;
|
||||
} List_adp expd_list = List_adp_.New();
|
||||
public Xob_fxt Fil_skip(Io_url... urls) {
|
||||
for (int i = 0; i < urls.length; i++)
|
||||
skip_list.Add(urls[i]);
|
||||
return this;
|
||||
} List_adp skip_list = List_adp_.New();
|
||||
public Xob_fxt doc_ary_(Xowd_page_itm... v) {doc_ary = v; return this;} private Xowd_page_itm[] doc_ary;
|
||||
public Xowd_page_itm doc_wo_date_(int id, String title, String text) {return doc_(id, "2012-01-02 13:14", title, text);}
|
||||
public Xowd_page_itm doc_(int id, String date, String title, String text) {
|
||||
Xowd_page_itm rv = new Xowd_page_itm().Id_(id).Ttl_(Bry_.new_u8(title), wiki.Ns_mgr()).Text_(Bry_.new_u8(text));
|
||||
int[] modified_on = new int[7];
|
||||
dateParser.Parse_iso8651_like(modified_on, date);
|
||||
rv.Modified_on_(DateAdp_.seg_(modified_on));
|
||||
return rv;
|
||||
}
|
||||
public Xob_fxt Run_id() {
|
||||
Xob_make_id_wkr wkr = new Xob_make_id_wkr(bldr, wiki);
|
||||
Run_page_wkrs(wkr);
|
||||
return this;
|
||||
}
|
||||
private void Run_wkr(Xob_page_wkr wkr) {
|
||||
wkr.Page_wkr__bgn();
|
||||
for (int i = 0; i < doc_ary.length; i++) {
|
||||
Xowd_page_itm page = doc_ary[i];
|
||||
wkr.Page_wkr__run(page);
|
||||
}
|
||||
wkr.Page_wkr__end();
|
||||
}
|
||||
private void tst_fils(Io_url[] ary) {
|
||||
Io_fil[] actls = Get_actl(ary);
|
||||
Io_fil_chkr[] expds = (Io_fil_chkr[])expd_list.To_ary(Io_fil_chkr.class);
|
||||
tst_mgr.Tst_ary("all", expds, actls);
|
||||
}
|
||||
Io_fil[] Get_actl(Io_url[] ary) {
|
||||
int len = ary.length;
|
||||
Io_fil[] rv = new Io_fil[len];
|
||||
for (int i = 0; i < len; i++) {
|
||||
Io_url url = ary[i];
|
||||
String data = Io_mgr.Instance.LoadFilStr(url);
|
||||
rv[i] = new Io_fil(url, data);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public Xob_fxt Run_tmpl_dump() {
|
||||
Xob_parse_dump_templates_cmd wkr = new Xob_parse_dump_templates_cmd(bldr, wiki);
|
||||
Run_wkr(wkr);
|
||||
tst_fils(wkr.Dump_url_gen().Prv_urls());
|
||||
return this;
|
||||
}
|
||||
public Xob_fxt Run(Xobd_parser_wkr... wkrs) {
|
||||
Xobd_parser parser_wkr = new Xobd_parser(bldr);
|
||||
int len = wkrs.length;
|
||||
for (int i = 0; i < len; i++)
|
||||
parser_wkr.Wkr_add(wkrs[i]);
|
||||
Run_page_wkrs(parser_wkr);
|
||||
return this;
|
||||
}
|
||||
public Xob_fxt Run_page_wkrs(Xob_page_wkr... wkrs) {
|
||||
int doc_ary_len = doc_ary.length;
|
||||
for (int j = 0; j < wkrs.length; j++) {
|
||||
Xob_page_wkr wkr = wkrs[j];
|
||||
wkr.Page_wkr__bgn();
|
||||
for (int i = 0; i < doc_ary_len; i++) {
|
||||
Xowd_page_itm page = doc_ary[i];
|
||||
wkr.Page_wkr__run(page);
|
||||
}
|
||||
wkr.Page_wkr__end();
|
||||
}
|
||||
Test_expd_files();
|
||||
return this;
|
||||
}
|
||||
public Xob_fxt Run_cmds(Xob_cmd... cmds) {
|
||||
for (int j = 0; j < cmds.length; j++) {
|
||||
Xob_cmd cmd = cmds[j];
|
||||
cmd.Cmd_bgn(bldr);
|
||||
cmd.Cmd_run();
|
||||
cmd.Cmd_end();
|
||||
}
|
||||
Test_expd_files();
|
||||
return this;
|
||||
}
|
||||
private void Test_expd_files() {
|
||||
if (expd_list.Count() > 0) {
|
||||
Io_fil_chkr[] expd = (Io_fil_chkr[])expd_list.To_ary(Io_fil_chkr.class);
|
||||
Io_fil[] actl = wiki_();
|
||||
tst_mgr.Tst_ary("all", expd, actl);
|
||||
}
|
||||
}
|
||||
Io_fil[] wiki_() {
|
||||
List_adp rv = List_adp_.New();
|
||||
wiki_fil_add(rv, wiki.Tdb_fsys_mgr().Ns_dir());
|
||||
wiki_fil_add(rv, wiki.Tdb_fsys_mgr().Site_dir());
|
||||
rv.Sort();
|
||||
return (Io_fil[])rv.To_ary(Io_fil.class);
|
||||
}
|
||||
private void wiki_fil_add(List_adp list, Io_url root_dir) {
|
||||
Io_url[] ary = Io_mgr.Instance.QueryDir_args(root_dir).Recur_().ExecAsUrlAry();
|
||||
for (int i = 0; i < ary.length; i++) {
|
||||
Io_url url = ary[i];
|
||||
Io_fil fil = new Io_fil(url, Io_mgr.Instance.LoadFilStr_args(url).MissingIgnored_().Exec());
|
||||
list.Add(fil);
|
||||
}
|
||||
}
|
||||
|
||||
public void Run_page_wkr(Xob_page_wkr wkr, Xowd_page_itm... pages) {
|
||||
int len = pages.length;
|
||||
wkr.Page_wkr__bgn();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Xowd_page_itm page = pages[i];
|
||||
wkr.Page_wkr__run(page);
|
||||
}
|
||||
wkr.Page_wkr__end();
|
||||
}
|
||||
}
|
||||
53
400_xowa/src/gplx/xowa/bldrs/Xobldr_fxt.java
Normal file
53
400_xowa/src/gplx/xowa/bldrs/Xobldr_fxt.java
Normal file
@ -0,0 +1,53 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
|
||||
import gplx.core.tests.*; import gplx.core.times.*;
|
||||
import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xobldr_fxt {
|
||||
private final DateAdp_parser dateParser = DateAdp_parser.new_();
|
||||
public Xoae_app App() {return app;} private Xoae_app app;
|
||||
public Xob_bldr Bldr() {return bldr;} private Xob_bldr bldr;
|
||||
public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki;
|
||||
public Xobldr_fxt Ctor_mem() {
|
||||
Io_mgr.Instance.InitEngine_mem();
|
||||
return Ctor(Io_url_.mem_dir_("mem/xowa/"));
|
||||
}
|
||||
private Xobldr_fxt Ctor(Io_url root_dir) {
|
||||
Db_conn_bldr.Instance.Reg_default_sqlite();
|
||||
app = Xoa_app_fxt.Make__app__edit("linux", root_dir);
|
||||
wiki = Xoa_app_fxt.Make__wiki__edit(app);
|
||||
bldr = Xoa_app_fxt.bldr_(app);
|
||||
return this;
|
||||
}
|
||||
public Xowd_page_itm New_page_wo_date(int id, String title, String text) {return New_page(id, "2012-01-02 13:14", title, text);}
|
||||
public Xowd_page_itm New_page(int id, String date, String title, String text) {
|
||||
Xowd_page_itm rv = new Xowd_page_itm().Id_(id).Ttl_(Bry_.new_u8(title), wiki.Ns_mgr()).Text_(Bry_.new_u8(text));
|
||||
int[] modified_on = new int[7];
|
||||
dateParser.Parse_iso8651_like(modified_on, date);
|
||||
rv.Modified_on_(DateAdp_.seg_(modified_on));
|
||||
return rv;
|
||||
}
|
||||
public void Run_page_wkr(Xob_page_wkr wkr, Xowd_page_itm... pages) {
|
||||
int len = pages.length;
|
||||
wkr.Page_wkr__bgn();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Xowd_page_itm page = pages[i];
|
||||
wkr.Page_wkr__run(page);
|
||||
}
|
||||
wkr.Page_wkr__end();
|
||||
}
|
||||
}
|
||||
@ -1,105 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
|
||||
import gplx.core.primitives.*; import gplx.core.ios.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.wtrs.*;
|
||||
import gplx.xowa.langs.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
public abstract class Srch_bldr_wkr_base extends Xob_itm_dump_base implements Xob_page_wkr {
|
||||
private final Ordered_hash list = Ordered_hash_.New(); private Xol_lang_itm lang;
|
||||
public abstract String Page_wkr__key();
|
||||
public void Page_wkr__bgn() {
|
||||
make_dir = wiki.Tdb_fsys_mgr().Ns_dir();
|
||||
this.Init_dump(this.Page_wkr__key(), make_dir);
|
||||
lang = wiki.Lang(); // wiki.Appe().Lang_mgr().Lang_en(); // NOTE: was .Lang_en which is wrong (should match lang of wiki); DATE:2013-05-11
|
||||
tmp_wtr_mgr = new Xob_tmp_wtr_mgr(new Xob_tmp_wtr_wkr__ttl(temp_dir, dump_fil_len));
|
||||
if (wiki.Db_mgr().Tid() == Xodb_mgr_sql.Tid_sql) // if sqlite, hard-code to ns_main; aggregates all ns into one
|
||||
ns_main = wiki.Ns_mgr().Ns_main();
|
||||
} private Xob_tmp_wtr_mgr tmp_wtr_mgr; private Xow_ns ns_main;
|
||||
public void Page_wkr__run(Xowd_page_itm page) {
|
||||
// if (page.Ns_id() != Xow_ns_.Tid__main) return; // limit to main ns for now
|
||||
try {
|
||||
byte[] ttl = page.Ttl_page_db();
|
||||
byte[][] words = Split_ttl_into_words(lang, list, dump_bfr, ttl);
|
||||
Xob_tmp_wtr wtr = tmp_wtr_mgr.Get_or_new(ns_main == null ? page.Ns() : ns_main);
|
||||
int words_len = words.length;
|
||||
int row_len = 0;
|
||||
for (int i = 0; i < words_len; i++) {
|
||||
byte[] word = words[i];
|
||||
row_len += word.length + 13; // 13=5(id) + 5(page_len) + 3(dlms)
|
||||
}
|
||||
if (wtr.FlushNeeded(row_len)) wtr.Flush(bldr.Usr_dlg());
|
||||
for (int i = 0; i < words_len; i++) {
|
||||
byte[] word = words[i];
|
||||
wtr.Bfr() .Add(word) .Add_byte(Byte_ascii.Pipe)
|
||||
.Add_base85_len_5(page.Id()) .Add_byte(Byte_ascii.Semic)
|
||||
.Add_base85_len_5(page.Text().length) .Add_byte(Byte_ascii.Nl);
|
||||
}
|
||||
} catch (Exception e) {bldr.Usr_dlg().Warn_many("", "", "search_index:fatal error: err=~{0}", Err_.Message_gplx_full(e));} // never let single page crash entire import
|
||||
}
|
||||
public void Page_wkr__run_cleanup() {}
|
||||
public void Page_wkr__end() {
|
||||
tmp_wtr_mgr.Flush_all(bldr.Usr_dlg());
|
||||
dump_bfr.ClearAndReset();
|
||||
Xobdc_merger.Ns(bldr.Usr_dlg(), tmp_wtr_mgr.Regy(), Xotdb_dir_info_.Name_search_ttl, temp_dir, make_dir, sort_mem_len, Io_line_rdr_key_gen_.first_pipe, this.Make_cmd_site());
|
||||
tmp_wtr_mgr.Rls_all();
|
||||
if (delete_temp) Io_mgr.Instance.DeleteDirDeep(temp_dir);
|
||||
}
|
||||
public abstract Io_make_cmd Make_cmd_site();
|
||||
public static byte[][] Split_ttl_into_words(Xol_lang_itm lang, Ordered_hash list, Bry_bfr bfr, byte[] ttl) {
|
||||
if (lang != null) // null lang passed in by searcher
|
||||
ttl = lang.Case_mgr().Case_build_lower(ttl);
|
||||
int ttl_len = ttl.length; Bry_obj_ref word_ref = Bry_obj_ref.New(Bry_.Empty);
|
||||
int i = 0; boolean word_done = false;
|
||||
while (true) {
|
||||
if (word_done || i == ttl_len) {
|
||||
if (bfr.Len() > 0) {
|
||||
byte[] word = bfr.To_bry_and_clear();
|
||||
word_ref.Val_(word);
|
||||
if (!list.Has(word_ref)) list.Add(word_ref, word); // don't add same word twice; EX: Title of "Can Can" should only have "Can" in index
|
||||
}
|
||||
if (i == ttl_len) break;
|
||||
word_done = false;
|
||||
}
|
||||
byte b = ttl[i];
|
||||
switch (b) {
|
||||
case Byte_ascii.Underline: // underline is word-breaking; EX: A_B -> A, B
|
||||
case Byte_ascii.Space: // should not occur, but just in case (only underscores)
|
||||
case Byte_ascii.Tab: case Byte_ascii.Nl: case Byte_ascii.Cr: // should not occur in titles, but just in case
|
||||
|
||||
case Byte_ascii.Dash: // treat hypenated words separately
|
||||
case Byte_ascii.Dot: // treat abbreviations as separate words; EX: A.B.C.
|
||||
case Byte_ascii.Bang: case Byte_ascii.Hash: case Byte_ascii.Dollar: case Byte_ascii.Percent:
|
||||
case Byte_ascii.Amp: case Byte_ascii.Paren_bgn: case Byte_ascii.Paren_end: case Byte_ascii.Star:
|
||||
case Byte_ascii.Comma: case Byte_ascii.Slash:
|
||||
case Byte_ascii.Colon: case Byte_ascii.Semic: case Byte_ascii.Gt:
|
||||
case Byte_ascii.Question: case Byte_ascii.At: case Byte_ascii.Brack_bgn: case Byte_ascii.Brack_end:
|
||||
case Byte_ascii.Pow: case Byte_ascii.Tick:
|
||||
case Byte_ascii.Curly_bgn: case Byte_ascii.Pipe: case Byte_ascii.Curly_end: case Byte_ascii.Tilde:
|
||||
case Byte_ascii.Quote: case Byte_ascii.Apos: // FUTURE: apos will split "Earth's" to Earth and s; should remove latter
|
||||
++i;
|
||||
word_done = true;
|
||||
break;
|
||||
default:
|
||||
bfr.Add_byte(b);
|
||||
++i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
byte[][] rv = (byte[][])list.To_ary(byte[].class);
|
||||
list.Clear(); list.Resize_bounds(16);
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
@ -1,113 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
|
||||
import gplx.xowa.bldrs.wkrs.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
import gplx.xowa.wikis.data.site_stats.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.tdbs.hives.*; import gplx.xowa.wikis.tdbs.xdats.*;
|
||||
public class Xob_calc_stats_cmd extends Xob_itm_basic_base implements Xob_cmd {
|
||||
public Xob_calc_stats_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
|
||||
public String Cmd_key() {return Xob_cmd_keys.Key_tdb_calc_stats;}
|
||||
public void Cmd_init(Xob_bldr bldr) {}
|
||||
public void Cmd_bgn(Xob_bldr bldr) {}
|
||||
public void Cmd_run() {Exec();}
|
||||
public void Cmd_end() {}
|
||||
public void Cmd_term() {}
|
||||
private void Exec() {
|
||||
int ns_len = wiki.Ns_mgr().Ords_len();
|
||||
int total = 0;
|
||||
for (int i = 0; i < ns_len; i++) {
|
||||
Xow_ns ns = wiki.Ns_mgr().Ords_ary()[i];
|
||||
int ns_count = Calc_counts(ns);
|
||||
ns.Count_(ns_count);
|
||||
total += ns_count;
|
||||
}
|
||||
int count_main = Calc_count_articles(wiki.Ns_mgr().Ns_main());
|
||||
int count_file = Calc_count_articles(wiki.Ns_mgr().Ns_file());
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
Gen_call(Bool_.Y, bfr, Xowe_wiki.Invk_stats);
|
||||
Gen_call(Bool_.N, bfr, Xowd_site_stats_mgr.Invk_number_of_articles_, count_main);
|
||||
Gen_call(Bool_.N, bfr, Xowd_site_stats_mgr.Invk_number_of_files_, count_file);
|
||||
Gen_call(Bool_.N, bfr, Xowd_site_stats_mgr.Invk_number_of_pages_, total);
|
||||
for (int i = 0; i < ns_len; i++) {
|
||||
Xow_ns ns = wiki.Ns_mgr().Ords_ary()[i];
|
||||
if (ns.Id() < 0) continue;
|
||||
bfr.Add_byte_nl();
|
||||
Gen_call(Bool_.N, bfr, Xowd_site_stats_mgr.Invk_number_of_articles_in_ns_, ns.Num_str(), Int_.To_str_pad_bgn_zero(ns.Count(), 10));
|
||||
}
|
||||
bfr.Add_byte_nl().Add_byte(Byte_ascii.Semic).Add_byte_nl();
|
||||
Io_url wiki_gfs = Wiki_gfs_url(wiki);
|
||||
Io_mgr.Instance.SaveFilBfr(wiki_gfs, bfr);
|
||||
}
|
||||
private void Gen_call(boolean first, Bry_bfr bfr, String key, Object... vals) {
|
||||
if (!first) bfr.Add_byte(Byte_ascii.Dot);
|
||||
bfr.Add_str_u8(key);
|
||||
int len = vals.length;
|
||||
if (len > 0) {
|
||||
bfr.Add_byte(Byte_ascii.Paren_bgn);
|
||||
for (int i = 0; i < len; i++) {
|
||||
if (i != 0) bfr.Add_byte(Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
|
||||
Object val = vals[i];
|
||||
bfr.Add_str_u8(Object_.Xto_str_strict_or_null_mark(val));
|
||||
}
|
||||
bfr.Add_byte(Byte_ascii.Paren_end);
|
||||
}
|
||||
}
|
||||
int Calc_counts(Xow_ns ns) {
|
||||
Io_url reg_url = wiki.Tdb_fsys_mgr().Url_ns_reg(ns.Num_str(), Xotdb_dir_info_.Tid_ttl);
|
||||
Xowd_regy_mgr reg_mgr = new Xowd_regy_mgr(reg_url);
|
||||
int files_ary_len = reg_mgr.Files_ary().length;
|
||||
int count = 0;
|
||||
for (int i = 0; i < files_ary_len; i++) {
|
||||
count += reg_mgr.Files_ary()[i].Count();
|
||||
}
|
||||
return count;
|
||||
}
|
||||
int Calc_count_articles(Xow_ns ns) {
|
||||
Io_url hive_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest(Xotdb_dir_info_.Name_ns, ns.Num_str(), Xotdb_dir_info_.Name_title);
|
||||
return Calc_count_articles_dir(ns, hive_dir);
|
||||
}
|
||||
int Calc_count_articles_dir(Xow_ns ns, Io_url dir) {
|
||||
Io_url[] subs = Io_mgr.Instance.QueryDir_args(dir).DirInclude_().ExecAsUrlAry();
|
||||
int count = 0;
|
||||
int subs_len = subs.length;
|
||||
bldr.Usr_dlg().Prog_one(GRP_KEY, "count", "calculating: ~{0}", dir.Raw());
|
||||
for (int i = 0; i < subs_len; i++) {
|
||||
Io_url sub = subs[i];
|
||||
if (sub.Type_dir())
|
||||
count += Calc_count_articles_dir(ns, sub);
|
||||
else
|
||||
count += Calc_count_articles_fil(ns, sub);
|
||||
}
|
||||
return count;
|
||||
}
|
||||
int Calc_count_articles_fil(Xow_ns ns, Io_url fil) {
|
||||
if (String_.Eq(fil.NameAndExt(), Xotdb_dir_info_.Name_reg_fil)) return 0;
|
||||
int rv = 0;
|
||||
byte[] bry = Io_mgr.Instance.LoadFilBry(fil);
|
||||
Xob_xdat_file xdat_file = new Xob_xdat_file().Parse(bry, bry.length, fil);
|
||||
Xowd_page_itm page = Xowd_page_itm.new_tmp();
|
||||
int count = xdat_file.Count();
|
||||
for (int i = 0; i < count; i++) {
|
||||
byte[] ttl_bry = xdat_file.Get_bry(i);
|
||||
Xotdb_page_itm_.Txt_ttl_load(page, ttl_bry);
|
||||
rv += page.Redirected() ? 0 : 1;
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
static final String GRP_KEY = "xowa.bldr.calc_stats";
|
||||
public static Io_url Wiki_gfs_url(Xowe_wiki wiki) {return wiki.Fsys_mgr().Root_dir().GenSubFil_nest("cfg", "wiki_stats.gfs");}
|
||||
}
|
||||
@ -15,6 +15,7 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
|
||||
import org.junit.*; import gplx.xowa.htmls.portal.*; import gplx.xowa.wikis.xwikis.*;
|
||||
import gplx.xowa.bldrs.cmds.texts.sqls.*;
|
||||
public class Xob_init_base_tst {
|
||||
@Before public void init() {fxt.Clear();} private Xob_init_base_fxt fxt = new Xob_init_base_fxt();
|
||||
@Test public void Dirty_wiki_itms() {
|
||||
@ -24,7 +25,7 @@ public class Xob_init_base_tst {
|
||||
Xow_xwiki_itm xwiki_itm = app.Usere().Wiki().Xwiki_mgr().Add_by_atrs("en.wikipedia.org", "en.wikipedia.org");
|
||||
xwiki_itm.Offline_(Bool_.Y); // simulate add via Available_from_fsys; DATE:2014-09-21
|
||||
Tfds.Eq("", wikis_list.Itms_as_html()); // still empty
|
||||
new Xob_init_tdb(app.Bldr(), wiki).Cmd_end(); // mock "init" task
|
||||
new Xob_init_cmd(app.Bldr(), wiki).Cmd_end(); // mock "init" task
|
||||
Tfds.Eq("\n <li><a href=\"/site/en.wikipedia.org/\" class='xowa-hover-off'>en.wikipedia.org</a></li>", wikis_list.Itms_as_html()); // no longer empty
|
||||
}
|
||||
}
|
||||
|
||||
@ -1,26 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
|
||||
import gplx.xowa.xtns.wbases.imports.*;
|
||||
public class Xob_init_tdb extends Xob_init_base {
|
||||
public Xob_init_tdb(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki);}
|
||||
@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_text_init;}
|
||||
@Override public void Cmd_ini_wdata(Xob_bldr bldr, Xowe_wiki wiki) {
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_wdata_qid);
|
||||
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_wdata_pid);
|
||||
}
|
||||
@Override public void Cmd_run_end(Xowe_wiki wiki) {}
|
||||
}
|
||||
@ -1,36 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
|
||||
import gplx.core.ios.*; import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xob_make_id_wkr extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
|
||||
public Xob_make_id_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
|
||||
public String Page_wkr__key() {return KEY;} public static final String KEY = "core.make_id";
|
||||
public void Page_wkr__bgn() {
|
||||
this.Init_dump(KEY, wiki.Tdb_fsys_mgr().Site_dir().GenSubDir(Xotdb_dir_info_.Name_id));
|
||||
}
|
||||
public void Page_wkr__run(Xowd_page_itm page) {
|
||||
byte[] ttl = page.Ttl_page_db();
|
||||
if (dump_bfr.Len() + row_fixed_len + ttl.length > dump_fil_len) Io_mgr.Instance.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
|
||||
Xotdb_page_itm_.Txt_id_save(dump_bfr, page);
|
||||
}
|
||||
public void Page_wkr__run_cleanup() {}
|
||||
public void Page_wkr__end() {
|
||||
this.Term_dump(new Xob_make_cmd_site(bldr.Usr_dlg(), make_dir, make_fil_len));
|
||||
if (delete_temp) Io_mgr.Instance.DeleteDirDeep(temp_dir);
|
||||
}
|
||||
static final int row_fixed_len = 25 + 1 + 7; // 25=5 base_85 flds; 1=Redirect; 7=dlm
|
||||
}
|
||||
@ -1,162 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.wikis.data; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
|
||||
import org.junit.*; import gplx.xowa.wikis.tdbs.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
public class Xow_data_mgr_tst {
|
||||
Xow_data_mgr_fxt fxt = new Xow_data_mgr_fxt();
|
||||
@Before public void init() {fxt.Clear(); Datetime_now.Manual_y_();}
|
||||
@After public void term() {Datetime_now.Manual_n_();}
|
||||
@Test public void Create() {
|
||||
fxt .Create("A1", "A1 data")
|
||||
.Create("B12", "B12 data")
|
||||
.Create("C123", "C123 data")
|
||||
.Tst_regy_title("0|A1|C123|3\n")
|
||||
.Tst_data_title(String_.Concat_lines_nl
|
||||
( "!!!!>|!!!!?|!!!!@|"
|
||||
, "!!!!!|!!!!!|!!!!!|0|!!!!(|A1"
|
||||
, "!!!!\"|!!!!!|!!!!\"|0|!!!!)|B12"
|
||||
, "!!!!#|!!!!!|!!!!#|0|!!!!*|C123"
|
||||
))
|
||||
.Tst_data_page(String_.Concat_lines_nl
|
||||
( "!!!!9|!!!!;|!!!!=|"
|
||||
, "!!!!!\t##PX+\tA1\tA1 data\t"
|
||||
, "!!!!\"\t##PX/\tB12\tB12 data\t"
|
||||
, "!!!!#\t##PX0\tC123\tC123 data\t"
|
||||
))
|
||||
;
|
||||
}
|
||||
@Test public void Update() {
|
||||
fxt .Create("A1", "A1 data")
|
||||
.Create("B12", "B12 data")
|
||||
.Create("C123", "C123 data")
|
||||
.Update("B12", "B12 changed")
|
||||
.Tst_regy_title("0|A1|C123|3\n")
|
||||
.Tst_data_title(String_.Concat_lines_nl
|
||||
( "!!!!>|!!!!?|!!!!@|"
|
||||
, "!!!!!|!!!!!|!!!!!|0|!!!!(|A1"
|
||||
, "!!!!\"|!!!!!|!!!!\"|0|!!!!,|B12"
|
||||
, "!!!!#|!!!!!|!!!!#|0|!!!!*|C123"
|
||||
))
|
||||
.Tst_data_page(String_.Concat_lines_nl
|
||||
( "!!!!9|!!!!>|!!!!=|"
|
||||
, "!!!!!\t##PX+\tA1\tA1 data\t"
|
||||
, "!!!!\"\t##PX/\tB12\tB12 changed\t"
|
||||
, "!!!!#\t##PX0\tC123\tC123 data\t"
|
||||
))
|
||||
;
|
||||
}
|
||||
@Test public void Update_zip() {
|
||||
// fxt.Wiki().Fsys_mgr().Dir_regy()[Xow_ns_.Tid__main].Ext_tid_(gplx.core.ios.streams.Io_stream_tid_.Tid__zip);
|
||||
// fxt.Wiki().Data_mgr().Zip_mgr_(new Io_zip_mgr_mok());
|
||||
// fxt .Create("A1", "A1 data")
|
||||
// .Create("B12", "B12 data")
|
||||
// .Create("C123", "C123 data")
|
||||
// .Update("B12", "B12 changed")
|
||||
// .Tst_regy_title("0|A1|C123|3\n")
|
||||
// .Tst_data_title(String_.Concat_lines_nl
|
||||
// ( "!!!!>|!!!!?|!!!!@|"
|
||||
// , "!!!!!|!!!!!|!!!!!|0|!!!!(|A1"
|
||||
// , "!!!!\"|!!!!!|!!!!\"|0|!!!!,|B12"
|
||||
// , "!!!!#|!!!!!|!!!!#|0|!!!!*|C123"
|
||||
// ))
|
||||
// .Tst_data_page(String_.Concat_lines_nl
|
||||
// ( "zipped:!!!!9|!!!!>|!!!!=|"
|
||||
// , "!!!!!\t##PX+\tA1\tA1 data\t"
|
||||
// , "!!!!\"\t##PX/\tB12\tB12 changed\t"
|
||||
// , "!!!!#\t##PX0\tC123\tC123 data\t"
|
||||
// ))
|
||||
// ;
|
||||
}
|
||||
@Test public void Create_out_of_order() {
|
||||
fxt .Create("C123", "C123 data")
|
||||
.Create("B12", "B12 data")
|
||||
.Create("A1", "A1 data")
|
||||
.Tst_regy_title("0|A1|C123|3\n")
|
||||
.Tst_data_title(String_.Concat_lines_nl
|
||||
( "!!!!>|!!!!?|!!!!@|"
|
||||
, "!!!!#|!!!!!|!!!!#|0|!!!!(|A1"
|
||||
, "!!!!\"|!!!!!|!!!!\"|0|!!!!)|B12"
|
||||
, "!!!!!|!!!!!|!!!!!|0|!!!!*|C123"
|
||||
))
|
||||
.Tst_data_page(String_.Concat_lines_nl
|
||||
( "!!!!=|!!!!;|!!!!9|"
|
||||
, "!!!!!\t##PX+\tC123\tC123 data\t"
|
||||
, "!!!!\"\t##PX/\tB12\tB12 data\t"
|
||||
, "!!!!#\t##PX0\tA1\tA1 data\t"
|
||||
))
|
||||
;
|
||||
}
|
||||
@Test public void Rename() {
|
||||
fxt .Create("A1", "A1 data")
|
||||
.Create("B12", "B12 data")
|
||||
.Create("C123", "C123 data")
|
||||
.Rename("C123", "C1234")
|
||||
.Tst_regy_title("0|A1|C123|3\n")
|
||||
.Tst_data_title(String_.Concat_lines_nl
|
||||
( "!!!!>|!!!!?|!!!!@|"
|
||||
, "!!!!!|!!!!!|!!!!!|0|!!!!(|A1"
|
||||
, "!!!!\"|!!!!!|!!!!\"|0|!!!!)|B12"
|
||||
, "!!!!#|!!!!!|!!!!#|0|!!!!*|C123"
|
||||
))
|
||||
.Tst_data_page(String_.Concat_lines_nl
|
||||
( "!!!!9|!!!!;|!!!!=|"
|
||||
, "!!!!!\t##PX+\tA1\tA1 data\t"
|
||||
, "!!!!\"\t##PX/\tB12\tB12 data\t"
|
||||
, "!!!!#\t##PX0\tC123\tC123 data\t"
|
||||
))
|
||||
;
|
||||
}
|
||||
}
|
||||
class Xow_data_mgr_fxt {
|
||||
Xoae_app app;
|
||||
public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki;
|
||||
public void Clear() {
|
||||
app = Xoa_app_fxt.Make__app__edit();
|
||||
wiki = Xoa_app_fxt.Make__wiki__edit(app);
|
||||
wiki.Db_mgr().Save_mgr().Page_id_next_(0);
|
||||
}
|
||||
public Xow_data_mgr_fxt Create(String ttl_str, String data) {
|
||||
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl_str));
|
||||
wiki.Db_mgr().Save_mgr().Data_create(wiki, ttl, Bry_.new_u8(data));
|
||||
return this;
|
||||
}
|
||||
public Xow_data_mgr_fxt Update(String ttl_str, String data) {
|
||||
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl_str));
|
||||
Xoae_page page = Xoae_page.New_test(wiki, ttl);
|
||||
wiki.Db_mgr().Save_mgr().Data_update(page, Bry_.new_u8(data));
|
||||
return this;
|
||||
}
|
||||
public Xow_data_mgr_fxt Rename(String old_ttl, String new_ttl) {
|
||||
Xoa_ttl ttl = Xoa_ttl.Parse(wiki, Bry_.new_u8(old_ttl));
|
||||
Xoae_page page = Xoae_page.New_test(wiki, ttl);
|
||||
wiki.Db_mgr().Save_mgr().Data_rename(page, ttl.Ns().Id(), Bry_.new_u8(new_ttl));
|
||||
return this;
|
||||
}
|
||||
public Xow_data_mgr_fxt Tst_regy_title(String expd) {return Tst_regy(Xotdb_dir_info_.Name_title, expd);}
|
||||
Xow_data_mgr_fxt Tst_regy(String name, String expd) {
|
||||
Io_url file_orig = Io_url_.mem_fil_("mem/xowa/wiki/en.wikipedia.org/ns/000/" + name + "/reg.csv");
|
||||
Tfds.Eq_str_lines(expd, Io_mgr.Instance.LoadFilStr(file_orig));
|
||||
return this;
|
||||
}
|
||||
public Xow_data_mgr_fxt Tst_data_page(String expd) {return Tst_data(Xotdb_dir_info_.Tid_page , Xow_ns_.Tid__main, 0, expd);}
|
||||
public Xow_data_mgr_fxt Tst_data_title(String expd) {return Tst_data(Xotdb_dir_info_.Tid_ttl, Xow_ns_.Tid__main, 0, expd);}
|
||||
public Xow_data_mgr_fxt Tst_data(byte dir_tid, int ns_id, int fil, String expd) {
|
||||
Io_url url = wiki.Tdb_fsys_mgr().Url_ns_fil(dir_tid, ns_id, fil);
|
||||
Tfds.Eq_str_lines(expd, Io_mgr.Instance.LoadFilStr(url));
|
||||
return this;
|
||||
}
|
||||
}
|
||||
@ -1,58 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.wikis.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
|
||||
import gplx.core.brys.*; import gplx.core.ios.*; import gplx.core.encoders.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
import gplx.xowa.wikis.tdbs.bldrs.*;
|
||||
public class Xotdb_page_raw_parser {
|
||||
public void Init(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, int load_len) {
|
||||
this.wiki = wiki; ns_mgr = wiki.Ns_mgr();
|
||||
rdr = new Io_line_rdr(usr_dlg, new Io_url[1]);
|
||||
rdr.Line_dlm_(Byte_ascii.Tab).Load_len_(load_len).Key_gen_(Io_line_rdr_key_gen_all.Instance);
|
||||
}
|
||||
public void Init_ns(Xow_ns ns_itm) {this.ns_itm = ns_itm;}
|
||||
public void Reset_one(Io_url url) {
|
||||
rdr.Reset_one(url);
|
||||
}
|
||||
public void Load(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xow_ns ns_itm, Io_url[] urls, int load_len) {
|
||||
this.wiki = wiki; ns_mgr = wiki.Ns_mgr(); this.ns_itm = ns_itm;
|
||||
rdr = new Io_line_rdr(usr_dlg, urls);
|
||||
rdr.Line_dlm_(Byte_ascii.Tab).Load_len_(load_len).Key_gen_(Io_line_rdr_key_gen_all.Instance);
|
||||
} Io_line_rdr rdr; Xowe_wiki wiki; Xow_ns_mgr ns_mgr; Xow_ns ns_itm;
|
||||
public void Skip_first_line() {
|
||||
rdr.Read_next();
|
||||
int pos = Bry_find_.Find_fwd(rdr.Bfr(), Byte_ascii.Nl);
|
||||
// rdr.Move(pos + 1);
|
||||
rdr.Truncate(pos + 1);
|
||||
}
|
||||
public boolean Read(Xowd_page_itm page) {
|
||||
boolean read = false;
|
||||
read = rdr.Read_next(); if (!read) return false;
|
||||
int id = Base85_.To_int_by_bry(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 2);
|
||||
page.Id_(id);
|
||||
read = rdr.Read_next(); if (!read) throw Err_.new_wo_type("could not read timestamp");
|
||||
int timestamp = Base85_.To_int_by_bry(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 1);
|
||||
page.Modified_on_(Int_flag_bldr_.To_date_short(timestamp));
|
||||
read = rdr.Read_next(); if (!read) throw Err_.new_wo_type("could not read ttl");
|
||||
byte[] ttl = Bry_.Mid(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 1);
|
||||
page.Ttl_(ttl, ns_mgr);
|
||||
read = rdr.Read_next(); if (!read) throw Err_.new_wo_type("could not read text");
|
||||
byte[] text = Bry_.Mid(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 1);
|
||||
page.Text_(text);
|
||||
rdr.Bfr_last_read_add(1);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@ -1,41 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.wikis.tdbs.stats; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*;
|
||||
import gplx.core.strings.*;
|
||||
public class Xob_stat_itm {
|
||||
public String Ns() {return ns;} private String ns;
|
||||
public int Fils;
|
||||
public long Size, SizeMax = Int_.Min_value, SizeMin = Int_.Max_value;
|
||||
public int SizeMaxIdx, SizeMinIdx;
|
||||
public void Tally(long size, int idx) {
|
||||
Fils++;
|
||||
Size += size;
|
||||
if (size > SizeMax) {SizeMax = size; SizeMaxIdx = idx;}
|
||||
if (size < SizeMin) {SizeMin = size; SizeMinIdx = idx;}
|
||||
}
|
||||
public void To_str(String_bldr sb) {
|
||||
XtoStr_fld(sb, ns).XtoStr_fld(sb, Fils).XtoStr_fld(sb, Size).XtoStr_fld(sb, SizeMax).XtoStr_fld(sb, SizeMaxIdx).XtoStr_fld(sb, SizeMin);
|
||||
sb.Add(Int_.To_str(SizeMinIdx));
|
||||
}
|
||||
Xob_stat_itm XtoStr_fld(String_bldr sb, long v) {sb.Add(Long_.To_str(v)).Add(Xob_stat_itm.Dlm); return this;}
|
||||
Xob_stat_itm XtoStr_fld(String_bldr sb, int v) {sb.Add(Int_.To_str(v)).Add(Xob_stat_itm.Dlm); return this;}
|
||||
Xob_stat_itm XtoStr_fld(String_bldr sb, String v) {sb.Add(v).Add(Xob_stat_itm.Dlm); return this;}
|
||||
public Xob_stat_itm(String ns) {
|
||||
this.ns = ns;
|
||||
}
|
||||
public Object NewByKey(Object o) {return new Xob_stat_itm((String)o);} public static final Xob_stat_itm Instance = new Xob_stat_itm(); Xob_stat_itm() {}
|
||||
public static final char Dlm = '|';
|
||||
}
|
||||
@ -1,67 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.wikis.tdbs.stats; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*;
|
||||
import gplx.core.strings.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
public class Xob_stat_mgr {
|
||||
public Xob_stat_type GetOrNew(byte tid) {
|
||||
Xob_stat_type rv = (Xob_stat_type)regy.Get_by(tid);
|
||||
if (rv == null) {
|
||||
rv = new Xob_stat_type(tid);
|
||||
regy.Add(tid, rv);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public String Print(Xow_ns_mgr nsMgr) {
|
||||
String_bldr sb = String_bldr_.new_();
|
||||
for (int i = 0; i < regy.Count(); i++) {
|
||||
Xob_stat_type typ = (Xob_stat_type)regy.Get_at(i);
|
||||
sb.Add(String_.PadEnd(Xotdb_dir_info_.Tid_name(typ.Tid()), 6, " "));
|
||||
}
|
||||
sb.Add_str_w_crlf("ns");
|
||||
String[] nsAry = GetNmsAry(nsMgr);
|
||||
for (String ns : nsAry) {
|
||||
for (int i = 0; i < regy.Count(); i++) {
|
||||
Xob_stat_type typ = (Xob_stat_type)regy.Get_at(i);
|
||||
Xob_stat_itm itm = (Xob_stat_itm)typ.GetOrNew(ns);
|
||||
sb.Add(Int_.To_str_pad_bgn_zero(itm.Fils, 5)).Add(" ");
|
||||
}
|
||||
sb.Add_str_w_crlf(ns);
|
||||
}
|
||||
return sb.To_str();
|
||||
}
|
||||
public String To_str() {
|
||||
String_bldr sb = String_bldr_.new_();
|
||||
for (int i = 0; i < regy.Count(); i++) {
|
||||
Xob_stat_type typ = (Xob_stat_type)regy.Get_at(i);
|
||||
typ.To_str(sb);
|
||||
}
|
||||
return sb.To_str();
|
||||
}
|
||||
String[] GetNmsAry(Xow_ns_mgr nsMgr) {
|
||||
Ordered_hash nsRegy = Ordered_hash_.New();
|
||||
for (int i = 0; i < regy.Count(); i++) {
|
||||
Xob_stat_type typ = (Xob_stat_type)regy.Get_at(i);
|
||||
for (int j = 0; j < typ.Count(); j++) {
|
||||
Xob_stat_itm itm = (Xob_stat_itm)typ.GetAt(j);
|
||||
if (!nsRegy.Has(itm.Ns()))
|
||||
nsRegy.Add_as_key_and_val(itm.Ns());
|
||||
}
|
||||
}
|
||||
return (String[])nsRegy.To_ary(String.class);
|
||||
}
|
||||
Ordered_hash regy = Ordered_hash_.New();
|
||||
}
|
||||
@ -1,41 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.wikis.tdbs.stats; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*;
|
||||
import gplx.core.strings.*; import gplx.xowa.wikis.tdbs.*;
|
||||
public class Xob_stat_type {
|
||||
public byte Tid() {return tid;} private byte tid;
|
||||
public Xob_stat_type(byte tid) {this.tid = tid;}
|
||||
public Xob_stat_itm GetOrNew(String ns) {
|
||||
Xob_stat_itm rv = (Xob_stat_itm)regy.Get_by(ns);
|
||||
if (rv == null) {
|
||||
rv = new Xob_stat_itm(ns);
|
||||
regy.Add(ns, rv);
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
public Xob_stat_itm GetAt(int i) {return (Xob_stat_itm)regy.Get_at(i);}
|
||||
public int Count() {return regy.Count();}
|
||||
public void To_str(String_bldr sb) {
|
||||
for (int i = 0; i < regy.Count(); i++) {
|
||||
Xob_stat_itm itm = (Xob_stat_itm)regy.Get_at(i);
|
||||
sb.Add(Xotdb_dir_info_.Tid_name(tid)).Add(Xob_stat_itm.Dlm);
|
||||
itm.To_str(sb);
|
||||
sb.Add(Byte_ascii.Nl);
|
||||
}
|
||||
}
|
||||
Ordered_hash regy = Ordered_hash_.New();
|
||||
public static final Xob_stat_type Instance = new Xob_stat_type(); Xob_stat_type() {}
|
||||
}
|
||||
@ -47,7 +47,6 @@ public class Pfunc_filepath_tst {
|
||||
}
|
||||
@Test public void Wiki_is_commons() {
|
||||
fxt.Init_page_create(commons_wiki, "File:A.png", "");
|
||||
commons_wiki.Tdb_fsys_mgr().Tdb_dir_regy()[Xotdb_dir_info_.Tid_page].Ext_tid_(gplx.core.ios.streams.Io_stream_tid_.Tid__raw);
|
||||
mock_wkr.Redirect_("A.png", "A.png").Repo_idx_(1);
|
||||
fxt.Test_parse_tmpl_str_test("{{filepath:A.png}}", "{{test}}", "file:///mem/xowa/file/commons/trg/orig/7/0/1/c/A.png");
|
||||
}
|
||||
|
||||
@ -50,8 +50,11 @@ public class Scrib_lib_wikibase_tst {
|
||||
fxt.Test_scrib_proc_bool(lib, Scrib_lib_wikibase.Invk_entityExists, Object_.Ary("Q1" ), false);
|
||||
}
|
||||
@Test public void GetEntityId() {
|
||||
wdata_fxt.Init_links_add("enwiki", "Earth", "q2");
|
||||
fxt.Test_scrib_proc_str(lib, Scrib_lib_wikibase.Invk_getEntityId, Object_.Ary("Earth" ), "q2");
|
||||
wdata_fxt.Init__docs__add(wdata_fxt.Wdoc("Q2")
|
||||
.Add_sitelink("enwiki", "Earth")
|
||||
);
|
||||
|
||||
fxt.Test_scrib_proc_str(lib, Scrib_lib_wikibase.Invk_getEntityId, Object_.Ary("Earth" ), "Q2");
|
||||
fxt.Test_scrib_proc_str(lib, Scrib_lib_wikibase.Invk_getEntityId, Object_.Ary("missing_page" ), "");
|
||||
fxt.Test_scrib_proc_str(lib, Scrib_lib_wikibase.Invk_getEntityId, Object_.Ary("" ), ""); // PAGE:en.w:Water_treader DATE:2018-07-01
|
||||
}
|
||||
|
||||
@ -32,7 +32,8 @@ public class Wdata_doc_bldr {
|
||||
public Wdata_doc_bldr Add_description(String lang, String text) {byte[] key = Bry_.new_u8(lang); descr_list.Add(key, new Wdata_langtext_itm(key, Bry_.new_u8(text))); return this;}
|
||||
public Wdata_doc_bldr Add_label(String lang, String text) {byte[] key = Bry_.new_u8(lang); label_list.Add(key, new Wdata_langtext_itm(key, Bry_.new_u8(text))); return this;}
|
||||
public Wdata_doc_bldr Add_sitelink(String site, String link, String... bdgs) {
|
||||
byte[] key = Bry_.new_u8(site); slink_list.Add(key, new Wdata_sitelink_itm(key, Bry_.new_u8(link), Bry_.Ary(bdgs)));
|
||||
byte[] key = Bry_.new_u8(site);
|
||||
slink_list.Add(key, new Wdata_sitelink_itm(key, Bry_.new_u8(link), Bry_.Ary(bdgs)));
|
||||
return this;
|
||||
}
|
||||
public Wdata_doc_bldr Add_alias(String lang, String... ary){byte[] key = Bry_.new_u8(lang); alias_list.Add(key, new Wdata_alias_itm (key, Bry_.Ary(ary))); return this;}
|
||||
|
||||
@ -15,12 +15,10 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
|
||||
import gplx.core.primitives.*; import gplx.langs.jsons.*;
|
||||
import gplx.xowa.wikis.*; import gplx.xowa.wikis.xwikis.*; import gplx.xowa.guis.*; import gplx.xowa.xtns.wbases.imports.*; import gplx.xowa.wikis.pages.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
import gplx.xowa.langs.*;
|
||||
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.domains.*;
|
||||
import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.pfuncs.*; import gplx.xowa.xtns.wbases.claims.*; import gplx.xowa.xtns.wbases.claims.enums.*; import gplx.xowa.xtns.wbases.claims.itms.*; import gplx.xowa.xtns.wbases.stores.*;
|
||||
import gplx.xowa.wikis.tdbs.hives.*; import gplx.xowa.wikis.tdbs.xdats.*;
|
||||
public class Wdata_wiki_mgr_fxt {
|
||||
private Xoae_app app; private Xowe_wiki wiki; private Wdata_doc_bldr wdoc_bldr;
|
||||
private final Wdata_xwiki_link_wtr wdata_lang_wtr = new Wdata_xwiki_link_wtr();
|
||||
private final Bry_bfr tmp_time_bfr = Bry_bfr_.New();
|
||||
public Xowe_wiki Wiki() {return parser_fxt.Wiki();}
|
||||
@ -39,15 +37,13 @@ public class Wdata_wiki_mgr_fxt {
|
||||
parser_fxt.Reset();
|
||||
}
|
||||
return this;
|
||||
} private Xoae_app app; private Xowe_wiki wiki; private Wdata_doc_bldr wdoc_bldr;
|
||||
}
|
||||
public Xoae_app App() {return app;}
|
||||
public Wdata_wiki_mgr Wdata_mgr() {return wdata_mgr;} private Wdata_wiki_mgr wdata_mgr;
|
||||
public Xop_fxt Parser_fxt() {return parser_fxt;} private Xop_fxt parser_fxt;
|
||||
public void Init_lang_fallbacks(String... fallbacks) {
|
||||
wiki.Lang().Fallback_bry_(Bry_.new_a7(String_.Concat_with_str(",", fallbacks)));
|
||||
}
|
||||
public Wdata_doc_bldr Wdoc_bldr(String qid) {return wdoc_bldr.Qid_(qid);}
|
||||
public Json_doc Make_json(String src) {return app.Utl__json_parser().Parse_by_apos(src);}
|
||||
|
||||
public Wbase_claim_base Make_claim_novalue(int pid) {return Wbase_claim_value.New_novalue(pid);}
|
||||
public Wbase_claim_base Make_claim_somevalue(int pid) {return Wbase_claim_value.New_somevalue(pid);}
|
||||
public Wbase_claim_base Make_claim_string(int pid, String val) {return Make_claim_string(pid, Bry_.new_u8(val));}
|
||||
@ -74,7 +70,16 @@ public class Wdata_wiki_mgr_fxt {
|
||||
return rv;
|
||||
}
|
||||
|
||||
public Wdata_doc_bldr Wdoc(String qid) {return wdoc_bldr.Qid_(qid);}
|
||||
public Wdata_doc doc_(String qid, Wbase_claim_base... props) {return wdoc_bldr.Qid_(qid).Add_claims(props).Xto_wdoc();}
|
||||
|
||||
public void Init_pids_add(String lang_key, String pid_name, int pid) {wdata_mgr.Pid_mgr.Add(Bry_.new_u8(lang_key + "|" + pid_name), pid);}
|
||||
public void Init_qids_add(String lang_key, int wiki_tid, String ttl, String qid) {
|
||||
Bry_bfr tmp_bfr = wiki.Utl__bfr_mkr().Get_b512();
|
||||
wdata_mgr.Qid_mgr.Add(tmp_bfr, Bry_.new_a7(lang_key), wiki_tid, Bry_.new_a7("000"), Bry_.new_a7(ttl), Bry_.new_a7(qid));
|
||||
tmp_bfr.Mkr_rls();
|
||||
}
|
||||
public void Init_lang_fallbacks(String... fallbacks) {wiki.Lang().Fallback_bry_(Bry_.new_a7(String_.Concat_with_str(",", fallbacks)));}
|
||||
public void Init_xwikis_add(String... prefixes) {
|
||||
int len = prefixes.length;
|
||||
for (int i = 0; i < len; i++) {
|
||||
@ -82,28 +87,6 @@ public class Wdata_wiki_mgr_fxt {
|
||||
wiki.Xwiki_mgr().Add_by_atrs(prefix, prefix + ".wikipedia.org");
|
||||
}
|
||||
}
|
||||
public void Init_qids_add(String lang_key, int wiki_tid, String ttl, String qid) {
|
||||
Bry_bfr tmp_bfr = wiki.Utl__bfr_mkr().Get_b512();
|
||||
wdata_mgr.Qid_mgr.Add(tmp_bfr, Bry_.new_a7(lang_key), wiki_tid, Bry_.new_a7("000"), Bry_.new_a7(ttl), Bry_.new_a7(qid));
|
||||
tmp_bfr.Mkr_rls();
|
||||
}
|
||||
public void Init_pids_add(String lang_key, String pid_name, int pid) {wdata_mgr.Pid_mgr.Add(Bry_.new_u8(lang_key + "|" + pid_name), pid);}
|
||||
public void Init_links_add(String wiki, String ttl, String qid) {Init_links_add(wiki, "000", ttl, qid);}
|
||||
public void Init_links_add(String wiki, String ns_num, String ttl, String qid) {
|
||||
byte[] ttl_bry = Bry_.new_u8(ttl);
|
||||
Xowd_regy_mgr regy_mgr = app.Hive_mgr().Regy_mgr();
|
||||
Io_url regy_fil = wdata_mgr.Wdata_wiki().Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "qid").GenSubFil_nest(wiki, ns_num, "reg.csv");
|
||||
regy_mgr.Init(regy_fil);
|
||||
regy_mgr.Create(ttl_bry);
|
||||
regy_mgr.Save();
|
||||
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
byte[] itm = bfr.Add(ttl_bry).Add_byte(Byte_ascii.Pipe).Add(Bry_.new_a7(qid)).Add_byte_nl().To_bry_and_clear();
|
||||
Xob_xdat_file xdat_file = new Xob_xdat_file();
|
||||
xdat_file.Insert(bfr, itm);
|
||||
Io_url file_orig = Xob_wdata_qid_base_tst.ttl_(app.Wiki_mgr().Wdata_mgr().Wdata_wiki(), wiki, ns_num, 0);
|
||||
xdat_file.Save(file_orig);
|
||||
}
|
||||
public void Init_external_links_mgr_clear() {wiki.Parser_mgr().Ctx().Page().Wdata_external_lang_links().Reset();}
|
||||
public void Init_external_links_mgr_add(String... langs) {
|
||||
Wdata_external_lang_links_data external_lang_links = wiki.Parser_mgr().Ctx().Page().Wdata_external_lang_links();
|
||||
@ -117,6 +100,23 @@ public class Wdata_wiki_mgr_fxt {
|
||||
external_lang_links.Langs_add(Bry_.new_a7(lang));
|
||||
}
|
||||
}
|
||||
|
||||
public void Init__docs__add(Wdata_doc_bldr bldr) {Init__docs__add(bldr.Xto_wdoc());}
|
||||
public void Init__docs__add(Wdata_doc wdoc) {
|
||||
wdata_mgr.Doc_mgr.Add(wdoc.Qid(), wdoc);
|
||||
|
||||
Bry_bfr tmp_bfr = Bry_bfr_.New();
|
||||
Wbase_qid_mgr qid_mgr = wdata_mgr.Qid_mgr;
|
||||
Ordered_hash slinks = wdoc.Slink_list();
|
||||
int slinks_len = slinks.Len();
|
||||
for (int i = 0; i < slinks_len; i++) {
|
||||
Wdata_sitelink_itm slink = (Wdata_sitelink_itm)slinks.Get_at(i);
|
||||
Xow_domain_itm domain = Xow_abrv_wm_.Parse_to_domain_itm(slink.Site());
|
||||
Xoa_ttl page_ttl = wiki.Ttl_parse(slink.Name());
|
||||
qid_mgr.Add(tmp_bfr, domain.Lang_actl_key(), domain.Domain_type_id(), page_ttl.Ns().Num_bry(), page_ttl.Page_db(), wdoc.Qid());
|
||||
}
|
||||
}
|
||||
|
||||
public void Test_link(String ttl_str, String expd) {Test_link(Xoa_ttl.Parse(wiki, Xow_ns_.Tid__main, Bry_.new_u8(ttl_str)), expd);}
|
||||
public void Test_link(Xoa_ttl ttl, String expd) {
|
||||
byte[] qid_ttl = wdata_mgr.Qid_mgr.Get_qid_or_null(wiki, ttl);
|
||||
@ -124,7 +124,6 @@ public class Wdata_wiki_mgr_fxt {
|
||||
}
|
||||
public void Test_parse_pid_null(String val) {Test_parse_pid(val, Wbase_pid.Id_null);}
|
||||
public void Test_parse_pid(String val, int expd) {Tfds.Eq(expd, Wbase_statement_mgr_.Parse_pid(num_parser, Bry_.new_a7(val)));} private Gfo_number_parser num_parser = new Gfo_number_parser();
|
||||
public void Init__docs__add(Wdata_doc page) {wdata_mgr.Doc_mgr.Add(page.Qid(), page);}
|
||||
public void Test_parse(String raw, String expd) {
|
||||
parser_fxt.Test_parse_page_tmpl_str(raw, expd);
|
||||
}
|
||||
@ -157,7 +156,7 @@ public class Wdata_wiki_mgr_fxt {
|
||||
tmp_langs.Clear();
|
||||
Wdata_xwiki_link_wtr.Write_wdata_links(tmp_langs, wiki, Xoa_ttl.Parse(wiki, Bry_.new_u8(ttl)), wiki.Parser_mgr().Ctx().Page().Wdata_external_lang_links());
|
||||
Tfds.Eq_ary_str(expd, Test_xwiki_links_xto_str_ary(tmp_langs));
|
||||
} List_adp tmp_langs = List_adp_.New();
|
||||
} private List_adp tmp_langs = List_adp_.New();
|
||||
String[] Test_xwiki_links_xto_str_ary(List_adp list) {
|
||||
int len = list.Count();
|
||||
String[] rv = new String[len];
|
||||
@ -175,4 +174,20 @@ public class Wdata_wiki_mgr_fxt {
|
||||
Wdata_wiki_mgr.Write_json_as_html(wdata_mgr.Jdoc_parser(), bfr, raw_bry);
|
||||
Tfds.Eq(expd, bfr.To_str_and_rls());
|
||||
}
|
||||
public static String New_json(String entity_id, String grp_key, String[] grp_vals) {
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
bfr.Add_str_a7("{ 'entity':'").Add_str_u8(entity_id).Add_byte(Byte_ascii.Apos).Add_byte_nl();
|
||||
bfr.Add_str_a7(", 'datatype':'commonsMedia'\n");
|
||||
bfr.Add_str_a7(", '").Add_str_u8(grp_key).Add_str_a7("':").Add_byte_nl();
|
||||
int len = grp_vals.length;
|
||||
for (int i = 0; i < len; i += 2) {
|
||||
bfr.Add_byte_repeat(Byte_ascii.Space, 2);
|
||||
bfr.Add_byte(i == 0 ? Byte_ascii.Curly_bgn : Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
|
||||
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i ]).Add_byte(Byte_ascii.Apos).Add_byte(Byte_ascii.Colon);
|
||||
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i + 1]).Add_byte(Byte_ascii.Apos).Add_byte_nl();
|
||||
}
|
||||
bfr.Add_str_a7(" }").Add_byte_nl();
|
||||
bfr.Add_str_a7("}").Add_byte_nl();
|
||||
return String_.Replace(bfr.To_str_and_clear(), "'", "\"");
|
||||
}
|
||||
}
|
||||
|
||||
@ -19,23 +19,31 @@ public class Wdata_wiki_mgr_tst {
|
||||
private final Wdata_prop_val_visitor__fxt fxt = new Wdata_prop_val_visitor__fxt();
|
||||
@Test public void Basic() {
|
||||
Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt().Init();
|
||||
fxt.Init_links_add("enwiki", "Q1", "Q1_en");
|
||||
fxt.Test_link("Q1", "Q1_en");
|
||||
fxt.Test_link("Q2", null);
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_sitelink("enwiki", "Q1_en")
|
||||
);
|
||||
|
||||
fxt.Test_link("Q1_en" , "Q1");
|
||||
fxt.Test_link("Q1_nil", null);
|
||||
}
|
||||
@Test public void Case_sensitive() { // PURPOSE: wikidata lkp should be case_sensitive; a vs A DATE:2013-09-03
|
||||
Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt().Init();
|
||||
fxt.Init_links_add("enwiki", "Page", "Page_data");
|
||||
fxt.Test_link("Page", "Page_data");
|
||||
fxt.Test_link("PAGE", null);
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_sitelink("enwiki", "Q1_EN")
|
||||
);
|
||||
|
||||
fxt.Test_link("Q1_EN", "Q1");
|
||||
fxt.Test_link("q1_en", null);
|
||||
}
|
||||
@Test public void Non_canonical_ns() { // PURPOSE: handle wikidata entries in non-canonical ns; EX:ukwikisource and Author; PAGE:uk.s:Автор:Богдан_Гаврилишин DATE:2014-07-23
|
||||
Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt().Init();
|
||||
Xowe_wiki wiki = fxt.Wiki();
|
||||
wiki.Ns_mgr().Add_new(124, "Test_ns");
|
||||
fxt.Init_links_add("enwiki", "000", "Test_ns:Test_page", "pass"); // NOTE: wdata will save to "000" ns, b/c "124" ns is not canonical
|
||||
Xoa_ttl ttl = Xoa_ttl.Parse(fxt.Wiki(), 124, Bry_.new_a7("Test_page"));
|
||||
fxt.Test_link(ttl, "pass");
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_sitelink("enwiki", "Test_ns:Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_link(Xoa_ttl.Parse(fxt.Wiki(), 124, Bry_.new_a7("Test_page")), "Q1"); // NOTE: wdata will save to "000" ns, b/c "124" ns is not canonical
|
||||
}
|
||||
@Test public void Write_json_as_html() {
|
||||
Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt().Init();
|
||||
|
||||
@ -34,7 +34,7 @@ public class Wbase_claim_type_ {
|
||||
, Tid__tabular_data = 14
|
||||
, Tid__lexeme = 15
|
||||
;
|
||||
public static final Wbase_enum_hash Reg = new Wbase_enum_hash("claim.data_type", 15);
|
||||
public static final Wbase_enum_hash Reg = new Wbase_enum_hash("claim.data_type", 16);
|
||||
public static final Wbase_enum_itm
|
||||
Itm__unknown = New(Tid__unknown , "unknown")
|
||||
, Itm__bad = New(Tid__bad , "bad") // NOTE: wikidata identifies several entries as "bad"; Q1615351|'s-Graveland, Q107538|Baco; DATE:2013-10-20
|
||||
|
||||
@ -1,40 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.langs.jsons.*; import gplx.core.ios.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
import gplx.xowa.langs.*; import gplx.xowa.langs.parsers.*;
|
||||
import gplx.xowa.xtns.wbases.imports.*;
|
||||
abstract class Wdata_idx_mgr_base {
|
||||
public void Ctor(Xob_itm_dump_base wkr, Xob_bldr bldr, Xowe_wiki wiki, int dump_fil_len) {
|
||||
this.wkr = wkr; this.wiki = wiki; this.bldr = bldr; this.dump_fil_len = dump_fil_len;
|
||||
} Xob_itm_dump_base wkr; protected Xowe_wiki wiki; Xob_bldr bldr; Xol_csv_parser csv_parser = Xol_csv_parser.Instance; protected Ordered_hash hash = Ordered_hash_.New(); protected int dump_fil_len;
|
||||
public void Flush() {
|
||||
int len = hash.Count();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Wdata_idx_wtr wtr = (Wdata_idx_wtr)hash.Get_at(i);
|
||||
wtr.Flush();
|
||||
}
|
||||
}
|
||||
public void Make() {
|
||||
int len = hash.Count();
|
||||
for (int i = 0; i < len; i++) {
|
||||
Wdata_idx_wtr wtr = (Wdata_idx_wtr)hash.Get_at(i);
|
||||
wtr.Make(bldr.Usr_dlg(), wkr.Make_fil_len());
|
||||
}
|
||||
if (wkr.Delete_temp()) Io_mgr.Instance.DeleteDirDeep(wkr.Temp_dir());
|
||||
}
|
||||
}
|
||||
@ -1,52 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.core.ios.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*;
|
||||
class Wdata_idx_wtr {
|
||||
public Wdata_idx_wtr(Io_url dump_dir, int dump_fil_max, Io_url make_dir) {
|
||||
this.dump_dir = dump_dir;
|
||||
this.dump_fil_max = dump_fil_max;
|
||||
dump_url_gen = Io_url_gen_.dir_(dump_dir);
|
||||
this.make_dir = make_dir;
|
||||
} private Bry_bfr dump_bfr = Bry_bfr_.Reset(2 * Io_mgr.Len_kb); int dump_fil_max; Io_url dump_dir, make_dir; Io_url_gen dump_url_gen;
|
||||
public void Write(byte[] ttl, byte[] qid) {
|
||||
if (dump_bfr.Len() + ttl.length + qid.length + 2 > dump_fil_max) Flush(); // 2 = "|" + "\n"; NOTE: all items have format of "data|qid\n"
|
||||
dump_bfr.Add(ttl).Add_byte_pipe().Add(qid).Add_byte_nl();
|
||||
}
|
||||
public void Flush() {
|
||||
Io_mgr.Instance.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
|
||||
}
|
||||
public void Make(Gfo_usr_dlg usr_dlg, int make_fil_len) {
|
||||
Xobdc_merger.Basic(usr_dlg, dump_url_gen, dump_dir.OwnerDir().GenSubDir("sort"), dump_fil_max, Io_line_rdr_key_gen_.first_pipe, new Xob_make_cmd_site(usr_dlg, make_dir, make_fil_len));
|
||||
}
|
||||
public static Wdata_idx_wtr new_qid_(Xowe_wiki wdata_wiki, String wiki_str, String ns_num, int dump_fil_max) {
|
||||
Io_url dump_dir = wdata_wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest("wdata.qid", "qid", wiki_str, ns_num, "dump"); // /xowa/wiki/www.wikidata.org/tmp/wdata_qid/ + enwiki/000/dump/
|
||||
Io_url make_dir = dir_qid_(wdata_wiki, wiki_str, ns_num); // /xowa/wiki/www.wikidata.org/site/data/qid/ + enwiki/000/
|
||||
return new Wdata_idx_wtr(dump_dir, dump_fil_max, make_dir);
|
||||
}
|
||||
public static Wdata_idx_wtr new_pid_(Xowe_wiki wdata_wiki, String lang_key, int dump_fil_max) {
|
||||
Io_url dump_dir = wdata_wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest("wdata.pid", "pid", lang_key, "dump"); // /xowa/wiki/www.wikidata.org/tmp/wdata_pid/ + en/
|
||||
Io_url make_dir = dir_pid_(wdata_wiki, lang_key); // /xowa/wiki/www.wikidata.org/site/data/pid/ + en/
|
||||
return new Wdata_idx_wtr(dump_dir, dump_fil_max, make_dir);
|
||||
}
|
||||
public static Io_url dir_qid_(Xowe_wiki wiki, String wiki_str, String ns_num) {
|
||||
return wiki.Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "qid", wiki_str, ns_num); // /xowa/wiki/www.wikidata.org/site/data/ + qid/enwiki/000/
|
||||
}
|
||||
public static Io_url dir_pid_(Xowe_wiki wiki, String lang_key) {
|
||||
return wiki.Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "pid", lang_key); // /xowa/wiki/www.wikidata.org/site/data/ + pid/en/
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,94 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.claims.enums.*; import gplx.xowa.xtns.wbases.parsers.*; import gplx.xowa.xtns.wbases.dbs.*;
|
||||
import gplx.langs.jsons.*;
|
||||
public class Xob_wdata_pid extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
|
||||
private Db_conn conn;
|
||||
private Wbase_pid_tbl tbl__pid;
|
||||
private Xowb_prop_tbl tbl__prop;
|
||||
private Json_parser jdoc_parser;
|
||||
private final Ordered_hash datatype_hash = Ordered_hash_.New_bry();
|
||||
public Xob_wdata_pid(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
}
|
||||
public Xob_wdata_pid Ctor(Xob_bldr bldr, Xowe_wiki wiki) {
|
||||
this.Cmd_ctor(bldr, wiki);
|
||||
this.jdoc_parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
|
||||
return this;
|
||||
}
|
||||
public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_pid;}
|
||||
public void Page_wkr__bgn() {this.Pid__bgn();}
|
||||
public void Page_wkr__run(Xowd_page_itm page) {
|
||||
if (page.Ns_id() != Wdata_wiki_mgr.Ns_property) return;
|
||||
|
||||
Json_doc jdoc = jdoc_parser.Parse(page.Text());
|
||||
if (jdoc == null) {
|
||||
bldr.Usr_dlg().Warn_many(GRP_KEY, "json.invalid", "json is invalid: ns=~{0} id=~{1}", page.Ns_id(), String_.new_u8(page.Ttl_page_db()));
|
||||
return;
|
||||
}
|
||||
Pid__run(jdoc);
|
||||
}
|
||||
public void Page_wkr__run_cleanup() {}
|
||||
public void Page_wkr__end() {this.Pid__end();}
|
||||
public void Pid__bgn() {
|
||||
if (conn == null) // conn will be null unless test
|
||||
conn = wiki.Data__core_mgr().Db__wbase().Conn();
|
||||
|
||||
// init datatype_hash
|
||||
Wbase_enum_hash enum_hash = Wbase_claim_type_.Reg;
|
||||
byte len = (byte)enum_hash.Len();
|
||||
for (byte i = 0; i < len; i++) {
|
||||
Wbase_claim_type claim_type = (Wbase_claim_type)enum_hash.Get_itm_or(i, null);
|
||||
datatype_hash.Add(Bry_.new_u8(claim_type.Key_for_scrib()), claim_type);
|
||||
}
|
||||
|
||||
// init wbase_pid
|
||||
tbl__pid = Wbase_pid_tbl.New_make(conn);
|
||||
tbl__pid.Create_tbl();
|
||||
tbl__pid.Insert_bgn();
|
||||
|
||||
// init wbase_prop
|
||||
tbl__prop = new Xowb_prop_tbl(conn);
|
||||
tbl__prop.Create_tbl();
|
||||
tbl__prop.Insert_bgn();
|
||||
}
|
||||
public void Pid__run(Json_doc jdoc) {
|
||||
Wdata_doc_parser wdoc_parser = app.Wiki_mgr().Wdata_mgr().Wdoc_parser(jdoc);
|
||||
byte[] pid = wdoc_parser.Parse_qid(jdoc);
|
||||
|
||||
// add datatype
|
||||
byte[] datatype = jdoc.Root_nde().Get_as_bry(Wdata_dict_mainsnak.Itm__datatype.Key_str());
|
||||
Wbase_claim_type claim_type = (Wbase_claim_type)datatype_hash.Get_by_or_fail(datatype);
|
||||
tbl__prop.Insert_cmd_by_batch(pid, claim_type.Tid());
|
||||
|
||||
// add langs
|
||||
Ordered_hash list = wdoc_parser.Parse_langvals(pid, jdoc, Bool_.Y);
|
||||
int len = list.Count();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Wdata_langtext_itm label = (Wdata_langtext_itm)list.Get_at(i);
|
||||
tbl__pid.Insert_cmd_by_batch(label.Lang(), label.Text(), pid);
|
||||
}
|
||||
}
|
||||
public void Pid__end() {
|
||||
tbl__pid.Insert_end();
|
||||
tbl__pid.Create_idx();
|
||||
tbl__prop.Insert_end();
|
||||
}
|
||||
private static final String GRP_KEY = "xowa.wdata.pid_wkr";
|
||||
}
|
||||
@ -1,62 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.langs.jsons.*; import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.parsers.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public abstract class Xob_wdata_pid_base extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
|
||||
private Json_parser parser;
|
||||
public Xob_wdata_pid_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); return this;}
|
||||
public abstract void Pid_bgn();
|
||||
public abstract void Pid_add(byte[] src_lang, byte[] src_ttl, byte[] trg_ttl);
|
||||
public abstract void Pid_datatype(byte[] pid, byte[] datatype_bry);
|
||||
public abstract void Pid_end();
|
||||
|
||||
public abstract String Page_wkr__key();
|
||||
public void Page_wkr__bgn() {
|
||||
this.Init_dump(this.Page_wkr__key(), wiki.Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "pid")); // NOTE: must pass in correct make_dir in order to delete earlier version (else make_dirs will append)
|
||||
this.parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
|
||||
this.Pid_bgn();
|
||||
}
|
||||
public void Page_wkr__run(Xowd_page_itm page) {
|
||||
if (page.Ns_id() != Wdata_wiki_mgr.Ns_property) return;
|
||||
|
||||
Json_doc jdoc = parser.Parse(page.Text());
|
||||
if (jdoc == null) {
|
||||
bldr.Usr_dlg().Warn_many(GRP_KEY, "json.invalid", "json is invalid: ns=~{0} id=~{1}", page.Ns_id(), String_.new_u8(page.Ttl_page_db()));
|
||||
return;
|
||||
}
|
||||
Parse_jdoc(jdoc);
|
||||
}
|
||||
public void Page_wkr__run_cleanup() {}
|
||||
public void Parse_jdoc(Json_doc jdoc) {
|
||||
Wdata_doc_parser wdoc_parser = app.Wiki_mgr().Wdata_mgr().Wdoc_parser(jdoc);
|
||||
byte[] qid = wdoc_parser.Parse_qid(jdoc);
|
||||
|
||||
// add datatype
|
||||
byte[] datatype = jdoc.Root_nde().Get_as_bry(Wdata_dict_mainsnak.Itm__datatype.Key_str());
|
||||
this.Pid_datatype(qid, datatype);
|
||||
|
||||
// add langs
|
||||
Ordered_hash list = wdoc_parser.Parse_langvals(qid, jdoc, Bool_.Y);
|
||||
int len = list.Count();
|
||||
for (int i = 0; i < len; ++i) {
|
||||
Wdata_langtext_itm label = (Wdata_langtext_itm)list.Get_at(i);
|
||||
this.Pid_add(label.Lang(), label.Text(), qid);
|
||||
}
|
||||
}
|
||||
public void Page_wkr__end() {this.Pid_end();}
|
||||
private static final String GRP_KEY = "xowa.wdata.pid_wkr";
|
||||
}
|
||||
@ -1,74 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import org.junit.*;
|
||||
import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*;
|
||||
import gplx.xowa.bldrs.*;
|
||||
public class Xob_wdata_pid_base_tst {
|
||||
private final Xob_fxt fxt = new Xob_fxt().Ctor_mem();
|
||||
Io_url reg_(Xowe_wiki wdata, String wiki) {return Wdata_idx_wtr.dir_pid_(wdata, wiki).GenSubFil(Xotdb_dir_info_.Name_reg_fil);}
|
||||
Io_url ttl_(Xowe_wiki wdata, String wiki, int fil_id) {
|
||||
Io_url root = Wdata_idx_wtr.dir_pid_(wdata, wiki);
|
||||
return Xotdb_fsys_mgr.Url_fil(root, fil_id, Xotdb_dir_info_.Bry_xdat);
|
||||
}
|
||||
@Test public void Basic() {
|
||||
fxt.Wiki().Ns_mgr().Add_new(Wdata_wiki_mgr.Ns_property, "Property");
|
||||
fxt.doc_ary_
|
||||
( fxt.doc_wo_date_(2, "Property:P2", json_("p2", "label", String_.Ary("en", "p2_en", "fr", "p2_fr")))
|
||||
, fxt.doc_wo_date_(1, "Property:P1", json_("p1", "label", String_.Ary("en", "p1_en", "fr", "p1_fr")))
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "en", 0)
|
||||
, "!!!!*|!!!!*|"
|
||||
, "p1_en|p1"
|
||||
, "p2_en|p2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "en")
|
||||
, "0|p1_en|p2_en|2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "fr", 0)
|
||||
, "!!!!*|!!!!*|"
|
||||
, "p1_fr|p1"
|
||||
, "p2_fr|p2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "fr")
|
||||
, "0|p1_fr|p2_fr|2"
|
||||
, ""
|
||||
)
|
||||
.Run_page_wkrs(new Xob_wdata_pid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
|
||||
;
|
||||
}
|
||||
public static String json_(String entity_id, String grp_key, String[] grp_vals) {
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
bfr.Add_str_a7("{ 'entity':'").Add_str_u8(entity_id).Add_byte(Byte_ascii.Apos).Add_byte_nl();
|
||||
bfr.Add_str_a7(", 'datatype':'commonsMedia'\n");
|
||||
bfr.Add_str_a7(", '").Add_str_u8(grp_key).Add_str_a7("':").Add_byte_nl();
|
||||
int len = grp_vals.length;
|
||||
for (int i = 0; i < len; i += 2) {
|
||||
bfr.Add_byte_repeat(Byte_ascii.Space, 2);
|
||||
bfr.Add_byte(i == 0 ? Byte_ascii.Curly_bgn : Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
|
||||
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i ]).Add_byte(Byte_ascii.Apos).Add_byte(Byte_ascii.Colon);
|
||||
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i + 1]).Add_byte(Byte_ascii.Apos).Add_byte_nl();
|
||||
}
|
||||
bfr.Add_str_a7(" }").Add_byte_nl();
|
||||
bfr.Add_str_a7("}").Add_byte_nl();
|
||||
return String_.Replace(bfr.To_str_and_clear(), "'", "\"");
|
||||
}
|
||||
}
|
||||
@ -1,62 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.xtns.wbases.dbs.*; import gplx.xowa.xtns.wbases.claims.enums.*;
|
||||
public class Xob_wdata_pid_sql extends Xob_wdata_pid_base {
|
||||
private Db_conn conn;
|
||||
private Wbase_pid_tbl tbl__pid;
|
||||
private Xowb_prop_tbl tbl__prop;
|
||||
private final Ordered_hash datatype_hash = Ordered_hash_.New_bry();
|
||||
public Xob_wdata_pid_sql(Db_conn conn, Xowe_wiki wiki) {
|
||||
this.conn = conn;
|
||||
this.wiki = wiki;
|
||||
}
|
||||
@Override public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_pid;}
|
||||
@Override public void Pid_bgn() {
|
||||
if (conn == null)
|
||||
conn = wiki.Data__core_mgr().Db__wbase().Conn();
|
||||
// init datatype_hash
|
||||
Wbase_enum_hash enum_hash = Wbase_claim_type_.Reg;
|
||||
byte len = (byte)enum_hash.Len();
|
||||
for (byte i = 0; i < len; i++) {
|
||||
Wbase_claim_type claim_type = (Wbase_claim_type)enum_hash.Get_itm_or(i, null);
|
||||
datatype_hash.Add(Bry_.new_u8(claim_type.Key_for_scrib()), claim_type);
|
||||
}
|
||||
|
||||
// init wbase_pid
|
||||
tbl__pid = Wbase_pid_tbl.New_make(conn);
|
||||
tbl__pid.Create_tbl();
|
||||
tbl__pid.Insert_bgn();
|
||||
|
||||
// init wbase_prop
|
||||
tbl__prop = new Xowb_prop_tbl(conn);
|
||||
tbl__prop.Create_tbl();
|
||||
tbl__prop.Insert_bgn();
|
||||
}
|
||||
@Override public void Pid_add(byte[] lang_key, byte[] ttl, byte[] pid) {
|
||||
tbl__pid.Insert_cmd_by_batch(lang_key, ttl, pid);
|
||||
}
|
||||
@Override public void Pid_datatype(byte[] pid, byte[] datatype_bry) {
|
||||
Wbase_claim_type claim_type = (Wbase_claim_type)datatype_hash.Get_by_or_fail(datatype_bry);
|
||||
tbl__prop.Insert_cmd_by_batch(pid, claim_type.Tid());
|
||||
}
|
||||
@Override public void Pid_end() {
|
||||
tbl__pid.Insert_end();
|
||||
tbl__pid.Create_idx();
|
||||
tbl__prop.Insert_end();
|
||||
}
|
||||
}
|
||||
@ -1,91 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import org.junit.*; import gplx.core.tests.*;
|
||||
import gplx.*;
|
||||
import gplx.dbs.*;
|
||||
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xob_wdata_pid_sql_tst {
|
||||
private Db_conn conn;
|
||||
private final Xob_fxt fxt = new Xob_fxt().Ctor_mem();
|
||||
private final Gfo_db_tester db_tester = new Gfo_db_tester();
|
||||
@Before public void init() {
|
||||
Io_mgr.Instance.InitEngine_mem();
|
||||
Db_conn_bldr.Instance.Reg_default_mem();
|
||||
this.conn = Db_conn_bldr.Instance.New(Io_url_.mem_fil_("mem/db/wbase.xowa"));
|
||||
}
|
||||
|
||||
@Test public void Basic() {
|
||||
fxt.Wiki().Ns_mgr().Add_new(Wdata_wiki_mgr.Ns_property, "Property");
|
||||
Xob_wdata_pid_sql wkr = new Xob_wdata_pid_sql(conn, null);
|
||||
wkr.Ctor(fxt.Bldr(), fxt.Wiki());
|
||||
|
||||
fxt.Run_page_wkr(wkr
|
||||
, fxt.doc_wo_date_(2, "Property:P2", Xob_wdata_tst_utl.Json("p2", "label", String_.Ary("en", "p2_en", "fr", "p2_fr")))
|
||||
, fxt.doc_wo_date_(1, "Property:P1", Xob_wdata_tst_utl.Json("p1", "label", String_.Ary("en", "p1_en", "fr", "p1_fr")))
|
||||
);
|
||||
|
||||
db_tester.Test__select_tbl(conn
|
||||
, "wbase_pid", new Dbmeta_fld_list().Bld_str("src_lang").Bld_str("src_ttl").Bld_str("trg_ttl")
|
||||
, String_.Ary("en", "p2_en", "p2")
|
||||
, String_.Ary("fr", "p2_fr", "p2")
|
||||
, String_.Ary("en", "p1_en", "p1")
|
||||
, String_.Ary("fr", "p1_fr", "p1")
|
||||
);
|
||||
}
|
||||
}
|
||||
class Gfo_db_tester {
|
||||
public void Test__select_tbl(Db_conn conn, String tbl, Dbmeta_fld_list flds, String[]... expd_rows) {
|
||||
Db_rdr rdr = Db_rdr_.Empty;
|
||||
try {
|
||||
rdr = conn.Stmt_select_all(tbl, flds).Exec_select__rls_auto();
|
||||
int row_idx = 0;
|
||||
while (rdr.Move_next()) {
|
||||
String[] actl_row = new String[flds.Len()];
|
||||
for (int i = 0; i < flds.Len(); i++)
|
||||
actl_row[i] = Object_.Xto_str_strict_or_null(rdr.Read_at(i));
|
||||
String[] expd_row = expd_rows[row_idx++];
|
||||
Gftest.Eq__ary(expd_row, actl_row, "row compare failed", "row", row_idx);
|
||||
}
|
||||
}
|
||||
finally {
|
||||
rdr.Rls();
|
||||
}
|
||||
}
|
||||
public void Test__objary(Object[][] expd, Object[][] actl) {
|
||||
// gplx.core.strings.String_bldr_.new_()
|
||||
}
|
||||
}
|
||||
|
||||
class Xob_wdata_tst_utl {
|
||||
public static String Json(String entity_id, String grp_key, String[] grp_vals) {
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
bfr.Add_str_a7("{ 'entity':'").Add_str_u8(entity_id).Add_byte(Byte_ascii.Apos).Add_byte_nl();
|
||||
bfr.Add_str_a7(", 'datatype':'commonsMedia'\n");
|
||||
bfr.Add_str_a7(", '").Add_str_u8(grp_key).Add_str_a7("':").Add_byte_nl();
|
||||
int len = grp_vals.length;
|
||||
for (int i = 0; i < len; i += 2) {
|
||||
bfr.Add_byte_repeat(Byte_ascii.Space, 2);
|
||||
bfr.Add_byte(i == 0 ? Byte_ascii.Curly_bgn : Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
|
||||
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i ]).Add_byte(Byte_ascii.Apos).Add_byte(Byte_ascii.Colon);
|
||||
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i + 1]).Add_byte(Byte_ascii.Apos).Add_byte_nl();
|
||||
}
|
||||
bfr.Add_str_a7(" }").Add_byte_nl();
|
||||
bfr.Add_str_a7("}").Add_byte_nl();
|
||||
return String_.Replace(bfr.To_str_and_clear(), "'", "\"");
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,127 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import org.junit.*; import gplx.core.tests.*;
|
||||
import gplx.*; import gplx.dbs.*;
|
||||
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xob_wdata_pid_tst {
|
||||
private Db_conn conn;
|
||||
private final Xobldr_fxt fxt = new Xobldr_fxt().Ctor_mem();
|
||||
private final Gfo_db_tester db_tester = new Gfo_db_tester();
|
||||
@Before public void init() {
|
||||
Io_mgr.Instance.InitEngine_mem();
|
||||
Db_conn_bldr.Instance.Reg_default_mem();
|
||||
this.conn = Db_conn_bldr.Instance.New(Io_url_.mem_fil_("mem/db/wbase.xowa"));
|
||||
}
|
||||
|
||||
@Test public void Basic() {
|
||||
fxt.Wiki().Ns_mgr().Add_new(Wdata_wiki_mgr.Ns_property, "Property");
|
||||
Xob_wdata_pid wkr = new Xob_wdata_pid(conn);
|
||||
wkr.Ctor(fxt.Bldr(), fxt.Wiki());
|
||||
|
||||
fxt.Run_page_wkr(wkr
|
||||
, fxt.New_page_wo_date(2, "Property:P2", Xob_wdata_tst_utl.Json("p2", "label", String_.Ary("en", "p2_en", "fr", "p2_fr")))
|
||||
, fxt.New_page_wo_date(1, "Property:P1", Xob_wdata_tst_utl.Json("p1", "label", String_.Ary("en", "p1_en", "fr", "p1_fr")))
|
||||
);
|
||||
|
||||
db_tester.Test__select_tbl(conn, "wbase_pid", new Dbmeta_fld_list().Bld_str("src_lang").Bld_str("src_ttl").Bld_str("trg_ttl")
|
||||
, Object_.Ary("en", "p2_en", "p2")
|
||||
, Object_.Ary("fr", "p2_fr", "p2")
|
||||
, Object_.Ary("en", "p1_en", "p1")
|
||||
, Object_.Ary("fr", "p1_fr", "p1")
|
||||
);
|
||||
|
||||
db_tester.Test__select_tbl(conn, "wbase_prop", new Dbmeta_fld_list().Bld_str("wbp_pid").Bld_int("wbp_datatype")
|
||||
, Object_.Ary("p2", 12)
|
||||
, Object_.Ary("p1", 12)
|
||||
);
|
||||
}
|
||||
}
|
||||
class Gfo_db_tester {
|
||||
public void Test__select_tbl(Db_conn conn, String tbl, Dbmeta_fld_list flds, Object[]... expd_rows) {
|
||||
// get actl
|
||||
List_adp actl_list = List_adp_.New();
|
||||
Db_rdr rdr = Db_rdr_.Empty;
|
||||
try {
|
||||
rdr = conn.Stmt_select_all(tbl, flds).Exec_select__rls_auto();
|
||||
while (rdr.Move_next()) {
|
||||
Object[] actl_row = new Object[flds.Len()];
|
||||
actl_list.Add(actl_row);
|
||||
for (int i = 0; i < flds.Len(); i++) {
|
||||
Object val = rdr.Read_at(i);
|
||||
int val_tid = Dbmeta_fld_tid.Get_by_obj(val);
|
||||
Dbmeta_fld_itm fld = flds.Get_at(i);
|
||||
if (val_tid != fld.Type().Tid_ansi())
|
||||
actl_row[i] = Object_.Xto_str_strict_or_null_mark(val) + "|shouldBe=" + String_.new_u8(fld.Type().Name()) + "|was=" + Type_.Name_by_obj(val);
|
||||
else
|
||||
actl_row[i] = val;
|
||||
}
|
||||
}
|
||||
}
|
||||
finally {
|
||||
rdr.Rls();
|
||||
}
|
||||
Object[][] actl_rows = (Object[][])actl_list.To_ary_and_clear(Object[].class);
|
||||
|
||||
// validate expd datatypes
|
||||
for (Object[] expd_row : expd_rows) {
|
||||
int len = expd_row.length;
|
||||
for (int i = 0; i < len; i++) {
|
||||
Object val = expd_row[i];
|
||||
int val_tid = Dbmeta_fld_tid.Get_by_obj(val);
|
||||
Dbmeta_fld_itm fld = flds.Get_at(i);
|
||||
if (val_tid != fld.Type().Tid_ansi())
|
||||
expd_row[i] = Object_.Xto_str_strict_or_null_mark(val) + "|shouldBe=" + String_.new_u8(fld.Type().Name()) + "|was=" + Type_.Name_by_obj(val);
|
||||
}
|
||||
}
|
||||
Gftest.Eq__ary(To_str_ary(expd_rows), To_str_ary(actl_rows), "rows mismatch");
|
||||
}
|
||||
public byte[][] To_str_ary(Object[][] rows) {
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
byte[][] rv = new byte[rows.length][];
|
||||
int rows_len = rows.length;
|
||||
for (int i = 0; i < rows_len; i++) {
|
||||
Object[] row = rows[i];
|
||||
for (int j = 0; j < row.length; j++) {
|
||||
if (j != 0) bfr.Add_byte_pipe();
|
||||
bfr.Add_obj(row[j]);
|
||||
}
|
||||
bfr.Add_byte_nl();
|
||||
rv[i] = bfr.To_bry_and_clear();
|
||||
}
|
||||
return rv;
|
||||
}
|
||||
}
|
||||
|
||||
class Xob_wdata_tst_utl {
|
||||
public static String Json(String entity_id, String grp_key, String[] grp_vals) {
|
||||
Bry_bfr bfr = Bry_bfr_.New();
|
||||
bfr.Add_str_a7("{ 'entity':'").Add_str_u8(entity_id).Add_byte(Byte_ascii.Apos).Add_byte_nl();
|
||||
bfr.Add_str_a7(", 'datatype':'commonsMedia'\n");
|
||||
bfr.Add_str_a7(", '").Add_str_u8(grp_key).Add_str_a7("':").Add_byte_nl();
|
||||
int len = grp_vals.length;
|
||||
for (int i = 0; i < len; i += 2) {
|
||||
bfr.Add_byte_repeat(Byte_ascii.Space, 2);
|
||||
bfr.Add_byte(i == 0 ? Byte_ascii.Curly_bgn : Byte_ascii.Comma).Add_byte(Byte_ascii.Space);
|
||||
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i ]).Add_byte(Byte_ascii.Apos).Add_byte(Byte_ascii.Colon);
|
||||
bfr.Add_byte(Byte_ascii.Apos).Add_str_u8(grp_vals[i + 1]).Add_byte(Byte_ascii.Apos).Add_byte_nl();
|
||||
}
|
||||
bfr.Add_str_a7(" }").Add_byte_nl();
|
||||
bfr.Add_str_a7("}").Add_byte_nl();
|
||||
return String_.Replace(bfr.To_str_and_clear(), "'", "\"");
|
||||
}
|
||||
}
|
||||
@ -1,48 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
public class Xob_wdata_pid_txt extends Xob_wdata_pid_base {
|
||||
@Override public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_tdb_text_wdata_pid;}
|
||||
@Override public void Pid_bgn() {
|
||||
pid_bldr = new Wdata_idx_bldr_pid(this, bldr, wiki, dump_fil_len);
|
||||
} Wdata_idx_bldr_pid pid_bldr;
|
||||
@Override public void Pid_add(byte[] lang_key, byte[] prop_key, byte[] qid) {
|
||||
pid_bldr.Add(lang_key, prop_key, qid);
|
||||
}
|
||||
@Override public void Pid_datatype(byte[] pid, byte[] datatype_bry) {}
|
||||
@Override public void Pid_end() {
|
||||
pid_bldr.Flush();
|
||||
pid_bldr.Make();
|
||||
}
|
||||
}
|
||||
class Wdata_idx_bldr_pid extends Wdata_idx_mgr_base {
|
||||
public Wdata_idx_bldr_pid(Xob_itm_dump_base wkr, Xob_bldr bldr, Xowe_wiki wiki, int dump_fil_len) {this.Ctor(wkr, bldr, wiki, dump_fil_len);}
|
||||
public void Add(byte[] lang, byte[] prop_key, byte[] pid) {
|
||||
Wdata_idx_wtr wtr = Get_or_new(lang);
|
||||
wtr.Write(prop_key, pid);
|
||||
}
|
||||
public Wdata_idx_wtr Get_or_new(byte[] lang_bry) {
|
||||
String lang = String_.Lower(String_.new_u8(lang_bry)); // NOTE: for some reason, both "en" and "En" can be added; normalize case
|
||||
Object rv = hash.Get_by(lang);
|
||||
if (rv == null) {
|
||||
Wdata_idx_wtr wtr = Wdata_idx_wtr.new_pid_(wiki, lang, dump_fil_len);
|
||||
hash.Add(lang, wtr);
|
||||
return wtr;
|
||||
}
|
||||
return (Wdata_idx_wtr)rv;
|
||||
}
|
||||
}
|
||||
@ -14,32 +14,44 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.langs.jsons.*; import gplx.core.ios.*; import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.parsers.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
import gplx.langs.jsons.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
import gplx.xowa.bldrs.wms.sites.*;
|
||||
public abstract class Xob_wdata_qid_base extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
|
||||
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.dbs.*; import gplx.xowa.xtns.wbases.parsers.*;
|
||||
public class Xob_wdata_qid extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
|
||||
private Db_conn conn;
|
||||
private Wbase_qid_tbl tbl;
|
||||
private final Object thread_lock = new Object();
|
||||
private Json_parser parser; private Xob_wbase_ns_parser ns_parser; private final Xob_wbase_ns_parser_rslt ns_parser_rslt = new Xob_wbase_ns_parser_rslt();
|
||||
public Xob_wdata_qid_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); return this;}
|
||||
public abstract String Page_wkr__key();
|
||||
public abstract void Qid_bgn();
|
||||
public abstract void Qid_add(byte[] wiki_key, int ns_id, byte[] ttl, byte[] qid);
|
||||
public abstract void Qid_end();
|
||||
public Xob_wdata_qid(Db_conn conn) {
|
||||
this.conn = conn;
|
||||
}
|
||||
public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_qid;}
|
||||
public Xob_wdata_qid Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); return this;}
|
||||
public void Page_wkr__bgn() {
|
||||
this.Init_dump(this.Page_wkr__key(), wiki.Tdb_fsys_mgr().Site_dir().GenSubDir_nest("data", "qid")); // NOTE: must pass in correct make_dir in order to delete earlier version (else make_dirs will append)
|
||||
this.parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
|
||||
this.ns_parser = new Xob_wbase_ns_parser(bldr.App().Fsys_mgr().Cfg_site_meta_fil());
|
||||
this.Qid_bgn();
|
||||
this.Qid__bgn();
|
||||
}
|
||||
public void Page_wkr__run(Xowd_page_itm page) {
|
||||
if (page.Ns_id() != Xow_ns_.Tid__main) return; // qid pages are only in the Main Srch_rslt_cbk
|
||||
Json_doc jdoc = parser.Parse(page.Text());
|
||||
if (jdoc == null) {bldr.Usr_dlg().Warn_many("", "", "json is invalid: ns=~{0} id=~{1}", page.Ns_id(), String_.new_u8(page.Ttl_page_db())); return;}
|
||||
this.Parse_jdoc(jdoc);
|
||||
this.Qid__run(jdoc);
|
||||
}
|
||||
public void Page_wkr__run_cleanup() {}
|
||||
public void Parse_jdoc(Json_doc jdoc) {
|
||||
public void Page_wkr__end() {this.Qid__end();}
|
||||
public void Qid__bgn() {
|
||||
if (conn == null) {
|
||||
Xow_db_file wbase_db = Make_wbase_db(wiki.Db_mgr_as_sql().Core_data_mgr());
|
||||
conn = wbase_db.Conn();
|
||||
}
|
||||
tbl = Wbase_qid_tbl.New_make(conn, false);
|
||||
tbl.Create_tbl();
|
||||
tbl.Insert_bgn();
|
||||
}
|
||||
public void Qid__run(Json_doc jdoc) {
|
||||
synchronized (thread_lock) {
|
||||
Wdata_doc_parser wdoc_parser = app.Wiki_mgr().Wdata_mgr().Wdoc_parser(jdoc);
|
||||
byte[] qid = wdoc_parser.Parse_qid(jdoc);
|
||||
@ -54,12 +66,21 @@ public abstract class Xob_wdata_qid_base extends Xob_itm_dump_base implements Xo
|
||||
if (sitelink_ns != Xow_ns_.Tid__main) // ttl not in main; chop off ns portion; EX:Aide:French_title -> French_title
|
||||
sitelink_ttl = Bry_.Mid(sitelink_ttl, ns_parser_rslt.Ttl_bgn(), sitelink_ttl.length);
|
||||
sitelink_ttl = wiki.Lang().Case_mgr().Case_build_1st_upper(tmp_bfr, sitelink_ttl, 0, sitelink_ttl.length);
|
||||
this.Qid_add(sitelink.Site(), sitelink_ns, Xoa_ttl.Replace_spaces(sitelink_ttl), qid); // NOTE: always convert spaces to underscores; EX: "A B" -> "A_B" DATE:2015-04-21
|
||||
tbl.Insert_cmd_by_batch(sitelink.Site(), sitelink_ns, Xoa_ttl.Replace_spaces(sitelink_ttl), qid); // NOTE: always convert spaces to underscores; EX: "A B" -> "A_B" DATE:2015-04-21
|
||||
}
|
||||
}
|
||||
}
|
||||
public void Page_wkr__end() {
|
||||
this.Qid_end();
|
||||
// wiki.Data__core_mgr().Db__wbase().Tbl__cfg().Insert_int("", "", 1);
|
||||
public void Qid__end() {
|
||||
tbl.Insert_end();
|
||||
tbl.Create_idx();
|
||||
}
|
||||
public static Xow_db_file Make_wbase_db(Xow_db_mgr db_mgr) {
|
||||
boolean db_is_all_or_few = db_mgr.Props().Layout_text().Tid_is_all_or_few();
|
||||
Xow_db_file wbase_db = db_is_all_or_few
|
||||
? db_mgr.Db__core()
|
||||
: db_mgr.Dbs__make_by_tid(Xow_db_file_.Tid__wbase);
|
||||
if (db_is_all_or_few)
|
||||
db_mgr.Db__wbase_(wbase_db);
|
||||
return wbase_db;
|
||||
}
|
||||
}
|
||||
@ -1,173 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import org.junit.*;
|
||||
import gplx.xowa.wikis.nss.*;
|
||||
import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*; import gplx.dbs.*;
|
||||
import gplx.xowa.bldrs.wms.sites.*;
|
||||
public class Xob_wdata_qid_base_tst {
|
||||
private gplx.xowa.bldrs.Xob_fxt fxt; // NOTE: reset memory instance (don't just call clear)
|
||||
@Before public void init() {
|
||||
this.fxt = new gplx.xowa.bldrs.Xob_fxt().Ctor_mem();
|
||||
gplx.dbs.Db_conn_bldr.Instance.Reg_default_mem();
|
||||
}
|
||||
@Test public void Basic() {
|
||||
fxt.doc_ary_
|
||||
( fxt.doc_wo_date_(2, "q2", Xob_wdata_pid_base_tst.json_("q2", "links", String_.Ary("enwiki", "q2_en", "frwiki", "q2_fr")))
|
||||
, fxt.doc_wo_date_(1, "q1", Xob_wdata_pid_base_tst.json_("q1", "links", String_.Ary("enwiki", "q1_en", "frwiki", "q1_fr")))
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "enwiki", "000", 0)
|
||||
, "!!!!*|!!!!*|"
|
||||
, "Q1_en|q1"
|
||||
, "Q2_en|q2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "enwiki", "000")
|
||||
, "0|Q1_en|Q2_en|2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "frwiki", "000", 0)
|
||||
, "!!!!*|!!!!*|"
|
||||
, "Q1_fr|q1"
|
||||
, "Q2_fr|q2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "frwiki", "000")
|
||||
, "0|Q1_fr|Q2_fr|2"
|
||||
, ""
|
||||
)
|
||||
.Run_page_wkrs(new Xob_wdata_qid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
|
||||
;
|
||||
}
|
||||
@Test public void Ns() {
|
||||
// setup db
|
||||
Site_core_db json_db = new Site_core_db(fxt.App().Fsys_mgr().Cfg_site_meta_fil());
|
||||
Site_namespace_tbl ns_tbl = json_db.Tbl__namespace();
|
||||
ns_tbl.Insert(Bry_.new_a7("en.w"), Xow_ns_.Tid__help, Xow_ns_case_.Bry__1st, Bry_.Empty, Bry_.new_a7("Help"), Bool_.N, Bool_.N, Bry_.Empty);
|
||||
ns_tbl.Insert(Bry_.new_a7("fr.w"), Xow_ns_.Tid__help, Xow_ns_case_.Bry__1st, Bry_.Empty, Bry_.new_a7("Aide"), Bool_.N, Bool_.N, Bry_.Empty);
|
||||
// run test
|
||||
fxt.doc_ary_
|
||||
( fxt.doc_wo_date_(1, "11", Xob_wdata_pid_base_tst.json_("q1", "links", String_.Ary("enwiki", "Help:Q1_en", "frwiki", "Aide:Q1_fr")))
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "enwiki", "012", 0)
|
||||
, "!!!!*|"
|
||||
, "Q1_en|q1"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "enwiki", "012")
|
||||
, "0|Q1_en|Q1_en|1"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "frwiki", "012", 0)
|
||||
, "!!!!*|"
|
||||
, "Q1_fr|q1"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "frwiki", "012")
|
||||
, "0|Q1_fr|Q1_fr|1"
|
||||
, ""
|
||||
)
|
||||
.Run_page_wkrs(new Xob_wdata_qid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
|
||||
;
|
||||
}
|
||||
@Test public void Links_w_name() { // PURPOSE: wikidata changed links node from "enwiki:A" to "enwiki:{name:A,badges:[]}"; DATE:2013-09-14
|
||||
String q1_str = String_.Concat_lines_nl
|
||||
( "{ \"entity\":\"q1\""
|
||||
, ", \"links\":"
|
||||
, " { \"enwiki\":\"q1_en\""
|
||||
, " , \"frwiki\":\"q1_fr\""
|
||||
, " }"
|
||||
, "}"
|
||||
);
|
||||
String q2_str = String_.Concat_lines_nl
|
||||
( "{ \"entity\":[\"item\",2]"
|
||||
, ", \"links\":"
|
||||
, " { \"enwiki\":{\"name\":\"q2_en\",\"badges\":[]}"
|
||||
, " , \"frwiki\":{\"name\":\"q2_fr\",\"badges\":[]}"
|
||||
, " }"
|
||||
, "}"
|
||||
);
|
||||
fxt.doc_ary_
|
||||
( fxt.doc_wo_date_(1, "q1", q1_str)
|
||||
, fxt.doc_wo_date_(2, "q2", q2_str)
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "enwiki", "000", 0)
|
||||
, "!!!!*|!!!!*|"
|
||||
, "Q1_en|q1"
|
||||
, "Q2_en|q2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "enwiki", "000")
|
||||
, "0|Q1_en|Q2_en|2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "frwiki", "000", 0)
|
||||
, "!!!!*|!!!!*|"
|
||||
, "Q1_fr|q1"
|
||||
, "Q2_fr|q2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "frwiki", "000")
|
||||
, "0|Q1_fr|Q2_fr|2"
|
||||
, ""
|
||||
)
|
||||
.Run_page_wkrs(new Xob_wdata_qid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
|
||||
;
|
||||
}
|
||||
@Test public void Spaces() { // PURPOSE: assert that ttls with spaces are converted to unders DATE:2015-04-21
|
||||
fxt.doc_ary_
|
||||
( fxt.doc_wo_date_(2, "q2", Xob_wdata_pid_base_tst.json_("q2", "links", String_.Ary("enwiki", "q2 en", "frwiki", "q2 fr"))) // note "q2 en" not "q2_en"
|
||||
, fxt.doc_wo_date_(1, "q1", Xob_wdata_pid_base_tst.json_("q1", "links", String_.Ary("enwiki", "q1 en", "frwiki", "q1 fr")))
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "enwiki", "000", 0)
|
||||
, "!!!!*|!!!!*|"
|
||||
, "Q1_en|q1"
|
||||
, "Q2_en|q2" // NOTE: q2_en
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "enwiki", "000")
|
||||
, "0|Q1_en|Q2_en|2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd(ttl_(fxt.Wiki(), "frwiki", "000", 0)
|
||||
, "!!!!*|!!!!*|"
|
||||
, "Q1_fr|q1"
|
||||
, "Q2_fr|q2"
|
||||
, ""
|
||||
)
|
||||
.Fil_expd
|
||||
( reg_(fxt.Wiki(), "frwiki", "000")
|
||||
, "0|Q1_fr|Q2_fr|2"
|
||||
, ""
|
||||
)
|
||||
.Run_page_wkrs(new Xob_wdata_qid_txt().Ctor(fxt.Bldr(), this.fxt.Wiki()))
|
||||
;
|
||||
}
|
||||
public static Io_url reg_(Xowe_wiki wdata, String wiki, String ns_id) {
|
||||
return Wdata_idx_wtr.dir_qid_(wdata, wiki, ns_id).GenSubFil(Xotdb_dir_info_.Name_reg_fil);
|
||||
}
|
||||
public static Io_url ttl_(Xowe_wiki wdata, String wiki, String ns_id, int fil_id) {
|
||||
Io_url root = Wdata_idx_wtr.dir_qid_(wdata, wiki, ns_id);
|
||||
return Xotdb_fsys_mgr.Url_fil(root, fil_id, Xotdb_dir_info_.Bry_xdat);
|
||||
}
|
||||
}
|
||||
@ -1,44 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
|
||||
import gplx.xowa.xtns.wbases.dbs.*;
|
||||
public class Xob_wdata_qid_sql extends Xob_wdata_qid_base {
|
||||
private Wbase_qid_tbl tbl;
|
||||
@Override public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_qid;}
|
||||
@Override public void Qid_bgn() {
|
||||
Xow_db_file wbase_db = Make_wbase_db(wiki.Db_mgr_as_sql().Core_data_mgr());
|
||||
tbl = Wbase_qid_tbl.New_make(wbase_db.Conn(), false);
|
||||
tbl.Create_tbl();
|
||||
tbl.Insert_bgn();
|
||||
}
|
||||
@Override public void Qid_add(byte[] wiki_key, int ns_id, byte[] ttl, byte[] qid) {
|
||||
tbl.Insert_cmd_by_batch(wiki_key, ns_id, ttl, qid);
|
||||
}
|
||||
@Override public void Qid_end() {
|
||||
tbl.Insert_end();
|
||||
tbl.Create_idx();
|
||||
}
|
||||
public static Xow_db_file Make_wbase_db(Xow_db_mgr db_mgr) {
|
||||
boolean db_is_all_or_few = db_mgr.Props().Layout_text().Tid_is_all_or_few();
|
||||
Xow_db_file wbase_db = db_is_all_or_few
|
||||
? db_mgr.Db__core()
|
||||
: db_mgr.Dbs__make_by_tid(Xow_db_file_.Tid__wbase);
|
||||
if (db_is_all_or_few)
|
||||
db_mgr.Db__wbase_(wbase_db);
|
||||
return wbase_db;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,107 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import org.junit.*; import gplx.core.tests.*;
|
||||
import gplx.*; import gplx.dbs.*;
|
||||
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.nss.*;
|
||||
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
|
||||
import gplx.xowa.bldrs.wms.sites.*;
|
||||
public class Xob_wdata_qid_tst {
|
||||
private Db_conn conn;
|
||||
private final Xobldr_fxt fxt = new Xobldr_fxt().Ctor_mem();
|
||||
private Xob_wdata_qid wkr;
|
||||
private final Gfo_db_tester db_tester = new Gfo_db_tester();
|
||||
private final Dbmeta_fld_list flds__wbase_qid = new Dbmeta_fld_list().Bld_str("src_wiki").Bld_int("src_ns").Bld_str("src_ttl").Bld_str("trg_ttl");
|
||||
@Before public void init() {
|
||||
Io_mgr.Instance.InitEngine_mem();
|
||||
Db_conn_bldr.Instance.Reg_default_mem();
|
||||
this.conn = Db_conn_bldr.Instance.New(Io_url_.mem_fil_("mem/db/wbase.xowa"));
|
||||
this.wkr = new Xob_wdata_qid(conn);
|
||||
wkr.Ctor(fxt.Bldr(), fxt.Wiki());
|
||||
}
|
||||
|
||||
@Test public void Basic() {
|
||||
fxt.Run_page_wkr(wkr
|
||||
, fxt.New_page_wo_date(2, "q2", Wdata_wiki_mgr_fxt.New_json("q2", "links", String_.Ary("enwiki", "q2_en", "frwiki", "q2_fr")))
|
||||
, fxt.New_page_wo_date(1, "q1", Wdata_wiki_mgr_fxt.New_json("q1", "links", String_.Ary("enwiki", "q1_en", "frwiki", "q1_fr")))
|
||||
);
|
||||
|
||||
db_tester.Test__select_tbl(conn, "wbase_qid", flds__wbase_qid
|
||||
, Object_.Ary("enwiki", 0, "Q2_en", "q2")
|
||||
, Object_.Ary("frwiki", 0, "Q2_fr", "q2")
|
||||
, Object_.Ary("enwiki", 0, "Q1_en", "q1")
|
||||
, Object_.Ary("frwiki", 0, "Q1_fr", "q1")
|
||||
);
|
||||
}
|
||||
@Test public void Ns() {
|
||||
Site_core_db json_db = new Site_core_db(fxt.App().Fsys_mgr().Cfg_site_meta_fil());
|
||||
Site_namespace_tbl ns_tbl = json_db.Tbl__namespace();
|
||||
ns_tbl.Insert(Bry_.new_a7("en.w"), Xow_ns_.Tid__help, Xow_ns_case_.Bry__1st, Bry_.Empty, Bry_.new_a7("Help"), Bool_.N, Bool_.N, Bry_.Empty);
|
||||
ns_tbl.Insert(Bry_.new_a7("fr.w"), Xow_ns_.Tid__help, Xow_ns_case_.Bry__1st, Bry_.Empty, Bry_.new_a7("Aide"), Bool_.N, Bool_.N, Bry_.Empty);
|
||||
|
||||
fxt.Run_page_wkr(wkr
|
||||
, fxt.New_page_wo_date(1, "11", Wdata_wiki_mgr_fxt.New_json("q1", "links", String_.Ary("enwiki", "Help:Q1_en", "frwiki", "Aide:Q1_fr")))
|
||||
);
|
||||
|
||||
db_tester.Test__select_tbl(conn, "wbase_qid", flds__wbase_qid
|
||||
, Object_.Ary("enwiki", 12, "Q1_en", "q1")
|
||||
, Object_.Ary("frwiki", 12, "Q1_fr", "q1")
|
||||
);
|
||||
}
|
||||
@Test public void Links_w_name() { // PURPOSE: wikidata changed links node from "enwiki:A" to "enwiki:{name:A,badges:[]}"; DATE:2013-09-14
|
||||
String q1_str = String_.Concat_lines_nl
|
||||
( "{ \"entity\":\"q1\""
|
||||
, ", \"links\":"
|
||||
, " { \"enwiki\":\"q1_en\""
|
||||
, " , \"frwiki\":\"q1_fr\""
|
||||
, " }"
|
||||
, "}"
|
||||
);
|
||||
String q2_str = String_.Concat_lines_nl
|
||||
( "{ \"entity\":[\"item\",2]"
|
||||
, ", \"links\":"
|
||||
, " { \"enwiki\":{\"name\":\"q2_en\",\"badges\":[]}"
|
||||
, " , \"frwiki\":{\"name\":\"q2_fr\",\"badges\":[]}"
|
||||
, " }"
|
||||
, "}"
|
||||
);
|
||||
|
||||
fxt.Run_page_wkr(wkr
|
||||
, fxt.New_page_wo_date(1, "q1", q1_str)
|
||||
, fxt.New_page_wo_date(2, "q2", q2_str)
|
||||
);
|
||||
|
||||
db_tester.Test__select_tbl(conn, "wbase_qid", flds__wbase_qid
|
||||
, Object_.Ary("enwiki", 0, "Q1_en", "q1")
|
||||
, Object_.Ary("frwiki", 0, "Q1_fr", "q1")
|
||||
, Object_.Ary("enwiki", 0, "Q2_en", "q2")
|
||||
, Object_.Ary("frwiki", 0, "Q2_fr", "q2")
|
||||
);
|
||||
}
|
||||
@Test public void Spaces() { // PURPOSE: assert that ttls with spaces are converted to unders DATE:2015-04-21
|
||||
fxt.Run_page_wkr(wkr
|
||||
, fxt.New_page_wo_date(2, "q2", Wdata_wiki_mgr_fxt.New_json("q2", "links", String_.Ary("enwiki", "q2 en", "frwiki", "q2 fr"))) // note "q2 en" not "q2_en"
|
||||
, fxt.New_page_wo_date(1, "q1", Wdata_wiki_mgr_fxt.New_json("q1", "links", String_.Ary("enwiki", "q1 en", "frwiki", "q1 fr")))
|
||||
);
|
||||
|
||||
db_tester.Test__select_tbl(conn, "wbase_qid", flds__wbase_qid
|
||||
, Object_.Ary("enwiki", 0, "Q2_en", "q2") // NOTE: q2_en
|
||||
, Object_.Ary("frwiki", 0, "Q2_fr", "q2")
|
||||
, Object_.Ary("enwiki", 0, "Q1_en", "q1")
|
||||
, Object_.Ary("frwiki", 0, "Q1_fr", "q1")
|
||||
);
|
||||
}
|
||||
}
|
||||
@ -1,46 +0,0 @@
|
||||
/*
|
||||
XOWA: the XOWA Offline Wiki Application
|
||||
Copyright (C) 2012-2017 gnosygnu@gmail.com
|
||||
|
||||
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
|
||||
or alternatively under the terms of the Apache License Version 2.0.
|
||||
|
||||
You may use XOWA according to either of these licenses as is most appropriate
|
||||
for your project on a case-by-case basis.
|
||||
|
||||
The terms of each license can be found in the source code repository:
|
||||
|
||||
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
|
||||
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
|
||||
*/
|
||||
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
|
||||
import gplx.xowa.bldrs.*;
|
||||
public class Xob_wdata_qid_txt extends Xob_wdata_qid_base {
|
||||
private Wdata_idx_bldr_qid qid_bldr;
|
||||
@Override public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_tdb_text_wdata_qid;}
|
||||
@Override public void Qid_bgn() {qid_bldr = new Wdata_idx_bldr_qid().Ctor(this, bldr, wiki, dump_fil_len);}
|
||||
@Override public void Qid_add(byte[] wiki_key, int ns_id, byte[] ttl, byte[] qid) {
|
||||
qid_bldr.Add(String_.new_u8(wiki_key), Int_.To_str_pad_bgn_zero(ns_id, 3), ttl, qid);
|
||||
}
|
||||
@Override public void Qid_end() {
|
||||
qid_bldr.Flush();
|
||||
qid_bldr.Make();
|
||||
}
|
||||
}
|
||||
class Wdata_idx_bldr_qid extends Wdata_idx_mgr_base {
|
||||
public Wdata_idx_bldr_qid Ctor(Xob_wdata_qid_base wkr, Xob_bldr bldr, Xowe_wiki wiki, int dump_fil_len) {super.Ctor(wkr, bldr, wiki, dump_fil_len); return this;}
|
||||
public void Add(String wiki_key, String ns_num_str, byte[] ttl, byte[] qid) {
|
||||
Wdata_idx_wtr wtr = Get_or_new(wiki_key, ns_num_str);
|
||||
wtr.Write(ttl, qid);
|
||||
}
|
||||
private Wdata_idx_wtr Get_or_new(String wiki_key, String ns_num_str) {
|
||||
String wtr_key = wiki_key + "|" + ns_num_str;
|
||||
Object rv = hash.Get_by(wtr_key);
|
||||
if (rv == null) {
|
||||
Wdata_idx_wtr wtr = Wdata_idx_wtr.new_qid_(wiki, wiki_key, ns_num_str, dump_fil_len);
|
||||
hash.Add(wtr_key, wtr);
|
||||
return wtr;
|
||||
}
|
||||
return (Wdata_idx_wtr)rv;
|
||||
}
|
||||
}
|
||||
@ -22,7 +22,7 @@ import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.parsers.*;
|
||||
class Xowb_json_dump_db {
|
||||
private final Xoae_app app; private final Gfo_usr_dlg usr_dlg; private final Xowe_wiki wiki; private final Xob_bldr bldr;
|
||||
private final Json_parser json_parser;
|
||||
private final Xob_wdata_pid_sql pid_cmd; private final Xob_wdata_qid_sql qid_cmd = new Xob_wdata_qid_sql();
|
||||
private final Xob_wdata_pid pid_cmd; private final Xob_wdata_qid qid_cmd;
|
||||
private Xow_ns_mgr ns_mgr; private Xow_db_mgr db_mgr;
|
||||
private Xowd_page_tbl page_tbl; private Xob_ns_to_db_mgr ns_to_db_mgr;
|
||||
private Io_stream_zip_mgr text_zip_mgr; private byte text_zip_tid;
|
||||
@ -32,7 +32,8 @@ class Xowb_json_dump_db {
|
||||
this.app = bldr.App(); this.usr_dlg = app.Usr_dlg(); this.wiki = wiki; this.bldr = bldr;
|
||||
this.json_parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
|
||||
this.ns_mgr = wiki.Ns_mgr();
|
||||
this.pid_cmd = new Xob_wdata_pid_sql(wiki.Data__core_mgr().Db__wbase().Conn(), null);
|
||||
this.pid_cmd = new Xob_wdata_pid(wiki.Data__core_mgr().Db__wbase().Conn());
|
||||
this.qid_cmd = new Xob_wdata_qid(wiki.Data__core_mgr().Db__wbase().Conn());
|
||||
}
|
||||
public void Parse_all_bgn(long src_fil_len, String src_fil_name) {
|
||||
// load wiki
|
||||
@ -57,7 +58,7 @@ class Xowb_json_dump_db {
|
||||
this.page_modified_on = Datetime_now.Get();
|
||||
page_tbl.Insert_bgn();
|
||||
qid_cmd.Page_wkr__bgn();
|
||||
pid_cmd.Pid_bgn();
|
||||
pid_cmd.Pid__bgn();
|
||||
}
|
||||
public void Parse_doc(byte[] json_bry) {
|
||||
// parse to jdoc
|
||||
@ -78,17 +79,17 @@ class Xowb_json_dump_db {
|
||||
|
||||
// insert text
|
||||
if (jdoc_is_qid) {
|
||||
qid_cmd.Parse_jdoc(jdoc);
|
||||
qid_cmd.Qid__run(jdoc);
|
||||
++page_count_main;
|
||||
}
|
||||
else
|
||||
pid_cmd.Parse_jdoc(jdoc);
|
||||
pid_cmd.Pid__run(jdoc);
|
||||
}
|
||||
public void Parse_all_end() {
|
||||
page_tbl.Insert_end();
|
||||
page_tbl.Create_idx();
|
||||
qid_cmd.Qid_end();
|
||||
pid_cmd.Pid_end();
|
||||
qid_cmd.Qid__end();
|
||||
pid_cmd.Pid__end();
|
||||
ns_to_db_mgr.Rls_all();
|
||||
|
||||
// cleanup core
|
||||
|
||||
@ -18,117 +18,189 @@ import org.junit.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*
|
||||
public class Wdata_pf_property__basic__tst {
|
||||
@Before public void init() {fxt.Init();} private final Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt();
|
||||
@Test public void String() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_string(1, "a")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_string(1, "a"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
fxt.Test_parse("{{#property:p1}}", "a");
|
||||
fxt.Test_parse("{{#property:p2}}", "");
|
||||
}
|
||||
@Test public void Entity() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "Q1");
|
||||
fxt.Init__docs__add(fxt.Wdoc_bldr("Q2").Add_label("en", "b").Xto_wdoc());
|
||||
fxt.Init__docs__add(fxt.doc_("Q1", fxt.Make_claim_entity_qid(1, 2)));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q2")
|
||||
.Add_label("en", "b")
|
||||
);
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_entity_qid(1, 2))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
fxt.Test_parse("{{#property:p1}}", "b");
|
||||
}
|
||||
@Test public void Entity_fr() { // PURPOSE: non-English wiki should default to English label if non-English label not available; DATE:2013-12-19
|
||||
fxt.Wiki().Wdata_wiki_lang_(Bry_.new_a7("fr")); // set wiki to French
|
||||
fxt.Init_links_add("frwiki", "Test_page", "Q1"); // create link for en:Test_page in wikidata
|
||||
fxt.Init__docs__add(fxt.doc_("Q1", fxt.Make_claim_entity_qid(1, 2))); // create wdata page Q1 with prop entity reference to Q2
|
||||
fxt.Init__docs__add(fxt.Wdoc_bldr("Q2").Add_label("en", "b").Xto_wdoc()); // create wdata page Q2 with label in en (not fr)
|
||||
fxt.Test_parse("{{#property:p1}}", "b"); // parse; should get en label
|
||||
// set wiki to French
|
||||
fxt.Wiki().Wdata_wiki_lang_(Bry_.new_a7("fr"));
|
||||
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_entity_qid(1, 2))
|
||||
.Add_sitelink("frwiki", "Test_page")
|
||||
);
|
||||
|
||||
// create wdata page Q2 with label in en (not fr)
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q2")
|
||||
.Add_label("en", "b")
|
||||
);
|
||||
|
||||
// parse; should get en label
|
||||
fxt.Test_parse("{{#property:p1}}", "b");
|
||||
}
|
||||
@Test public void Entity_missing() { // PURPOSE: wiki may refer to entity that no longer exists; EX: {{#property:p1}} which links to Q1, but p1 links to Q2 and Q2 was deleted; DATE:2014-02-01
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1"); // create link for en:Test_page in wikidata
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_entity_qid(1, 2))); // create wdata page Q1 with prop entity reference to Q2; note that Q2 is not created
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_entity_qid(1, 2)) // create wdata page Q1 with prop entity reference to Q2; note that Q2 is not created
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", ""); // parse; get ""
|
||||
}
|
||||
@Test public void Time() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_time(1, "2012-01-02 03:04:05")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_time(1, "2012-01-02 03:04:05"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "30405 2 Jan 2012"); // NOTE: format is missing ":" b/c test does not init messages for html_wtr; DATE:2015-08-03
|
||||
}
|
||||
@Test public void Geodata() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_geo(1, "6.789", "1.2345")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_geo(1, "6.789", "1.2345"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "1°14'4.2"N, 6°47'20.4"E");
|
||||
}
|
||||
@Test public void Quantity__plus_minus__y() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234", "1", "+1236", "+1232")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_quantity(1, "+1234", "1", "+1236", "+1232"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "1,234±2");
|
||||
}
|
||||
@Test public void Quantity__plus_minus__n() { // PURPOSE:do not output ± if lbound == val == ubound; PAGE:en.w:Tintinan DATE:2015-08-02
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234", "1", "+1234", "+1234")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_quantity(1, "+1234", "1", "+1234", "+1234"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "1,234");
|
||||
}
|
||||
@Test public void Quantity__range() { // PURPOSE:do not output ± if lbound == val == ubound; PAGE:en.w:Tintinan DATE:2015-08-02
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234", "1", "+1236", "+1233")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_quantity(1, "+1234", "1", "+1236", "+1233"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "1,233-1,236");
|
||||
}
|
||||
@Test public void Quantity__long() { // PURPOSE: must cast to long for large numbers; EX:{{#property:P1082}} PAGE:en.w:Earth; DATE:2015-08-02
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+4321000000", "1", "4321000000", "4321000000")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_quantity(1, "+4321000000", "1", "4321000000", "4321000000"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "4,321,000,000");
|
||||
}
|
||||
@Test public void Quantity__unit__entity() {// PURPOSE: get entity name; EX:{{#invoke:Wikidata|getUnits|P2386|FETCH_WIKIDATA}} PAGE:en.w:Arecibo_Observatory; DATE:2016-10-11
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234", "http://www.wikidata.org/entity/q2", "+1236", "+1232")));
|
||||
Wdata_doc wdoc = fxt.doc_("q2", fxt.Make_claim_string(2, "a"));
|
||||
wdoc.Label_list().Add(Bry_.new_a7("en"), new gplx.xowa.xtns.wbases.core.Wdata_langtext_itm(Bry_.new_a7("en"), Bry_.new_a7("meter")));
|
||||
fxt.Init__docs__add(wdoc);
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_quantity(1, "+1234", "http://www.wikidata.org/entity/Q2", "+1236", "+1232"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q2")
|
||||
.Add_claims(fxt.Make_claim_string(2, "a"))
|
||||
.Add_label("en", "meter")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "1,234±2 meter");
|
||||
}
|
||||
@Test public void Quantity__decimal() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_quantity(1, "+1234.50", "1", "+1236.75", "+1232.25")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_quantity(1, "+1234.50", "1", "+1236.75", "+1232.25"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "1,234.5±2.25");
|
||||
}
|
||||
@Test public void Monolingualtext() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_monolingual(1, "la", "Lorem ipsum dolor sit amet")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_monolingual(1, "la", "Lorem ipsum dolor sit amet"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "Lorem ipsum dolor sit amet");
|
||||
}
|
||||
@Test public void Novalue() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_novalue(1)));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_novalue(1))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "novalue");
|
||||
}
|
||||
@Test public void Somevalue() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_somevalue(1)));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_somevalue(1))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "somevalue");
|
||||
}
|
||||
@Test public void Multiple() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_string(1, "a"), fxt.Make_claim_string(1, "b")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_string(1, "a"), fxt.Make_claim_string(1, "b"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1}}", "a"); // only take first; DATE:2015-08-02
|
||||
}
|
||||
@Test public void Q() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q2");
|
||||
fxt.Init__docs__add(fxt.doc_("q2", fxt.Make_claim_string(1, "a")));
|
||||
fxt.Test_parse("{{#property:p1|q=q2}}", "a");
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q2")
|
||||
.Add_claims(fxt.Make_claim_string(1, "a"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1|q=Q2}}", "a");
|
||||
}
|
||||
@Test public void Of() {
|
||||
fxt.Init_links_add("enwiki", "Of_page", "q2");
|
||||
fxt.Init__docs__add(fxt.doc_("q2", fxt.Make_claim_string(1, "a")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q2")
|
||||
.Add_claims(fxt.Make_claim_string(1, "a"))
|
||||
.Add_sitelink("enwiki", "Of_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1|of=Of_page}}", "a");
|
||||
}
|
||||
@Test public void From() {
|
||||
fxt.Init__docs__add(fxt.doc_("Property:p2", fxt.Make_claim_string(1, "a")));
|
||||
fxt.Test_parse("{{#property:p1|from=p2}}", "a");
|
||||
fxt.Init__docs__add(fxt.Wdoc("Property:P2")
|
||||
.Add_claims(fxt.Make_claim_string(1, "a"))
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#property:p1|from=P2}}", "a");
|
||||
fxt.Test_parse("{{#property:p1|from=}}", "");
|
||||
}
|
||||
@Test public void Pid_as_name() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q2");
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q2")
|
||||
.Add_claims(fxt.Make_claim_string(1, "a"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
fxt.Init_pids_add("en", "astronomic symbol", 1);
|
||||
fxt.Init__docs__add(fxt.doc_("q2", fxt.Make_claim_string(1, "a")));
|
||||
|
||||
fxt.Test_parse("{{#property:astronomic symbol}}", "a");
|
||||
}
|
||||
@Test public void Empty_arg() { // PURPOSE: {{#property:p1|}} should not fail / warn; DATE:2013-11-15
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q2");
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q2")
|
||||
.Add_claims(fxt.Make_claim_string(1, "a"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
fxt.Init_pids_add("en", "astronomic symbol", 1);
|
||||
fxt.Init__docs__add(fxt.doc_("q2", fxt.Make_claim_string(1, "a")));
|
||||
|
||||
fxt.Test_parse("{{#property:p1|}}", "a");
|
||||
}
|
||||
@Test public void Parse_pid() {
|
||||
|
||||
@ -18,14 +18,23 @@ import org.junit.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*
|
||||
public class Wdata_pf_statements__basic__tst {
|
||||
@Before public void init() {fxt.Init();} private final Wdata_wiki_mgr_fxt fxt = new Wdata_wiki_mgr_fxt();
|
||||
@Test public void String() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "q1");
|
||||
fxt.Init__docs__add(fxt.doc_("q1", fxt.Make_claim_string(1, "a")));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_string(1, "a"))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#statements:p1}}", "a");
|
||||
}
|
||||
@Test public void Entity() {
|
||||
fxt.Init_links_add("enwiki", "Test_page", "Q1");
|
||||
fxt.Init__docs__add(fxt.Wdoc_bldr("Q2").Add_label("en", "b").Xto_wdoc());
|
||||
fxt.Init__docs__add(fxt.doc_("Q1", fxt.Make_claim_entity_qid(1, 2)));
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q1")
|
||||
.Add_claims(fxt.Make_claim_entity_qid(1, 2))
|
||||
.Add_sitelink("enwiki", "Test_page")
|
||||
);
|
||||
|
||||
fxt.Init__docs__add(fxt.Wdoc("Q2")
|
||||
.Add_label("en", "b")
|
||||
);
|
||||
|
||||
fxt.Test_parse("{{#statements:p1}}", "[[b]]");
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user