Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.9.2.1'

Browse Source
This commit is contained in:
gnosygnu
2016-09-11 21:49:20 -04:00
parent 232838c732
commit 35d78f6106
310 changed files with 4358 additions and 5116 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
400_xowa/src/gplx/xowa/bldrs/Db_mgr_fxt.java
View File

@@ -17,10 +17,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.primitives.*; import gplx.core.strings.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.ctgs.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.addons.wikis.ctgs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.infos.*;
import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*;
public class Db_mgr_fxt {
public Db_mgr_fxt Ctor_fsys() {bldr_fxt = new Xob_fxt().Ctor(Xoa_test_.Url_root().GenSubDir("root")); return this;}
public Db_mgr_fxt Ctor_mem() {bldr_fxt = new Xob_fxt().Ctor_mem(); return this;} private Xob_fxt bldr_fxt;
@@ -78,47 +79,27 @@ public class Db_mgr_fxt {
public void Test_category_v1(String ctg_name_str, int... expd) {
Xowe_wiki wiki = bldr_fxt.Wiki();
byte[] ctg_name_bry = Bry_.new_a7(ctg_name_str);
Xoctg_view_ctg ctg = new Xoctg_view_ctg();
Xoctg_catpage_ctg ctg = new Xoctg_catpage_ctg(ctg_name_bry);
wiki.Db_mgr_as_sql().Load_mgr().Load_ctg_v1(ctg, ctg_name_bry);
Tfds.Eq_ary(expd, Xto_int_ary(ctg));
}
int[] Xto_int_ary(Xoctg_view_ctg ctg) {
int[] Xto_int_ary(Xoctg_catpage_ctg ctg) {
List_adp list = List_adp_.New();
byte tid_max = Xoa_ctg_mgr.Tid__max;
for (byte tid = 0; tid < tid_max; tid++) {
Xoctg_view_grp grp = ctg.Grp_by_tid(tid); if (grp == null) continue;
Xoctg_catpage_grp grp = ctg.Grp_by_tid(tid); if (grp == null) continue;
int len = grp.Itms_list().Count();
for (int i = 0; i < len; i++) {
Xoctg_view_itm itm = (Xoctg_view_itm)grp.Itms_list().Get_at(i);
Xoctg_catpage_itm itm = (Xoctg_catpage_itm)grp.Itms_list().Get_at(i);
list.Add(itm.Page_id());
}
}
return (int[])list.To_ary_and_clear(int.class);
}
public void Test_category_v2(String ctg_name_str, int... expd) {
Xowe_wiki wiki = bldr_fxt.Wiki();
byte[] ctg_name_bry = Bry_.new_a7(ctg_name_str);
Xoctg_data_ctg ctg = new Xoctg_data_ctg(ctg_name_bry);
wiki.Db_mgr_as_sql().Load_mgr().Load_ctg_v2(ctg, ctg_name_bry);
Tfds.Eq_ary(expd, Xto_int_ary(ctg));
}
public void Test_file(String url, String expd) {
String actl = Io_mgr.Instance.LoadFilStr(url);
Tfds.Eq_str_lines(expd, actl);
}
int[] Xto_int_ary(Xoctg_data_ctg ctg) {
List_adp list = List_adp_.New();
byte tid_max = Xoa_ctg_mgr.Tid__max;
for (byte tid = 0; tid < tid_max; tid++) {
Xoctg_idx_mgr grp = ctg.Grp_by_tid(tid); if (grp == null) continue;
int len = grp.Itms_len();
for (int i = 0; i < len; i++) {
Xoctg_idx_itm itm = grp.Itms_get_at(i);
list.Add(itm.Id());
}
}
return (int[])list.To_ary_and_clear(int.class);
}
public void Init_db_sqlite() {
Xowe_wiki wiki = this.Wiki();
Db_conn_pool.Instance.Rls_all();

11
400_xowa/src/gplx/xowa/bldrs/Xob_cmd_keys.java
View File

@@ -23,10 +23,6 @@ public class Xob_cmd_keys {
, Key_text_css = "text.css"
, Key_text_search_cmd = "text.search.cmd" // "import.sql.search_title.cmd"
, Key_text_search_wkr = "text.search" // "import.sql.search_title.wkr"
, Key_text_cat_core_v1 = "text.cat.core.v1" // "import.sql.category_v1"
, Key_text_cat_core = "text.cat.core" // "import.sql.category_registry"
, Key_text_cat_link = "text.cat.link" // "import.sql.categorylinks"
, Key_text_cat_hidden = "text.cat.hidden" // "import.sql.hiddencat"
, Key_text_term = "text.term" // "import.sql.term"
, Key_html_redlinks = "html.redlinks"
, Key_util_cleanup = "util.cleanup" // "core.cleanup"
@@ -46,19 +42,12 @@ public class Xob_cmd_keys {
, Key_tdb_text_init = "tdb.text.init" // "core.init"
, Key_tdb_make_page = "tdb.text.page" // "core.make_page"
, Key_tdb_make_id = "core.make_id"
, Key_tdb_make_search_title = "core.make_search_title"
, Key_tdb_make_category = "core.make_category"
, Key_tdb_calc_stats = "core.calc_stats"
, Key_tdb_core_term = "tdb.text.term" // "core.term"
, Key_tdb_text_cat_link = "ctg.link_sql"
, Key_tdb_ctg_link_idx = "ctg.link_idx"
, Key_tdb_cat_hidden_sql = "ctg.hiddencat_sql"
, Key_tdb_cat_hidden_ttl = "ctg.hiddencat_ttl"
, Key_tdb_text_wdata_qid = "tdb.text.wdata.qid"
, Key_tdb_text_wdata_pid = "tdb.text.wdata.pid"
, Key_exec_sql = "import.sql.exec_sql"
, Key_deploy_zip = "deploy.zip"
, Key_deploy_copy = "deploy.copy"
, Key_decompress_bz2 = "core.decompress_bz2"
;
}

20
400_xowa/src/gplx/xowa/bldrs/Xob_cmd_mgr.java
View File

@@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.primitives.*;
import gplx.xowa.wikis.*; import gplx.xowa.xtns.wbases.imports.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.cmds.texts.*; import gplx.xowa.bldrs.cmds.texts.sqls.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.bldrs.cmds.ctgs.*; import gplx.xowa.bldrs.cmds.utils.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.cmds.texts.*; import gplx.xowa.bldrs.cmds.texts.sqls.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.wikis.ctgs.bldrs.*; import gplx.xowa.bldrs.cmds.utils.*;
import gplx.xowa.bldrs.cmds.diffs.*;
import gplx.xowa.files.origs.*; import gplx.xowa.htmls.core.bldrs.*;
import gplx.xowa.addons.wikis.searchs.bldrs.*;
@@ -43,10 +43,6 @@ public class Xob_cmd_mgr implements Gfo_invk {
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_css)) return Add(new Xob_css_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_search_wkr)) return Xml_rdr_direct_add(wiki, new gplx.xowa.addons.wikis.searchs.bldrs.Srch_bldr_wkr(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_search_cmd)) return Add(new Srch_bldr_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_core_v1)) return Xml_rdr_parser_add(wiki, new Xob_ctg_v1_sql().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_core)) return Add(new Xob_category_registry_sql(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_link)) return Add(new Xob_categorylinks_sql(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_hidden)) return Add(new Xoctg_hiddencat_parser_sql(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_term)) return Add(new Xob_term_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_delete_page)) return Add(new Xob_page_delete_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_html_redlinks)) return Add(new Xob_redlink_mkr_cmd(bldr, wiki));
@@ -63,13 +59,7 @@ public class Xob_cmd_mgr implements Gfo_invk {
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_init)) return Add(new Xob_init_tdb(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_page)) return Xml_rdr_direct_add(wiki, new Xob_page_txt(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_id)) return Xml_rdr_direct_add(wiki, new Xob_make_id_wkr(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_search_title)) return Xml_rdr_direct_add(wiki, new Xob_search_tdb(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_category)) return Xml_rdr_parser_add(wiki, new Xob_ctg_v1_txt().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_calc_stats)) return Add(new Xob_calc_stats_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_cat_link)) return Add(new Xob_categorylinks_txt(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_ctg_link_idx)) return Add(new Xoctg_link_idx_wkr(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_cat_hidden_sql)) return Add(new Xoctg_hiddencat_parser_txt(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_cat_hidden_ttl)) return Add(new Xoctg_hiddencat_ttl_wkr(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_core_term)) return Add(new Xob_term_txt(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_qid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_qid_txt().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_txt().Ctor(bldr, wiki));
@@ -77,19 +67,13 @@ public class Xob_cmd_mgr implements Gfo_invk {
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_exec_sql)) return Add(new Xob_exec_sql_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_decompress_bz2)) return Add(new Xob_decompress_bz2_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_deploy_zip)) return Add(new Xob_deploy_zip_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_deploy_copy)) return Add(new Xob_deploy_copy_cmd(bldr, wiki));
else throw Err_.new_unhandled(cmd_key);
else throw Err_.new_unimplemented_w_msg("builder command is not supported: " + cmd_key);
}
private Xob_page_wkr Xml_rdr_direct_add(Xowe_wiki wiki, Xob_page_wkr wkr) {
Xob_page_wkr_cmd dump_rdr = Xml_rdr_get(wiki);
dump_rdr.Wkr_add(wkr);
return wkr;
}
private Xobd_parser_wkr Xml_rdr_parser_add(Xowe_wiki wiki, Xobd_parser_wkr wkr) {
Xob_page_wkr_cmd dump_rdr = Xml_rdr_get(wiki);
dump_rdr.Page_parser_assert().Wkr_add(wkr);
return wkr;
}
private Xob_page_wkr_cmd Xml_rdr_get(Xowe_wiki wiki) {
byte[] wiki_key = wiki.Domain_bry();
Xob_page_wkr_cmd rv = (Xob_page_wkr_cmd)dump_rdrs.Get_by(dump_rdrs_ref.Val_(wiki_key));

8
400_xowa/src/gplx/xowa/bldrs/Xob_fxt.java
View File

@@ -61,14 +61,6 @@ public class Xob_fxt {
rv.Modified_on_(DateAdp_.seg_(modified_on));
return rv;
}
public Xob_fxt Run_ctg() {
Xobd_parser parser = new Xobd_parser(bldr);
gplx.xowa.bldrs.cmds.ctgs.Xob_ctg_v1_base ctg_wkr = new gplx.xowa.bldrs.cmds.ctgs.Xob_ctg_v1_txt().Ctor(bldr, wiki);
byte[] bry = Bry_.new_a7("[[Category:");
ctg_wkr.Wkr_hooks().Add(bry, bry);
parser.Wkr_add(ctg_wkr);
return Run(parser);
}
public Xob_fxt Run_id() {
Xob_make_id_wkr wkr = new Xob_make_id_wkr(bldr, wiki);
Run(wkr);

1076
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Uca_trie.java
View File

File diff suppressed because it is too large Load Diff

45
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Uca_trie_tst.java
View File

@@ -1,45 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.xowa.bldrs.*;
public class Uca_trie_tst {
@Before public void init() {fxt.Clear();} private Xob_base_fxt fxt = new Xob_base_fxt();
@Test public void Basic() {
Uca_trie_fxt fxt = new Uca_trie_fxt();
fxt.Clear();
fxt.Init_trie_itm("a", Bry_.New_by_ints(10, 11));
fxt.Init_trie_itm("b", Bry_.New_by_ints(20, 21));
fxt.Init_trie_itm("c", Bry_.New_by_ints(30, 31));
fxt.Test_decode(Bry_.New_by_ints(10, 11), "a");
fxt.Test_decode(Bry_.New_by_ints(10, 11, 20, 21, 30, 31), "abc");
}
}
class Uca_trie_fxt {
public void Clear() {
if (trie == null) {
trie = new Uca_trie();
bfr = Bry_bfr_.New();
}
trie.Clear();
} Uca_trie trie; Bry_bfr bfr;
public void Init_trie_itm(String charAsStr, byte[] uca) {trie.Init_itm(gplx.core.intls.Utf16_.Decode_to_int(Bry_.new_u8(charAsStr), 0), uca);}
public void Test_decode(byte[] bry, String expd) {
trie.Decode(bfr, bry, 0, bry.length);
Tfds.Eq(expd, bfr.To_str_and_clear());
}
}

58
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_category_registry_sql.java
View File

@@ -1,58 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.ios.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.dbs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.bldrs.wtrs.*;
public class Xob_category_registry_sql implements Xob_cmd {
public Xob_category_registry_sql(Xob_bldr bldr, Xowe_wiki wiki) {this.wiki = wiki;} private Xowe_wiki wiki;
public String Cmd_key() {return Xob_cmd_keys.Key_text_cat_core;}
public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public void Cmd_end() { // NOTE: placing in end, b/c must run *after* page_sql
// setup
wiki.Html_mgr().Importing_ctgs_(Bool_.Y);
Io_url rslt_dir = Xob_category_registry_sql.Tmp_dir(wiki);
Io_mgr.Instance.DeleteDirDeep(rslt_dir);
Xob_tmp_wtr rslt_wtr = Xob_tmp_wtr.new_wo_ns_(Io_url_gen_.dir_(rslt_dir), Io_mgr.Len_mb);
// read data
Gfo_usr_dlg usr_dlg = wiki.Appe().Usr_dlg();
wiki.Init_db_mgr();
Xowd_page_tbl page_core_tbl = wiki.Db_mgr_as_sql().Core_data_mgr().Tbl__page();
Db_rdr rdr = page_core_tbl.Conn().Stmt_select_order(page_core_tbl.Tbl_name(), String_.Ary(page_core_tbl.Fld_page_title(), page_core_tbl.Fld_page_id()), String_.Ary(page_core_tbl.Fld_page_ns()), page_core_tbl.Fld_page_title())
.Crt_int(page_core_tbl.Fld_page_ns(), Xow_ns_.Tid__category)
.Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
byte[] page_ttl = rdr.Read_bry_by_str(page_core_tbl.Fld_page_title());
int page_id = rdr.Read_int(page_core_tbl.Fld_page_id());
if (rslt_wtr.FlushNeeded(page_ttl.length + 2 + 5)) rslt_wtr.Flush(usr_dlg);
rslt_wtr.Bfr().Add(page_ttl).Add_byte_pipe().Add_base85_len_5(page_id).Add_byte_nl();
}
} finally {rdr.Rls();}
// cleanup
rslt_wtr.Flush(usr_dlg);
wiki.Html_mgr().Importing_ctgs_(Bool_.N);
}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_run() {}
public void Cmd_term() {}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return Gfo_invk_.Rv_unhandled;}
public static Io_url Tmp_dir(Xowe_wiki wiki) {return wiki.Fsys_mgr().Tmp_dir().GenSubDir(Xob_cmd_keys.Key_text_cat_core);}
}

66
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_category_registry_sql_tst.java
View File

@@ -1,66 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.core.primitives.*; import gplx.core.encoders.*; import gplx.xowa.bldrs.*; import gplx.xowa.wikis.nss.*;
public class Xob_category_registry_sql_tst {
@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Clear();} private Xob_category_registry_sql_fxt fxt = new Xob_category_registry_sql_fxt();
@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();}
@Test public void Basic() {
if (Xoa_test_.Db_skip()) return;
fxt.Init_page_insert(String_.Ary("Ctg3", "Ctg2", "Ctg1"));
fxt.Exec_category_registry_cmd();
fxt.Test_ids(Int_.Ary(3, 2, 1)); // note that Ctg1 is page_id 3
}
}
class Xob_category_registry_sql_fxt {
Db_mgr_fxt fxt; Xowe_wiki wiki; Xoae_app app; Int_obj_ref page_id_next = Int_obj_ref.New(1);
public void Clear() {
if (fxt == null) {
fxt = new Db_mgr_fxt().Ctor_fsys();
fxt.Init_db_sqlite();
wiki = fxt.Wiki();
app = wiki.Appe();
}
}
public void Rls() {fxt.Rls();}
public void Init_page_insert(String[] ttls) {
fxt.Init_page_insert(page_id_next, Xow_ns_.Tid__category, ttls);
}
public void Exec_category_registry_cmd() {
app.Bldr().Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core);
app.Bldr().Run();
}
public void Test_ids(int[] expd) {
Io_url rslts_dir = Xob_category_registry_sql.Tmp_dir(wiki);
String rslts_txt = Io_mgr.Instance.LoadFilStr(Io_mgr.Instance.QueryDir_fils(rslts_dir)[0]);
int[] actl = Parse_rslts_txt(rslts_txt);
Tfds.Eq_ary(expd, actl);
}
int[] Parse_rslts_txt(String rslts_txt) {
String[] lines = String_.SplitLines_nl(rslts_txt);
List_adp list = List_adp_.New();
int len = lines.length;
for (int i = 0; i < len; i++) {
String line = lines[i];
if (String_.Len_eq_0(line)) break; // ignore blank line
String[] flds = String_.Split(line, '|');
list.Add(Base85_.To_int_by_str(flds[1]));
}
return (int[])list.To_ary_and_clear(int.class);
}
}

91
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_base.java
View File

@@ -1,91 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.brys.*; import gplx.core.ios.*; import gplx.core.times.*;
import gplx.xowa.wikis.ctgs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
public abstract class Xob_categorylinks_base extends Xob_sql_dump_base implements Sql_file_parser_cmd {
private DateAdp_parser date_parser = DateAdp_parser.new_(); private Sql_file_parser sql_parser; Uca_trie trie; private Bry_bfr uca_bfr = Bry_bfr_.Reset(255);
public abstract Io_sort_cmd Make_sort_cmd(Sql_file_parser sql_parser);
@Override public String Sql_file_name() {return "categorylinks";}
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
this.sql_parser = parser;
wiki.Html_mgr().Importing_ctgs_(Bool_.Y);
parser.Fld_cmd_(this).Flds_req_(Fld_cl_from, Fld_cl_to, Fld_cl_timestamp, Fld_cl_collation, Fld_cl_sortkey, Fld_cl_type);
}
public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
if (Bry_.Eq(fld_key, Fld_cl_from)) cur_id = Bry_.To_int_or(src, fld_bgn, fld_end, -1);
else if (Bry_.Eq(fld_key, Fld_cl_to)) cur_ctg = Bry_.Mid(src, fld_bgn, fld_end);
else if (Bry_.Eq(fld_key, Fld_cl_collation)) cur_collation_is_uca = Bry_.Has_at_bgn(src, Collation_uca, fld_bgn, fld_end);
else if (Bry_.Eq(fld_key, Fld_cl_timestamp)) {
date_parser.Parse_iso8651_like(cur_modified_on, src, fld_bgn, fld_end);
cur_date = fld_end - fld_bgn == 0 // ignore null dates added by ctg_v1
? 0 : Int_flag_bldr_.To_int_date_short(cur_modified_on);
}
else if (Bry_.Eq(fld_key, Fld_cl_sortkey)) {
int nl_pos = Bry_find_.Find_fwd(src, Byte_ascii.Nl, fld_bgn, fld_end);
if (nl_pos != Bry_find_.Not_found) // sortkey sometimes has format of "sortkey\ntitle"; EX: "WALES, JIMMY\nJIMMY WALES"; discard 2nd to conserve hard-disk space
fld_end = nl_pos;
cur_sortkey = Bry_.Mid(src, fld_bgn, fld_end);
}
else if (Bry_.Eq(fld_key, Fld_cl_type)) {
byte char_0 = src[fld_bgn];
switch (char_0) {
case Byte_ascii.Ltr_f:
case Byte_ascii.Ltr_p: cur_tid = char_0; break;
case Byte_ascii.Ltr_s: cur_tid = Byte_ascii.Ltr_c; break;
default: throw Err_.new_unhandled(char_0);
}
if (cur_collation_is_uca) {
if (trie == null) {
trie = new Uca_trie();
trie.Init();
}
trie.Decode(uca_bfr, cur_sortkey, 0, cur_sortkey.length);
cur_sortkey = uca_bfr.Len() == 0 ? Sortkey_space : uca_bfr.To_bry_and_clear();
}
fld_wtr.Bfr_(file_bfr);
fld_wtr.Write_bry_escape_fld(cur_ctg);
fld_wtr.Write_byte_fld(cur_tid);
fld_wtr.Write_bry_escape_fld(cur_sortkey);
fld_wtr.Write_int_base85_len5_fld(cur_id);
fld_wtr.Write_int_base85_len5_fld(cur_date);
}
} int cur_id = -1, cur_date = -1; byte[] cur_ctg = null, cur_sortkey = null; byte cur_tid = Byte_.Max_value_127; boolean cur_collation_is_uca; int[] cur_modified_on = new int[7];
@Override public void Cmd_end() {
Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, temp_dir.GenSubDir("sort"), sort_mem_len, Xoctg_link_sql_sorter.Instance, Io_line_rdr_key_gen_.noop, Make_sort_cmd(sql_parser));
wiki.Html_mgr().Importing_ctgs_(Bool_.N);
gplx.xowa.bldrs.wkrs.Xob_io_utl_.Delete_sql_files(wiki.Fsys_mgr().Root_dir(), this.Sql_file_name());
}
private static final byte[] Fld_cl_from = Bry_.new_a7("cl_from"), Fld_cl_to = Bry_.new_a7("cl_to"), Fld_cl_timestamp = Bry_.new_a7("cl_timestamp"), Fld_cl_collation = Bry_.new_a7("cl_collation"), Fld_cl_sortkey = Bry_.new_a7("cl_sortkey"), Fld_cl_type = Bry_.new_a7("cl_type");
private static final byte[] Collation_uca = Bry_.new_a7("uca"), Sortkey_space = new byte[] {Byte_ascii.Space};
}
class Xoctg_link_sql_sorter implements gplx.core.lists.ComparerAble {
public int compare(Object lhsObj, Object rhsObj) {
Io_sort_split_itm lhs = (Io_sort_split_itm)lhsObj, rhs = (Io_sort_split_itm)rhsObj;
byte[] lhs_bry = lhs.Bfr(); int lhs_bgn = 0, lhs_end = lhs.Row_bgn() - 1;
byte[] rhs_bry = rhs.Bfr(); int rhs_bgn = 0, rhs_end = rhs.Row_bgn() - 1;
for (int i = 0; i < 3; i++) {
lhs_bgn = lhs_end + 1; lhs_end = Bry_find_.Find_fwd(lhs_bry, Byte_ascii.Pipe, lhs_bgn);
rhs_bgn = rhs_end + 1; rhs_end = Bry_find_.Find_fwd(rhs_bry, Byte_ascii.Pipe, rhs_bgn);
int comp = Bry_.Compare(lhs_bry, lhs_bgn, lhs_end, rhs_bry, rhs_bgn, rhs_end);
if (comp != CompareAble_.Same) return comp;
}
return CompareAble_.Same;
}
public static final Xoctg_link_sql_sorter Instance = new Xoctg_link_sql_sorter(); Xoctg_link_sql_sorter() {}
}

70
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_base_tst.java
View File

@@ -1,70 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.xowa.bldrs.*;
public class Xob_categorylinks_base_tst {
@Before public void init() {fxt.Clear();} private Xob_base_fxt fxt = new Xob_base_fxt();
@Test public void Basic() {
Io_url src_fil = Io_url_.new_fil_("mem/temp/sql_dump.sql");
fxt .Init_fil(src_fil, String_.Concat
( Xob_categorylinks_sql.Sql_categorylinks
, "INSERT INTO `categorylinks` VALUES"
, " (1,'Ctg_2','PAGE_2a','2013-04-15 01:02:03','','uppercase','page')"
, ",(2,'Ctg_1','PAGE_1b','2013-04-15 01:02:03','','uppercase','page')"
, ",(3,'Ctg_1','PAGE_1a','2013-04-15 01:02:03','','uppercase','page')"
, ";"
))
.Exec_cmd(Xob_cmd_keys.Key_tdb_text_cat_link, GfoMsg_.basic_(Xob_categorylinks_base.Invk_src_fil_, src_fil))
.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.link_sql/make/0000000000.csv", String_.Concat_lines_nl
( "Ctg_1|p|PAGE_1a|!!!!$|#8DDL|"
, "Ctg_1|p|PAGE_1b|!!!!#|#8DDL|"
, "Ctg_2|p|PAGE_2a|!!!!\"|#8DDL|"
))
;
}
@Test public void Super_earths() { // PURPOSE: handle multi-field sort
Io_url src_fil = Io_url_.new_fil_("mem/temp/sql_dump.sql");
fxt .Init_fil(src_fil, String_.Concat
( Xob_categorylinks_sql.Sql_categorylinks
, "INSERT INTO `categorylinks` VALUES"
, " (1,'Super-Earths','PAGE_1a','2013-04-15 01:02:03','','uppercase','page')"
, ",(2,'Super-Earths_in_the_habitable_zone','PAGE_1b','2013-04-15 01:02:03','','uppercase','page')"
, ";"
))
.Exec_cmd(Xob_cmd_keys.Key_tdb_text_cat_link, GfoMsg_.basic_(Xob_categorylinks_base.Invk_src_fil_, src_fil))
.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.link_sql/make/0000000000.csv", String_.Concat_lines_nl
( "Super-Earths|p|PAGE_1a|!!!!\"|#8DDL|"
, "Super-Earths_in_the_habitable_zone|p|PAGE_1b|!!!!#|#8DDL|"
))
;
}
@Test public void Sortkey_has_newline() { // PURPOSE: sortkey sometimes has format of "sortkey\ntitle"; EX: "WALES, JIMMY\nJIMMY WALES"; discard 2nd for hard-disk space savings
Io_url src_fil = Io_url_.new_fil_("mem/temp/sql_dump.sql");
fxt .Init_fil(src_fil, String_.Concat
( Xob_categorylinks_sql.Sql_categorylinks
, "INSERT INTO `categorylinks` VALUES"
, " (1,'Ctg_1','LAST,FIRST\\nFIRST LAST','2013-04-15 01:02:03','','uppercase','page')"
, ";"
))
.Exec_cmd(Xob_cmd_keys.Key_tdb_text_cat_link, GfoMsg_.basic_(Xob_categorylinks_base.Invk_src_fil_, src_fil))
.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.link_sql/make/0000000000.csv", String_.Concat_lines_nl
( "Ctg_1|p|LAST,FIRST|!!!!\"|#8DDL|"
))
;
}
}

44
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_sql.java
View File

@@ -1,44 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.ios.*;
import gplx.xowa.bldrs.sqls.*;
public class Xob_categorylinks_sql extends Xob_categorylinks_base {
private Db_idx_mode idx_mode = Db_idx_mode.Itm_end;
public Xob_categorylinks_sql(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_text_cat_link;}
@Override public Io_sort_cmd Make_sort_cmd(Sql_file_parser sql_parser) {return new Xob_categorylinks_sql_make(sql_parser, wiki, idx_mode);}
public static final String Sql_categorylinks = String_.Concat_lines_nl
( "CREATE TABLE `categorylinks` ("
, " `cl_from` int(10) unsigned NOT NULL DEFAULT '0',"
, " `cl_to` varbinary(255) NOT NULL DEFAULT '',"
, " `cl_sortkey` varbinary(230) NOT NULL DEFAULT '',"
, " `cl_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,"
, " `cl_sortkey_prefix` varbinary(255) NOT NULL DEFAULT '',"
, " `cl_collation` varbinary(32) NOT NULL DEFAULT '',"
, " `cl_type` enum('page','subcat','file') NOT NULL DEFAULT 'page',"
, " UNIQUE KEY `cl_from` (`cl_from`,`cl_to`),"
, ");"
);
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_idx_mode_)) idx_mode = Db_idx_mode.Xto_itm(m.ReadStr("v"));
else return super.Invk(ctx, ikey, k, m);
return this;
}
private static final String Invk_idx_mode_ = "idx_mode_";
}

153
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_sql_make.java
View File

@@ -1,153 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.flds.*; import gplx.core.ios.*; import gplx.core.encoders.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.ctgs.*;
import gplx.xowa.bldrs.sqls.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_categorylinks_sql_make implements Io_make_cmd {
private Gfo_usr_dlg usr_dlg; private final Xowe_wiki wiki; private final Db_idx_mode idx_mode; private Xow_db_mgr core_db_mgr;
private final Sql_file_parser sql_parser; private final Gfo_fld_rdr fld_rdr = Gfo_fld_rdr.xowa_(); private Io_line_rdr name_id_rdr;
private Xowd_cat_core_tbl cat_core_tbl; private Xowd_cat_link_tbl cat_link_tbl;
private long cat_db_size, cat_db_max; private int cur_row_count;
private int[] cur_cat_counts = new int[Xoa_ctg_mgr.Tid__max]; private byte[] cur_cat_ttl = Ttl_first; private int cur_cat_id; private int cur_cat_file_idx;
public Xob_categorylinks_sql_make(Sql_file_parser sql_parser, Xowe_wiki wiki, Db_idx_mode idx_mode) {
this.wiki = wiki; this.sql_parser = sql_parser; this.idx_mode = idx_mode; usr_dlg = wiki.Appe().Usr_dlg();
}
public void Sort_bgn() {
this.core_db_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
this.name_id_rdr = New_registry_rdr(wiki, usr_dlg);
this.cat_db_max = wiki.Appe().Api_root().Bldr().Wiki().Import().Cat_link_db_max();
boolean one_file = core_db_mgr.Props().Layout_text().Tid_is_all_or_few();
if (!one_file) // cat is in its own dbs: delete dbs
core_db_mgr.Dbs__delete_by_tid(Xow_db_file_.Tid__cat, Xow_db_file_.Tid__cat_core, Xow_db_file_.Tid__cat_link); // delete existing category files else upgrade won't work
Db_make(Bool_.Y);
if (one_file) { // cat is in single db; delete tbls;
cat_core_tbl.Delete_all();
cat_link_tbl.Delete_all();
}
}
public void Sort_do(Io_line_rdr rdr) {
byte[] new_cat_ttl = Bry_.Empty;
try {
fld_rdr.Ini(rdr.Bfr(), rdr.Itm_pos_bgn());
new_cat_ttl = fld_rdr.Read_bry_escape();
if (!Bry_.Eq(new_cat_ttl, cur_cat_ttl)) // ttl changed
cur_cat_id = Save_ctg(new_cat_ttl);
if (cur_cat_id == Cur_cat_id_null) return;
byte cat_tid = Convert_cat_tid_to_byte(fld_rdr.Read_byte());
byte[] cat_sortkey = fld_rdr.Read_bry_escape();
int page_id = fld_rdr.Read_int_base85_len5();
int cat_timestamp = fld_rdr.Read_int_base85_len5();
cat_link_tbl.Insert_cmd_by_batch(page_id, cur_cat_id, cat_tid, cat_sortkey, cat_timestamp);
cat_db_size += 34 + 20 + 12 + (cat_sortkey.length * 2); // NOTE_1: categorylinks row size
++cur_cat_counts[cat_tid];
++cur_row_count;
if (cur_row_count % 100000 == 0) usr_dlg.Prog_one("", "", "inserting category row: ~{0}", cur_row_count);
if (cur_row_count % 1000000 == 0) {cat_core_tbl.Conn().Txn_sav(); cat_link_tbl.Conn().Txn_sav();}
} catch (Exception e) {usr_dlg.Warn_many("", "", "ctg_links.insert failed: name=~{0} err=~{1}", String_.new_u8(new_cat_ttl), Err_.Message_gplx_full(e));}
}
public void Sort_end() {
Save_ctg(Ttl_last);
Db_close();
Xodb_mgr_sql db_mgr_sql = wiki.Db_mgr_as_sql();
if (db_mgr_sql.Category_version() == Xoa_ctg_mgr.Version_null) // NOTE: ctg_v1 wkr will set this to v1; only set to v2 if null
db_mgr_sql.Category_version_update(false);
usr_dlg.Log_many("", "", "import.category.v2: insert done; committing; rows=~{0}", cur_row_count);
name_id_rdr.Rls();
if (String_.Eq(sql_parser.Src_fil().NameAndExt(), Xob_ctg_v1_sql_make.Url_sql)) // delete temp xowa_categorylinks.sql file created by cat_v1
Io_mgr.Instance.DeleteFil(sql_parser.Src_fil());
}
private int Save_ctg(byte[] new_ctg_ttl) {
if (cur_cat_ttl != Bry_.Empty && cur_cat_id != -1)
cat_core_tbl.Insert_cmd_by_batch(cur_cat_id, cur_cat_counts[Xoa_ctg_mgr.Tid_page], cur_cat_counts[Xoa_ctg_mgr.Tid_subc], cur_cat_counts[Xoa_ctg_mgr.Tid_file], Xoa_ctg_mgr.Hidden_n, cur_cat_file_idx);
if (new_ctg_ttl == Ttl_last) return Cur_cat_id_null; // last ttl; called by this.End(); exit early else will fail in Cur_cat_id_find()
if (cat_db_size > cat_db_max) {Db_close(); Db_make(Bool_.N);}
cur_cat_id = Cur_cat_id_find(new_ctg_ttl);
for (int i = 0; i < Xoa_ctg_mgr.Tid__max; i++)
cur_cat_counts[i] = 0;
cur_cat_ttl = new_ctg_ttl;
return cur_cat_id;
}
private void Db_make(boolean first) {
boolean one_file = core_db_mgr.Props().Layout_text().Tid_is_all_or_few();
if (first) { // create cat_core
Xow_db_file cat_core_db = one_file ? core_db_mgr.Db__cat_core() : core_db_mgr.Dbs__make_by_tid(Xow_db_file_.Tid__cat_core);
this.cat_core_tbl = cat_core_db.Tbl__cat_core().Create_tbl();
}
Xow_db_file cat_link_db = one_file ? core_db_mgr.Db__core() : core_db_mgr.Dbs__make_by_tid(Xow_db_file_.Tid__cat_link);
this.cat_link_tbl = cat_link_db.Tbl__cat_link();
if ( (one_file && first)
|| !one_file)
cat_link_tbl.Create_tbl();
this.cur_cat_file_idx = cat_link_db.Id();
cat_core_tbl.Insert_bgn();
cat_link_tbl.Insert_bgn();
if (idx_mode.Tid_is_bgn()) cat_link_tbl.Create_idx();
}
private void Db_close() {
cat_core_tbl.Insert_end();
cat_link_tbl.Insert_end();
cat_db_size = 0;
if (idx_mode.Tid_is_end()) cat_link_tbl.Create_idx();
}
private static byte Convert_cat_tid_to_byte(byte ltr) {
switch (ltr) {
case Byte_ascii.Ltr_f: return Xoa_ctg_mgr.Tid_file;
case Byte_ascii.Ltr_p: return Xoa_ctg_mgr.Tid_page;
case Byte_ascii.Ltr_c: return Xoa_ctg_mgr.Tid_subc;
default: throw Err_.new_unhandled(ltr);
}
}
public Io_sort_cmd Make_dir_(Io_url v) {return this;}
private int Cur_cat_id_find(byte[] ttl) {
while (true) {
int compare = Bry_.Compare(ttl, 0, ttl.length, name_id_rdr.Bfr(), name_id_rdr.Key_pos_bgn(), name_id_rdr.Key_pos_end());
switch (compare) {
case CompareAble_.Same: return Base85_.To_int_by_bry(name_id_rdr.Bfr(), name_id_rdr.Key_pos_end() + 1, name_id_rdr.Itm_pos_end() - 2);
case CompareAble_.More:
boolean reading = name_id_rdr.Read_next();
if (!reading) return Cur_cat_id_null; // eof return
break;
case CompareAble_.Less: return Cur_cat_id_null; // stop
}
}
}
private static final int Cur_cat_id_null = -1;
private static Io_line_rdr New_registry_rdr(Xowe_wiki wiki, Gfo_usr_dlg usr_dlg) {
Io_url make_dir = Xob_category_registry_sql.Tmp_dir(wiki);
usr_dlg.Prog_many("", "", "loading category_registry files: ~{0}", make_dir.Raw());
Io_url[] urls = Io_mgr.Instance.QueryDir_args(make_dir).ExecAsUrlAry();
return new Io_line_rdr(usr_dlg, urls).Key_gen_(Io_line_rdr_key_gen_.first_pipe);
}
private static final byte[] Ttl_last = null, Ttl_first = Bry_.Empty;
}
/*
NOTE_1: categorylinks row size: 34 + 20 + 12 + (cat_sortkey.length * 2)
row length (data) : 34=8+4+4+14+4 ROWID, cl_from, cl_to_id, cl_timestamp, cl_type_id
cl_main length (idx) : 20=8+4+4+4 ROWID, cl_from, cl_to_id, cl_type_id
cl_from length (idx) : 12=8+4 ROWID, cl_from
variable_data length : cat_sortkey.length * 2 sortkey is used for row and cl_main
Note the following
. ints are 4 bytes
. tinyint is assumed to be 4 bytes (should be 1, but sqlite only has one numeric datatype, so import all 4?)
. varchar(14) is assumed to be 14 bytes (should be 15? +1 for length of varchar?)
. calculations work out "too well". comparing 4 databases gets +/- .25 bytes per row. however
.. - bytes should not be possible
.. +.25 bytes is too low (18 MB out of 5.5 GB).*; there must be other bytes used for page breaks / fragmentation
*/

142
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_sql_tst.java
View File

@@ -1,142 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.core.primitives.*; import gplx.core.stores.*;
import gplx.dbs.*; import gplx.dbs.sqls.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.ctgs.*; import gplx.xowa.bldrs.*; import gplx.xowa.wikis.nss.*;
public class Xob_categorylinks_sql_tst {
@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Ctor_fsys();} Db_mgr_fxt fxt = new Db_mgr_fxt();
@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();}
@Test public void Basic() {
if (Xoa_test_.Db_skip()) return;
fxt.Init_db_sqlite();
fxt.Init_page_insert(Int_obj_ref.New(1), Xow_ns_.Tid__category, String_.Ary("Ctg_1", "Ctg_2"));
fxt.Init_fil(Xoa_test_.Url_wiki_enwiki().GenSubFil("xowa_categorylinks.sql"), String_.Concat
( Xob_categorylinks_sql.Sql_categorylinks
, "INSERT INTO `categorylinks` VALUES"
, " (3,'Ctg_2','File:2a','2013-04-15 01:02:03','','uppercase','file')"
, ",(4,'Ctg_1','1b','2013-04-15 01:02:03','','uppercase','page')"
, ",(5,'Ctg_1','1a','2013-04-15 01:02:03','','uppercase','page')"
, ";"
));
fxt.Exec_run(new Xob_category_registry_sql(fxt.Bldr(), fxt.Wiki()));
fxt.Exec_run(new Xob_categorylinks_sql(fxt.Bldr(), fxt.Wiki()));
Db_conn conn = fxt.Wiki().Db_mgr_as_sql().Core_data_mgr().Db__cat_core().Conn();
Db_tst_qry.tbl_("cat_core", "cat_id")
.Cols_("cat_id", "cat_subcats", "cat_files", "cat_pages")
.Rows_add_vals(1, 0, 0, 2)
.Rows_add_vals(2, 0, 1, 0)
.Test(conn);
Db_tst_qry.tbl_("cat_link", "cl_from")
.Cols_("cl_from", "cl_to_id", "cl_sortkey", "cl_type_id")
.Rows_add_vals(3, 2, "File:2a" , Xoa_ctg_mgr.Tid_file)
.Rows_add_vals(4, 1, "1b" , Xoa_ctg_mgr.Tid_page)
.Rows_add_vals(5, 1, "1a" , Xoa_ctg_mgr.Tid_page)
.Test(conn);
}
}
class Db_tst_val {
public Db_tst_val(int idx, String key, Object val) {this.idx = idx; this.key = key; this.val = val;}
public int Idx() {return idx;} public Db_tst_val Idx_(int val) {idx = val; return this;} private int idx = -1;
public String Key() {return key;} public Db_tst_val Key_(String v) {this.key = v; return this;} private String key;
public Object Val() {return val;} public Db_tst_val Val_(Object v) {this.val = v; return this;} Object val;
public static final int Idx_null = -1;
public static final String Key_null = null;
}
class Db_tst_row {
public int Idx() {return idx;} public Db_tst_row Idx_(int val) {idx = val; return this;} private int idx = -1;
public Db_tst_val[] Vals_ary() {return vals_ary;} Db_tst_val[] vals_ary = null;
public static Db_tst_row vals_(Object... ary) {
Db_tst_row rv = new Db_tst_row();
int len = ary.length;
Db_tst_val[] vals_ary = new Db_tst_val[len];
for (int i = 0; i < len; i++)
vals_ary[i] = new Db_tst_val(Db_tst_val.Idx_null, Db_tst_val.Key_null, ary[i]);
rv.vals_ary = vals_ary;
return rv;
}
public static Db_tst_row kvs_(String[] cols, Object[] vals) {
int cols_len = cols.length;
int vals_len = vals.length;
if (cols_len != vals_len) throw Err_.new_wo_type("mismatch in cols / vals");
Db_tst_row rv = new Db_tst_row();
Db_tst_val[] vals_ary = new Db_tst_val[cols_len];
for (int i = 0; i < cols_len; i++) {
String col = cols[i];
Object val = vals[i];
vals_ary[i] = new Db_tst_val(Db_tst_val.Idx_null, col, val);
}
rv.vals_ary = vals_ary;
return rv;
}
}
class Db_tst_qry {
private final Sql_qry_wtr qry_wtr = Sql_qry_wtr_.New__sqlite();
public Db_qry Qry() {return qry;} Db_qry qry;
public String[] Cols() {return cols;} public Db_tst_qry Cols_(String... v) {this.cols = v; return this;} private String[] cols;
public List_adp Rows() {return rows;} List_adp rows = List_adp_.New();
public Db_tst_qry Rows_add_vals(Object... ary) {
Db_tst_row row = Db_tst_row.kvs_(cols, ary);
rows.Add(row);
return this;
}
public void Test(Db_conn conn) {
DataRdr rdr = DataRdr_.Null;
Bry_bfr bfr = Bry_bfr_.New();
try {
rdr = conn.Exec_qry_as_old_rdr(qry);
int expd_row_idx = 0, expd_row_max = rows.Count();
while (rdr.MoveNextPeer()) {
if (expd_row_idx == expd_row_max) break;
Db_tst_row expd_row = (Db_tst_row)rows.Get_at(expd_row_idx);
Test_row(bfr, expd_row_idx, expd_row, rdr);
++expd_row_idx;
}
} finally {rdr.Rls();}
}
private void Test_row(Bry_bfr bfr, int expd_row_idx, Db_tst_row expd_row, DataRdr rdr) {
Db_tst_val[] expd_vals = expd_row.Vals_ary();
int len = expd_vals.length;
int pad_max = 16;
boolean pass = true;
bfr.Clear().Add_byte_nl(); // add prefix nl for JUnit formatting
for (int i = 0; i < len; i++) {
Db_tst_val expd_val = expd_vals[i];
String expd_key = expd_val.Key();
String expd_str = Object_.Xto_str_strict_or_empty(expd_val.Val());
String actl_str = Object_.Xto_str_strict_or_empty(rdr.Read(expd_key));
boolean eq = String_.Eq(expd_str, actl_str); // NOTE: always compare strings, not objs; problems with comparing byte to int
bfr.Add_str_pad_space_end(expd_key, pad_max);
bfr.Add_str_pad_space_bgn(expd_str, pad_max);
bfr.Add(eq ? Lbl_eq_y : Lbl_eq_n);
bfr.Add_str_pad_space_end(actl_str, pad_max);
bfr.Add_byte_nl();
if (!eq) pass = false;
}
if (!pass) {
bfr.Add(Lbl_row_hdr).Add_int_variable(expd_row_idx).Add_byte_nl();
bfr.Add_str_u8(qry.To_sql__exec(qry_wtr)).Add_byte(Byte_ascii.Semic);
throw Err_.new_wo_type(bfr.To_str_and_clear());
}
} private static final byte[] Lbl_row_hdr = Bry_.new_a7("row: "), Lbl_eq_y = Bry_.new_a7(" == "), Lbl_eq_n = Bry_.new_a7(" != ");
public static Db_tst_qry tbl_(String tbl_name, String order_by) {return new_(Db_qry_.select_tbl_(tbl_name).Order_asc_(order_by));}
public static Db_tst_qry new_(Db_qry qry) {
Db_tst_qry rv = new Db_tst_qry();
rv.qry = qry;
return rv;
}
}

25
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_categorylinks_txt.java
View File

@@ -1,25 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.ios.*;
import gplx.xowa.bldrs.sqls.*;
public class Xob_categorylinks_txt extends Xob_categorylinks_base {
public Xob_categorylinks_txt(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_text_cat_link;}
@Override public Io_sort_cmd Make_sort_cmd(Sql_file_parser sql_parser) {return new Io_sort_fil_basic(bldr.Usr_dlg(), this.Make_url_gen(), make_fil_len);}
}

128
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_base.java
View File

@@ -1,128 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.primitives.*; import gplx.core.btries.*; import gplx.core.flds.*; import gplx.core.ios.*; import gplx.xowa.wikis.tdbs.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.bldrs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.data.tbls.*;
public abstract class Xob_ctg_v1_base extends Xob_itm_dump_base implements Xobd_parser_wkr, Gfo_invk {
protected Xob_ctg_v1_base() {} // TEST:needed for fxt
public Xob_ctg_v1_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); return this;}
public abstract String Wkr_key();
public abstract Io_sort_cmd Make_sort_cmd();
public Ordered_hash Wkr_hooks() {return wkr_hooks;} private Ordered_hash wkr_hooks = Ordered_hash_.New_bry();
public void Wkr_bgn(Xob_bldr bldr) {
this.Init_dump(this.Wkr_key(), wiki.Tdb_fsys_mgr().Site_dir().GenSubDir(Xotdb_dir_info_.Name_category));
Bry_bfr tmp_bfr = wiki.Utl__bfr_mkr().Get_b512();
Xol_lang_itm lang = wiki.Lang();
wkr_hooks_add(tmp_bfr, lang.Ns_names());
wkr_hooks_add(tmp_bfr, lang.Ns_aliases());
wkr_hooks_add(tmp_bfr, Xow_ns_canonical_.Ary);
tmp_bfr.Mkr_rls();
fld_wtr.Bfr_(dump_bfr);
}
public int Wkr_run(Xowd_page_itm page, byte[] src, int src_len, int bgn, int end) {
int ttl_bgn = end, ttl_end = -1;
int pos = end;
while (true) {
if (pos == src_len) { // fail: EOS
return end;
}
Object o = trie.Match_bgn(src, pos, src_len);
if (o != null) {
Btrie_itm_stub stub = (Btrie_itm_stub)o;
byte[] bry = stub.Val();
switch (stub.Tid()) {
case Tid_brack_end: case Tid_pipe:
ttl_bgn = Move_fwd_while_space(src, src_len, ttl_bgn);
ttl_end = Move_bwd_while_space(src, src_len, pos - 1);
if (ttl_end > ttl_bgn) // NOTE: ignore examples like [[Category: ]]
Process_ctg(page, src, src_len, ttl_bgn, ttl_end);
break;
case Tid_brack_bgn: // fail: [[ is invalid
return pos;
case Tid_nl: // fail: \n is invalid
return pos;
}
return pos + bry.length;
}
++pos;
}
}
@gplx.Virtual public void Process_ctg(Xowd_page_itm page, byte[] src, int src_len, int bgn, int end) {
Process_ctg_row(fld_wtr, dump_fil_len, dump_url_gen, page.Id(), src, src_len, bgn, end);
}
public static void Process_ctg_row(Gfo_fld_wtr fld_wtr, int dump_fil_len, Io_url_gen dump_url_gen, int page_id, byte[] src, int src_len, int bgn, int end) {
int len = end - bgn;
Bry_bfr dump_bfr = fld_wtr.Bfr();
if (dump_bfr.Len() + row_fixed_len + len > dump_fil_len) Io_mgr.Instance.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
byte[] ttl = Bry_.Mid(src, bgn, end);
Bry_.Replace_reuse(ttl, Byte_ascii.Space, Byte_ascii.Underline);
fld_wtr.Write_bry_escape_fld(ttl).Write_int_base85_len5_row(page_id);
}
public void Wkr_end() {
this.Term_dump(this.Make_sort_cmd());
if (delete_temp) Io_mgr.Instance.DeleteDirDeep(temp_dir);
}
private Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
Btrie_fast_mgr trie = Btrie_fast_mgr.cs().Add_stub(Tid_brack_end, "]]").Add_stub(Tid_pipe, "|").Add_stub(Tid_nl, "\n").Add_stub(Tid_brack_bgn, "[[");
static final int row_fixed_len = 5 + 1 + 1; // 5=rowId; 1=|; 1=\n
List_adp category_list = List_adp_.New(); Int_obj_ref cur_pos = Int_obj_ref.New_zero();
static final byte Tid_eos = 0, Tid_brack_end = 1, Tid_pipe = 2, Tid_nl = 3, Tid_brack_bgn = 4;
private static int Move_fwd_while_space(byte[] src, int src_len, int pos) {
while (true) {
if (pos == src_len) return pos;
switch (src[pos]) {
case Byte_ascii.Space: break;
default: return pos;
}
++pos;
}
}
private static int Move_bwd_while_space(byte[] src, int src_len, int pos) {
while (true) {
if (pos == -1) return 0;
switch (src[pos]) {
case Byte_ascii.Space: break;
default: return pos + 1;
}
--pos;
}
}
private void wkr_hooks_add(Bry_bfr tmp_bfr, Xow_ns[] ary) {
int len = ary.length;
for (int i = 0; i < len; i++) {
Xow_ns ns = ary[i];
if (ns.Id_is_ctg()) wkr_hooks_add(tmp_bfr, ns.Name_db());
}
}
private void wkr_hooks_add(Bry_bfr tmp_bfr, Xol_ns_grp ns_grp) {
int len = ns_grp.Len();
for (int i = 0; i < len; i++) {
Xow_ns ns = ns_grp.Get_at(i);
if (ns.Id_is_ctg()) wkr_hooks_add(tmp_bfr, ns.Name_db());
}
}
private void wkr_hooks_add(Bry_bfr tmp_bfr, byte[] word) {
tmp_bfr.Add_byte(Byte_ascii.Brack_bgn).Add_byte(Byte_ascii.Brack_bgn).Add(word).Add_byte(Byte_ascii.Colon);
byte[] key = tmp_bfr.To_bry_and_clear();
if (!wkr_hooks.Has(key)) wkr_hooks.Add(key, key);
}
static final String GRP_KEY = "xowa.bldr.make_ctg";
}

59
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_base_tst.java
View File

@@ -1,59 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.core.ios.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_ctg_v1_base_tst {
Xowd_page_wkr_ctg_fxt fxt = new Xowd_page_wkr_ctg_fxt();
@Test public void One() {fxt.ini_("[[Category:A]]").tst_("A");}
@Test public void Many() {fxt.ini_("[[Category:A]] [[Category:B]] [[Category:C]]").tst_("A", "B", "C");}
@Test public void Pipe() {fxt.ini_("[[Category:A|B]]").tst_("A");}
@Test public void Nl() {fxt.ini_("[[Category:A\nB]]").tst_();}
@Test public void Brack_bgn() {fxt.ini_("[[Category:A[[Category:B]]").tst_("B");}
@Test public void Mix() {fxt.ini_("[[Category:A]] [[Category:B\nc]][[Category:D|E]]").tst_("A", "D");}
@Test public void Ws() {fxt.ini_("[[Category: A ]]").tst_("A");}
@Test public void Eos() {fxt.ini_("[[Category:abcdef").tst_();}
}
class Xowd_page_wkr_ctg_fxt {
byte[] src;
public Xowd_page_wkr_ctg_fxt ini_(String s) {src = Bry_.new_u8(s); return this;}
public Xowd_page_wkr_ctg_fxt tst_(String... expd) {
Xoae_app app = Xoa_app_fxt.Make__app__edit();
Xowe_wiki wiki = Xoa_app_fxt.Make__wiki__edit(app);
Xob_bldr bldr = Xoa_app_fxt.bldr_(app);
Xobd_parser mgr = new Xobd_parser(bldr);
Xobd_parser_wkr_ctg_tstr wkr = (Xobd_parser_wkr_ctg_tstr)new Xobd_parser_wkr_ctg_tstr().Ctor(bldr, wiki);
byte[] bry = Bry_.new_a7("[[Category:");
wkr.Wkr_hooks().Add(bry, bry);
mgr.Wkr_add(wkr);
Xowd_page_itm page = new Xowd_page_itm().Text_(src);//.Ttl_(Bry_.new_a7("Test"), new Xow_ns_mgr());
mgr.Page_wkr__bgn();
mgr.Page_wkr__run(page);
byte[][] ttl = (byte[][])wkr.Found().To_ary(byte[].class);
String[] actl = new String[ttl.length];
for (int i = 0; i < actl.length; i++) {
actl[i] = String_.new_u8(ttl[i]);
}
Tfds.Eq_ary_str(expd, actl);
return this;
}
}
class Xobd_parser_wkr_ctg_tstr extends Xob_ctg_v1_txt { public List_adp Found() {return found;} List_adp found = List_adp_.New();
@Override public void Process_ctg(Xowd_page_itm page, byte[] src, int src_len, int bgn, int end) {
found.Add(Bry_.Mid(src, bgn, end));
}
}

79
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_sql.java
View File

@@ -1,79 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.brys.fmtrs.*;
import gplx.core.flds.*; import gplx.core.ios.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.ctgs.*;
import gplx.xowa.bldrs.wtrs.*;
public class Xob_ctg_v1_sql extends Xob_ctg_v1_base {
@Override public String Wkr_key() {return Xob_cmd_keys.Key_text_cat_core_v1;}
@Override public Io_sort_cmd Make_sort_cmd() {return new Xob_ctg_v1_sql_make(wiki);}
}
class Xob_ctg_v1_sql_make implements Io_make_cmd {
private Gfo_fld_rdr fld_rdr = Gfo_fld_rdr.xowa_(); private Xob_tmp_wtr sql_wtr; private Gfo_usr_dlg usr_dlg; private boolean is_first = true;
private byte[] prv_ctg_name = Bry_.Empty; private int prv_page_id = 0;
private Xowe_wiki wiki; private Xodb_mgr_sql db_mgr; private int page_count = 0; private int progress_interval = 10000;
private final Bry_fmtr fmtr = Bry_fmtr.new_("(~{page_id},'~{cat_name}','','','','','~{cat_type}')\n", "page_id", "cat_name", "cat_type");
public Xob_ctg_v1_sql_make(Xowe_wiki wiki) {this.wiki = wiki; db_mgr = wiki.Db_mgr_as_sql();}
public Io_sort_cmd Make_dir_(Io_url v) {return this;} // ignore
public void Sort_bgn() {
usr_dlg = wiki.Appe().Usr_dlg();
Io_url sql_url = wiki.Fsys_mgr().Root_dir().GenSubFil(Url_sql);
Io_mgr.Instance.DeleteFil_args(sql_url).MissingFails_off().Exec();
sql_wtr = Xob_tmp_wtr.new_wo_ns_(Io_url_gen_.fil_(sql_url), Io_mgr.Len_mb);
sql_wtr.Bfr().Add_str_u8(Xob_categorylinks_sql.Sql_categorylinks).Add(Sql_hdr);
}
public void Sort_do(Io_line_rdr rdr) {
if (page_count++ % progress_interval == 0) usr_dlg.Prog_many("", "", "building category; count=~{0}", page_count);
fld_rdr.Ini(rdr.Bfr(), rdr.Itm_pos_bgn());
byte[] ctg_name = fld_rdr.Read_bry_escape();
ctg_name = Escape_for_sql(wiki, ctg_name);
int page_id = fld_rdr.Read_int_base85_len5();
if (Bry_.Eq(prv_ctg_name, ctg_name) && page_id == prv_page_id) return;
if (sql_wtr.FlushNeeded(5 + 2 + ctg_name.length)) sql_wtr.Flush(usr_dlg); // 5=base85; 2=dlms
byte row_dlm = is_first ? Byte_ascii.Space : Byte_ascii.Comma; // handle " (" or ",("
is_first = false;
sql_wtr.Bfr().Add_byte(row_dlm);
fmtr.Bld_bfr_many(sql_wtr.Bfr(), page_id, ctg_name, Byte_ascii.Ltr_p); // 'p' b/c page_ttl is not available; limitation of ctg_v1
prv_page_id = page_id;
prv_ctg_name = ctg_name;
}
public void Sort_end() {
sql_wtr.Bfr().Add_byte(Byte_ascii.Semic); // close INSERT with ";"
sql_wtr.Flush(usr_dlg);
db_mgr.Category_version_update(true);
}
private static final byte[] Sql_hdr = Bry_.new_a7("INSERT INTO 'categorylinks' VALUES");
public static final String Url_sql = "xowa_categorylinks.sql";
private static byte[] Escape_for_sql(Xowe_wiki wiki, byte[] bry) {
Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b512();
int len = bry.length;
boolean dirty = false;
for (int i = 0; i < len; i++) {
byte b = bry[i];
switch (b) {
case Byte_ascii.Apos: if (!dirty) {bfr.Add_mid(bry, 0, i); dirty = true;} bfr.Add_byte(Byte_ascii.Backslash).Add_byte(Byte_ascii.Apos); break;
case Byte_ascii.Backslash: if (!dirty) {bfr.Add_mid(bry, 0, i); dirty = true;} bfr.Add_byte(Byte_ascii.Backslash).Add_byte(Byte_ascii.Backslash); break;
default:
if (dirty) bfr.Add_byte(b);
break;
}
}
bfr.Mkr_rls();
return dirty ? bfr.To_bry_and_clear() : bry;
}
}

84
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_sql_tst.java
View File

@@ -1,84 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.dbs.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.xowa.bldrs.cmds.texts.sqls.*;
public class Xob_ctg_v1_sql_tst {
@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Ctor_fsys();} Db_mgr_fxt fxt = new Db_mgr_fxt();
@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();}
@Test public void Basic() {
if (Xoa_test_.Db_skip()) return;
fxt.Init_db_sqlite();
fxt.doc_ary_
( fxt.doc_wo_date_(1, "A" , "[[Category:Ctg_0]] [[Category:Ctg_1]]")
, fxt.doc_wo_date_(2, "B" , "[[Category:Ctg_1]] [[Category:Ctg_2]]")
, fxt.doc_wo_date_(3, "File:C" , "[[Category:Ctg_0]]")
, fxt.doc_wo_date_(4, "Category:D" , "[[Category:Ctg_0]]")
, fxt.doc_wo_date_(5, "E" , "[[Category:Ctg_a'b]]")
, fxt.doc_wo_date_(6, "F" , "[[Category:Ctg_a\\]]")
)
.Exec_run(new Xob_page_cmd(fxt.Bldr(), fxt.Wiki()))
.Exec_run(new Xob_ctg_v1_sql().Ctor(fxt.Bldr(), fxt.Wiki()))
;
fxt.Test_file(Xoa_test_.Url_root().GenSubFil_nest("root", "wiki", "en.wikipedia.org", "xowa_categorylinks.sql").Raw(), String_.Concat_lines_nl
( "CREATE TABLE `categorylinks` ("
, " `cl_from` int(10) unsigned NOT NULL DEFAULT '0',"
, " `cl_to` varbinary(255) NOT NULL DEFAULT '',"
, " `cl_sortkey` varbinary(230) NOT NULL DEFAULT '',"
, " `cl_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,"
, " `cl_sortkey_prefix` varbinary(255) NOT NULL DEFAULT '',"
, " `cl_collation` varbinary(32) NOT NULL DEFAULT '',"
, " `cl_type` enum('page','subcat','file') NOT NULL DEFAULT 'page',"
, " UNIQUE KEY `cl_from` (`cl_from`,`cl_to`),"
, ");"
, "INSERT INTO 'categorylinks' VALUES (1,'Ctg_0','','','','','p')"
, ",(3,'Ctg_0','','','','','p')"
, ",(4,'Ctg_0','','','','','p')"
, ",(1,'Ctg_1','','','','','p')"
, ",(2,'Ctg_1','','','','','p')"
, ",(2,'Ctg_2','','','','','p')"
, ",(5,'Ctg_a\\'b','','','','','p')"
, ",(6,'Ctg_a\\\\','','','','','p')"
, ";"
));
}
@Test public void Ignore_dupes() { // PURPOSE: ignore dupe ctgs
if (Xoa_test_.Db_skip()) return;
fxt.Init_db_sqlite();
fxt.doc_ary_
( fxt.doc_wo_date_(1, "A" , "[[Category:Ctg_0]] [[Category:Ctg_0]]")
)
.Exec_run(new Xob_page_cmd(fxt.Bldr(), fxt.Wiki()))
.Exec_run(new Xob_ctg_v1_sql().Ctor(fxt.Bldr(), fxt.Wiki()))
;
fxt.Test_file(Xoa_test_.Url_root().GenSubFil_nest("root", "wiki", "en.wikipedia.org", "xowa_categorylinks.sql").Raw(), String_.Concat_lines_nl
( "CREATE TABLE `categorylinks` ("
, " `cl_from` int(10) unsigned NOT NULL DEFAULT '0',"
, " `cl_to` varbinary(255) NOT NULL DEFAULT '',"
, " `cl_sortkey` varbinary(230) NOT NULL DEFAULT '',"
, " `cl_timestamp` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,"
, " `cl_sortkey_prefix` varbinary(255) NOT NULL DEFAULT '',"
, " `cl_collation` varbinary(32) NOT NULL DEFAULT '',"
, " `cl_type` enum('page','subcat','file') NOT NULL DEFAULT 'page',"
, " UNIQUE KEY `cl_from` (`cl_from`,`cl_to`),"
, ");"
, "INSERT INTO 'categorylinks' VALUES (1,'Ctg_0','','','','','p')"
, ";"
));
}
}

24
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_ctg_v1_txt.java
View File

@@ -1,24 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.ios.*;
import gplx.xowa.bldrs.cmds.texts.tdbs.*;
public class Xob_ctg_v1_txt extends Xob_ctg_v1_base {
@Override public String Wkr_key() {return Xob_cmd_keys.Key_tdb_make_category;}
@Override public Io_sort_cmd Make_sort_cmd() {return new Xob_make_cmd_site(bldr.Usr_dlg(), make_dir, make_fil_len);}
}

36
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_sql_join_mgr.java
View File

@@ -1,36 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.ios.*;
interface Xob_sql_join_wkr {
Io_line_rdr New_main_rdr();
Io_line_rdr New_join_rdr();
void Process_match(Io_line_rdr main, Io_line_rdr join, byte[] key_bry);
}
class Xob_sql_join_mgr {
public static void Exec_join(Xob_sql_join_wkr wkr) {
Io_line_rdr main_rdr = wkr.New_main_rdr();
Io_line_rdr join_rdr = wkr.New_join_rdr();
while (main_rdr.Read_next()) {
byte[] key_bry = Bry_.Mid(main_rdr.Bfr(), main_rdr.Key_pos_bgn(), main_rdr.Key_pos_end());
if (join_rdr.Match(key_bry)) {
wkr.Process_match(main_rdr, join_rdr, key_bry);
}
}
}
}

44
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_base.java
View File

@@ -1,44 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.ios.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
public abstract class Xoctg_hiddencat_parser_base extends Xob_sql_dump_base implements Sql_file_parser_cmd {
public Xoctg_hiddencat_parser_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb; return this;}
@Override public String Sql_file_name() {return "page_props";}
private static final byte Fld_id = 0, Fld_key = 1, Fld_val = 2;
private int cur_id = -1;
private boolean cur_is_hiddencat = false;
private int rows = 0;
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
parser.Fld_cmd_(this).Flds_req_idx_(4, 0, 1, 2); // NOTE: 4 b/c MW added fld_3:pp_sortkey; DATE:2014-04-28
}
public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
switch (fld_idx) {
case Fld_id: cur_id = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld_key: cur_is_hiddencat = Bry_.Eq(src, fld_bgn, fld_end, Key_hiddencat); break;
case Fld_val:
if (!cur_is_hiddencat) {data.Cancel_row_y_(); return;}
Exec_hook(file_bfr, cur_id, cur_is_hiddencat);
if (++rows % 100000 == 0) usr_dlg.Prog_many("", "", "reading row ~{0}", Int_.To_str_fmt(rows, "#,##0"));
break;
}
}
public abstract void Exec_hook(Bry_bfr file_bfr, int cur_id, boolean cur_is_hiddencat);
public static final byte[] Key_hiddencat = Bry_.new_a7("hiddencat");
}

47
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_sql.java
View File

@@ -1,47 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.envs.*;
import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.sqls.*;
import gplx.xowa.wikis.data.*;
public class Xoctg_hiddencat_parser_sql extends Xoctg_hiddencat_parser_base {
private Xowd_cat_core_tbl tbl;
public Xoctg_hiddencat_parser_sql(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki);}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_text_cat_hidden;}
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
super.Cmd_bgn_hook(bldr, parser);
Xodb_mgr_sql db_mgr = wiki.Db_mgr_as_sql();
tbl = db_mgr.Core_data_mgr().Db__cat_core().Tbl__cat_core();
tbl.Update_bgn();
}
@Override public void Exec_hook(Bry_bfr file_bfr, int cur_id, boolean cur_is_hiddencat) {
if (cur_is_hiddencat)
tbl.Update_by_batch(cur_id, cur_is_hiddencat ? Bool_.Y_byte : Bool_.N_byte);
}
@Override public void Cmd_end() {
tbl.Update_end();
if (!Env_.Mode_testing()) // NOTE: do not delete when testing
Io_mgr.Instance.DeleteDirDeep(wiki.Fsys_mgr().Tmp_dir()); // delete /wiki/wiki_name/tmp
// cleanup; delete files;
Io_url wiki_root_dir = wiki.Fsys_mgr().Root_dir();
gplx.xowa.bldrs.wkrs.Xob_io_utl_.Delete_sql_files(wiki_root_dir, this.Sql_file_name());
Io_mgr.Instance.DeleteFil_args(wiki_root_dir.GenSubFil("xowa_categorylinks.sql")).MissingFails_off().Exec();
}
}

61
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_sql_tst.java
View File

@@ -1,61 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.*;
public class Xoctg_hiddencat_parser_sql_tst {
@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Ctor_fsys();} Db_mgr_fxt fxt = new Db_mgr_fxt();
@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();}
@Test public void Basic() {
if (Xoa_test_.Db_skip()) return;
fxt.Init_db_sqlite();
fxt.Wiki().Db_mgr_as_sql().Core_data_mgr().Db__cat_core().Tbl__cat_core().Create_tbl();
Init_ctgs(1, 2, 3);
Io_url page_props_url = Xoa_test_.Url_root().GenSubFil_nest("root", "wiki", "en.wikipedia.org", "page_props.sql");
fxt .Init_fil(page_props_url, String_.Concat
( "INSERT INTO `page_props` VALUES"
, " (1,'hiddencat','')"
, ",(2,'pageimage','A.png')"
, ",(3,'hiddencat','')"
, ";"
))
.Exec_run(new Xoctg_hiddencat_parser_sql(fxt.Bldr(), fxt.Wiki()))
;
Tst_ctg_hidden(Bool_.Y, 1, 3);
Tst_ctg_hidden(Bool_.N, 2);
}
private void Init_ctgs(int... ctgs) {
int len = ctgs.length;
Xowd_cat_core_tbl tbl = fxt.Wiki().Db_mgr_as_sql().Core_data_mgr().Db__cat_core().Tbl__cat_core().Create_tbl();
tbl.Insert_bgn();
try {
for (int i = 0; i < len; i++) {
int ctg_id = ctgs[i];
tbl.Insert_cmd_by_batch(ctg_id, 0, 0, 0, Bool_.N_byte, 0);
}
} finally {tbl.Insert_end();}
}
private void Tst_ctg_hidden(boolean expd_hidden, int... ctgs) {
int len = ctgs.length;
Xowd_cat_core_tbl tbl = fxt.Wiki().Db_mgr_as_sql().Core_data_mgr().Db__cat_core().Tbl__cat_core();
for (int i = 0; i < len; i++) {
int ctg_id = ctgs[i];
Xowd_category_itm ctg_itm = tbl.Select(ctg_id);
Tfds.Eq(expd_hidden, ctg_itm.Hidden(), Int_.To_str(ctg_id));
}
}
}

33
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_parser_txt.java
View File

@@ -1,33 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.ios.*; import gplx.xowa.bldrs.sqls.*;
public class Xoctg_hiddencat_parser_txt extends Xoctg_hiddencat_parser_base {
public Xoctg_hiddencat_parser_txt(Xob_bldr bldr, Xowe_wiki wiki) {this.Ctor(bldr, wiki);}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_cat_hidden_sql;}
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
super.Cmd_bgn_hook(bldr, parser);
}
@Override public void Exec_hook(Bry_bfr file_bfr, int cur_id, boolean cur_is_hiddencat) {
fld_wtr.Bfr_(file_bfr);
fld_wtr.Write_int_base85_len5_fld(cur_id);
}
@Override public void Cmd_end() {
Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, temp_dir.GenSubDir("sort"), sort_mem_len, Xoctg_link_sql_sorter.Instance, Io_line_rdr_key_gen_.noop, new Io_sort_fil_basic(bldr.Usr_dlg(), this.Make_url_gen(), make_fil_len));
}
}

69
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_ttl_wkr.java
View File

@@ -1,69 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.ios.*; import gplx.xowa.wikis.tdbs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xoctg_hiddencat_ttl_wkr extends Xob_itm_dump_base implements Xob_cmd, Gfo_invk {
public Xoctg_hiddencat_ttl_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;} private Xob_sql_join_wkr_ctg_hidden join_wkr;
public String Cmd_key() {return Xob_cmd_keys.Key_tdb_cat_hidden_ttl;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
this.Init_dump(Xob_cmd_keys.Key_tdb_cat_hidden_ttl);
src_sql_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xob_cmd_keys.Key_tdb_cat_hidden_sql, "make");
join_wkr = new Xob_sql_join_wkr_ctg_hidden(bldr.App(), wiki, temp_dir, src_sql_dir);
} Io_url src_sql_dir;
public void Cmd_run() {
Xob_sql_join_mgr.Exec_join(join_wkr);
}
public void Cmd_end() {
join_wkr.Flush();
Io_url_gen make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
Xobdc_merger.Basic(bldr.Usr_dlg(), join_wkr.Dump_url_gen(), temp_dir.GenSubDir("sort"), sort_mem_len, Io_sort_split_itm_sorter.Instance, Io_line_rdr_key_gen_.first_pipe, new Io_sort_fil_basic(bldr.Usr_dlg(), make_url_gen, make_fil_len));
if (delete_temp) Io_mgr.Instance.DeleteDirDeep(src_sql_dir);
}
public void Cmd_term() {}
}
class Xob_sql_join_wkr_ctg_hidden implements Xob_sql_join_wkr {
public Xob_sql_join_wkr_ctg_hidden(Xoae_app app, Xowe_wiki wiki, Io_url temp_dir, Io_url src_sql_dir) {
this.app = app; this.wiki = wiki;
this.dump_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("dump"));
this.src_sql_dir = src_sql_dir;
} private Xoae_app app = null; Xowe_wiki wiki = null; Io_url src_sql_dir;
public Io_url_gen Dump_url_gen() {return dump_url_gen;} Io_url_gen dump_url_gen;
public Io_line_rdr New_main_rdr() {
Io_url[] urls = Io_mgr.Instance.QueryDir_fils(src_sql_dir);
return new Io_line_rdr(app.Usr_dlg(), urls).Key_gen_(Io_line_rdr_key_gen_.first_pipe);
}
public Io_line_rdr New_join_rdr() {
Io_url make_dir = wiki.Tdb_fsys_mgr().Url_site_dir(Xotdb_dir_info_.Tid_id);
app.Usr_dlg().Prog_many("", "", "getting id files: ~{0}", make_dir.Raw());
Io_url[] urls = Io_mgr.Instance.QueryDir_args(make_dir).Recur_().FilPath_("*.xdat").ExecAsUrlAry();
return new Io_line_rdr(app.Usr_dlg(), urls).Key_gen_(Io_line_rdr_key_gen_.first_pipe).File_skip_line0_(true);
}
public void Process_match(Io_line_rdr main, Io_line_rdr join, byte[] key_bry) {
byte[] src = join.Bfr();
int itm_end = join.Itm_pos_end();
int pipe_pos = Bry_find_.Find_bwd(src, Byte_ascii.Pipe, itm_end);
if (pipe_pos == Bry_find_.Not_found) throw Err_.new_wo_type("failed to find pipe for name", "excerpt", String_.new_u8(src, join.Itm_pos_bgn(), join.Itm_pos_end()));
file_bfr.Add_mid(src, pipe_pos + 1, itm_end - 1).Add_byte_pipe();
file_bfr.Add(key_bry).Add_byte_nl();
} private Bry_bfr file_bfr = Bry_bfr_.New();
public void Flush() {
Io_mgr.Instance.SaveFilBfr(dump_url_gen.Nxt_url(), file_bfr);
}
}

45
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_hiddencat_ttl_wkr_tst.java
View File

@@ -1,45 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.xowa.bldrs.*;
public class Xoctg_hiddencat_ttl_wkr_tst {
@Before public void init() {fxt.Clear();} private Xob_base_fxt fxt = new Xob_base_fxt();
@Test public void Basic() {
fxt
.Init_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.hiddencat_sql/make/0000000000.csv", String_.Concat_lines_nl
( "!!!!#|"
, "!!!!$|"
, "!!!!%|"
))
.Init_fil("mem/xowa/wiki/en.wikipedia.org/site/id/00/00/00/00/0000000000.xdat", String_.Concat_lines_nl
( "header_line"
, "!!!!#|C"
, "!!!!%|A"
))
.Exec_cmd(Xob_cmd_keys.Key_tdb_cat_hidden_ttl)
.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.hiddencat_ttl/dump/0000000000.csv", String_.Concat_lines_nl
( "C|!!!!#"
, "A|!!!!%"
))
.Test_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.hiddencat_ttl/make/0000000000.csv", String_.Concat_lines_nl
( "A|!!!!%"
, "C|!!!!#"
))
;
}
}

186
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_link_idx_wkr.java
View File

@@ -1,186 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.flds.*; import gplx.core.ios.*; import gplx.xowa.wikis.ctgs.*; import gplx.xowa.wikis.tdbs.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*;
public class Xoctg_link_idx_wkr extends Xob_idx_base { // NOTE: similar functionality to Xob_make_cmd_site, but more complicated due to p,f,s; not inheriting
Io_url src_link_dir; int make_fil_max = Int_.Min_value;
public Xoctg_link_idx_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_tdb_ctg_link_idx;}
@Override public void Cmd_bgn_hook() {
this.fld_rdr = this.Fld_rdr();
if (src_link_dir == null) src_link_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xob_cmd_keys.Key_tdb_text_cat_link, "make");
if (make_fil_max == Int_.Min_value) make_fil_max = Io_mgr.Len_mb;
make_link_mgr = new Xoctg_make_link_mgr(usr_dlg, make_fil_max, wiki.Tdb_fsys_mgr());
make_main_mgr = new Xoctg_make_main_mgr(usr_dlg, make_fil_max, wiki.Tdb_fsys_mgr());
Io_mgr.Instance.DeleteDirDeep_ary(make_link_mgr.Make_dir(), make_main_mgr.Make_dir());
} Gfo_fld_rdr fld_rdr; Xoctg_make_link_mgr make_link_mgr;
@Override public void Cmd_run() {
Xoctg_make_link_grp cur_grp = null;
Io_line_rdr link_rdr = rdr_(src_link_dir);
byte[] prv_name = Bry_.Empty; byte prv_tid = Xoa_ctg_mgr.Tid_null;
make_link_mgr.Write_bgn();
make_main_mgr.Write_bgn();
while (link_rdr.Read_next()) {
link_data.Parse_from_sql_dump(fld_rdr, link_rdr);
byte[] cur_name = link_data.Name();
if (!Bry_.Eq(cur_name, prv_name)) { // grp changed
if (prv_name != Bry_.Empty) // ignore change from 1st row
Write_grp(prv_name, make_link_mgr);
prv_name = cur_name;
}
byte cur_tid = link_data.Tid();
if (cur_tid != prv_tid) { // tid changed
cur_grp = make_link_mgr.Grp_by_tid(cur_tid);
prv_tid = cur_tid;
}
cur_grp.Add_itm(link_data);
}
Write_grp(prv_name, make_link_mgr);
make_link_mgr.Flush();
make_main_mgr.Flush();
} private Xoctg_idx_data_link link_data = new Xoctg_idx_data_link();
@Override public void Cmd_end() {
if (delete_temp) Io_mgr.Instance.DeleteDirDeep_ary(src_link_dir, make_main_mgr.Src_dir());
} boolean delete_temp = true;
private void Write_grp(byte[] cur_name, Xoctg_make_link_mgr make_link_mgr) {
make_main_mgr.Grps_write(cur_name, make_link_mgr.Subc_grp().Count(), make_link_mgr.File_grp().Count(), make_link_mgr.Page_grp().Count());
make_link_mgr.Grps_write(cur_name);
} private Xoctg_make_main_mgr make_main_mgr;
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_src_link_dir_)) src_link_dir = m.ReadIoUrl("v");
else if (ctx.Match(k, Invk_make_fil_max_)) make_fil_max = m.ReadInt("v");
else if (ctx.Match(k, Invk_make_fil_max_)) make_fil_max = m.ReadInt("v");
else if (ctx.Match(k, Invk_delete_temp_)) delete_temp = m.ReadBool("v");
else return super.Invk(ctx, ikey, k, m);
return this;
}
public static final String Invk_src_link_dir_ = "src_link_dir_", Invk_make_fil_max_ = "make_fil_max_", Invk_delete_temp_ = "delete_temp_";
}
class Xoctg_make_link_mgr {
public Xoctg_make_link_mgr(Gfo_usr_dlg usr_dlg, int make_fil_max, Xotdb_fsys_mgr fsys_mgr) {
this.make_fil_max = make_fil_max;
subc_grp = new Xoctg_make_link_grp(Xoa_ctg_mgr.Tid_subc, make_fil_max);
file_grp = new Xoctg_make_link_grp(Xoa_ctg_mgr.Tid_file, make_fil_max);
page_grp = new Xoctg_make_link_grp(Xoa_ctg_mgr.Tid_page, make_fil_max);
make_fil_bfr = Bry_bfr_.Reset(make_fil_max);
make_fld_wtr = Gfo_fld_wtr.xowa_().Bfr_(make_fil_bfr);
make_dir = fsys_mgr.Url_site_dir(Xotdb_dir_info_.Tid_category2_link);
make_cmd = new Xob_make_cmd_site(usr_dlg, make_dir, make_fil_max);
} Gfo_fld_wtr make_fld_wtr; Bry_bfr make_fil_bfr; int make_fil_max; Xob_make_cmd_site make_cmd;
public Io_url Make_dir() {return make_dir;} Io_url make_dir;
public Xoctg_make_link_grp Subc_grp() {return subc_grp;} private Xoctg_make_link_grp subc_grp;
public Xoctg_make_link_grp File_grp() {return file_grp;} private Xoctg_make_link_grp file_grp;
public Xoctg_make_link_grp Page_grp() {return page_grp;} private Xoctg_make_link_grp page_grp;
public Xoctg_make_link_grp Grp_by_tid(byte tid) {
switch (tid) {
case Byte_ascii.Ltr_c: return subc_grp;
case Byte_ascii.Ltr_f: return file_grp;
case Byte_ascii.Ltr_p: return page_grp;
default: throw Err_.new_unhandled(tid);
}
}
public int Grps_bfr_len() {return subc_grp.Bfr_len() + file_grp.Bfr_len() + page_grp.Bfr_len();}
public void Grps_reset() {subc_grp.Reset(); file_grp.Reset(); page_grp.Reset();}
public void Grps_write(byte[] cur_name) {
make_fld_wtr.Write_bry_escape_fld(cur_name); // write name
make_fld_wtr.Write_int_base85_len5_fld(subc_grp.Bfr_len()).Write_int_base85_len5_fld(file_grp.Bfr_len()).Write_int_base85_len5_fld(page_grp.Bfr_len());
Write_grp(make_fil_bfr, subc_grp); Write_grp(make_fil_bfr, file_grp); Write_grp(make_fil_bfr, page_grp);
make_fil_bfr.Add_byte_nl();
byte[] bry = make_fld_wtr.Bfr().Bfr();
make_cmd.Do_bry(bry, 0, cur_name.length, 0, make_fld_wtr.Bfr().Len());
make_fil_bfr.Clear();
this.Grps_reset();
}
public void Write_bgn() {
make_cmd.Sort_bgn();
make_cmd.Line_dlm_(Byte_ascii.Nl);
}
public void Flush() {
make_cmd.Sort_end();
}
private void Write_grp(Bry_bfr make_fil_bfr, Xoctg_make_link_grp grp) {
if (grp.Bfr_len() == 0) return;
make_fil_bfr.Add_bfr_and_clear(grp.Bfr()); // NOTE: should have trailing pipe
}
static final int Itm_len_fixed
= 1 // name: 1 pipe
+ (4 + (5 * 3)) // meta: 3 semis; 1 pipe; 3 base85 ints
+ (2 * 3) // sect: c| f| p|
+ 1 // nl
;
}
class Xoctg_make_link_grp {
public Xoctg_make_link_grp(byte tid, int bfr_max) {this.tid = tid; bfr = Bry_bfr_.Reset(bfr_max);} Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
public byte Tid() {return tid;} private byte tid;
public int Bfr_len() {return bfr.Len();}
public Bry_bfr Bfr() {return bfr;} Bry_bfr bfr;
public int Count() {return count;} private int count;
public void Reset() {count = 0;}
public void Add_itm(Xoctg_idx_data_link link_data) {
bfr .Add_base85_len_5(link_data.Id()).Add_byte(Byte_ascii.Semic) // add id; "!!!!;"
.Add_base85_len_5(link_data.Timestamp()).Add_byte(Byte_ascii.Semic); // add timestamp; "!!!!;"
fld_wtr.Bfr_(bfr).Write_bry_escape_fld(link_data.Sortkey()); // add sortkey; "A|"
++count;
}
}
class Xoctg_idx_data_link {
public byte[] Name() {return name;} private byte[] name;
public byte Tid() {return tid;} private byte tid;
public int Id() {return id;} private int id;
public int Timestamp() {return timestamp;} private int timestamp;
public byte[] Sortkey() {return sortkey;} private byte[] sortkey;
public void Parse_from_sql_dump(Gfo_fld_rdr fld_rdr, Io_line_rdr rdr) {
fld_rdr.Ini(rdr.Bfr(), rdr.Itm_pos_bgn());
name = fld_rdr.Read_bry_escape();
tid = (byte)(fld_rdr.Read_byte());// - Byte_ascii.Num_0);
sortkey = fld_rdr.Read_bry_escape();
id = fld_rdr.Read_int_base85_len5();
timestamp = fld_rdr.Read_int_base85_len5();
}
}
class Xoctg_make_main_mgr {
public Xoctg_make_main_mgr(Gfo_usr_dlg usr_dlg, int make_fil_max, Xotdb_fsys_mgr fsys_mgr) {
this.make_fil_max = make_fil_max;
make_fil_bfr = Bry_bfr_.Reset(make_fil_max);
make_fld_wtr = Gfo_fld_wtr.xowa_().Bfr_(make_fil_bfr);
make_dir = fsys_mgr.Url_site_dir(Xotdb_dir_info_.Tid_category2_main);
make_cmd = new Xob_make_cmd_site(usr_dlg, make_dir, make_fil_max);
src_dir = fsys_mgr.Tmp_dir().GenSubDir_nest(Xob_cmd_keys.Key_tdb_cat_hidden_ttl, "make");
hidden_rdr = new Io_line_rdr(usr_dlg, Io_mgr.Instance.QueryDir_fils(src_dir));
} Gfo_fld_wtr make_fld_wtr; Bry_bfr make_fil_bfr; int make_fil_max; Xob_make_cmd_site make_cmd; Io_line_rdr hidden_rdr;
public Io_url Src_dir() {return src_dir;} Io_url src_dir;
public Io_url Make_dir() {return make_dir;} Io_url make_dir;
public void Grps_write(byte[] cur_name, int count_subc, int count_file, int count_page) {
boolean hidden = hidden_rdr.Match(cur_name);
make_fld_wtr.Write_bry_escape_fld(cur_name); // write name
make_fld_wtr.Write_byte_fld(hidden ? Byte_ascii.Ltr_y : Byte_ascii.Ltr_n);
make_fld_wtr.Write_int_base85_len5_fld(count_subc).Write_int_base85_len5_fld(count_file).Write_int_base85_len5_fld(count_page);
make_fil_bfr.Add_byte_nl();
byte[] bry = make_fld_wtr.Bfr().Bfr();
make_cmd.Do_bry(bry, 0, cur_name.length, 0, make_fld_wtr.Bfr().Len());
make_fil_bfr.Clear();
}
public void Write_bgn() {
make_cmd.Sort_bgn();
make_cmd.Line_dlm_(Byte_ascii.Nl);
}
public void Flush() {
make_cmd.Sort_end();
}
}

63
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xoctg_link_idx_wkr_tst.java
View File

@@ -1,63 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.xowa.wikis.ctgs.*; import gplx.xowa.bldrs.*;
public class Xoctg_link_idx_wkr_tst {
@Before public void init() {fxt.Clear();} private Xob_base_fxt fxt = new Xob_base_fxt();
@Test public void Basic() {
fxt
.Init_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.link_sql/make/0000000000.csv", String_.Concat_lines_nl
( "Ctg_1|c|Ctg_1a|!!!!@|#8DDL|"
, "Ctg_1|f|A.png|!!!!#|#8DDL|"
, "Ctg_1|p|A0|!!!!$|#8DDL|"
, "Ctg_1|p|A1|!!!!%|#8DDL|"
, "Ctg_2|p|B1|!!!!%|#8DDL|"
))
.Init_fil("mem/xowa/wiki/en.wikipedia.org/tmp/ctg.hiddencat_ttl/make/0000000000.csv", String_.Concat_lines_nl
( "Ctg_2|!!!!%|"
))
.Exec_cmd(Xob_cmd_keys.Key_tdb_ctg_link_idx, GfoMsg_.basic_(Xoctg_link_idx_wkr.Invk_make_fil_max_, 72))
.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/link/00/00/00/00/0000000000.xdat", String_.Concat_lines_nl
( "!!!\"(|"
, "Ctg_1|!!!!4|!!!!3|!!!!?|!!!!@;#8DDL;Ctg_1a|!!!!#;#8DDL;A.png|!!!!$;#8DDL;A0|!!!!%;#8DDL;A1|"
))
.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/link/00/00/00/00/0000000001.xdat", String_.Concat_lines_nl
( "!!!!I|"
, "Ctg_2|!!!!!|!!!!!|!!!!0|!!!!%;#8DDL;B1|"
))
.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/link/reg.csv", String_.Concat_lines_nl
( "0|Ctg_1|Ctg_1|1"
, "1|Ctg_2|Ctg_2|1"
))
.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/main/00/00/00/00/0000000000.xdat", String_.Concat_lines_nl
( "!!!!<|!!!!<|"
, "Ctg_1|n|!!!!\"|!!!!\"|!!!!#|"
, "Ctg_2|y|!!!!!|!!!!!|!!!!\"|"
))
.Test_fil("mem/xowa/wiki/en.wikipedia.org/site/category2/main/reg.csv", String_.Concat_lines_nl
( "0|Ctg_1|Ctg_2|2"
))
;
byte[] ctg_name = Bry_.new_a7("Ctg_1");
Xoctg_data_ctg main = new Xoctg_data_ctg(ctg_name);
fxt.Wiki().Db_mgr().Load_mgr().Load_ctg_v2(main, ctg_name);
Tfds.Eq(1, main.Grp_by_tid(Xoa_ctg_mgr.Tid_subc).Total());
Tfds.Eq(1, main.Grp_by_tid(Xoa_ctg_mgr.Tid_file).Total());
Tfds.Eq(2, main.Grp_by_tid(Xoa_ctg_mgr.Tid_page).Total());
}
}

2
400_xowa/src/gplx/xowa/bldrs/cmds/texts/Xob_init_base.java
View File

@@ -29,7 +29,7 @@ public abstract class Xob_init_base implements Xob_cmd, Gfo_invk {
public abstract void Cmd_run_end(Xowe_wiki wiki);
@gplx.Virtual public void Cmd_init(Xob_bldr bldr) { // add other cmds; EX: wikidata
bldr.Import_marker().Bgn(wiki);
if (wbase_enabled == Bool_.__byte) wbase_enabled = wiki.Domain_tid() == Xow_domain_tid_.Int__wikidata ? Bool_.Y_byte : Bool_.N_byte; // if wbase_enabled not explicitly set, set it to y if wiki is "www.wikidata.org"
if (wbase_enabled == Bool_.__byte) wbase_enabled = wiki.Domain_tid() == Xow_domain_tid_.Tid__wikidata ? Bool_.Y_byte : Bool_.N_byte; // if wbase_enabled not explicitly set, set it to y if wiki is "www.wikidata.org"
if (wbase_enabled == Bool_.Y_byte) // if wbase_enabled, auto-add wdata_wkrs bldr
this.Cmd_ini_wdata(bldr, wiki);
}

2
400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_page_cmd_tst.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import org.junit.*; import gplx.dbs.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.ctgs.*; import gplx.xowa.wikis.nss.*;
import org.junit.*; import gplx.dbs.*; import gplx.xowa.bldrs.*; import gplx.xowa.addons.wikis.ctgs.bldrs.*; import gplx.xowa.wikis.nss.*;
public class Xob_page_cmd_tst {
@Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Ctor_fsys();} Db_mgr_fxt fxt = new Db_mgr_fxt();
@After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();}

57
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_search_base_tst.java
View File

@@ -1,57 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import org.junit.*; import gplx.xowa.wikis.tdbs.*; import gplx.xowa.bldrs.*; import gplx.xowa.wikis.nss.*;
import gplx.xowa.addons.wikis.searchs.bldrs.*;
public class Xob_search_base_tst {
@Test public void Split() {
tst_Split("a", "a");
tst_Split("a b", "a", "b");
tst_Split("a b c", "a", "b", "c");
tst_Split("a-b.c", "a", "b", "c");
tst_Split("a A", "a");
tst_Split("a_b", "a", "b");
tst_Split("a (b)", "a", "b");
}
@Test public void Title_word() {
fxt.doc_ary_
( fxt.doc_wo_date_(2, "A b", "text")
, fxt.doc_wo_date_(3, "B a", "text")
)
.Fil_expd(fxt.fil_ns_sttl(Xow_ns_.Tid__main, 0)
, "!!!!;|!!!!;|"
, "a|!!!!#;!!!!%|!!!!$;!!!!%"
, "b|!!!!#;!!!!%|!!!!$;!!!!%"
, ""
)
.Fil_expd(fxt.fil_reg(Xow_ns_.Tid__main, Xotdb_dir_info_.Tid_search_ttl)
, "0|a|b|2"
, ""
)
.Run(new Xob_search_tdb(fxt.Bldr(), this.fxt.Wiki()))
;
} private Xob_fxt fxt = new Xob_fxt().Ctor_mem();
private void tst_Split(String raw, String... expd) {
Ordered_hash list = Ordered_hash_.New(); Bry_bfr bfr = Bry_bfr_.New();
byte[][] actl_bry = Srch_bldr_wkr_base.Split_ttl_into_words(fxt.App().Lang_mgr().Lang_en(), list, bfr, Bry_.new_u8(raw));
String[] actl = new String[actl_bry.length];
for (int i = 0; i < actl_bry.length; i++)
actl[i] = String_.new_u8(actl_bry[i]);
Tfds.Eq_ary(expd, actl);
}
}

26
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_search_tdb.java
View File

@@ -1,26 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.xowa.addons.wikis.searchs.bldrs.*;
public class Xob_search_tdb extends Srch_bldr_wkr_base {
public Xob_search_tdb(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
@Override public String Page_wkr__key() {return Xob_cmd_keys.Key_tdb_make_search_title;}
@Override public gplx.core.ios.Io_make_cmd Make_cmd_site() {
return new Xob_make_cmd_site(bldr.Usr_dlg(), this.make_dir, this.make_fil_len);
}
}

36
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_tst.java
View File

@@ -111,42 +111,6 @@ public class Xob_tst {
.Run_id()
;
}
@Test public void Category() {
fxt.doc_ary_
( fxt.doc_wo_date_(2, "A", "[[Category:Y]] [[Category:Z]]")
, fxt.doc_wo_date_(3, "B", "[[Category:Y]]")
)
.Fil_expd(fxt.fil_site_ctg(0)
, "!!!!/|!!!!)|"
, "Y|!!!!#|!!!!$"
, "Z|!!!!#"
, ""
)
.Fil_expd(fxt.fil_reg(Xotdb_dir_info_.Tid_category)
, "0|Y|Z|2"
, ""
)
.Run_ctg()
;
}
@Test public void Category2() {
fxt.doc_ary_
( fxt.doc_wo_date_(2, "A", "[[Category:X]] [[Category:Y]]")
, fxt.doc_wo_date_(3, "B", "[[Category:Y]]")
)
.Fil_expd(fxt.fil_site_ctg(0)
, "!!!!)|!!!!/|"
, "X|!!!!#"
, "Y|!!!!#|!!!!$"
, ""
)
.Fil_expd(fxt.fil_reg(Xotdb_dir_info_.Tid_category)
, "0|X|Y|2"
, ""
)
.Run_ctg()
;
}
@Test public void Tmpl_dump() {
fxt.doc_ary_
( fxt.doc_wo_date_(2, "Template:A", "a")

70
400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_deploy_copy_cmd.java
View File

@@ -1,70 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.wikis.tdbs.*;
public class Xob_deploy_copy_cmd extends Xob_itm_basic_base implements Xob_cmd, Gfo_invk {
public Xob_deploy_copy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_deploy_copy;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_end() {}
public void Cmd_run() {
Io_url src_root_dir = wiki.Fsys_mgr().Root_dir();
Xotdb_fsys_mgr url_mgr = wiki.Tdb_fsys_mgr();
Copy_dir_root(src_root_dir, Xotdb_dir_info_.Name_cfg);
Copy_dir_ns(url_mgr.Ns_dir(), Xotdb_dir_info_.Name_page);
Copy_dir_ns(url_mgr.Ns_dir(), Xotdb_dir_info_.Name_title);
Copy_dir_root(src_root_dir, Xotdb_dir_info_.Name_site, Xotdb_dir_info_.Name_category);
Copy_dir_root(src_root_dir, Xotdb_dir_info_.Name_site, Xotdb_dir_info_.Name_id);
Copy_dir_root(src_root_dir, Xotdb_dir_info_.Name_site, Xotdb_dir_info_.Name_search_ttl);
}
public void Cmd_term() {}
private void Copy_dir_ns(Io_url root, String name) {
Io_url[] ns_dirs = Io_mgr.Instance.QueryDir_args(root).DirOnly_().ExecAsUrlAry();
for (int i = 0; i < ns_dirs.length; i++) {
Io_url ns_dir = ns_dirs[i];
Io_url src_sub_dir = ns_dir.GenSubDir(name);
String dir_name = name;
if (zip) {
Io_url src_zip_dir = ns_dir.GenSubDir(name + Xob_deploy_zip_cmd.Dir_zip_suffix);
if (Io_mgr.Instance.ExistsDir(src_zip_dir)) {
src_sub_dir = src_zip_dir;
dir_name = name + Xob_deploy_zip_cmd.Dir_zip_suffix;
}
}
Copy_dir(src_sub_dir, trg_root_dir.GenSubDir_nest(Xotdb_dir_info_.Name_ns, ns_dir.NameOnly(), dir_name));
}
}
private void Copy_dir_root(Io_url src_root_dir, String... sub_dirs) {
Io_url src = src_root_dir.GenSubDir_nest(sub_dirs);
Io_url trg = trg_root_dir.GenSubDir_nest(sub_dirs);
Copy_dir(src, trg);
}
private void Copy_dir(Io_url src, Io_url trg) {
bldr.Usr_dlg().Prog_many(GRP_KEY, "copy", "copying to ~{1}", src.Xto_api(), trg.Xto_api());
Io_mgr.Instance.CopyDirDeep(src, trg);
}
boolean zip; Io_url trg_root_dir;
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_trg_dir_)) {trg_root_dir = m.ReadIoUrl("val");}
else if (ctx.Match(k, Invk_zip_)) {zip = m.ReadBoolOrFalse("val");}
else return super.Invk(ctx, ikey, k, m);
return this;
} private static final String Invk_trg_dir_ = "trg_dir_", Invk_zip_ = "zip_";
static final String GRP_KEY = "xowa.bldr.copy";
}

2
400_xowa/src/gplx/xowa/bldrs/css/Xoa_css_extractor.java
View File

@@ -53,7 +53,7 @@ public class Xoa_css_extractor {
Io_url css_comm_fil = wiki_html_dir.GenSubFil(Css_common_name);
Io_url css_wiki_fil = wiki_html_dir.GenSubFil(Css_wiki_name);
wiki.Html__wtr_mgr().Init_css_urls(css_comm_fil, css_wiki_fil);
if (wiki.Domain_tid() == Xow_domain_tid_.Int__home || Env_.Mode_testing()) return; // NOTE: do not download if home_wiki; also needed for TEST
if (wiki.Domain_tid() == Xow_domain_tid_.Tid__home || Env_.Mode_testing()) return; // NOTE: do not download if home_wiki; also needed for TEST
if (Io_mgr.Instance.ExistsFil(css_wiki_fil)) return; // css file exists; nothing to generate
if (wiki.Html__css_installing()) return;
wiki.Html__css_installing_(true);

18
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_base.java
View File

@@ -30,15 +30,11 @@ abstract class Xoi_cmd_base implements Gfo_thread_cmd {
public boolean Async_prog_enabled() {return false;}
public void Async_prog_run(int async_sleep_sum) {}
public byte Async_init() {return Gfo_thread_cmd_.Init_ok;}
public boolean Async_term() {
// install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.end", "import.end ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
return true;
}
public boolean Async_term() {return true;}
public Gfo_invk Owner() {return owner;} public Xoi_cmd_base Owner_(Gfo_invk v) {owner = v; return this;} Gfo_invk owner;
public Gfo_thread_cmd Async_next_cmd() {return next_cmd;} public void Async_next_cmd_(Gfo_thread_cmd v) {next_cmd = v;} Gfo_thread_cmd next_cmd;
public void Async_run() {
running = true;
// install_mgr.App().Gui_wtr().Log_many(GRP_KEY, "import.bgn", "import.bgn ~{0} ~{1} ~{2}", wiki_key, wiki_date, dump_type);
Thread_adp_.Start_by_key(this.Async_key(), this, Invk_process_async);
}
public boolean Async_running() {return running;} private boolean running;
@@ -82,16 +78,12 @@ class Xoi_cmd_category2_build extends Xoi_cmd_base {
public Xoi_cmd_category2_build(Xoi_setup_mgr install_mgr, String wiki_key) {this.Ctor(install_mgr, wiki_key); this.app = install_mgr.App(); this.wiki_key = wiki_key;} private Xoae_app app; private String wiki_key;
@Override public void Cmd_ctor() {
Xowe_wiki wiki = app.Wiki_mgr().Get_by_or_make(Bry_.new_u8(wiki_key));
wiki.Import_cfg().Category_version_(gplx.xowa.wikis.ctgs.Xoa_ctg_mgr.Version_2);
wiki.Import_cfg().Category_version_(gplx.xowa.addons.wikis.ctgs.Xoa_ctg_mgr.Version_2);
}
@Override public String Async_key() {return KEY;} public static final String KEY = "wiki.category2.build";
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_text_cat_core, Xob_cmd_keys.Key_text_cat_link, Xob_cmd_keys.Key_text_cat_hidden);
}
else
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_tdb_cat_hidden_sql, Xob_cmd_keys.Key_tdb_cat_hidden_ttl, Xob_cmd_keys.Key_text_cat_link, Xob_cmd_keys.Key_tdb_ctg_link_idx);
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
wiki.Db_mgr_as_sql().Category_version_update(false);
bldr.Cmd_mgr().Add_many(wiki, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.BLDR_CMD_KEY, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.BLDR_CMD_KEY);
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
app.Usr_dlg().Prog_many("", "", "category2 setup done");

14
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_import.java
View File

@@ -19,6 +19,7 @@ package gplx.xowa.bldrs.installs; import gplx.*; import gplx.xowa.*; import gplx
import gplx.core.threads.*; import gplx.xowa.bldrs.*; import gplx.xowa.guis.views.*; import gplx.xowa.bldrs.cmds.utils.*;
import gplx.xowa.htmls.hrefs.*;
class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
private boolean running;
public Xoi_cmd_wiki_import(Xoi_setup_mgr install_mgr, String wiki_key, String wiki_date, String dump_type) {this.install_mgr = install_mgr; this.Owner_(install_mgr); this.wiki_key = wiki_key; this.wiki_date = wiki_date; this.dump_type = dump_type;} private Xoi_setup_mgr install_mgr; String wiki_key, wiki_date, dump_type;
public static final String KEY = "wiki.import";
public void Cmd_ctor() {}
@@ -41,16 +42,12 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
public boolean Async_running() {
return running;
}
boolean running;
public boolean Import_move_bz2_to_done() {return import_move_bz2_to_done;} public Xoi_cmd_wiki_import Import_move_bz2_to_done_(boolean v) {import_move_bz2_to_done = v; return this;} private boolean import_move_bz2_to_done = true;
private void Process_txt(Xob_bldr bldr) {
((Xob_cleanup_cmd)bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_util_cleanup)).Delete_tdb_(true).Delete_sqlite3_(true);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_id);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_search_title);
if (wiki.Import_cfg().Category_version() == gplx.xowa.wikis.ctgs.Xoa_ctg_mgr.Version_1)
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_make_category);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_calc_stats);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_tdb_core_term);
}
@@ -59,14 +56,13 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_init);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_page);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_css);
if (wiki.Import_cfg().Category_version() == gplx.xowa.wikis.ctgs.Xoa_ctg_mgr.Version_1) {
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core_v1);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_link);
}
if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.addons.wikis.searchs.specials.Srch_special_page.Version_2)
gplx.xowa.addons.wikis.searchs.bldrs.Srch_bldr_mgr_.Setup(wiki);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term);
bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.Dump_file_name));
bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.Dump_file_name));
bldr.Cmd_mgr().Add_many(wiki, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_pageprop_cmd.BLDR_CMD_KEY, gplx.xowa.addons.wikis.ctgs.bldrs.Xob_catlink_cmd.BLDR_CMD_KEY);
}
private void Process_async() {
Xoae_app app = install_mgr.App();

2
400_xowa/src/gplx/xowa/bldrs/setups/upgrades/Xoa_upgrade_mgr.java
View File

@@ -22,7 +22,7 @@ public class Xoa_upgrade_mgr {
Upgrade_history(app);
}
public static void Check(Xowe_wiki wiki) {
if (wiki.Domain_tid() == Xow_domain_tid_.Int__home) return; // home wiki never needs to be migrated
if (wiki.Domain_tid() == Xow_domain_tid_.Tid__home) return; // home wiki never needs to be migrated
try {
if (Bry_.Eq(wiki.Props().Bldr_version(), Bry_.Empty)) { // version is ""; wiki must be created prior to v0.2.1; create wiki_core.gfs
Upgrader_v00_02_01 mgr = new Upgrader_v00_02_01();

33
400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser.java
View File

@@ -18,14 +18,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.flds.*; import gplx.core.ios.*;
public class Sql_file_parser {
public Io_url Src_fil() {return src_fil;} public Sql_file_parser Src_fil_(Io_url v) {src_fil = v; return this;} Io_url src_fil;
public int Src_len() {return src_len;} public Sql_file_parser Src_len_(int v) {src_len = v; return this;} private int src_len = 8 * Io_mgr.Len_mb;
public Io_url_gen Trg_fil_gen() {return trg_fil_gen;} public Sql_file_parser Trg_fil_gen_(Io_url_gen v) {trg_fil_gen = v; return this;} Io_url_gen trg_fil_gen;
public int Trg_len() {return trg_len;} public Sql_file_parser Trg_len_(int v) {trg_len = v; return this;} private int trg_len = 4 * Io_mgr.Len_mb;
private final Gfo_fld_rdr sql_parser = Gfo_fld_rdr.sql_(); private final Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
private Sql_fld_itm[] flds_all; private int flds_all_len;
Gfo_fld_rdr sql_parser = Gfo_fld_rdr.sql_(); Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
static final byte Mode_sql_bgn = 0, Mode_row_bgn = 1, Mode_row_end = 2, Mode_fld = 3, Mode_quote = 4, Mode_escape = 5;
public Sql_file_parser Fld_cmd_(Sql_file_parser_cmd v) {this.fld_cmd = v; return this;} Sql_file_parser_cmd fld_cmd;
public boolean Save_csv() {return save_csv;} private boolean save_csv = true; public Sql_file_parser Save_csv_n_() {save_csv = false; return this;}
public Sql_file_parser Fld_cmd_(Sql_file_parser_cmd v) {this.fld_cmd = v; return this;} private Sql_file_parser_cmd fld_cmd;
public Sql_file_parser Flds_req_(byte[]... v) {flds_req = v; return this;} private byte[][] flds_req;
public Sql_file_parser Flds_req_idx_(int flds_all_len, int... idxs) {
new_flds_all(flds_all_len);
@@ -132,7 +128,10 @@ public class Sql_file_parser {
default: throw Err_.new_unhandled(mode);
}
}
Io_mgr.Instance.AppendFilByt(trg_fil_gen.Nxt_url(), fil_bfr.To_bry_and_clear());
if (save_csv)
Io_mgr.Instance.AppendFilByt(trg_fil_gen.Nxt_url(), fil_bfr.To_bry_and_clear());
else
fil_bfr.Clear();
}
finally {rdr.Rls();}
}
@@ -140,8 +139,11 @@ public class Sql_file_parser {
fil_bfr.Add_byte(Byte_ascii.Nl);
if (fil_bfr.Len() > trg_len) {
Io_url trg_fil = trg_fil_gen.Nxt_url();
usr_dlg.Prog_one(GRP_KEY, "make", "making ~{0}", trg_fil.NameAndExt());
Io_mgr.Instance.AppendFilByt(trg_fil, fil_bfr.To_bry_and_clear());
usr_dlg.Prog_one("", "", "making ~{0}", trg_fil.NameAndExt());
if (save_csv)
Io_mgr.Instance.AppendFilByt(trg_fil, fil_bfr.To_bry_and_clear());
else
fil_bfr.Clear();
}
}
private void Commit_fld(int fld_idx, Bry_bfr val_bfr, Bry_bfr fil_bfr, Sql_file_parser_data data) {
@@ -157,6 +159,15 @@ public class Sql_file_parser {
}
val_bfr.Clear();
}
private Io_url_gen trg_fil_gen; private int trg_len = 4 * Io_mgr.Len_mb;
public Io_url Src_fil() {return src_fil;} private Io_url src_fil;
public int Src_len() {return src_len;} private int src_len = 8 * Io_mgr.Len_mb;
public Sql_file_parser Src_fil_(Io_url v) {src_fil = v; return this;}
public Sql_file_parser Src_len_(int v) {src_len = v; return this;}
public Sql_file_parser Trg_fil_gen_(Io_url_gen v) {trg_fil_gen = v; return this;}
public Sql_file_parser Trg_len_(int v) {trg_len = v; return this;}
private static final byte[] Bry_insert_into = Bry_.new_a7("INSERT INTO "), Bry_values = Bry_.new_a7(" VALUES (");
private static final String GRP_KEY = "xowa.bldr.sql";
private static final byte Mode_sql_bgn = 0, Mode_row_bgn = 1, Mode_row_end = 2, Mode_fld = 3, Mode_quote = 4, Mode_escape = 5;
}

6
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java
View File

@@ -57,4 +57,10 @@ public abstract class Xob_sql_dump_base extends Xob_itm_dump_base implements Xob
return this;
}
public static final String Invk_src_fil_ = "src_fil_";
public void Cmd_cleanup_sql() {
// get dump files to delete; EX: "*-categorylinks.sql*" matches "simplewiki-latest-categorylinks.sql" and "simplewiki-latest-categorylinks.sql.gz"
Io_url[] dump_files = Io_mgr.Instance.QueryDir_args(wiki.Fsys_mgr().Root_dir()).FilPath_("*-" + this.Sql_file_name() + ".sql*").ExecAsUrlAry();
for (Io_url dump_file : dump_files)
Io_mgr.Instance.DeleteFil(dump_file);
}
}

2
400_xowa/src/gplx/xowa/bldrs/xmls/Xob_import_cfg.java
View File

@@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.ios.*; import gplx.core.ios.streams.*; import gplx.core.envs.*;
import gplx.xowa.wikis.ctgs.*; import gplx.xowa.wikis.tdbs.*;
import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.wikis.tdbs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_import_cfg {
public Xob_import_cfg(Xowe_wiki wiki) {this.wiki = wiki;} private Xowe_wiki wiki; private boolean src_fil_is_bz2 = true;