Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v1.6.5.1

Browse Source
This commit is contained in:
gnosygnu
2014-06-30 00:04:32 -04:00
parent 85594d3cdd
commit bae88e739c
2482 changed files with 198730 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
400_xowa/src/gplx/xowa/dbs/Xodb_file.java Normal file
View File

@@ -0,0 +1,75 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.dbs.*;
public class Xodb_file {
public Xodb_file(int id, byte tid) {this.id = id; this.tid = tid;}
public byte Cmd_mode() {return cmd_mode;} public Xodb_file Cmd_mode_(byte v) {cmd_mode = v; return this;} private byte cmd_mode;
public int Id() {return id;} private int id;
public byte Tid() {return tid;} private byte tid;
public Io_url Url() {return url;} public Xodb_file Url_(Io_url v) {url = v; return this;} private Io_url url;
public String Url_rel() {return url_rel;} public Xodb_file Url_rel_(String v) {this.url_rel = v; return this;} private String url_rel;
public Db_connect Connect() {return connect;} public Xodb_file Connect_(Db_connect v) {connect = v; return this;} Db_connect connect;
public long File_len() {return file_len;} public void File_len_add(int v) {file_len += v;} long file_len;
public long File_max() {return file_max;} public Xodb_file File_max_(long v) {file_max = v; return this;} long file_max;
public Db_provider Provider() {
if (provider == null) provider = Db_provider_pool._.FetchOrNew(connect);
return provider;
} Db_provider provider;
public void Provider_(Db_provider p) {provider = p;}
public void Rls() {
if (provider == null) return;
try {
provider.Txn_mgr().Txn_end_all(); // close any open transactions
provider.Rls();
} finally {provider = null;}
}
public static final byte Tid_core = 1, Tid_text = 2, Tid_category = 3, Tid_search = 4, Tid_wikidata = 5, Tid_temp = 6;
public static String Tid_to_name(byte v) {
switch (v) {
case Tid_core: return "core";
case Tid_text: return "text";
case Tid_category: return "category";
case Tid_wikidata: return "wikidata";
case Tid_temp: return "temp";
case Tid_search: return "search";
default: throw Err_.unhandled(v);
}
}
public static Xodb_file load_(int id, byte tid, String url) {return new Xodb_file(id, tid).Url_rel_(url).Cmd_mode_(Db_cmd_mode.Ignore);}
public static Xodb_file make_(int id, byte tid, String url) {return new Xodb_file(id, tid).Url_rel_(url).Cmd_mode_(Db_cmd_mode.Create);}
public void Index_create(Gfo_usr_dlg usr_dlg, Db_idx_itm[] idxs) {
int len = idxs.length;
provider.Txn_mgr().Txn_end_all(); // commit any pending transactions
for (int i = 0; i < len; i++) {
provider.Txn_mgr().Txn_bgn_if_none();
String index = idxs[i].Xto_sql();
usr_dlg.Prog_many("", "", "creating index: ~{0} ~{1}", id, index);
provider.Exec_sql(index);
provider.Txn_mgr().Txn_end_all();
}
}
public static final Db_idx_itm
Indexes_page_title = Db_idx_itm.sql_("CREATE UNIQUE INDEX IF NOT EXISTS page__title ON page (page_namespace, page_title, page_id, page_len, page_is_redirect);") // PERF:page_id for general queries; PERF: page_len for search_suggest; PREF:page_is_redirect for oimg
, Indexes_page_random = Db_idx_itm.sql_("CREATE UNIQUE INDEX IF NOT EXISTS page__name_random ON page (page_namespace, page_random_int);")
, Indexes_categorylinks_main = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS categorylinks__cl_main ON categorylinks (cl_to_id, cl_type_id, cl_sortkey, cl_from);")
, Indexes_categorylinks_from = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS categorylinks__cl_from ON categorylinks (cl_from);")
, Indexes_wikidata_qids = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS wdata_qids__src ON wdata_qids (wq_src_wiki, wq_src_ns, wq_src_ttl);")
, Indexes_wikidata_pids = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS wdata_pids__src ON wdata_pids (wp_src_lang, wp_src_ttl);")
;
}

132
400_xowa/src/gplx/xowa/dbs/Xodb_fsys_mgr.java Normal file
View File

@@ -0,0 +1,132 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.dbs.*;
public class Xodb_fsys_mgr {
public Xodb_fsys_mgr Ctor(Io_url src_dir, Io_url trg_dir, String trg_name) {
this.src_dir = src_dir;
this.trg_dir = trg_dir;
this.trg_name = trg_name;
return this;
} private Io_url src_dir; private String trg_name;
public Db_provider Core_provider() {return core_provider;} Db_provider core_provider;
public Db_provider Page_provider() {return page_provider;} Db_provider page_provider;
public Db_provider Category_provider() {return category_provider;} Db_provider category_provider;
public Db_provider Wdata_provider() {return wikidata_provider;} Db_provider wikidata_provider;
public Io_url Trg_dir() {return trg_dir;} private Io_url trg_dir;
public int Tid_text_idx() {return tid_text_idx;} public Xodb_fsys_mgr Tid_text_idx_(int v) {tid_text_idx = v; return this;} private int tid_text_idx = File_id_core;
public long Tid_text_max() {return tid_text_max;} long tid_text_max = Heap_max_infinite;
static final int File_id_core = 0;
public Xodb_file[] Ary() {return files_ary;}
public void Init_by_files(Db_provider p, Xodb_file[] v) {
files_ary = v; files_ary_len = v.length;
boolean category_provider_core_null = true;
for (int i = 0; i < files_ary_len; i++) {
Xodb_file file = files_ary[i];
Io_url url = trg_dir.GenSubFil(file.Url_rel()); // relative name only
file.Connect_(Db_connect_.sqlite_(url)).Url_(url);
switch (file.Tid()) {
case Xodb_file.Tid_core : file.Provider_(p); Init_by_tid_core(file); break;
case Xodb_file.Tid_category : if (category_provider_core_null) {Init_by_tid_category(file); category_provider_core_null = false;}break;
case Xodb_file.Tid_wikidata : Init_by_tid_wikidata(file); break;
case Xodb_file.Tid_text : Init_by_tid_text(file); break;
}
}
}
public void Init_make(Xow_ns_mgr ns_mgr, String ns_map_str, long text_max) {
Init_by_tid_core(Make(Xodb_file.Tid_core));
Init_by_ns_map(ns_mgr, Xodb_ns_map_mgr.Parse(Bry_.new_ascii_(ns_map_str)));
if (text_max > 0)
Init_by_tid_text(Init_make_file(Xodb_file.Tid_text, text_max));
}
public Xodb_file Init_make_file(byte tid, long max) {
if (max == Max_core_db) return files_ary[File_id_core];
return Make(tid).File_max_(max);
}
private void Init_by_tid_core(Xodb_file file) {core_provider = page_provider = category_provider = wikidata_provider = file.Provider();}
public void Init_by_tid_category(Xodb_file file) {category_provider = file.Provider();}
public void Init_by_tid_wikidata(Xodb_file file) {wikidata_provider = file.Provider();}
private void Init_by_ns_map(Xow_ns_mgr ns_mgr, Xodb_ns_map_mgr ns_map) {
Xodb_ns_map_itm[] ns_map_itms = ns_map.Itms();
int ns_map_itms_len = ns_map_itms.length;
for (int i = 0; i < ns_map_itms_len; i++) {
Xodb_ns_map_itm itm = ns_map_itms[i];
int[] ns_ids = itm.Ns_ids();
int ns_ids_len = ns_ids.length;
Xodb_file file = Make(Xodb_file.Tid_text);
for (int j = 0; j < ns_ids_len; j++) {
int ns_id = ns_ids[j];
Xow_ns ns = ns_mgr.Ids_get_or_null(ns_id); if (ns == null) continue; // some dumps may not have ns; for example, pre-2013 dumps won't have Module (828)
ns.Bldr_file_idx_(file.Id());
}
}
}
private void Init_by_tid_text(Xodb_file file) {
tid_text_idx = file.Id();
tid_text_max = file.File_max();
}
public Xodb_file Get_by_db_idx(int db_idx) {return files_ary[db_idx];}
public Xodb_file Get_or_make(byte file_tid, int file_idx) {return file_idx < files_ary_len ? files_ary[file_idx] : Make(file_tid);}
public Io_url Get_url(byte file_tid) {
Xodb_file file = Get_tid_root(file_tid);
return trg_dir.GenSubFil(file.Url_rel());
}
public Xodb_file Get_tid_root(byte file_tid) {
for (int i = 0; i < files_ary_len; i++) {
Xodb_file file = files_ary[i];
if (file.Tid() == file_tid) return file; // assume 1st file is root
}
return null;
}
public void Index_create(Gfo_usr_dlg usr_dlg, byte[] tids, Db_idx_itm... idxs) {
for (int i = 0; i < files_ary_len; i++) {
Xodb_file file = files_ary[i];
if (Byte_.In(file.Tid(), tids))
file.Index_create(usr_dlg, idxs);
}
}
public void Overwrite(int file_idx) {Create_sqlite3(src_dir, trg_dir, trg_name, file_idx);}
public Xodb_file Get_or_make(String name) {
Io_url url = trg_dir.GenSubFil(name + ".sqlite3");
if (!Io_mgr._.ExistsFil(url))
Io_mgr._.CopyFil(src_dir.GenSubFil("xowa.sqlite3"), url, true);
return Xodb_file.make_(-1, Xodb_file.Tid_temp, url.NameAndExt()).Connect_(Db_connect_.sqlite_(url));
}
public Xodb_file Make(byte file_tid) {
int file_idx = files_ary_len;
Io_url url = Create_sqlite3(src_dir, trg_dir, trg_name, file_idx);
Xodb_file rv = Xodb_file.make_(file_idx, file_tid, url.NameAndExt()).Connect_(Db_connect_.sqlite_(url));
gplx.xowa.dbs.tbls.Xodb_xowa_cfg_tbl.Insert_str(rv.Provider(), "db.meta", "type_name", Xodb_file.Tid_to_name(file_tid));
files_ary = (Xodb_file[])Array_.Resize(files_ary, files_ary_len + 1);
files_ary[files_ary_len] = rv;
++files_ary_len;
return rv;
} private Xodb_file[] files_ary = new Xodb_file[0]; int files_ary_len = 0;
public void Rls() {
for (int i = 0; i < files_ary_len; i++)
files_ary[i].Rls();
}
private static Io_url Create_sqlite3(Io_url src_dir, Io_url trg_dir, String trg_name, int file_idx) {
Io_url src_fil = src_dir.GenSubFil("xowa.sqlite3"); // /bin/any/sql/xowa/xowa.sqlite3
Io_url trg_fil = trg_dir.GenSubFil_ary(trg_name, ".", Int_.XtoStr_PadBgn(file_idx, 3), ".sqlite3"); // /wiki/en.wikipedia.org/en.wikipedia.org.000.sqlite3
Io_mgr._.CopyFil(src_fil, trg_fil, true);
return trg_fil;
}
public static final int Heap_max_infinite = 0;
public static final long Max_core_db = -1;
}

39
400_xowa/src/gplx/xowa/dbs/Xodb_load_mgr.java Normal file
View File

@@ -0,0 +1,39 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.xowa.bldrs.imports.ctgs.*; import gplx.xowa.ctgs.*; import gplx.xowa.specials.search.*;
public interface Xodb_load_mgr {
void Load_init (Xow_wiki wiki);
void Load_page (Xodb_page rv, Xow_ns ns, boolean timestamp_enabled);
boolean Load_by_id (Xodb_page rv, int id);
void Load_by_ids (Cancelable cancelable, ListAdp rv, int bgn, int end);
boolean Load_by_ttl (Xodb_page rv, Xow_ns ns, byte[] ttl);
void Load_by_ttls (Cancelable cancelable, OrderedHash rv, boolean fill_idx_fields_only, int bgn, int end);
int Load_ctg_count (byte[] ttl);
boolean Load_ctg_v1 (Xoctg_view_ctg rv, byte[] ttl);
boolean Load_ctg_v2 (Xoctg_data_ctg rv, byte[] ttl);
void Load_ctg_v2a (Xoctg_view_ctg rv, Xoctg_url url_ctg, byte[] ttl_bry, int limit);
Xodb_page[] Load_ctg_list (byte[][] ctg_ttls);
void Load_search (Cancelable cancelable, ListAdp rv, byte[] search, int results_max);
void Load_ttls_for_all_pages (Cancelable cancelable, ListAdp rslt_list, Xodb_page rslt_nxt, Xodb_page rslt_prv, Int_obj_ref rslt_count, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item);
void Load_ttls_for_search_suggest(Cancelable cancelable, ListAdp rslt_list, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item);
byte[] Find_random_ttl (Xow_ns ns);
void Clear(); // TEST:helper function
byte[] Load_qid (byte[] wiki_alias, byte[] ns_num, byte[] ttl);
int Load_pid (byte[] lang_key, byte[] pid_name);
}

172
400_xowa/src/gplx/xowa/dbs/Xodb_load_mgr_sql.java Normal file
View File

@@ -0,0 +1,172 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.dbs.*;
import gplx.xowa.apps.*; import gplx.xowa.bldrs.imports.ctgs.*; import gplx.xowa.ctgs.*; import gplx.xowa.specials.search.*; import gplx.xowa.dbs.tbls.*;
public class Xodb_load_mgr_sql implements Xodb_load_mgr {
public Xodb_load_mgr_sql(Xodb_mgr_sql db_mgr) {this.db_mgr = db_mgr; this.fsys_mgr = db_mgr.Fsys_mgr();} private Xodb_mgr_sql db_mgr; Xodb_fsys_mgr fsys_mgr;
public byte Search_version() {
if (search_provider == null) Search_version_init();
return search_version;
} private byte search_version = gplx.xowa.specials.search.Xosrh_core.Version_null;
public void Search_version_refresh() {
search_provider = null;
Search_version_init();
}
public void Load_init(Xow_wiki wiki) {
Load_init_cfg(wiki);
db_mgr.Tbl_site_stats().Select(wiki);
db_mgr.Tbl_xowa_ns().Load(wiki.Ns_mgr());
}
private void Load_init_cfg(Xow_wiki wiki) {
String_obj_ref version_val = String_obj_ref.null_();
String version_key = Xoa_gfs_mgr.Build_code(Xow_wiki.Invk_props, Xow_wiki_props.Invk_bldr_version);
KeyVal[] kv_ary = db_mgr.Tbl_xowa_cfg().Select_kvs(Xodb_mgr_sql.Grp_wiki_init, version_key, version_val);
Xodb_upgrade_mgr.Upgrade(db_mgr, kv_ary, version_key, version_val.Val());
Bry_bfr bfr = wiki.Utl_bry_bfr_mkr().Get_k004();
Xoa_gfs_mgr gfs_mgr = wiki.App().Gfs_mgr();
try {
int len = kv_ary.length;
for (int i = 0; i < len; i++) {
KeyVal kv = kv_ary[i];
gfs_mgr.Build_prop_set(bfr, Bry_.new_utf8_(kv.Key()), Bry_.new_utf8_(kv.Val_to_str_or_empty()));
}
gfs_mgr.Run_str_for(wiki, bfr.XtoStrAndClear());
} finally {bfr.Mkr_rls();}
}
public boolean Load_by_ttl(Xodb_page rv, Xow_ns ns, byte[] ttl) {return db_mgr.Tbl_page().Select_by_ttl(rv, ns, ttl);}
public void Load_by_ttls(Cancelable cancelable, OrderedHash rv, boolean fill_idx_fields_only, int bgn, int end) {db_mgr.Tbl_page().Select_by_ttl_in(cancelable, rv, db_mgr.Wiki(), fill_idx_fields_only, bgn, end);}
public void Load_page(Xodb_page rv, Xow_ns ns, boolean timestamp_enabled) {rv.Text_(db_mgr.Tbl_text().Select(rv.Db_file_idx(), rv.Id()));}
public boolean Load_by_id (Xodb_page rv, int id) {return db_mgr.Tbl_page().Select_by_id(rv, id);}
public void Load_by_ids(Cancelable cancelable, ListAdp rv, int bgn, int end) {db_mgr.Tbl_page().Select_by_id_list(cancelable, false, rv, bgn, end);}
public boolean Load_ctg_v1(Xoctg_view_ctg rv, byte[] ctg_bry) {
int cat_page_id = db_mgr.Tbl_page().Select_id(Xow_ns_.Id_category, ctg_bry); if (cat_page_id == Xodb_mgr_sql.Page_id_null) return false;
Xodb_category_itm ctg = db_mgr.Tbl_category().Select(fsys_mgr.Category_provider(), cat_page_id); if (ctg == Xodb_category_itm.Null) return false;
Db_provider p = fsys_mgr.Get_by_db_idx(ctg.File_idx()).Provider();
return db_mgr.Ctg_select_v1(rv, p, ctg);
}
public boolean Load_ctg_v2(Xoctg_data_ctg rv, byte[] ctg_bry) {throw Err_.not_implemented_();}
public void Load_ctg_v2a(Xoctg_view_ctg rv, Xoctg_url ctg_url, byte[] ctg_ttl, int load_max) {
int cat_page_id = db_mgr.Tbl_page().Select_id(Xow_ns_.Id_category, ctg_ttl); if (cat_page_id == Xodb_mgr_sql.Page_id_null) return;
Xodb_category_itm ctg = db_mgr.Tbl_category().Select(fsys_mgr.Category_provider(), cat_page_id); if (ctg == Xodb_category_itm.Null) return;
Db_provider p = fsys_mgr.Get_by_db_idx(ctg.File_idx()).Provider();
ListAdp list = ListAdp_.new_();
Load_ctg_v2a_db_retrieve(rv, ctg_url, cat_page_id, load_max, p, list);
Load_ctg_v2a_ui_sift(rv, ctg, list);
}
private void Load_ctg_v2a_db_retrieve(Xoctg_view_ctg rv, Xoctg_url ctg_url, int cat_page_id, int load_max, Db_provider p, ListAdp list) {
int len = Xoa_ctg_mgr.Tid__max;
for (byte i = Xoa_ctg_mgr.Tid_subc; i < len; i++) {
boolean arg_is_from = ctg_url.Grp_fwds()[i] == Bool_.N_byte;
byte[] arg_sortkey = ctg_url.Grp_idxs()[i];
int found = db_mgr.Tbl_categorylinks().Select_by_type(p, list, cat_page_id, i, arg_sortkey, arg_is_from, load_max);
if (found > 0 && found == load_max + 1) {
Xodb_page last_page = (Xodb_page)ListAdp_.Pop(list);
Xoctg_page_xtn last_ctg = (Xoctg_page_xtn)last_page.Xtn();
rv.Grp_by_tid(i).Itms_last_sortkey_(last_ctg.Sortkey());
}
}
db_mgr.Tbl_page().Select_by_id_list(Cancelable_.Never, list);
}
private void Load_ctg_v2a_ui_sift(Xoctg_view_ctg rv, Xodb_category_itm ctg, ListAdp list) {
int len = list.Count();
Xow_wiki wiki = this.db_mgr.Wiki();
byte prv_tid = Byte_.MaxValue_127;
Xoctg_view_grp view_grp = null;
for (int i = 0; i < len; i++) {
Xodb_page db_page = (Xodb_page)list.FetchAt(i);
if (db_page.Ns_id() == Int_.MinValue) continue; // HACK: page not found; ignore
Xoctg_page_xtn db_ctg = (Xoctg_page_xtn)db_page.Xtn();
byte cur_tid = db_ctg.Tid();
if (prv_tid != cur_tid) {
view_grp = rv.Grp_by_tid(cur_tid);
prv_tid = cur_tid;
}
Xoa_ttl ttl = Xoa_ttl.parse_(wiki, db_page.Ns_id(), db_page.Ttl_wo_ns());
Xoctg_view_itm view_itm = new Xoctg_view_itm().Sortkey_(db_ctg.Sortkey()).Ttl_(ttl);
view_itm.Load_by_ttl_data(cur_tid, db_page.Id(), Xodb_page.Timestamp_null, db_page.Text_len());
view_grp.Itms_add(view_itm);
}
len = Xoa_ctg_mgr.Tid__max;
for (byte i = Xoa_ctg_mgr.Tid_subc; i < len; i++) {
view_grp = rv.Grp_by_tid(i);
view_grp.Itms_make();
view_grp.Total_(ctg.Count_by_tid(i));
}
}
private Db_provider search_provider = null;
private void Search_version_init() {
if (search_provider == null) {
Xodb_file search_file = db_mgr.Fsys_mgr().Get_tid_root(Xodb_file.Tid_search);
if (search_file == null) {
search_provider = Db_provider_.Null;
search_version = gplx.xowa.specials.search.Xosrh_core.Version_1;
}
else {
search_provider = search_file.Provider();
search_version = gplx.xowa.specials.search.Xosrh_core.Version_2;
}
}
}
public void Load_search(Cancelable cancelable, ListAdp rv, byte[] search, int results_max) {
if (search_provider == null) Search_version_init();
if (search_version == gplx.xowa.specials.search.Xosrh_core.Version_1)
db_mgr.Tbl_page().Select_by_search(cancelable, rv, search, results_max);
else {
Xodb_search_title_word_tbl.Select_by_word(cancelable, rv, search, results_max, db_mgr.Fsys_mgr().Get_tid_root(Xodb_file.Tid_search).Provider());
db_mgr.Tbl_page().Select_by_id_list(cancelable, true, rv);
}
}
public void Load_ttls_for_all_pages(Cancelable cancelable, ListAdp rslt_list, Xodb_page rslt_nxt, Xodb_page rslt_prv, Int_obj_ref rslt_count, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item) {
db_mgr.Tbl_page().Load_ttls_for_all_pages(cancelable, rslt_list, rslt_nxt, rslt_prv, rslt_count, ns, key, max_results, min_page_len, browse_len, include_redirects, fetch_prv_item);
}
public void Load_ttls_for_search_suggest(Cancelable cancelable, ListAdp rslt_list, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item) {
db_mgr.Tbl_page().Load_ttls_for_search_suggest(cancelable, rslt_list, ns, key, max_results, min_page_len, browse_len, include_redirects, fetch_prv_item);
}
public int Load_ctg_count(byte[] ttl) {
int page_id = db_mgr.Tbl_page().Select_id(Xow_ns_.Id_category, ttl);
if (page_id == Xodb_mgr_sql.Page_id_null) return 0; // title not found; return 0;
return db_mgr.Tbl_category().Select(fsys_mgr.Category_provider(), page_id).Count_all();
}
public byte[] Load_qid(byte[] wiki_alias, byte[] ns_num, byte[] ttl) {return db_mgr.Tbl_wdata_qids().Select_qid(fsys_mgr.Wdata_provider(), wiki_alias, ns_num, ttl);}
public int Load_pid(byte[] lang_key, byte[] pid_name) {return db_mgr.Tbl_wdata_pids().Select_pid(fsys_mgr.Wdata_provider(), lang_key, pid_name);}
public byte[] Find_random_ttl(Xow_ns ns) {return db_mgr.Tbl_page().Select_random(ns);}
public void Clear() {}
public Xodb_page[] Load_ctg_list(byte[][] ctg_ttls) {
int len = ctg_ttls.length;
OrderedHash hash = OrderedHash_.new_bry_();
for (int i = 0; i < len; i++) {
Xodb_page page = new Xodb_page();
byte[] ttl = Xoa_ttl.Replace_spaces(ctg_ttls[i]); // NOTE: ctg_ttls has spaces since v1 rendered it literally;
page.Ttl_wo_ns_(ttl);
if (!hash.Has(ttl))
hash.Add(ttl, page);
}
len = hash.Count(); // must update len (!hash.Has() may have skipped titles)
db_mgr.Tbl_page().Select_by_ttl_in(Cancelable_.Never, hash, Xow_ns_.Id_category, 0, len);
OrderedHash hash2 = OrderedHash_.new_();
for (int i = 0; i < len; i++) {
Xodb_page page = (Xodb_page)hash.FetchAt(i);
if (!hash2.Has(page.Id_val()))
hash2.Add(page.Id_val(), page);
}
len = hash2.Count(); // must update len (!hash2.Has() may have skipped titles)
db_mgr.Tbl_category().Select_by_cat_id_in(Cancelable_.Never, hash2, fsys_mgr.Category_provider(), 0, len);
return (Xodb_page[])hash.XtoAry(Xodb_page.class);
}
}

193
400_xowa/src/gplx/xowa/dbs/Xodb_load_mgr_sql_tst.java Normal file
View File

@@ -0,0 +1,193 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import org.junit.*; import gplx.xowa.bldrs.*; import gplx.xowa.ctgs.*; import gplx.dbs.*; import gplx.xowa.dbs.tbls.*;
public class Xodb_load_mgr_sql_tst {
@Before public void init() {fxt.Clear();} private Xodb_load_mgr_sql_fxt fxt = new Xodb_load_mgr_sql_fxt();
@Test public void Load_ctg_ttls() {
Xodb_page[] ctgs = fxt.pages_
( fxt.ctg_(1, "Ctg_1", Bool_.Y, 10, 11, 12)
, fxt.ctg_(2, "Ctg_2", Bool_.N, 20, 21, 22)
, fxt.ctg_(3, "Ctg_3", Bool_.Y, 30, 31, 32)
);
fxt.Init_save_ctgs(ctgs);
fxt.Test_load_ctg_list(ctgs);
}
}
class Xoctg_url_mok extends Xoctg_url { public Xoctg_url_mok Page_bgn_(String v) {return Grp(Xoa_ctg_mgr.Tid_page, Bool_.Y, v);}
public Xoctg_url_mok Page_end_(String v) {return Grp(Xoa_ctg_mgr.Tid_page, Bool_.N, v);}
Xoctg_url_mok Grp(byte tid, boolean v, String bmk) {
this.Grp_fwds()[tid] = v ? Bool_.Y_byte : Bool_.N_byte;
this.Grp_idxs()[tid] = Bry_.new_ascii_(bmk);
return this;
}
}
class Xodb_load_mgr_sql_fxt {
public void Clear() {
if (fxt == null) {
fxt = new Db_mgr_fxt();
fxt.Ctor_fsys();
fxt.Init_db_sqlite();
wiki = fxt.Wiki();
app = wiki.App();
}
} Db_mgr_fxt fxt; Int_obj_ref next_id = Int_obj_ref.new_(1); Xoa_app app; Xow_wiki wiki;
public Xodb_page[] pages_(Xodb_page... ary) {return ary;}
public Xodb_page ctg_(int id, String ttl, boolean hidden, int count_subcs, int count_files, int count_pages) {
Xodb_page rv = new Xodb_page().Ns_id_(Xow_ns_.Id_category).Id_(id).Ttl_wo_ns_(Bry_.new_ascii_(ttl));
Xodb_category_itm ctg = Xodb_category_itm.load_(id, 0, hidden, count_subcs, count_files, count_pages);
rv.Xtn_(ctg);
return rv;
}
public void Init_save_ctgs(Xodb_page[] ary) {
int len = ary.length;
Xodb_mgr_sql db_mgr = wiki.Db_mgr_as_sql();
Db_provider p = db_mgr.Fsys_mgr().Core_provider();
p.Txn_mgr().Txn_bgn_if_none();
Db_stmt page_stmt = db_mgr.Tbl_page().Insert_stmt(p);
Db_stmt category_stmt = db_mgr.Tbl_category().Insert_stmt(p);
DateAdp modified = DateAdp_.Now();
for (int i = 0; i < len; i++) {
Xodb_page page = ary[i];
db_mgr.Tbl_page().Insert(page_stmt, page.Id(), page.Ns_id(), page.Ttl_wo_ns(), false, modified, 10, page.Id(), 0);
Xodb_category_itm ctg_itm = (Xodb_category_itm)page.Xtn();
db_mgr.Tbl_category().Insert(category_stmt, ctg_itm.Id(), ctg_itm.Count_pages(), ctg_itm.Count_subcs(), ctg_itm.Count_files(), Bool_.Xto_byte(ctg_itm.Hidden()), 0);
}
p.Txn_mgr().Txn_end_all();
}
public void Test_load_ctg_list(Xodb_page[] ary) {
int len = ary.length;
byte[][] ttls = new byte[len][];
for (int i = 0; i < len; i++) {
ttls[i] = ary[i].Ttl_wo_ns();
}
Xodb_page[] actl = wiki.Db_mgr_as_sql().Load_mgr().Load_ctg_list(ttls);
Tfds.Eq_str_lines(Xto_str(ary), Xto_str(actl));
}
private static String Xto_str(Xodb_page[] ary) {
Bry_bfr bfr = Bry_bfr.new_();
int len = ary.length;
for (int i = 0; i < len; i++) {
Xodb_page page = ary[i];
Xodb_category_itm ctg_itm = (Xodb_category_itm)page.Xtn();
bfr.Add_int_variable(page.Id()).Add_byte_pipe();
bfr.Add(page.Ttl_wo_ns()).Add_byte_pipe();
bfr.Add_byte(Bool_.Xto_byte(ctg_itm.Hidden())).Add_byte_nl();
}
return bfr.XtoStrAndClear();
}
public Xoctg_url_mok ctg_url_() {return new Xoctg_url_mok();}
public Xodb_load_mgr_sql_fxt Init_limit_(int v) {limit = v; return this;} private int limit = 3;
// public void Init_ctg_insert(Xoctg_mok_ctg ctg) {
// Xodb_mgr_sql db_mgr = wiki.Db_mgr_as_sql();
// Db_provider p = db_mgr.Fsys_mgr().Core_provider();
// p.Txn_mgr().Txn_bgn_if_none();
// Xodb_categorylinks_tbl ctg_tbl = db_mgr.Tbl_categorylinks();
// Db_stmt ctg_stmt = ctg_tbl.Insert_stmt(p);
// Xodb_page_tbl page_tbl = db_mgr.Tbl_page();
// Db_stmt page_stmt = page_tbl.Insert_stmt();
// DateAdp modified_on = Tfds.Now_time0_add_min(0);
// int page_id = next_id.Val_add_post();
// page_tbl.Insert(page_stmt, page_id, Xow_ns_.Id_category, ctg.Ttl(), false, modified_on, 0, page_id, 0);
// for (byte i = 0; i < Xoa_ctg_mgr.Tid__max; i++) {
// Xoctg_mok_grp grp = ctg.Grps_get_or_new(i);
// int grp_len = grp.Itms().Count();
// for (int j = 0; j < grp_len; j++) {
// Xodb_page page = (Xodb_page)grp.Itms().FetchAt(j);
// Xoctg_page_xtn page_ctg = (Xoctg_page_xtn)page.Xtn();
// page_id = page.Id();
// ctg_tbl.Insert(ctg_stmt, page_id, ctg.Ttl(), page_ctg.Sortkey(), 0, i);
// page_tbl.Insert(page_stmt, page_id, page.Ns_id(), page.Ttl_wo_ns(), false, modified_on, 0, page_id, 0);
// }
// }
// p.Txn_mgr().Txn_end_all();
// }
public void Test_select(Xoctg_url ctg_url, Xoctg_mok_ctg expd) {
Xoctg_view_ctg view_ctg = new Xoctg_view_ctg();
wiki.Db_mgr_as_sql().Load_mgr().Load_ctg_v2a(view_ctg, ctg_url, expd.Ttl(), limit);
for (byte i = 0; i < Xoa_ctg_mgr.Tid__max; i++) {
Xoctg_view_grp view_grp = view_ctg.Grp_by_tid(i);
Xoctg_mok_grp mok_grp = expd.Grps_get_or_new(i);
Tfds.Eq_ary_str(Xto_str(mok_grp), Xto_str(view_grp));
Tfds.Eq(String_.new_ascii_(mok_grp.Last_plus_one_sortkey()), String_.new_ascii_(view_grp.Itms_last_sortkey()));
}
}
String[] Xto_str(Xoctg_view_grp grp) {
Xoctg_view_itm[] ary = grp.Itms();
int len = ary.length;
String[] rv = new String[len];
for (int i = 0; i< len; i++) {
Xoctg_view_itm itm = ary[i];
rv[i] = itm.Ttl().Page_db_as_str();
}
return rv;
}
String[] Xto_str(Xoctg_mok_grp grp) {
ListAdp list = grp.Itms();
int len = list.Count();
String[] rv = new String[len];
for (int i = 0; i< len; i++) {
Xodb_page itm = (Xodb_page)list.FetchAt(i);
rv[i] = String_.new_ascii_(itm.Ttl_wo_ns());
}
return rv;
}
public Xoctg_mok_ctg ctg_() {
Xoctg_mok_ctg rv = new Xoctg_mok_ctg(next_id);
return rv;
}
}
class Xoctg_mok_grp {
public byte Tid() {return tid;} public Xoctg_mok_grp Tid_(byte v) {this.tid = v; return this;} private byte tid;
public byte[] Last_plus_one_sortkey() {return last_plus_one_sortkey;} public Xoctg_mok_grp Last_plus_one_sortkey_(byte[] v) {this.last_plus_one_sortkey = v; return this;} private byte[] last_plus_one_sortkey;
public ListAdp Itms() {return itms;} ListAdp itms = ListAdp_.new_();
}
class Xoctg_mok_ctg {
public Xoctg_mok_ctg(Int_obj_ref next_id) {this.next_id = next_id;} Int_obj_ref next_id;
public byte[] Ttl() {return ttl;}
public Xoctg_mok_ctg Ttl_(String v) {return Ttl_(Bry_.new_ascii_(v));}
public Xoctg_mok_ctg Ttl_(byte[] v) {this.ttl = v; return this;} private byte[] ttl;
public Xoctg_mok_grp[] Grps() {return grps;} private Xoctg_mok_grp[] grps = new Xoctg_mok_grp[3];
public Xoctg_mok_grp Grps_get_or_new(byte tid) {
Xoctg_mok_grp rv = grps[tid];
if (rv == null) {
rv = new Xoctg_mok_grp().Tid_(tid);
grps[tid] = rv;
}
return rv;
}
public Xoctg_mok_ctg Grp_pages_(int count) {return Grp_pages_(0, count, null);}
public Xoctg_mok_ctg Grp_pages_(int bgn, int end, String last_itm_plus_one_sortkey) {
Xoctg_mok_grp grp = Grps_get_or_new(Xoa_ctg_mgr.Tid_page);
byte[] ttl_prefix = Bry_.new_ascii_("Page_");
int ns_id = Xow_ns_.Id_main;
byte ctg_tid = Xoa_ctg_mgr.Tid_page;
for (int i = bgn; i < end; i++) {
byte[] ttl = Bry_.Add(ttl_prefix, Bry_.new_ascii_(Int_.XtoStr_PadBgn(i, 3)));
Xoctg_page_xtn db_ctg = new Xoctg_page_xtn(ctg_tid, ttl);
Xodb_page page = new Xodb_page();
int page_id = next_id.Val_add_post();
page.Id_(page_id).Ns_id_(ns_id).Ttl_wo_ns_(ttl).Xtn_(db_ctg);
grp.Itms().Add(page);
}
grp.Last_plus_one_sortkey_(Bry_.new_ascii_(last_itm_plus_one_sortkey));
return this;
}
}

549
400_xowa/src/gplx/xowa/dbs/Xodb_load_mgr_txt.java Normal file
View File

@@ -0,0 +1,549 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.xowa.bldrs.imports.ctgs.*; import gplx.xowa.ctgs.*; import gplx.xowa.dbs.tbls.*; import gplx.xowa.specials.search.*;
public class Xodb_load_mgr_txt implements Xodb_load_mgr {
public Xodb_load_mgr_txt(Xow_wiki wiki) {
this.wiki = wiki;
this.fsys_mgr = wiki.Fsys_mgr();
} private Xow_wiki wiki; Xow_fsys_mgr fsys_mgr;
Xob_xdat_file tmp_xdat_file = new Xob_xdat_file(); Xob_xdat_itm tmp_xdat_itm = new Xob_xdat_itm();
public void Load_init (Xow_wiki wiki) {}
public void Load_page(Xodb_page rv, Xow_ns ns, boolean timestamp_enabled) {Load_page(rv, rv.Db_file_idx(), rv.Db_row_idx(), ns, timestamp_enabled, tmp_xdat_file, tmp_xdat_itm);}
public void Load_page(Xodb_page rv, int txtdb_fil_idx, int txtdb_row_idx, Xow_ns ns, boolean timestamp_enabled, Xob_xdat_file xdat_file, Xob_xdat_itm xdat_itm) {
Io_url file = fsys_mgr.Url_ns_fil(Xow_dir_info_.Tid_page, ns.Id(), txtdb_fil_idx);
byte[] bry = gplx.ios.Io_stream_rdr_.Load_all(file); int bry_len = bry.length;
xdat_file.Clear().Parse(bry, bry_len, file).GetAt(xdat_itm, txtdb_row_idx);
Load_page_parse(rv, bry, bry_len, xdat_itm.Itm_bgn(), xdat_itm.Itm_end(), timestamp_enabled);
}
public boolean Load_by_ttl(Xodb_page rv, Xow_ns ns, byte[] ttl) { // NOTE: ttl must be correct case; EX: "Example title"
if (!Env_.Mode_testing() && wiki.Init_needed()) wiki.Init_assert(); // NOTE: need to call assert as wiki_finder (and possibly elsewhere) may call load on commons_wiki without ever asserting; DATE:2013-03-19
if (!Load_xdat_itm(tmp_xdat_itm, ns, Xow_dir_info_.Tid_ttl, ttl, Xodb_page_.Txt_ttl_pos, Byte_ascii.Tab, true)) return false;
Xodb_page_.Txt_ttl_load(rv, tmp_xdat_itm.Itm_bry());
rv.Exists_(true);
return Bry_.Eq(rv.Ttl_wo_ns(), ttl);
}
public void Load_by_ttls(Cancelable cancelable, OrderedHash rv, boolean fill_idx_fields_only, int bgn, int end) {// NOTE: Load_by_ttls just a wrapper around Load_by_ttl; for xdat, Load_by_ttl is fast enough
for (int i = bgn; i < end; i++) {
if (cancelable.Canceled()) return;
Xodb_page page = (Xodb_page)rv.FetchAt(i);
Load_by_ttl(page, page.Ns(), page.Ttl_wo_ns());
}
}
public void Load_by_ids(Cancelable cancelable, ListAdp list, int bgn, int end) {
int prv_fil_idx = -1;
byte[] id_bry = new byte[5];
int len = end - bgn;
Gfo_usr_dlg_fmt msg_wtr = Gfo_usr_dlg_fmt.fmt_(GRP_KEY, "search2_ids", "resolving ids: ~{0} of ~{1} (~{2})", len, 10f);
for (int i = 0; i < len; i++) {
if (cancelable.Canceled()) return;
Xodb_page itm = (Xodb_page)list.FetchAt(i + bgn);
Base85_utl.XtoStrByAry(itm.Id(), id_bry, 0, 5);
int cur_fil_idx = this.Find_file_idx_by_site(Xow_dir_info_.Tid_id, id_bry);
if (cur_fil_idx != prv_fil_idx) {
if (!this.Load_xdat_file(cancelable, tmp_xdat_file, Xow_dir_info_.Tid_id, cur_fil_idx)) continue; // file not found; ignore
prv_fil_idx = cur_fil_idx;
}
if (!this.Load_by_id(tmp_page, tmp_xdat_file, id_bry)) continue; // id not found in file; ignore
itm.Ns_id_(tmp_page.Ns_id()).Ttl_wo_ns_(tmp_page.Ttl_wo_ns());
msg_wtr.Write_prog_cur(i, wiki.App().Usr_dlg());
}
} Xodb_page tmp_page = new Xodb_page();
public void Load_search(Cancelable cancelable, ListAdp rv, byte[] search, int results_max) {
Xow_ns ns = wiki.Ns_mgr().Ns_main();
int search_len = search.length;
byte match_tid = Xosrh_core.Match_tid_all;
if (search_len > 0 && search[search_len - 1] == Byte_ascii.Asterisk) {
search = Bry_.Mid(search, 0, search_len - 1);
match_tid = Xosrh_core.Match_tid_bgn;
}
int bgn_idx = this.Find_file_idx_by_ns(Xow_dir_info_.Tid_search_ttl, ns, search);
if (bgn_idx == Xow_data_mgr.File_idx_unknown) return;
if (match_tid == Xosrh_core.Match_tid_all) {
if (!this.Load_xdat_file(cancelable, tmp_xdat_file, Xow_dir_info_.Tid_search_ttl, ns, bgn_idx)) return;
tmp_xdat_file.Find(tmp_xdat_itm, search, 0, Byte_ascii.Pipe, true);
if (tmp_xdat_itm.Missing()) return;
Find_ttls__add_itms(rv, tmp_xdat_file, tmp_xdat_itm);
}
else {
byte[] end_ttl = Bry_.Increment_last(Bry_.Copy(search));
int end_idx = this.Find_file_idx_by_ns(Xow_dir_info_.Tid_search_ttl, ns, end_ttl);
for (int i = bgn_idx; i <= end_idx; i++) {
if (cancelable.Canceled()) return;
this.Load_xdat_file(cancelable, tmp_xdat_file, Xow_dir_info_.Tid_search_ttl, ns, i);
if (cancelable.Canceled()) return;
int itm_bgn_idx = 0;
if (i == bgn_idx) {
tmp_xdat_file.Find(tmp_xdat_itm, search, 0, Byte_ascii.Pipe, false);
itm_bgn_idx = tmp_xdat_itm.Itm_idx();
}
int itm_end_idx = tmp_xdat_file.Count();
if (i == end_idx) {
tmp_xdat_file.Find(tmp_xdat_itm, end_ttl, 0, Byte_ascii.Pipe, false);
itm_end_idx = tmp_xdat_itm.Itm_idx();
}
for (int j = itm_bgn_idx; j < itm_end_idx; j++) {
tmp_xdat_file.GetAt(tmp_xdat_itm, j);
Find_ttls__add_itms(rv, tmp_xdat_file, tmp_xdat_itm);
}
}
}
}
private void Find_ttls__add_itms(ListAdp rv, Xob_xdat_file rdr, Xob_xdat_itm xdat_itm) {
byte[] raw = rdr.Src();
int itm_bgn = xdat_itm.Itm_bgn(), itm_end = xdat_itm.Itm_end();
int pos = Bry_finder.Find_fwd(raw, Byte_ascii.Pipe, itm_bgn, raw.length);
if (pos == Bry_.NotFound) throw wiki.App().Usr_dlg().Fail_many(GRP_KEY, "invalid_search_file", "search file is invalid");
pos += Int_.Const_dlm_len; // pipe
while (pos < itm_end) {
int page_id = Base85_utl.XtoIntByAry(raw, pos, pos + 4);
pos += 6; // 5 + 1 for semic;
int page_len = Base85_utl.XtoIntByAry(raw, pos, pos + 4);
rv.Add(Xodb_page.srch_(page_id, page_len));
pos += 6; // 5 + 1 for pipe
// if (match.Itms_len() == max_results) break;
}
}
public boolean Load_ctg_v2(Xoctg_data_ctg ctg, byte[] name) {
boolean rv = Load_xdat_itm(tmp_xdat_itm, Xow_dir_info_.Tid_category2_link, name, true); if (!rv) return false;
byte[] src = tmp_xdat_itm.Itm_bry();
Load_ctg_v2_data(wiki.App().Usr_dlg(), wiki.App().Utl_fld_rdr(), ctg, name, src, tmp_ctg_grp_lens);
Load_ctg_v2_main(ctg, name);
return true;
} int[] tmp_ctg_grp_lens = new int[3];
private void Load_ctg_v2_data(Gfo_usr_dlg usr_dlg, Gfo_fld_rdr fld_rdr, Xoctg_data_ctg ctg, byte[] ctg_name, byte[] src, int[] tmp_grp_lens) { // Name|subc_len|file_len|page_len|subc_bfr|file_bfr|page_bfr
fld_rdr.Data_(src); fld_rdr.Read_bry_escape(); // 1st field is name; skip;
Load_ctg_v2_data_lens(fld_rdr, tmp_grp_lens);
Load_ctg_v2_data_mgrs(usr_dlg, ctg_name, src, fld_rdr.Pos(), tmp_grp_lens, ctg.Grp_mgrs());
}
private void Load_ctg_v2_data_lens(Gfo_fld_rdr fld_rdr, int[] grp_lens) {
for (int i = 0; i < 3; i++)
grp_lens[i] = fld_rdr.Read_int_base85_len5();
}
private void Load_ctg_v2_data_mgrs(Gfo_usr_dlg usr_dlg, byte[] ctg_name, byte[] src, int bgn, int[] grp_lens, Xoctg_idx_mgr[] grp_mgrs) {
for (int i = 0; i < 3; i++) {
int grp_len = grp_lens[i]; if (grp_len == 0) continue;
int end = bgn + grp_len;
Xoctg_idx_mgr grp_mgr = new Xoctg_idx_mgr();
byte[] grp_src = Bry_.Mid(src, bgn, end);
grp_mgr.Src_(grp_src);
grp_mgr.Index(usr_dlg, ctg_name, grp_src);
grp_mgrs[i] = grp_mgr;
bgn = end;
}
}
private void Load_ctg_v2_main(Xoctg_data_ctg rv, byte[] name) {
if (!Load_xdat_itm(tmp_xdat_itm, Xow_dir_info_.Tid_category2_main, name, true)) return;
if (tmp_xdat_itm.Missing()) return;
byte[] bry = tmp_xdat_itm.Itm_bry();
int bgn = name.length + 1;
boolean hidden = bry[bgn] == Byte_ascii.Ltr_y;
int count_subcs = Base85_utl.XtoIntByAry(bry, bgn + 2, bgn + 6);
int count_files = Base85_utl.XtoIntByAry(bry, bgn + 8, bgn + 12);
int count_pages = Base85_utl.XtoIntByAry(bry, bgn + 14, bgn + 18);
rv.Hidden_(hidden);
for (byte i = 0; i < Xoa_ctg_mgr.Tid__max; i++) {
Xoctg_idx_mgr idx_mgr = rv.Grp_by_tid(i);
if (idx_mgr == null) continue;
int count = 0;
switch (i) {
case Xoa_ctg_mgr.Tid_subc: count = count_subcs; break;
case Xoa_ctg_mgr.Tid_file: count = count_files; break;
case Xoa_ctg_mgr.Tid_page: count = count_pages; break;
default: throw Err_.unhandled(i);
}
idx_mgr.Total_(count);
}
}
public boolean Load_by_id(Xodb_page page, int id) {Base85_utl.XtoStrByAry(id, tmp_id_bry, 0, 5); return Load_by_id(page, tmp_id_bry);} private byte[] tmp_id_bry = new byte[5];
boolean Load_by_id(Xodb_page page, byte[] id_bry) {
if (!Load_xdat_itm(tmp_xdat_itm, Xow_dir_info_.Tid_id, id_bry, true)) return false;;
Xodb_page_.Txt_id_load(page, tmp_xdat_itm.Itm_bry());
return true;
}
boolean Load_by_id(Xodb_page page, Xob_xdat_file xdat_file, byte[] id_bry) {
xdat_file.Find(tmp_xdat_itm, id_bry, 0, Byte_ascii.Pipe, true);
if (tmp_xdat_itm.Missing()) return false;
Xodb_page_.Txt_id_load(page, tmp_xdat_itm.Itm_bry());
return true;
}
boolean Load_xdat_itm(Xob_xdat_itm xdat_itm, byte regy_tid, byte[] key, boolean exact) {return Load_xdat_itm(xdat_itm, null, regy_tid, key, 0, Byte_ascii.Pipe, exact);}
boolean Load_xdat_itm(Xob_xdat_itm xdat_itm, Xow_ns ns, byte regy_tid, byte[] key, int parse_bgn, byte parse_dlm, boolean exact) {
// get regy
Xowd_regy_mgr regy = null;
if (ns == null)
regy = Get_regy_by_site(regy_tid);
else {
regy = Get_regy_by_ns(ns);
if (regy == null) return false;
}
// find file
int fil_idx = regy.Files_find(key);
if (fil_idx == Xowd_regy_mgr.Regy_null) return false; // NOTE: must check for -1, not 0; else defect in which entries in file 0 are ignored; DATE:2013-04-11
// load file
Io_url fil = ns == null ? fsys_mgr.Url_site_fil(regy_tid, fil_idx) : fsys_mgr.Url_ns_fil(regy_tid, ns.Id(), fil_idx);
Load_xdat_file(Cancelable_.Never, tmp_xdat_file, fil);
// find itm by key
tmp_xdat_file.Find(xdat_itm, key, parse_bgn, parse_dlm, exact);
return !xdat_itm.Missing();
} Int_obj_ref tmp_len = Int_obj_ref.zero_();
public boolean Load_xdat_file(Cancelable cancelable, Xob_xdat_file xdat_file, byte regy_tid, int fil_idx) {return Load_xdat_file(cancelable, xdat_file, regy_tid, null, fil_idx);}
boolean Load_xdat_file(Cancelable cancelable, Xob_xdat_file xdat_file, byte regy_tid, Xow_ns ns, int fil_idx) {
Io_url fil = ns == null ? fsys_mgr.Url_site_fil(regy_tid, fil_idx) : fsys_mgr.Url_ns_fil(regy_tid, ns.Id(), fil_idx);
return Load_xdat_file(cancelable, xdat_file, fil);
}
public boolean Load_xdat_file(Cancelable cancelable, Xob_xdat_file xdat_file, Io_url url) {
boolean rv = false;
if (cancelable.Canceled()) return false;
Bry_bfr tmp_bfr = wiki.Utl_bry_bfr_mkr().Get_m001();
byte[] tmp_bry = tmp_bfr.Bfr();
if (cancelable.Canceled()) return false;
tmp_bry = Io_mgr._.LoadFilBry_reuse(url, tmp_bry, tmp_len);
if (cancelable.Canceled()) return false;
if (tmp_bry.length == 0)
wiki.App().Usr_dlg().Warn_many("", "file.empty", "hive file is empty: ~{0}", url.Raw());
else {
int src_len = tmp_len.Val();
xdat_file.Clear().Parse(tmp_bry, src_len, url);
xdat_file.Src_len_(src_len);
rv = true;
}
if (cancelable.Canceled()) return false;
tmp_bfr.Mkr_rls().Clear();
return rv;
}
int Find_file_idx_by_ns(byte regy_tid, Xow_ns ns, byte[] key) {
Xowd_regy_mgr regy = new Xowd_regy_mgr(fsys_mgr.Url_ns_reg(ns.Num_str(), regy_tid));
return regy.Files_find(key);
}
public int Find_file_idx_by_site(byte regy_tid, byte[] key) {
Xowd_regy_mgr regy = site_regys[regy_tid];
if (regy == null) {
regy = new Xowd_regy_mgr(fsys_mgr.Url_site_reg(regy_tid));
site_regys[regy_tid] = regy;
}
return regy.Files_find(key);
}
public void Clear() {
int len = ns_regys.length;
for (int i = 0; i < len; i++)
ns_regys[i] = null;
len = site_regys.length;
for (int i = 0; i < len; i++)
site_regys[i] = null;
}
public byte[] Find_random_ttl(Xow_ns ns) {
Xowd_regy_mgr regy_mgr = this.Get_regy_by_ns(ns);
Int_obj_ref count = Int_obj_ref.zero_();
Xob_random_itm[] files = Build_random_itms(regy_mgr, count);
int random_idx = RandomAdp_.new_().Next(count.Val() - 1); // get a random idx; -1 since count is super 1 (EX: count of 1 mil; random_idx of 0 - 999,999)
int file_idx = CompareAble_.FindSlot(Xob_random_itm_comparer._, files, new Xob_random_itm(-1, random_idx, -1));
Io_url file_url = fsys_mgr.Url_ns_fil(Xow_dir_info_.Tid_ttl, ns.Id(), file_idx);
Load_xdat_file(Cancelable_.Never, tmp_xdat_file, file_url);
Xob_random_itm file = files[file_idx];
tmp_xdat_file.GetAt(tmp_xdat_itm, random_idx - file.Bgn()); // get nth row; EX: random_idx=120; .Bgn=103 -> get 17th
Xodb_page page = Xodb_page_.Txt_ttl_load(tmp_xdat_itm.Itm_bry());
return page.Ttl_wo_ns();
}
private static Xob_random_itm[] Build_random_itms(Xowd_regy_mgr mgr, Int_obj_ref count) {
// convert regy to list of random_itms (similar to regy_itms, but has integer bgn / end; EX: [0]:0,50; [1]:51-102; [2]:103-130)
Xowd_hive_regy_itm[] files_ary = mgr.Files_ary();
int len = files_ary.length;
Xob_random_itm[] rv = new Xob_random_itm[len];
int tmp_count = 0;
for (int i = 0; i < len; i++) {
Xowd_hive_regy_itm file = files_ary[i];
rv[i] = new Xob_random_itm(i, tmp_count, file.Count());
tmp_count += file.Count();
}
count.Val_(tmp_count);
return rv;
}
public boolean Load_ctg_v1(Xoctg_view_ctg view_ctg, byte[] ctg_ttl) {return Load_ctg_v1_wkr(view_ctg, ctg_ttl, null);}
private boolean Load_ctg_v1_wkr(Xoctg_view_ctg view_ctg, byte[] ctg_ttl, Int_obj_ref count_only) {
Xowd_regy_mgr ctg_regy = Get_regy_by_site(Xow_dir_info_.Tid_category);
int fil_idx = ctg_regy.Files_find(ctg_ttl);
if (fil_idx == Xowd_regy_mgr.Regy_null) return false; // NOTE: must check for -1, not 0; else defect in which entries in file 0 are ignored; DATE:2013-04-11
Io_url fil = fsys_mgr.Url_site_fil(Xow_dir_info_.Tid_category, fil_idx);
Load_xdat_file(Cancelable_.Never, tmp_xdat_file, fil);
tmp_xdat_file.Find(tmp_xdat_itm, ctg_ttl, 0, Byte_ascii.Pipe, true);
if (tmp_xdat_itm.Missing()) return false;
ListAdp ctgs = ListAdp_.new_();
Load_ctg_v1_parse(ctgs, wiki.App().Usr_dlg(), tmp_xdat_itm.Itm_bry());
ctgs.SortBy(Xodb_page_sorter.IdAsc);
this.Load_by_ids(Cancelable_.Never, ctgs, 0, ctgs.Count());
ctgs.SortBy(Xodb_page_sorter.Ns_id_TtlAsc);
int ctgs_len = ctgs.Count(); if (ctgs_len == 0) return false;
if (count_only != null) { // Ctg_count specificed
count_only.Val_(ctgs_len);
return true;
}
for (int i = 0; i < ctgs_len; i++) {
Xodb_page itm = (Xodb_page)ctgs.FetchAt(i);
byte itm_tid = Load_ctg_v1_tid(itm.Ns_id());
Xoctg_view_itm sub = Load_ctg_v1_sub(itm_tid, itm);
sub.Ttl_(Xoa_ttl.parse_(wiki, itm.Ns_id(), itm.Ttl_wo_ns())).Sortkey_(itm.Ttl_wo_ns());
view_ctg.Grp_by_tid(itm_tid).Itms_add(sub);
}
for (byte i = 0; i < Xoa_ctg_mgr.Tid__max; i++) {
Xoctg_view_grp grp = view_ctg.Grp_by_tid(i);
grp.Itms_make();
grp.Total_(grp.Itms().length);
}
return true;
}
private static void Load_ctg_v1_parse(ListAdp rv, Gfo_usr_dlg usr_dlg, byte[] ary) {
int aryLen = ary.length;
int pos = Bry_finder.Find_fwd(ary, Byte_ascii.Pipe, 0, aryLen);
int rowCount = (aryLen - pos + 1) / (Base85_utl.Len_int + gplx.xowa.apps.fsys.Launcher_app_mgr.Len_dlm_fld);
rv.Clear();
boolean garbage = false;
for (int i = 0; i < rowCount; i++) {
Xodb_page row = new Xodb_page();
rv.Add(row);
if (garbage) continue;
int bgn = pos + 1 + (i * (Base85_utl.Len_int + gplx.xowa.apps.fsys.Launcher_app_mgr.Len_dlm_fld));
try {
int id = Base85_utl.XtoIntByAry(ary, bgn, bgn + Base85_utl.Len_int - 1);
if (id < 0) throw Err_.new_("invalid id").Add("id", id);
row.Id_(id);
}
catch (Exception e) {
row.Id_(0);
usr_dlg.Warn_many("xowa.ttl_request", "parse", "failed to parse row: row=~{0} err=~{1}", String_.new_utf8_(ary), Err_.Message_lang(e));
garbage = true;
continue; // NOTE: must break b/c rest of ids are garbage
}
}
}
public static byte Load_ctg_v1_tid(int ns_id) {
switch (ns_id) {
case Xow_ns_.Id_category: return Xoa_ctg_mgr.Tid_subc;
case Xow_ns_.Id_file: return Xoa_ctg_mgr.Tid_file;
default: return Xoa_ctg_mgr.Tid_page;
}
}
private static Xoctg_view_itm Load_ctg_v1_sub(byte tid, Xodb_page data) {
Xoctg_view_itm rv = new Xoctg_view_itm();
rv.Load_by_ttl_data(tid, data.Id(), 0, data.Text_len());
return rv;
}
private void Load_page_parse(Xodb_page page, byte[] src, int src_len, int row_bgn, int row_end, boolean timestamp_enabled) { // \n\tdate5\tpage_title\tpage_text
int timestamp_bgn = row_bgn + 5 + 1;
int timestamp_end = timestamp_bgn + 5;
if (timestamp_enabled) {
int timestamp = Base85_utl.XtoIntByAry(src, timestamp_bgn, timestamp_end - 1);
page.Modified_on_(Bit_.Xto_date_short(timestamp));
}
int name_bgn = timestamp_end + 1;
int name_end = Bry_finder.Find_fwd(src, Xodb_page_.Txt_page_dlm, name_bgn, src_len);
page.Text_(Bry_.Mid(src, name_end + 1, row_end - 1)); // +1 to skip dlm
}
Xowd_regy_mgr Get_regy_by_site(byte regy_tid) {
Xowd_regy_mgr rv = site_regys[regy_tid];
if (rv == null) {
rv = new Xowd_regy_mgr(fsys_mgr.Url_site_reg(regy_tid));
site_regys[regy_tid] = rv;
}
return rv;
} private Xowd_regy_mgr[] site_regys = new Xowd_regy_mgr[Xow_dir_info_.Regy_tid_max];
Xowd_regy_mgr Get_regy_by_ns(Xow_ns ns) {
int ns_ord = ns.Ord();
Xowd_regy_mgr rv = ns_regys[ns_ord];
if (rv == null) {
Io_url file = fsys_mgr.Url_ns_reg(ns.Num_str(), Xow_dir_info_.Tid_ttl);
if (!Io_mgr._.ExistsFil(file)) return null;
rv = new Xowd_regy_mgr(file);
ns_regys[ns_ord] = rv;
}
return rv;
} private Xowd_regy_mgr[] ns_regys = new Xowd_regy_mgr[Xow_ns_mgr_.Ordinal_max];
private Xodb_page tmp_rslt_nxt = new Xodb_page(), tmp_rslt_prv = new Xodb_page(); private Int_obj_ref tmp_rslt_count = Int_obj_ref.zero_();
public void Load_ttls_for_search_suggest(Cancelable cancelable, ListAdp rslt_list, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item) {
this.Load_ttls_for_all_pages(cancelable, rslt_list, tmp_rslt_nxt, tmp_rslt_prv, tmp_rslt_count, ns, key, max_results, min_page_len, browse_len, include_redirects, fetch_prv_item);
}
public void Load_ttls_for_all_pages(Cancelable cancelable, ListAdp rslt_list, Xodb_page rslt_nxt, Xodb_page rslt_prv, Int_obj_ref rslt_count, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item) {
byte dir_tid = Xow_dir_info_.Tid_ttl;
Xob_xdat_file cur_xdat_file = new Xob_xdat_file();
Xob_xdat_itm cur_xdat_itm = new Xob_xdat_itm();
Xowd_regy_mgr regy = new Xowd_regy_mgr(fsys_mgr.Url_ns_reg(ns.Num_str(), dir_tid));
int fil_idx = regy.Files_find(key); if (fil_idx == Xowd_regy_mgr.Regy_null) return;
if (!this.Load_xdat_file(Cancelable_.Never, cur_xdat_file, dir_tid, ns, fil_idx)) return;
cur_xdat_file.Find(cur_xdat_itm, key, Xodb_page_.Txt_ttl_pos, Byte_ascii.Tab, false);
int itm_idx = cur_xdat_itm.Itm_idx();
if (itm_idx == -1) itm_idx = 0; // nothing found; return;
Special_allpages_query_fwd(rslt_list, rslt_nxt, rslt_count , dir_tid, ns, include_redirects, browse_len, fil_idx, itm_idx , cur_xdat_file, cur_xdat_itm, regy);
Special_allpages_query_bwd(rslt_list, rslt_prv , dir_tid, ns, include_redirects, browse_len, fil_idx, itm_idx - 1, cur_xdat_file, cur_xdat_itm);
}
private void Special_allpages_query_fwd(ListAdp rslt_list, Xodb_page rslt_nxt, Int_obj_ref rslt_count, byte dir_tid, Xow_ns ns, boolean include_redirects, int total, int fil_idx, int row_idx, Xob_xdat_file xdat_file, Xob_xdat_itm xdat_itm, Xowd_regy_mgr regy) {
int count = 0; ++total;
boolean loop = true;
int regy_len = regy.Files_ary().length;
int rslt_list_len = rslt_count.Val();
Xodb_page nxt_itm = null;
while (loop) {
if (fil_idx == regy_len) break;
if (xdat_file == null) {
xdat_file = new Xob_xdat_file();
this.Load_xdat_file(Cancelable_.Never, xdat_file, dir_tid, ns, fil_idx);
row_idx = 0;
}
int rows_len = xdat_file.Count();
for (; row_idx < rows_len; row_idx++) {
xdat_file.GetAt(xdat_itm, row_idx);
Xodb_page ttl_itm = Xodb_page_.Txt_ttl_load(Bry_.Mid(xdat_itm.Src(), xdat_itm.Itm_bgn(), xdat_itm.Itm_end()));
if (!include_redirects && ttl_itm.Type_redirect()) continue;
++count;
nxt_itm = ttl_itm;
if (count == total) {
loop = false;
break;
}
else {
rslt_list.Add(ttl_itm);
++rslt_list_len;
}
}
xdat_file = null;
++fil_idx;
}
rslt_count.Val_(rslt_list_len);
if (rslt_nxt != null)
rslt_nxt.Copy(nxt_itm);
}
private void Special_allpages_query_bwd(ListAdp rslt_list, Xodb_page rslt_prv, byte dir_tid, Xow_ns ns, boolean include_redirects, int total, int fil_idx, int row_idx, Xob_xdat_file xdat_file, Xob_xdat_itm xdat_itm) {
if (row_idx < 0) {
--fil_idx;
row_idx = -1;
}
int count = 0;
boolean loop = true;
Xodb_page prv_itm = null;
while (loop) {
if (fil_idx == -1) break;
if (xdat_file == null) {
xdat_file = new Xob_xdat_file();
this.Load_xdat_file(Cancelable_.Never, xdat_file, dir_tid, ns, fil_idx);
row_idx = -1;
}
if (row_idx == -1)
row_idx = xdat_file.Count() - 1;
for (; row_idx > -1; row_idx--) {
xdat_file.GetAt(xdat_itm, row_idx);
Xodb_page ttl_itm = Xodb_page_.Txt_ttl_load(Bry_.Mid(xdat_itm.Src(), xdat_itm.Itm_bgn(), xdat_itm.Itm_end()));
if (!include_redirects && ttl_itm.Type_redirect()) continue;
// list.Add(ttl_itm);
++count;
prv_itm = ttl_itm;
if (count == total) {
loop = false;
break;
}
else {
// rslt_list_ttls[rslt_list_len++] = ttl_itm;
}
}
xdat_file = null;
--fil_idx;
}
if (prv_itm == null && rslt_list.Count() > 0) {
prv_itm = (Xodb_page)rslt_list.FetchAt(0);
}
if (rslt_prv != null)
rslt_prv.Copy(prv_itm);
}
public byte[] Load_qid(byte[] wiki_alias, byte[] ns_num, byte[] ttl) {
String xwiki_key = String_.new_ascii_(wiki_alias);
if (qids_root == null)
qids_root = wiki.App().Wiki_mgr().Wdata_mgr().Wdata_wiki().Fsys_mgr().Site_dir().GenSubDir_nest("data", "qid");
Xob_xdat_itm qid_itm = Load_xdat_itm_by_dir(qids_root.GenSubDir_nest(xwiki_key, String_.new_ascii_(ns_num)), ttl); if (qid_itm == null) return null;
return Bry_.Mid(qid_itm.Src(), qid_itm.Itm_bgn() + ttl.length + 1, qid_itm.Itm_end()); // extract qid; note that all itms have format of "ttl|qid"
} Io_url qids_root;
public int Load_pid(byte[] lang_key, byte[] pid_name) {
if (pids_root == null)
pids_root = wiki.App().Wiki_mgr().Wdata_mgr().Wdata_wiki().Fsys_mgr().Site_dir().GenSubDir_nest("data", "pid");
Xob_xdat_itm pid_itm = Load_xdat_itm_by_dir(pids_root.GenSubDir(String_.new_utf8_(lang_key)), pid_name); if (pid_itm == null) return gplx.xowa.xtns.wdatas.Wdata_wiki_mgr.Pid_null;
return Bry_.X_to_int_or(pid_itm.Src(), pid_itm.Itm_bgn() + pid_name.length + 1 + 1, pid_itm.Itm_end(), gplx.xowa.xtns.wdatas.Wdata_wiki_mgr.Pid_null); // extract pid; note that all itms have format of "ttl|pid"; +1=skip pipe; +1 skip p
} Io_url pids_root;
public int Load_ctg_count(byte[] ttl) {return wiki.Db_mgr().Category_version() == Xoa_ctg_mgr.Version_1 ? Load_ctg_count_v1(ttl) : Load_ctg_count_v2(ttl);}
int Load_ctg_count_v1(byte[] ttl) {
Xoctg_view_ctg view_ctg = new Xoctg_view_ctg();
Int_obj_ref count = Int_obj_ref.zero_();
Load_ctg_v1_wkr(view_ctg, ttl, count);
return count.Val();
}
int Load_ctg_count_v2(byte[] ttl) {
Xoctg_data_ctg data_ctg = new Xoctg_data_ctg(ttl);
Load_ctg_v2(data_ctg, ttl);
return data_ctg.Total_count();
}
Xob_xdat_itm Load_xdat_itm_by_dir(Io_url dir, byte[] ttl) {
Xoa_hive_mgr hive_mgr = wiki.App().Hive_mgr();
int fil_idx = hive_mgr.Find_fil(dir, ttl); if (fil_idx == Xowd_regy_mgr.Regy_null) return null; // sub_dir not found; EX: commonswiki if qid; fr if pid;
Xob_xdat_file rdr = hive_mgr.Get_rdr(dir, Xow_dir_info_.Bry_xdat, fil_idx);
rdr.Find(tmp_xdat_itm, ttl, 0, Byte_ascii.Pipe, true);
return tmp_xdat_itm.Found_exact() ? tmp_xdat_itm : null;
}
public void Load_ctg_v2a(Xoctg_view_ctg rv, Xoctg_url url_ctg, byte[] ttl_bry, int limit) {
Xoctg_html_mgr ctg_mgr = wiki.Html_mgr().Ns_ctg();
Xoctg_data_cache data_cache = ctg_mgr.Data_cache();
Xoctg_data_ctg data_ctg = data_cache.Get_or_null(ttl_bry);
if (data_ctg == null) {
data_ctg = data_cache.Load_or_null(wiki, ttl_bry);
if (data_ctg == null) return;
}
rv.Fill(url_ctg, data_ctg);
ctg_mgr.Get_titles(wiki.App().Usr_dlg(), wiki, rv);
rv.Num_(data_ctg);
}
public Xodb_page[] Load_ctg_list(byte[][] ttls) {
int len = ttls.length;
Xodb_page[] rv = new Xodb_page[len];
Xow_ns ns = wiki.Ns_mgr().Ns_category();
Xoctg_data_ctg ctg_temp = new Xoctg_data_ctg(Bry_.Empty);
for (int i = 0; i < len; i++) {
byte[] ttl = Xoa_ttl.Replace_spaces(ttls[i]); // NOTE: ctg_ttls has spaces since v1 rendered it literally;
Xodb_page page = new Xodb_page();
this.Load_by_ttl(page, ns, ttl);
Load_ctg_v2_main(ctg_temp, page.Ttl_wo_ns());
Xodb_category_itm ctg_itm = Xodb_category_itm.load_(page.Id(), page.Db_file_idx(), ctg_temp.Hidden(), ctg_temp.Total_by_tid(Xoa_ctg_mgr.Tid_subc), ctg_temp.Total_by_tid(Xoa_ctg_mgr.Tid_file), ctg_temp.Total_by_tid(Xoa_ctg_mgr.Tid_page));
page.Xtn_(ctg_itm);
rv[i] = page;
}
return rv;
}
static final String GRP_KEY = "xowa.wiki.db.load";
}
class Xob_random_itm {
public int Idx() {return idx;} private int idx;
public int Bgn() {return bgn;} private int bgn;
public int End() {return bgn + len;}
public int Len() {return len;} private int len;
public Xob_random_itm(int idx, int bgn, int len) {this.idx = idx; this.bgn = bgn; this.len = len;}
}
class Xob_random_itm_comparer implements gplx.lists.ComparerAble {
public int compare(Object lhsObj, Object rhsObj) {
return Int_.Compare(((Xob_random_itm)lhsObj).End(), ((Xob_random_itm)rhsObj).End());
}
public static final Xob_random_itm_comparer _ = new Xob_random_itm_comparer(); Xob_random_itm_comparer() {}
}

29
400_xowa/src/gplx/xowa/dbs/Xodb_mgr.java Normal file
View File

@@ -0,0 +1,29 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.dbs.*;
public interface Xodb_mgr extends GfoInvkAble, RlsAble {
byte Tid();
String Tid_name();
byte Data_storage_format(); void Data_storage_format_(byte v);
byte Category_version();
byte Search_version(); void Search_version_refresh();
DateAdp Dump_date_query();
Xodb_load_mgr Load_mgr();
Xodb_save_mgr Save_mgr();
}

190
400_xowa/src/gplx/xowa/dbs/Xodb_mgr_sql.java Normal file
View File

@@ -0,0 +1,190 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.dbs.*;
import gplx.xowa.apps.*; import gplx.xowa.dbs.tbls.*; import gplx.xowa.ctgs.*;
public class Xodb_mgr_sql implements Xodb_mgr, GfoInvkAble {
public Xodb_mgr_sql(Xow_wiki wiki) {
this.wiki = wiki;
load_mgr = new Xodb_load_mgr_sql(this);
save_mgr = new Xodb_save_mgr_sql(this);
tbl_text = new Xodb_text_tbl(this);
fsys_mgr.Ctor(wiki.App().Fsys_mgr().Bin_db_dir(), wiki.Fsys_mgr().Root_dir(), wiki.Domain_str());
}
public byte Tid() {return Tid_sql;} public static final byte Tid_sql = 1;
public String Tid_name() {return "sqlite3";}
public Xow_wiki Wiki() {return wiki;} private Xow_wiki wiki;
public byte Data_storage_format() {return data_storage_format;} public void Data_storage_format_(byte v) {data_storage_format = v;} private byte data_storage_format = gplx.ios.Io_stream_.Tid_gzip;
public byte Category_version() {return category_version;} private byte category_version = Xoa_ctg_mgr.Version_null;
public byte Search_version() {return load_mgr.Search_version();}
public void Search_version_refresh() {load_mgr.Search_version_refresh();}
public Xodb_fsys_mgr Fsys_mgr() {return fsys_mgr;} private Xodb_fsys_mgr fsys_mgr = new Xodb_fsys_mgr();
public Xodb_load_mgr Load_mgr() {return load_mgr;} private Xodb_load_mgr_sql load_mgr;
public Xodb_save_mgr Save_mgr() {return save_mgr;} private Xodb_save_mgr_sql save_mgr;
public Xodb_xowa_cfg_tbl Tbl_xowa_cfg() {return tbl_cfg;} private Xodb_xowa_cfg_tbl tbl_cfg = new Xodb_xowa_cfg_tbl();
public Xodb_xowa_ns_tbl Tbl_xowa_ns() {return tbl_ns;} private Xodb_xowa_ns_tbl tbl_ns = new Xodb_xowa_ns_tbl();
public Xodb_xowa_db_tbl Tbl_xowa_db() {return tbl_db;} private Xodb_xowa_db_tbl tbl_db = new Xodb_xowa_db_tbl();
public Xodb_page_tbl Tbl_page() {return tbl_page;} private Xodb_page_tbl tbl_page = new Xodb_page_tbl();
public Xodb_text_tbl Tbl_text() {return tbl_text;} private Xodb_text_tbl tbl_text;
public Xodb_site_stats_tbl Tbl_site_stats() {return tbl_site_stats;} private Xodb_site_stats_tbl tbl_site_stats = new Xodb_site_stats_tbl();
public Xodb_wdata_qids_tbl Tbl_wdata_qids() {return tbl_wdata_qids;} private Xodb_wdata_qids_tbl tbl_wdata_qids = new Xodb_wdata_qids_tbl();
public Xodb_wdata_pids_tbl Tbl_wdata_pids() {return tbl_wdata_pids;} private Xodb_wdata_pids_tbl tbl_wdata_pids = new Xodb_wdata_pids_tbl();
public Xodb_category_tbl Tbl_category() {return tbl_category;} private Xodb_category_tbl tbl_category = new Xodb_category_tbl();
public Xodb_categorylinks_tbl Tbl_categorylinks() {return tbl_categorylinks;} private Xodb_categorylinks_tbl tbl_categorylinks = new Xodb_categorylinks_tbl();
public Xodb_search_title_word_tbl Tbl_search_title_word() {return tbl_search_title_word;} private Xodb_search_title_word_tbl tbl_search_title_word = new Xodb_search_title_word_tbl();
public Xodb_search_title_page_tbl Tbl_search_title_page() {return tbl_search_title_page;} private Xodb_search_title_page_tbl tbl_search_title_page = new Xodb_search_title_page_tbl();
public byte State() {return state;} private byte state = State_init; public static final byte State_init = 0, State_make = 1, State_load = 2;
public DateAdp Dump_date_query() {
DateAdp rv = wiki.Props().Modified_latest();
if (rv != null) return rv;
Io_url url = fsys_mgr.Get_url(Xodb_file.Tid_core);
return Io_mgr._.QueryFil(url).ModifiedTime();
}
public void Init_make(String ns_map) {
Xoi_dump_mgr dump_mgr = wiki.App().Setup_mgr().Dump_mgr();
data_storage_format = dump_mgr.Data_storage_format();
fsys_mgr.Init_make(wiki.Ns_mgr(), ns_map, dump_mgr.Db_text_max());
Core_provider_(fsys_mgr.Core_provider());
state = State_make;
}
public void Init_load(Db_connect connect) {
Db_provider provider = Db_provider_pool._.FetchOrNew(connect);
Xodb_file[] files = tbl_db.Select_all(provider);
fsys_mgr.Init_by_files(provider, files);
Core_provider_(provider);
state = State_load;
}
private void Core_provider_(Db_provider provider) {
tbl_cfg.Provider_(provider);
tbl_ns.Provider_(provider);
tbl_db.Provider_(provider);
tbl_page.Provider_(provider);
tbl_site_stats.Provider_(provider);
}
public void Page_create(Db_stmt page_stmt, Db_stmt text_stmt, int page_id, int ns_id, byte[] ttl_wo_ns, boolean redirect, DateAdp modified_on, byte[] text, int random_int, int file_idx) {
tbl_page.Insert(page_stmt, page_id, ns_id, ttl_wo_ns, redirect, modified_on, text.length, random_int, file_idx);
tbl_text.Insert(text_stmt, page_id, text, data_storage_format);
}
public boolean Ctg_select_v1(Xoctg_view_ctg view_ctg, Db_provider ctg_provider, Xodb_category_itm ctg) {
Db_qry_select qry = Db_qry_.select_().Cols_(Xodb_categorylinks_tbl.Fld_cl_from)
.From_(Xodb_categorylinks_tbl.Tbl_name)
.Where_(Db_crt_.eq_(Xodb_categorylinks_tbl.Fld_cl_to_id, ctg.Id()))
;
ListAdp rslts = ListAdp_.new_();
DataRdr rdr = DataRdr_.Null;
try {
rdr = ctg_provider.Exec_qry_as_rdr(qry);
while (rdr.MoveNextPeer()) {
int page_id = rdr.ReadInt(Xodb_categorylinks_tbl.Fld_cl_from);
Xodb_page itm = new Xodb_page().Id_(page_id);
rslts.Add(itm);
}
} finally {rdr.Rls();}
int rslts_len = rslts.Count();
rslts.SortBy(Xodb_page_sorter.IdAsc);
tbl_page.Select_by_id_list(Cancelable_.Never, false, rslts, 0, rslts_len);
rslts.SortBy(Xodb_page_sorter.Ns_id_TtlAsc);
boolean rv = false;
for (int i = 0; i < rslts.Count(); i++) {
Xodb_page page = (Xodb_page)rslts.FetchAt(i);
if (page.Ns_id() == Int_.MinValue) continue; // HACK: page not found; ignore
byte ctg_tid = Xodb_load_mgr_txt.Load_ctg_v1_tid(page.Ns_id());
Xoctg_view_grp ctg_grp = view_ctg.Grp_by_tid(ctg_tid);
Xoctg_view_itm ctg_itm = new Xoctg_view_itm();
ctg_itm.Load_by_ttl_data(ctg_tid, page.Id(), 0, page.Text_len());
ctg_itm.Ttl_(Xoa_ttl.parse_(wiki, page.Ns_id(), page.Ttl_wo_ns()));
ctg_itm.Sortkey_(page.Ttl_wo_ns());
ctg_grp.Itms_add(ctg_itm);
rv = true;
}
for (byte i = 0; i < Xoa_ctg_mgr.Tid__max; i++) {
Xoctg_view_grp ctg_grp = view_ctg.Grp_by_tid(i);
ctg_grp.Itms_make();
ctg_grp.Total_(ctg_grp.Itms().length);
}
return rv;
}
public void Rls() {
fsys_mgr.Rls();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_data_storage_format)) return Xoi_dump_mgr.Wtr_tid_to_str(data_storage_format);
else if (ctx.Match(k, Invk_data_storage_format_)) data_storage_format = Xoi_dump_mgr.Wtr_tid_parse(m.ReadStr("v"));
else if (ctx.Match(k, Invk_category_version)) return category_version;
else if (ctx.Match(k, Invk_category_version_)) category_version = m.ReadByte("v");
else if (ctx.Match(k, Invk_search_version)) return this.Search_version();
else if (ctx.Match(k, Invk_tid_name)) return this.Tid_name();
return this;
}
public static final String
Invk_data_storage_format = "data_storage_format", Invk_data_storage_format_ = "data_storage_format_"
, Invk_category_version = "category_version", Invk_category_version_ = "category_version_"
, Invk_search_version = "search_version"
, Invk_tid_name = "tid_name"
;
public void Category_version_update(boolean version_is_1) {
String grp = Xodb_mgr_sql.Grp_wiki_init;
String key = Xoa_gfs_mgr.Build_code(Xow_wiki.Invk_db_mgr, Xodb_mgr_sql.Invk_category_version);
// if (category_version != Xoa_ctg_mgr.Version_null)
tbl_cfg.Delete(grp, key);// always delete ctg version
category_version = version_is_1 ? Xoa_ctg_mgr.Version_1 : Xoa_ctg_mgr.Version_2;
tbl_cfg.Insert_str(grp, key, Byte_.XtoStr(category_version));
}
public void Delete_by_tid(byte tid) {
Xodb_file[] ary = fsys_mgr.Ary();
int len = ary.length;
for (int i = 0; i < len; i++) {
Xodb_file file = ary[i] ;
if (file.Tid() != tid) continue;
file.Rls();
gplx.dbs.Db_connect_sqlite sqlite = (gplx.dbs.Db_connect_sqlite)file.Connect();
Io_mgr._.DeleteFil_args(sqlite.Url()).MissingFails_off().Exec();
file.Cmd_mode_(Db_cmd_mode.Delete);
}
tbl_db.Commit_all(fsys_mgr.Core_provider(), ary);
this.Init_load(fsys_mgr.Core_provider().ConnectInfo());
}
public static final String Grp_wiki_init = "wiki.init";
public static final int Page_id_null = -1;
public static Io_url Find_core_url(Xow_wiki wiki) {
Io_url[] ary = Io_mgr._.QueryDir_args(wiki.Fsys_mgr().Root_dir()).FilPath_("*.sqlite3").ExecAsUrlAry();
int ary_len = ary.length; if (ary_len == 0) return null;
if (ary_len == 1) return ary[0]; // only 1 file; assume it is core
String v0_str = wiki.Domain_str() + ".000";
String v1_str = wiki.Domain_str() + "core.000";
for (int i = 0; i < ary_len; i++) {
Io_url itm = ary[i];
if (String_.Eq(itm.NameOnly(), v0_str)) return itm; // EX: "en.wikipedia.org.000"
else if (String_.Eq(itm.NameOnly(), v1_str)) return itm; // EX: "en.wikipedia.org.core.000"
}
return null;
}
public static Xodb_mgr_sql Get_or_load(Xow_wiki wiki) {
Xodb_mgr db_mgr = wiki.Db_mgr();
Xodb_mgr_sql rv = db_mgr.Tid() == Xodb_mgr_txt.Tid_txt ? wiki.Db_mgr_create_as_sql() : wiki.Db_mgr_as_sql();
byte state = rv.State();
switch (state) {
case Xodb_mgr_sql.State_init: rv.Init_load(Db_connect_.sqlite_(Xodb_mgr_sql.Find_core_url(wiki))); break; // load
case Xodb_mgr_sql.State_make: break; // noop; being made; don't load from db;
case Xodb_mgr_sql.State_load: break; // noop; already loaded;
default: throw Err_.unhandled(state);
}
return rv;
}
}

62
400_xowa/src/gplx/xowa/dbs/Xodb_mgr_txt.java Normal file
View File

@@ -0,0 +1,62 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.dbs.*; import gplx.xowa.ctgs.*;
public class Xodb_mgr_txt implements Xodb_mgr {
public Xodb_mgr_txt(Xow_wiki wiki, Xow_data_mgr data_mgr) {
this.wiki = wiki;
load_mgr = new Xodb_load_mgr_txt(wiki);
save_mgr = new Xodb_save_mgr_txt(wiki, load_mgr);
} private Xow_wiki wiki;
public byte Tid() {return Tid_txt;} public static final byte Tid_txt = 0;
public String Tid_name() {return "xdat";}
public byte Data_storage_format() {return data_storage_format;} public void Data_storage_format_(byte v) {data_storage_format = v;} private byte data_storage_format = gplx.ios.Io_stream_.Tid_file;
public Xodb_load_mgr Load_mgr() {return load_mgr;} private Xodb_load_mgr_txt load_mgr;
public Xodb_save_mgr Save_mgr() {return save_mgr;} private Xodb_save_mgr_txt save_mgr;
public DateAdp Dump_date_query() {
Io_url url = wiki.Fsys_mgr().Url_ns_fil(Xow_dir_info_.Tid_page, Xow_ns_.Id_main, 0);
return Io_mgr._.QueryFil(url).ModifiedTime();
}
public byte Category_version() {
if (category_version == Xoa_ctg_mgr.Version_null) {
if (Io_mgr._.ExistsDir(wiki.Fsys_mgr().Url_site_dir(Xow_dir_info_.Tid_category2_link)))
category_version = Xoa_ctg_mgr.Version_2;
else
category_version = Xoa_ctg_mgr.Version_1;
}
return category_version;
} byte category_version = Xoa_ctg_mgr.Version_null;
public byte Search_version() {return gplx.xowa.specials.search.Xosrh_core.Version_2;}
public void Search_version_refresh() {throw Err_.not_implemented_();}
public void Rls() {}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_data_storage_format)) return Xoi_dump_mgr.Wtr_tid_to_str(data_storage_format);
else if (ctx.Match(k, Invk_data_storage_format_)) data_storage_format = Xoi_dump_mgr.Wtr_tid_parse(m.ReadStr("v"));
else if (ctx.Match(k, Invk_category_version)) return this.Category_version();
else if (ctx.Match(k, Invk_category_version_)) category_version = m.ReadByte("v");
else if (ctx.Match(k, Invk_search_version)) return this.Search_version();
else if (ctx.Match(k, Invk_tid_name)) return Tid_name();
return this;
}
public static final String
Invk_data_storage_format = "data_storage_format", Invk_data_storage_format_ = "data_storage_format_"
, Invk_category_version = "category_version", Invk_category_version_ = "category_version_"
, Invk_search_version = "search_version"
, Invk_tid_name = "tid_name"
;
}

51
400_xowa/src/gplx/xowa/dbs/Xodb_ns_map_mgr.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
class Xodb_ns_map_mgr {
public Xodb_ns_map_itm[] Itms() {return itms;} private Xodb_ns_map_itm[] itms;
public static Xodb_ns_map_mgr Parse(byte[] src) {
byte[][] lines = Bry_.Split(src, Byte_ascii.NewLine);
int lines_len = lines.length;
Xow_ns_mgr canonical_ns_mgr = Xow_ns_mgr_.default_();
Xodb_ns_map_mgr rv = new Xodb_ns_map_mgr();
ListAdp itms = ListAdp_.new_();
for (int i = 0; i < lines_len; i++) {
byte[] line = lines[i];
if (line.length == 0) continue; // ignore blank lines
Xodb_ns_map_itm itm = Parse_itm(src, line, canonical_ns_mgr);
itms.Add(itm);
}
rv.itms = (Xodb_ns_map_itm[])itms.XtoAryAndClear(Xodb_ns_map_itm.class);
return rv;
}
private static Xodb_ns_map_itm Parse_itm(byte[] src, byte[] line, Xow_ns_mgr ns_mgr) {
byte[][] ns_names = Bry_.Split(line, Byte_ascii.Tilde);
int len = ns_names.length;
int[] ns_ids = new int[len];
for (int i = 0; i < len; i++) {
byte[] ns_name = ns_names[i];
Xow_ns ns = ns_mgr.Names_get_or_null(ns_name, 0, ns_name.length);
ns_ids[i] = ns.Id();
}
return new Xodb_ns_map_itm(ns_ids);
}
}
class Xodb_ns_map_itm {
public Xodb_ns_map_itm(int[] ns_ids) {this.ns_ids = ns_ids;}
public int[] Ns_ids() {return ns_ids;} private int[] ns_ids;
}

57
400_xowa/src/gplx/xowa/dbs/Xodb_ns_map_mgr_tst.java Normal file
View File

@@ -0,0 +1,57 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import org.junit.*; import gplx.dbs.*;
public class Xodb_ns_map_mgr_tst {
@Before public void init() {fxt.Clear();} private Xodb_ns_map_mgr_fxt fxt = new Xodb_ns_map_mgr_fxt();
@Test public void Basic() {
fxt.Test_parse("");
}
@Test public void Ns() {
fxt.Test_parse(String_.Concat_lines_nl
( ""
, "Template"
, ""
, "File~Category"
, ""
)
, fxt.itm_(Xow_ns_.Id_template), fxt.itm_(Xow_ns_.Id_file, Xow_ns_.Id_category));
}
}
class Xodb_ns_map_mgr_fxt {
public void Clear() {} Bry_bfr bfr = Bry_bfr.new_();
public Xodb_ns_map_itm itm_(int... ary) {return new Xodb_ns_map_itm(ary);}
public void Test_parse(String src_str, Xodb_ns_map_itm... expd) {
byte[] src_bry = Bry_.new_ascii_(src_str);
Xodb_ns_map_mgr actl_mgr = Xodb_ns_map_mgr.Parse(src_bry);
Tfds.Eq_str_lines(Xto_str(expd), Xto_str(actl_mgr.Itms()));
}
String Xto_str(Xodb_ns_map_itm[] ary) {
int len = ary.length;
for (int i = 0; i < len; i++) {
Xodb_ns_map_itm itm = ary[i];
int[] ns_ids = itm.Ns_ids();
int ns_ids_len = ns_ids.length;
for (int j = 0; j < ns_ids_len; j++) {
bfr.Add_int_variable(ns_ids[j]).Add_byte_pipe();
}
bfr.Add_byte_nl();
}
return bfr.XtoStrAndClear();
}
}

27
400_xowa/src/gplx/xowa/dbs/Xodb_save_mgr.java Normal file
View File

@@ -0,0 +1,27 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
public interface Xodb_save_mgr {
boolean Create_enabled(); void Create_enabled_(boolean v);
boolean Update_modified_on_enabled(); void Update_modified_on_enabled_(boolean v);
int Page_id_next(); void Page_id_next_(int v);
void Data_create(Xoa_ttl ttl, byte[] text);
void Data_update(Xoa_page page, byte[] text);
void Data_rename(Xoa_page page, int trg_ns, byte[] trg_ttl);
void Clear();
}

102
400_xowa/src/gplx/xowa/dbs/Xodb_save_mgr_sql.java Normal file
View File

@@ -0,0 +1,102 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.dbs.*; import gplx.ios.*;
public class Xodb_save_mgr_sql implements Xodb_save_mgr {
public Xodb_save_mgr_sql(Xodb_mgr_sql db_mgr) {
this.db_mgr = db_mgr; zip_mgr = db_mgr.Wiki().App().Zip_mgr();
} Xodb_mgr_sql db_mgr; Io_stream_zip_mgr zip_mgr;
public boolean Create_enabled() {return create_enabled;} public void Create_enabled_(boolean v) {create_enabled = v;} private boolean create_enabled;
public boolean Update_modified_on_enabled() {return update_modified_on_enabled;} public void Update_modified_on_enabled_(boolean v) {update_modified_on_enabled = v;} private boolean update_modified_on_enabled;
public int Page_id_next() {return page_id_next;} public void Page_id_next_(int v) {page_id_next = v;} private int page_id_next;
public void Data_create(Xoa_ttl ttl, byte[] text) {
int ns_id = ttl.Ns().Id();
int ns_count = db_mgr.Tbl_xowa_ns().Select_ns_count(ns_id) + 1;
String page_id = db_mgr.Tbl_xowa_cfg().Select_val_or("db", "page.id_nxt", null);
int page_id_int = -1;
if (page_id == null) {
DataRdr rdr = db_mgr.Tbl_page().Provider().Exec_sql_as_rdr("SELECT (Max(page_id) + 1) AS max_page_id FROM page;");
if (rdr.MoveNextPeer()) {
page_id = Int_.XtoStr(rdr.ReadInt("max_page_id"));
page_id_int = Int_.parse_(page_id);
db_mgr.Tbl_xowa_cfg().Insert_int("db", "page.id_next", page_id_int);
}
rdr.Rls();
}
else
page_id_int = Int_.parse_(page_id);
Xodb_fsys_mgr fsys_mgr = db_mgr.Fsys_mgr();
int file_idx = fsys_mgr.Tid_text_idx();
boolean redirect = db_mgr.Wiki().Redirect_mgr().Is_redirect(text, text.length);
Db_stmt page_stmt = db_mgr.Tbl_page().Insert_stmt(fsys_mgr.Page_provider());
Db_provider text_provider = db_mgr.Fsys_mgr().Get_by_db_idx(file_idx).Provider();
Db_stmt text_stmt = db_mgr.Tbl_text().Insert_stmt(text_provider);
text = zip_mgr.Zip(db_mgr.Data_storage_format(), text);
try {
db_mgr.Page_create(page_stmt, text_stmt, page_id_int, ns_id, text, redirect, DateAdp_.Now(), text, ns_count, file_idx);
db_mgr.Tbl_xowa_ns().Update_ns_count(ns_id, ns_count);
db_mgr.Tbl_xowa_cfg().Update("db", "page.id_next", page_id + 1);
} finally {
page_stmt.Rls();
text_stmt.Rls();
}
}
public void Data_update(Xoa_page page, byte[] text) {
boolean redirect = db_mgr.Wiki().Redirect_mgr().Is_redirect(text, text.length);
DateAdp modified = update_modified_on_enabled ? DateAdp_.Now() : page.Revision_data().Modified_on();
boolean redirect_changed = redirect != db_mgr.Wiki().Redirect_mgr().Is_redirect(page.Data_raw(), page.Data_raw().length);
boolean modified_changed = !modified.Eq(page.Revision_data().Modified_on());
int kv_len = 0;
if (redirect_changed) ++kv_len;
if (modified_changed) ++kv_len;
Db_qry qry = null;
if (kv_len > 0) {
KeyVal[] kv_ary = new KeyVal[kv_len];
int kv_idx = 0;
if (redirect_changed) kv_ary[kv_idx++] = KeyVal_.new_("page_is_redirect", redirect_changed);
if (modified_changed) kv_ary[kv_idx++] = KeyVal_.new_("page_touched", Xto_touched_str(modified));
qry = Db_qry_.update_common_("page", Db_crt_.eq_("page_id", page.Revision_data().Id()), kv_ary);
Db_provider provider = db_mgr.Fsys_mgr().Core_provider();
provider.Txn_mgr().Txn_bgn_if_none();
provider.Exec_qry(qry);
provider.Txn_mgr().Txn_end_all();
}
Xodb_page db_page = new Xodb_page();
db_mgr.Load_mgr().Load_by_id(db_page, page.Revision_data().Id());
text = zip_mgr.Zip(db_mgr.Data_storage_format(), text);
db_mgr.Tbl_text().Update(db_page.Db_file_idx(), page.Revision_data().Id(), text);
}
public void Data_rename(Xoa_page page, int trg_ns, byte[] trg_ttl) {
Db_qry qry = Db_qry_.update_common_("page", Db_crt_.eq_("page_id", page.Revision_data().Id())
, KeyVal_.new_("page_namespace", trg_ns)
, KeyVal_.new_("page_title", String_.new_utf8_(trg_ttl))
);
try {
db_mgr.Fsys_mgr().Core_provider().Exec_qry(qry);
} catch (Exception exc) {
if (String_.Has(Err_.Message_gplx_brief(exc), "columns page_namespace, page_random_int are not unique")) { // HACK: terrible hack, but moving pages across ns will break UNIQUE index
db_mgr.Fsys_mgr().Core_provider().Exec_sql("DROP INDEX page__name_random;"); // is UNIQUE by default
db_mgr.Fsys_mgr().Core_provider().Exec_sql("CREATE INDEX page__name_random ON page (page_namespace, page_random_int);");
db_mgr.Fsys_mgr().Core_provider().Exec_qry(qry);
}
}
}
public void Clear() {}
private static String Xto_touched_str(DateAdp v) {return v.XtoStr_fmt("yyyyMMddHHmmss");}
}

111
400_xowa/src/gplx/xowa/dbs/Xodb_save_mgr_txt.java Normal file
View File

@@ -0,0 +1,111 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.xowa.wikis.*;
public class Xodb_save_mgr_txt implements Xodb_save_mgr {
public Xodb_save_mgr_txt(Xow_wiki wiki, Xodb_load_mgr_txt load_mgr) {
this.wiki = wiki;
this.load_mgr = load_mgr;
this.fsys_mgr = wiki.Fsys_mgr();
this.redirect_mgr = wiki.Redirect_mgr();
} private Xow_wiki wiki; private Xow_fsys_mgr fsys_mgr; private Xodb_load_mgr_txt load_mgr; private Xop_redirect_mgr redirect_mgr;
public boolean Create_enabled() {return create_enabled;} public void Create_enabled_(boolean v) {create_enabled = v;} private boolean create_enabled;
public boolean Update_modified_on_enabled() {return update_modified_on_enabled;} public void Update_modified_on_enabled_(boolean v) {update_modified_on_enabled = v;} private boolean update_modified_on_enabled;
public int Page_id_next() {return page_id_next;} public void Page_id_next_(int v) {page_id_next = v;} private int page_id_next = 0;
public void Clear() {page_id_next = 0;} // TEST: needed for ctg_test
public void Data_create(Xoa_ttl ttl, byte[] text) {
Xow_ns ns_itm = ttl.Ns(); byte[] ttl_bry = ttl.Page_db();
Xodb_page db_page = Xodb_page.tmp_();
boolean found = load_mgr.Load_by_ttl(db_page, ns_itm, ttl_bry);
if (found) throw Err_mgr._.fmt_(GRP_KEY, "title_exists", "create requested but title already exists: ~{0}", String_.new_utf8_(ttl_bry));
int text_len = text.length;
Bry_bfr tmp = wiki.Utl_bry_bfr_mkr().Get_m001();
int page_id = page_id_next++;
int fil_idx = 0;
int ns_id = ttl.Ns().Id();
Xodb_page_.Txt_page_save(tmp, page_id, DateAdp_.Now(), ttl_bry, text, true);
Io_url page_rdr_url = fsys_mgr.Url_ns_fil(Xow_dir_info_.Tid_page, ns_id, fil_idx);
byte[] page_rdr_bry = gplx.ios.Io_stream_rdr_.Load_all(page_rdr_url);
Xob_xdat_file page_rdr = new Xob_xdat_file();
if (Bry_.Len_gt_0(page_rdr_bry)) page_rdr.Parse(page_rdr_bry, page_rdr_bry.length, page_rdr_url);
int row_idx = page_rdr.Count();
Bry_bfr tmp_bfr = wiki.Utl_bry_bfr_mkr().Get_b512();
page_rdr.Insert(tmp_bfr, tmp.XtoAryAndClear());
this.Data_save(Xow_dir_info_.Tid_page, page_rdr, page_rdr_url, tmp_bfr);
tmp_bfr.Mkr_rls();
Xoa_ttl redirect_ttl = redirect_mgr.Extract_redirect(text, text_len);
db_page.Set_all_(page_id, fil_idx, row_idx, redirect_ttl != null, text_len, ttl.Page_db());
Xodb_page_.Txt_ttl_save(tmp, db_page);
byte[] ttl_row_bry = tmp.Mkr_rls().XtoAryAndClear();
Xowd_hive_mgr ttl_hive = new Xowd_hive_mgr(wiki, Xow_dir_info_.Tid_ttl);
ttl_hive.Create(ttl.Ns(), ttl.Page_db(), ttl_row_bry, Bry_comparer_fld_last._);
wiki.Db_mgr().Load_mgr().Clear(); // NOTE: need to clear cached regy_ary in load_mgr
}
public void Data_update(Xoa_page page, byte[] text) {Data_update_under(page, text, null);}
public void Data_rename(Xoa_page page, int trg_ns, byte[] trg_ttl) {
if (wiki.Domain_tid() != Xow_wiki_domain_.Tid_home) {
wiki.App().Gui_wtr().Warn_many("", "", "Only pages in the home wiki can be renamed");
return;
}
Data_update_under(page, null, trg_ttl);
}
private void Data_update_under(Xoa_page page, byte[] text, byte[] new_ttl) {
Xoa_ttl ttl = page.Ttl();
Xow_ns ns = ttl.Ns(); byte[] ttl_bry = ttl.Page_db();
Xodb_page db_page = Xodb_page.tmp_();
if (!load_mgr.Load_by_ttl(db_page, ns, ttl_bry)) throw Err_mgr._.fmt_(GRP_KEY, "title_missing", "update requested but title does not exist: ~{0}", String_.new_utf8_(ttl_bry));
byte[] old_ttl = ttl_bry;
if (new_ttl != null) {
ttl_bry = new_ttl;
db_page.Ttl_wo_ns_(new_ttl);
}
// update page
Xob_xdat_file page_rdr = new Xob_xdat_file(); Xob_xdat_itm page_itm = new Xob_xdat_itm();
load_mgr.Load_page(tmp_page, db_page.Db_file_idx(), db_page.Db_row_idx(), ns, true, page_rdr, page_itm);
Bry_bfr tmp_bfr = wiki.Utl_bry_bfr_mkr().Get_b512();
if (text == null) text = tmp_page.Text();
int text_len = text.length;
DateAdp modified_on = tmp_page.Modified_on();
if (update_modified_on_enabled) {
modified_on = DateAdp_.Now();
page.Revision_data().Modified_on_(modified_on);
}
Xodb_page_.Txt_page_save(tmp_bfr, db_page.Id(), modified_on, ttl_bry, text, true);
page_rdr.Update(tmp_bfr, page_itm, tmp_bfr.XtoAryAndClear());
Io_url page_rdr_url = fsys_mgr.Url_ns_fil(Xow_dir_info_.Tid_page, ttl.Ns().Id(), db_page.Db_file_idx());
this.Data_save(Xow_dir_info_.Tid_page, page_rdr, page_rdr_url, tmp_bfr);
tmp_bfr.Mkr_rls();
// update ttl
Xoa_ttl redirect_ttl = redirect_mgr.Extract_redirect(text, text_len);
db_page.Text_len_(text_len);
db_page.Type_redirect_(redirect_ttl != null);
Bry_bfr tmp = wiki.Utl_bry_bfr_mkr().Get_b512();
Xodb_page_.Txt_ttl_save(tmp, db_page);
byte[] ttl_row_bry = tmp.XtoAryAndClear();
tmp.Mkr_rls();
Xowd_hive_mgr ttl_hive = new Xowd_hive_mgr(wiki, Xow_dir_info_.Tid_ttl);
ttl_hive.Update(ns, old_ttl, new_ttl, ttl_row_bry, Xodb_page_.Txt_ttl_pos, Byte_ascii.Pipe, true, true);
}
private void Data_save(byte dir_tid, Xob_xdat_file xdat_file, Io_url url, Bry_bfr tmp_bfr) {
xdat_file.Save(url);
}
private Xodb_page tmp_page = new Xodb_page();
public static final int File_idx_unknown = -1;
private static final String GRP_KEY = "xowa.wiki.db.save";
}

68
400_xowa/src/gplx/xowa/dbs/Xodb_upgrade_mgr.java Normal file
View File

@@ -0,0 +1,68 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs; import gplx.*; import gplx.xowa.*;
import gplx.dbs.*; import gplx.xowa.dbs.tbls.*;
class Xodb_upgrade_mgr {
public static void Upgrade(Xodb_mgr_sql db_mgr, KeyVal[] kv_ary, String version_key, String version_val) {
// String version_new = null;
// if (String_.Eq(version_val, "0.6.2.0")) {
// Xodb_upgrade_mgr_v0_6_2_0.Upgrade(db_mgr, kv_ary);
// version_new = "0.6.2.1";
// }
// if (version_new != null) {
// db_mgr.Tbl_xowa_cfg().Update(Xodb_mgr_sql.Grp_wiki_init, version_key, version_new);
// }
}
}
// class Xodb_upgrade_mgr_v0_6_2_0 {
// public static void Upgrade(Xodb_mgr_sql db_mgr, KeyVal[] kv_ary) {
// Db_provider p = db_mgr.Fsys_mgr().Core_provider();
// Fix_storage_format(p, db_mgr, kv_ary);
// Fix_category_version(p, db_mgr);
// }
// private static void Fix_storage_format(Db_provider p, Xodb_mgr_sql db_mgr, KeyVal[] kv_ary) { // storage_format saved incorrectly as int
// int len = kv_ary.length;
// String gfs_data_storage_format = Xoa_gfs_mgr.Build_code(Xow_wiki.Invk_db_mgr, Xodb_mgr_sql.Invk_data_storage_format);
// for (int i = 0; i < len; i++) {
// KeyVal kv = kv_ary[i];
// String kv_key = kv.Key();
// if (String_.Eq(kv_key, gfs_data_storage_format)) {
// byte data_storage_format_byte = Byte_.parse_(kv.Val_to_str_or_empty());
// String data_storage_format_name = Xoi_dump_mgr.Wtr_tid_to_str(data_storage_format_byte);
// kv.Val_(data_storage_format_name); // update memory
// db_mgr.Tbl_xowa_cfg().Update(Xodb_mgr_sql.Grp_wiki_init, gfs_data_storage_format, data_storage_format_name); // update_database
// break;
// }
// }
// }
// private static void Fix_category_version(Db_provider p, Xodb_mgr_sql db_mgr) {
// Db_qry qry = Db_qry_.select_().From_(Xodb_categorylinks_tbl.Tbl_name).Cols_(Xodb_categorylinks_tbl.Fld_cl_type_id).Where_(Db_crt_.eq_(Xodb_categorylinks_tbl.Fld_cl_type_id, ));
// Db_stmt stmt = Db_stmt_.Null;
// DataRdr rdr = DataRdr_.Null;
// int types = 0;
// try {
// stmt = db_mgr.Fsys_mgr().Category_provider().Prepare(qry);
// rdr = stmt.Exec_select();
// while (rdr.MoveNextPeer()) {
// ++types;
// }
// } finally {rdr.Rls(); stmt.Rls();}
// boolean version_is_1 = types <= 1; // if 0 or 1 types assume version_1 (1=page only; 0=not set up)
// db_mgr.Category_version_update(true); // assume version_1; will be wrong if user actually did version_2, but currently version_1 vs version_2 has no
// }
// }

44
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_category_itm.java Normal file
View File

@@ -0,0 +1,44 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.xowa.ctgs.*;
public class Xodb_category_itm {
public int Id() {return id;} private int id;
public Int_obj_val Id_val() {if (id_val == null) id_val = Int_obj_val.new_(id); return id_val;} Int_obj_val id_val;
public int File_idx() {return file_idx;} private int file_idx;
public boolean Hidden() {return hidden;} private boolean hidden;
public int Count_all() {return count_subcs + count_files + count_pages;}
public int Count_subcs() {return count_subcs;} private int count_subcs;
public int Count_files() {return count_files;} private int count_files;
public int Count_pages() {return count_pages;} private int count_pages;
public int Count_by_tid(byte tid) {
switch (tid) {
case Xoa_ctg_mgr.Tid_subc: return count_subcs;
case Xoa_ctg_mgr.Tid_page: return count_pages;
case Xoa_ctg_mgr.Tid_file: return count_files;
default: throw Err_.unhandled(tid);
}
}
public static Xodb_category_itm load_(int id, int file_idx, boolean hidden, int count_subcs, int count_files, int count_pages) {
Xodb_category_itm rv = new Xodb_category_itm();
rv.id = id; rv.file_idx = file_idx; rv.hidden = hidden;
rv.count_subcs = count_subcs; rv.count_files = count_files; rv.count_pages = count_pages;
return rv;
}
public static final Xodb_category_itm Null = new Xodb_category_itm(); Xodb_category_itm() {}
}

76
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_category_tbl.java Normal file
View File

@@ -0,0 +1,76 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*;
public class Xodb_category_tbl {
public Db_stmt Update_stmt(Db_provider p) {return Db_stmt_.new_update_(p, Tbl_name, String_.Ary(Fld_cat_id), Fld_cat_hidden);}
public void Update(Db_stmt stmt, int cat_id, byte cat_hidden) {
stmt.Clear()
.Val_byte_(cat_hidden)
.Val_int_(cat_id)
.Exec_update()
;
}
public Db_stmt Insert_stmt(Db_provider p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_cat_id, Fld_cat_pages, Fld_cat_subcats, Fld_cat_files, Fld_cat_hidden, Fld_cat_file_idx);}
public void Insert(Db_stmt stmt, int cat_id, int cat_pages, int cat_subcats, int cat_files, byte cat_hidden, int cat_file_idx) {
stmt.Clear()
.Val_int_(cat_id)
.Val_int_(cat_pages)
.Val_int_(cat_subcats)
.Val_int_(cat_files)
.Val_byte_(cat_hidden)
.Val_int_(cat_file_idx)
.Exec_insert()
;
}
public Xodb_category_itm Select(Db_provider p, int cat_page_id) {
Db_stmt stmt = Db_stmt_.Null;
DataRdr rdr = DataRdr_.Null;
try {
stmt = Db_stmt_.new_select_(p, Tbl_name, String_.Ary(Fld_cat_id));
rdr = stmt.Val_int_(cat_page_id).Exec_select();
if (rdr.MoveNextPeer()) {
return Xodb_category_itm.load_
( cat_page_id
, rdr.ReadInt(Fld_cat_file_idx)
, rdr.ReadByte(Fld_cat_hidden) == Bool_.Y_byte
, rdr.ReadInt(Fld_cat_subcats)
, rdr.ReadInt(Fld_cat_files)
, rdr.ReadInt(Fld_cat_pages)
);
}
} finally {stmt.Rls(); rdr.Rls();}
return Xodb_category_itm.Null;
}
public void Select_by_cat_id_in(Cancelable cancelable, OrderedHash rv, Db_provider p, int bgn, int end) {
Xodb_in_wkr_category_id wkr = new Xodb_in_wkr_category_id();
wkr.Init(rv);
wkr.Select_in(p, cancelable, bgn, end);
}
public static Xodb_category_itm Read_ctg(DataRdr rdr) {
return Xodb_category_itm.load_
( rdr.ReadInt(Xodb_category_tbl.Fld_cat_id)
, rdr.ReadInt(Xodb_category_tbl.Fld_cat_file_idx)
, rdr.ReadByte(Xodb_category_tbl.Fld_cat_hidden) == Bool_.Y_byte
, rdr.ReadInt(Xodb_category_tbl.Fld_cat_subcats)
, rdr.ReadInt(Xodb_category_tbl.Fld_cat_files)
, rdr.ReadInt(Xodb_category_tbl.Fld_cat_pages)
);
}
public static final String Tbl_name = "category", Fld_cat_id = "cat_id", Fld_cat_pages = "cat_pages", Fld_cat_subcats = "cat_subcats", Fld_cat_files = "cat_files", Fld_cat_hidden = "cat_hidden", Fld_cat_file_idx = "cat_file_idx";
}

68
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_categorylinks_tbl.java Normal file
View File

@@ -0,0 +1,68 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*; import gplx.xowa.ctgs.*;
public class Xodb_categorylinks_tbl {
public void Delete_all(Db_provider p) {p.Exec_qry(Db_qry_.delete_tbl_(Tbl_name));}
public Db_stmt Insert_stmt(Db_provider p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_cl_from, Fld_cl_to_id, Fld_cl_sortkey, Fld_cl_timestamp, Fld_cl_type_id);}
public void Insert(Db_stmt stmt, int page_id, int ctg_page_id, byte[] sortkey, int timestamp, byte ctg_tid) {
stmt.Clear()
.Val_int_(page_id)
.Val_int_(ctg_page_id)
.Val_str_(String_.new_utf8_(sortkey))
.Val_int_(timestamp)
.Val_byte_(ctg_tid)
.Exec_insert()
;
}
Xoctg_idx_mgr Grp_by_tid(Xoctg_data_ctg ctg, byte tid) {
Xoctg_idx_mgr ctg_grp = ctg.Grp_by_tid(tid);
if (ctg_grp == null) {
ctg_grp = new Xoctg_idx_mgr();
ctg.Grp_mgrs()[tid] = ctg_grp;
}
return ctg_grp;
}
public int Select_by_type(Db_provider p, ListAdp list, int cat_page_id, byte arg_tid, byte[] arg_sortkey, boolean arg_is_from, int limit) {
String arg_sortkey_str = arg_sortkey == null ? "" : String_.new_utf8_(arg_sortkey);
gplx.criterias.Criteria comp_crt = !arg_is_from
? Db_crt_.mte_(Fld_cl_sortkey, arg_sortkey_str) // from: sortkey >= 'val'
: Db_crt_.lte_(Fld_cl_sortkey, arg_sortkey_str); // until: sortkey <= 'val'
Db_qry_select qry = Db_qry_.select_().Cols_(Fld_cl_from, Fld_cl_sortkey).From_(Tbl_name)
.Where_(gplx.criterias.Criteria_.And_many(Db_crt_.eq_(Fld_cl_to_id, -1), Db_crt_.eq_(Fld_cl_type_id, arg_tid), comp_crt))
.OrderBy_(Fld_cl_sortkey, !arg_is_from)
.Limit_(limit + 1); // + 1 to get last_plus_one for next page / previous page
Db_stmt stmt = Db_stmt_.Null;
DataRdr rdr = DataRdr_.Null;
int count = 0;
try {
stmt = p.Prepare(qry);
rdr = stmt.Val_int_(cat_page_id).Val_byte_(arg_tid).Val_str_(arg_sortkey_str).Exec_select();
while (rdr.MoveNextPeer()) {
int itm_page_id = rdr.ReadInt(Fld_cl_from);
byte[] itm_sortkey = rdr.ReadBryByStr(Fld_cl_sortkey);
Xodb_page itm = new Xodb_page().Id_(itm_page_id).Xtn_(new Xoctg_page_xtn(arg_tid, itm_sortkey));
list.Add(itm);
++count;
}
} finally {rdr.Rls(); stmt.Rls();}
list.SortBy(Xodb_page_sorter.Ctg_tid_sortkey_asc);
return count;
}
public static final String Tbl_name = "categorylinks", Fld_cl_from = "cl_from", Fld_cl_to = "cl_to", Fld_cl_to_id = "cl_to_id", Fld_cl_sortkey = "cl_sortkey", Fld_cl_timestamp = "cl_timestamp", Fld_cl_type_id = "cl_type_id";
}

47
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_in_wkr_base.java Normal file
View File

@@ -0,0 +1,47 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*;
public abstract class Xodb_in_wkr_base {
public abstract int Interval();
public abstract void Fill_stmt(Db_stmt stmt, int bgn, int end);
public abstract Db_qry Build_qry(int bgn, int end);
public abstract void Eval_rslts(Cancelable cancelable, DataRdr rdr);
public void Select_in(Db_provider provider, Cancelable cancelable, int full_bgn, int full_end) {
DataRdr rdr = DataRdr_.Null;
Db_stmt stmt = Db_stmt_.Null;
int interval = Interval();
for (int i = full_bgn; i < full_end; i += interval) {
int part_end = i + interval;
if (part_end > full_end) part_end = full_end;
try {
stmt = provider.Prepare(Build_qry(i, part_end));
Fill_stmt(stmt, i, part_end);
rdr = stmt.Exec_select();
Eval_rslts(cancelable, rdr);
}
finally {rdr.Rls(); stmt.Rls();}
}
}
public static Object[] In_ary(int len) {
Object[] rv = new Object[len];
for (int i = 0; i < len; i++)
rv[i] = "";
return rv;
}
}

413
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_page_tbl.java Normal file
View File

@@ -0,0 +1,413 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*; import gplx.criterias.*;
public class Xodb_page_tbl {
public Db_provider Provider() {return provider;} public void Provider_(Db_provider provider) {this.provider = provider;} private Db_provider provider;
public void Delete_all() {provider.Exec_qry(Db_qry_.delete_tbl_(Tbl_name));}
public Db_stmt Insert_stmt(Db_provider p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_page_id, Fld_page_ns, Fld_page_title, Fld_page_is_redirect, Fld_page_touched, Fld_page_len, Fld_page_random_int, Fld_page_file_idx);}
public void Insert(Db_stmt stmt, int page_id, int ns_id, byte[] ttl_wo_ns, boolean redirect, DateAdp modified_on, int page_len, int random_int, int file_idx) {
stmt.Clear()
.Val_int_(page_id)
.Val_int_(ns_id)
.Val_str_(String_.new_utf8_(ttl_wo_ns))
.Val_byte_((byte)(redirect ? 1 : 0))
.Val_str_(Xto_touched_str(modified_on))
.Val_int_(page_len)
.Val_int_(random_int)
.Val_int_(file_idx)
.Exec_insert();
}
public int Select_id(int ns_id, byte[] ttl) {
DataRdr rdr = DataRdr_.Null;
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_select_(provider, Tbl_name, String_.Ary(Fld_page_ns, Fld_page_title), Fld_page_id);
rdr = stmt
.Val_int_(ns_id)
.Val_str_by_bry_(ttl)
.Exec_select();
while (rdr.MoveNextPeer()) {
return rdr.ReadInt(Fld_page_id);
}
} finally {rdr.Rls(); stmt.Rls();}
return Xodb_mgr_sql.Page_id_null;
}
public DataRdr Select_all(Db_provider p) {
Db_qry_select qry = Db_qry_select.new_().From_(Tbl_name).Cols_(Fld_page_id, Fld_page_title).OrderBy_asc_(Fld_page_id);
return p.Exec_qry_as_rdr(qry);
}
public boolean Select_by_id(Xodb_page rv, int page_id) {
DataRdr rdr = DataRdr_.Null;
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_select_(provider, Tbl_name, String_.Ary(Fld_page_id));
rdr = stmt
.Val_int_(page_id)
.Exec_select();
while (rdr.MoveNextPeer()) {
Read_page(rv, rdr);
return true;
}
} finally {rdr.Rls(); stmt.Rls();}
return false;
}
public static void Read_page(Xodb_page page, DataRdr rdr) {
page.Id_ (rdr.ReadInt(Fld_page_id));
page.Ns_id_ (rdr.ReadInt(Fld_page_ns));
page.Ttl_wo_ns_ (rdr.ReadBryByStr(Fld_page_title));
page.Modified_on_ (DateAdp_.parse_fmt(rdr.ReadStr(Fld_page_touched), Page_touched_fmt));
page.Type_redirect_ (rdr.ReadByte(Fld_page_is_redirect) == 1);
page.Text_len_ (rdr.ReadInt(Fld_page_len));
page.Db_file_idx_ (rdr.ReadInt(Fld_page_file_idx));
}
public static void Read_page_for_search_suggest(Xodb_page page, DataRdr rdr) {
page.Id_ (rdr.ReadInt(Fld_page_id));
page.Ns_id_ (rdr.ReadInt(Fld_page_ns));
page.Ttl_wo_ns_ (rdr.ReadBryByStr(Fld_page_title));
page.Text_len_ (rdr.ReadInt(Fld_page_len));
}
private DataRdr Load_ttls_starting_with_rdr(int ns_id, byte[] ttl_frag, boolean include_redirects, int max_results, int min_page_len, int browse_len, boolean fwd, boolean search_suggest) {
Criteria crt_ttl = fwd ? Db_crt_.mte_(Fld_page_title, String_.new_utf8_(ttl_frag)) : Db_crt_.lt_(Fld_page_title, String_.new_utf8_(ttl_frag));
Criteria crt = Criteria_.And_many(Db_crt_.eq_(Fld_page_ns, ns_id), crt_ttl, Db_crt_.mte_(Fld_page_len, min_page_len));
if (!include_redirects)
crt = Criteria_.And(crt, Db_crt_.eq_(Fld_page_is_redirect, Byte_.Zero));
String[] cols = search_suggest ? Flds_ary_search_suggest : Flds_ary_all;
int limit = fwd ? max_results + 1 : max_results; // + 1 to get next item
Db_qry_select select = Db_qry_.select_cols_(Tbl_name, crt, cols).Limit_(limit).OrderBy_(Fld_page_title, fwd);
return select.Exec_qry_as_rdr(provider);
}
public void Load_ttls_for_all_pages(Cancelable cancelable, ListAdp rslt_list, Xodb_page rslt_nxt, Xodb_page rslt_prv, Int_obj_ref rslt_count, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item) {
DataRdr rdr = DataRdr_.Null;
Xodb_page nxt_itm = null;
int rslt_idx = 0;
boolean max_val_check = max_results == Int_.MaxValue;
try {
rdr = Load_ttls_starting_with_rdr(ns.Id(), key, include_redirects, max_results, min_page_len, browse_len, true, true);
while (rdr.MoveNextPeer()) {
if (cancelable.Canceled()) return;
Xodb_page page = new Xodb_page();
Read_page_for_search_suggest(page, rdr);
if (max_val_check && !Bry_.HasAtBgn(page.Ttl_wo_ns(), key)) break;
nxt_itm = page;
if (rslt_idx == max_results) {} // last item which is not meant for rslts, but only for nxt itm
else {
rslt_list.Add(page);
++rslt_idx;
}
}
if (rslt_nxt != null && nxt_itm != null) // occurs when range is empty; EX: "Module:A" in simplewikibooks
rslt_nxt.Copy(nxt_itm);
if (fetch_prv_item) { // NOTE: Special:AllPages passes in true, but Search_suggest passes in false
if (cancelable.Canceled()) return;
rdr = Load_ttls_starting_with_rdr(ns.Id(), key, include_redirects, max_results, min_page_len, browse_len, false, false);
Xodb_page prv_itm = new Xodb_page();
boolean found = false;
while (rdr.MoveNextPeer()) {
Read_page(prv_itm, rdr);
found = true;
}
if (found)
rslt_prv.Copy(prv_itm);
else { // at beginning of range, so no items found; EX: "Module:A" is search, but 1st Module is "Module:B"
if (rslt_list.Count() > 0) // use 1st item
rslt_prv.Copy((Xodb_page)rslt_list.FetchAt(0));
}
}
}
finally {rdr.Rls();}
rslt_count.Val_(rslt_idx);
}
public void Load_ttls_for_search_suggest(Cancelable cancelable, ListAdp rslt_list, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item) {
String search_bgn = String_.new_utf8_(key);
String search_end = String_.new_utf8_(gplx.intl.Utf8_.Increment_char_at_last_pos(key));
Db_qry qry = Db_qry_sql.rdr_("SELECT page_id, page_namespace, page_title, page_len FROM page INDEXED BY page__title WHERE page_namespace = " + Int_.XtoStr(ns.Id()) + " AND page_title BETWEEN '" + search_bgn + "' AND '" + search_end + "' ORDER BY page_len DESC LIMIT " + Int_.XtoStr(max_results) + ";");
DataRdr rdr = DataRdr_.Null;
try {
rdr = provider.Exec_qry_as_rdr(qry);
while (rdr.MoveNextPeer()) {
if (cancelable.Canceled()) return;
Xodb_page page = new Xodb_page();
Read_page_for_search_suggest(page, rdr);
rslt_list.Add(page);
}
rslt_list.SortBy(Xodb_page_sorter.TitleAsc);
}
finally {rdr.Rls();}
}
public boolean Select_by_id_list(Cancelable cancelable, ListAdp rv) {return Select_by_id_list(cancelable, false, rv, 0, rv.Count());}
public boolean Select_by_id_list(Cancelable cancelable, boolean skip_table_read, ListAdp rv) {return Select_by_id_list(cancelable, skip_table_read, rv, 0, rv.Count());}
public boolean Select_by_id_list(Cancelable cancelable, boolean skip_table_read, ListAdp rv, int bgn, int end) {
Xodb_page[] page_ary = (Xodb_page[])rv.XtoAry(Xodb_page.class);
int len = page_ary.length; if (len == 0) return false;
OrderedHash hash = OrderedHash_.new_();
for (int i = 0; i < len; i++) {
if (cancelable.Canceled()) return false;
Xodb_page p = page_ary[i];
if (!hash.Has(p.Id_val())) // NOTE: must check if file already exists b/c dynamicPageList currently allows dupes; DATE:2013-07-22
hash.Add(p.Id_val(), p);
}
Xodb_in_wkr_page_id wkr = new Xodb_in_wkr_page_id();
wkr.Init(rv, hash);
wkr.Select_in(provider, cancelable, bgn, end);
return true;
}
public boolean Select_by_ttl(Xodb_page rv, Xow_ns ns, byte[] ttl) {
DataRdr rdr = DataRdr_.Null;
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_select_(provider, Tbl_name, String_.Ary(Fld_page_ns, Fld_page_title));
rdr = stmt
.Val_int_(ns.Id())
.Val_str_(String_.new_utf8_(ttl))
.Exec_select();
if (rdr.MoveNextPeer()) {
Read_page(rv, rdr);
return true;
}
} finally {rdr.Rls(); stmt.Rls();}
return false;
}
public byte[] Select_random(Xow_ns ns) {// ns should be ns_main
int random_int = RandomAdp_.new_().Next(ns.Count());
DataRdr rdr = DataRdr_.Null;
Db_stmt stmt = Db_stmt_.Null;
byte[] rv = null;
try {
stmt = Db_stmt_.new_select_(provider, Tbl_name, String_.Ary(Fld_page_ns, Fld_page_random_int), Fld_page_title);
rdr = stmt
.Val_int_(ns.Id())
.Val_int_(random_int)
.Exec_select();
if (rdr.MoveNextPeer()) {
rv = rdr.ReadBryByStr(Fld_page_title);
}
} finally {rdr.Rls(); stmt.Rls();}
return rv;
}
public void Select_by_search(Cancelable cancelable, ListAdp rv, byte[] search, int results_max) {
if (Bry_.Len_eq_0(search)) return; // do not allow empty search
Criteria crt = gplx.criterias.Criteria_.And_many(Db_crt_.eq_(Fld_page_ns, Xow_ns_.Id_main), Db_crt_.like_(Fld_page_title, ""));
Db_qry_select qry = Db_qry_.select_().From_(Tbl_name).Cols_(Fld_page_id, Fld_page_len, Fld_page_ns, Fld_page_title).Where_(crt); // NOTE: use fields from main index only
DataRdr rdr = DataRdr_.Null;
Db_stmt stmt = Db_stmt_.Null;
search = Bry_.Replace(search, Byte_ascii.Asterisk, Byte_ascii.Percent);
try {
stmt = provider.Prepare(qry);
rdr = stmt.Clear().Val_int_(Xow_ns_.Id_main).Val_str_by_bry_(search).Exec_select();
while (rdr.MoveNextPeer()) {
if (cancelable.Canceled()) return;
Xodb_page page = new Xodb_page();
page.Id_ (rdr.ReadInt(Fld_page_id));
page.Ns_id_ (rdr.ReadInt(Fld_page_ns));
page.Ttl_wo_ns_ (rdr.ReadBryByStr(Fld_page_title));
page.Text_len_ (rdr.ReadInt(Fld_page_len));
rv.Add(page);
}
} finally {rdr.Rls(); stmt.Rls();}
}
public static byte[] Find_search_end(byte[] orig) {
byte[] rv = Bry_.Copy(orig);
int rv_len = rv.length;
int increment_pos = rv[rv_len - 1] == Byte_ascii.Percent ? rv_len - 2 : rv_len - 1; // increment last char, unless it is %; if %, increment one before it
return Bry_.Increment_last(rv, increment_pos);
}
public Db_stmt Select_for_parse_all_stmt(Db_provider p, int limit, byte redirect) {
Criteria crt = gplx.criterias.Criteria_.And_many(Db_crt_.eq_(Fld_page_ns, -1), Db_crt_.mt_(Fld_page_title, ""));
if (redirect != Bool_.__byte)
crt = gplx.criterias.Criteria_.And(crt, Db_crt_.eq_(Fld_page_is_redirect, redirect));
Db_qry_select qry = Db_qry_.select_().From_(Tbl_name).Cols_(Flds_all)
.Where_(crt)
.Limit_(limit);
return p.Prepare(qry);
}
public void Select_for_parse_all(Cancelable cancelable, OrderedHash rv, Db_stmt stmt, int ns, byte[] ttl, byte redirect) {
String ttl_str = String_.new_utf8_(ttl);
DataRdr rdr = DataRdr_.Null;
try {
stmt.Clear().Val_int_(ns).Val_str_(ttl_str);
if (redirect != Bool_.__byte) stmt.Val_byte_(redirect);
rdr = stmt.Exec_select();
while (rdr.MoveNextPeer()) {
if (cancelable.Canceled()) return;
Xodb_page page = new Xodb_page();
Read_page(page, rdr);
rv.Add(page.Id_val(), page);
}
} finally {rdr.Rls();}
}
public void Select_by_ttl_in(Cancelable cancelable, OrderedHash rv, Xow_wiki wiki, boolean fill_idx_fields_only, int bgn, int end) {
Xodb_in_wkr_page_title_ns wkr = new Xodb_in_wkr_page_title_ns();
wkr.Fill_idx_fields_only_(fill_idx_fields_only);
wkr.Init(wiki, rv);
wkr.Select_in(provider, cancelable, bgn, end);
}
public void Select_by_ttl_in(Cancelable cancelable, OrderedHash rv, int ns_id, int bgn, int end) {
Xodb_in_wkr_page_title wkr = new Xodb_in_wkr_page_title();
wkr.Init(rv, ns_id);
wkr.Select_in(provider, cancelable, bgn, end);
}
private static final String Page_touched_fmt = "yyyyMMddHHmmss";
private static String Xto_touched_str(DateAdp v) {return v.XtoStr_fmt(Page_touched_fmt);}
public static final String Tbl_name = "page", Fld_page_id = "page_id", Fld_page_ns = "page_namespace", Fld_page_title = "page_title", Fld_page_is_redirect = "page_is_redirect", Fld_page_touched = "page_touched", Fld_page_len = "page_len", Fld_page_random_int = "page_random_int", Fld_page_file_idx = "page_file_idx";
private static final String[] Flds_all = new String[] {Fld_page_id, Fld_page_ns, Fld_page_title, Fld_page_touched, Fld_page_is_redirect, Fld_page_len, Fld_page_file_idx};
public static String[] Flds_ary_all = String_.Ary(Flds_all), Flds_ary_search_suggest = String_.Ary(Fld_page_id, Fld_page_ns, Fld_page_title, Fld_page_len);
public static final boolean Load_idx_flds_only_y = true;
}
class Xodb_in_wkr_page_id extends Xodb_in_wkr_page_base {
private ListAdp list; // list is original list of ids which may have dupes; needed to fill statement (which takes range of bgn - end); DATE:2013-12-08
private OrderedHash hash; // hash is unique list of ids; needed for fetch from rdr (which indexes by id)
public void Init(ListAdp list, OrderedHash hash) {this.list = list; this.hash = hash; this.Fill_idx_fields_only_(true);}
@Override public int Interval() {return 990;}
@Override public String In_fld_name() {return Xodb_page_tbl.Fld_page_id;}
@Override public Criteria In_filter(Object[] part_ary) {
return Db_crt_.in_(this.In_fld_name(), part_ary);
}
@Override public void Fill_stmt(Db_stmt stmt, int bgn, int end) {
for (int i = bgn; i < end; i++) {
Xodb_page page = (Xodb_page)list.FetchAt(i);
stmt.Val_int_(page.Id());
}
}
@Override public Xodb_page Eval_rslts_key(Xodb_page rdr_page) {return (Xodb_page)hash.Fetch(rdr_page.Id_val());}
}
class Xodb_in_wkr_page_title extends Xodb_in_wkr_page_base {
private OrderedHash hash;
private int in_ns;
@Override public int Interval() {return 64;} // NOTE: 96+ overflows; EX: w:Space_Liability_Convention; DATE:2013-10-24
public void Init(OrderedHash hash, int in_ns) {this.hash = hash; this.in_ns = in_ns;}
@Override public String In_fld_name() {return Xodb_page_tbl.Fld_page_title;}
@Override public Criteria In_filter(Object[] part_ary) {
int len = part_ary.length;
Criteria[] crt_ary = new Criteria[len];
for (int i = 0; i < len; i++)
crt_ary[i] = Criteria_.And(Db_crt_.eq_(Xodb_page_tbl.Fld_page_ns, in_ns), Db_crt_.eq_(Xodb_page_tbl.Fld_page_title, Bry_.Empty));
return Criteria_.Or_many(crt_ary);
}
@Override public void Fill_stmt(Db_stmt stmt, int bgn, int end) {
for (int i = bgn; i < end; i++) {
Xodb_page page = (Xodb_page)hash.FetchAt(i);
stmt.Val_int_(in_ns);
stmt.Val_str_by_bry_(page.Ttl_wo_ns());
}
}
@Override public Xodb_page Eval_rslts_key(Xodb_page rdr_page) {return (Xodb_page)hash.Fetch(rdr_page.Ttl_wo_ns());}
}
class Xodb_in_wkr_page_title_ns extends Xodb_in_wkr_page_base {
private Xow_wiki wiki;
private OrderedHash hash;
@Override public int Interval() {return 64;} // NOTE: 96+ overflows; EX: w:Space_Liability_Convention; DATE:2013-10-24
public void Init(Xow_wiki wiki, OrderedHash hash) {this.wiki = wiki; this.hash = hash;}
@Override public String In_fld_name() {return Xodb_page_tbl.Fld_page_title;}
@Override public Criteria In_filter(Object[] part_ary) {
int len = part_ary.length;
Criteria[] crt_ary = new Criteria[len];
for (int i = 0; i < len; i++)
crt_ary[i] = Criteria_.And(Db_crt_.eq_(Xodb_page_tbl.Fld_page_ns, 0), Db_crt_.eq_(Xodb_page_tbl.Fld_page_title, Bry_.Empty));
return Criteria_.Or_many(crt_ary);
}
@Override public void Fill_stmt(Db_stmt stmt, int bgn, int end) {
for (int i = bgn; i < end; i++) {
Xodb_page page = (Xodb_page)hash.FetchAt(i);
stmt.Val_int_(page.Ns_id());
stmt.Val_str_by_bry_(page.Ttl_wo_ns());
}
}
// public override Criteria In_filter(Object[] part_ary) {
// int len = part_ary.length;
// Object[] vals = new Object[len];
// for (int i = 0; i < len; i++)
// vals[i] = Bry_.Empty;
// Db_obj_ary_crt crt_x = Db_obj_ary_crt.new_(new Db_fld(Xodb_page_tbl.Fld_page_title, ClassAdp_.Tid_str));
// crt_x.Vals_(new Object[][] {vals});
// return Criteria_.And(Db_crt_.eq_(Xodb_page_tbl.Fld_page_ns, 0), crt_x);
//// return Criteria_.And(Db_crt_.eq_(Xodb_page_tbl.Fld_page_ns, 0), Criteria_.Or_many(crt_ary));
// }
// public override void Fill_stmt(Db_stmt stmt, int bgn, int end) {
// stmt.Val_int_(0);
// for (int i = bgn; i < end; i++) {
// Xodb_page page = (Xodb_page)hash.FetchAt(i);
// stmt.Val_str_by_bry_(page.Ttl_wo_ns());
// }
// }
@Override public Xodb_page Eval_rslts_key(Xodb_page rdr_page) {
Xow_ns ns = wiki.Ns_mgr().Ids_get_or_null(rdr_page.Ns_id());
if (ns == null) return null; // NOTE: ns seems to "randomly" be null when threading during redlinks; guard against null; DATE:2014-01-03
byte[] ttl_wo_ns = rdr_page.Ttl_wo_ns();
rdr_page.Ttl_(ns, ttl_wo_ns);
return (Xodb_page)hash.Fetch(rdr_page.Ttl_w_ns());
}
}
abstract class Xodb_in_wkr_page_base extends Xodb_in_wkr_base {
public String Tbl_name() {return Xodb_page_tbl.Tbl_name;}
public abstract String In_fld_name();
public abstract Criteria In_filter(Object[] part_ary);
public abstract Xodb_page Eval_rslts_key(Xodb_page rdr_page);
public boolean Fill_idx_fields_only() {return fill_idx_fields_only;} public Xodb_in_wkr_page_base Fill_idx_fields_only_(boolean v) {fill_idx_fields_only = v; return this;} private boolean fill_idx_fields_only;
@Override public Db_qry Build_qry(int bgn, int end) {
Object[] part_ary = Xodb_in_wkr_base.In_ary(end - bgn);
return Db_qry_.select_cols_
( this.Tbl_name()
, In_filter(part_ary)
, fill_idx_fields_only ? Xodb_page_tbl.Flds_ary_search_suggest : Xodb_page_tbl.Flds_ary_all
)
;
}
@Override public void Eval_rslts(Cancelable cancelable, DataRdr rdr) {
Xodb_page temp = new Xodb_page();
while (rdr.MoveNextPeer()) {
if (cancelable.Canceled()) return;
if (fill_idx_fields_only)
Xodb_page_tbl.Read_page_for_search_suggest(temp, rdr);
else
Xodb_page_tbl.Read_page(temp, rdr);
Xodb_page page = Eval_rslts_key(temp);
if (page == null) continue; // page not found
temp.Exists_(true);
page.Copy(temp);
}
}
}
class Xodb_in_wkr_category_id extends Xodb_in_wkr_base {
private OrderedHash hash;
@Override public int Interval() {return 990;}
public void Init(OrderedHash hash) {this.hash = hash;}
@Override public Db_qry Build_qry(int bgn, int end) {
Object[] part_ary = Xodb_in_wkr_base.In_ary(end - bgn);
String in_fld_name = Xodb_category_tbl.Fld_cat_id;
return Db_qry_.select_cols_
( Xodb_category_tbl.Tbl_name
, Db_crt_.in_(in_fld_name, part_ary)
)
;
}
@Override public void Fill_stmt(Db_stmt stmt, int bgn, int end) {
for (int i = bgn; i < end; i++) {
Xodb_page page = (Xodb_page)hash.FetchAt(i);
stmt.Val_int_(page.Id());
}
}
@Override public void Eval_rslts(Cancelable cancelable, DataRdr rdr) {
while (rdr.MoveNextPeer()) {
if (cancelable.Canceled()) return;
Xodb_category_itm ctg_data = Xodb_category_tbl.Read_ctg(rdr);
Xodb_page page = (Xodb_page)hash.Fetch(ctg_data.Id_val());
page.Xtn_(ctg_data);
}
}
}

29
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_page_tbl_tst.java Normal file
View File

@@ -0,0 +1,29 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import org.junit.*; import gplx.xowa.bldrs.*; import gplx.xowa.ctgs.*; import gplx.dbs.*; import gplx.xowa.dbs.tbls.*;
public class Xodb_page_tbl_tst {
private Xodb_page_tbl_fxt fxt = new Xodb_page_tbl_fxt();
@Test public void Find_search_end() {
fxt.Test_find_search_end("ab", "ac");
fxt.Test_find_search_end("ab%", "ac%");
}
}
class Xodb_page_tbl_fxt {
public void Test_find_search_end(String val, String expd) {Tfds.Eq(expd, String_.new_utf8_(Xodb_page_tbl.Find_search_end(Bry_.new_utf8_(val))));}
}

42
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_search_title_page_tbl.java Normal file
View File

@@ -0,0 +1,42 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*;
public class Xodb_search_title_page_tbl {
public static void Create_table(Db_provider p) {Sqlite_engine_.Tbl_create(p, Tbl_name, Tbl_sql);}
public static void Create_index_unique(Gfo_usr_dlg usr_dlg, Db_provider p) {Sqlite_engine_.Idx_create(usr_dlg, p, "search", Indexes_main_unique);}
public static void Create_index_non_unique(Gfo_usr_dlg usr_dlg, Db_provider p) {Sqlite_engine_.Idx_create(usr_dlg, p, "search", Indexes_main_non_unique);}
public static Db_stmt Insert_stmt(Db_provider p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_stp_word_id, Fld_stp_page_id);}
public static void Insert(Db_stmt stmt, int word_id, int page_id) {
stmt.Clear()
.Val_int_(word_id)
.Val_int_(page_id)
.Exec_insert();
}
public static final String Tbl_name = "search_title_page", Fld_stp_word_id = "stp_word_id", Fld_stp_page_id = "stp_page_id";
private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS search_title_page"
, "( stp_word_id integer NOT NULL"
, ", stp_page_id integer NOT NULL"
, ");"
);
private static final Db_idx_itm
Indexes_main_unique = Db_idx_itm.sql_("CREATE UNIQUE INDEX IF NOT EXISTS search_title_page__main ON search_title_page (stp_word_id, stp_page_id);")
, Indexes_main_non_unique = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS search_title_page__main ON search_title_page (stp_word_id, stp_page_id);")
;
}

96
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_search_title_word_tbl.java Normal file
View File

@@ -0,0 +1,96 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*;
public class Xodb_search_title_word_tbl {
public static void Create_table(Db_provider p) {Sqlite_engine_.Tbl_create(p, Tbl_name, Tbl_sql);}
public static void Create_index(Gfo_usr_dlg usr_dlg, Db_provider p) {Sqlite_engine_.Idx_create(usr_dlg, p, "search", Indexes_main);}
public static Db_stmt Insert_stmt(Db_provider p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_stw_word_id, Fld_stw_word);}
public static void Insert(Db_stmt stmt, int word_id, byte[] word) {
stmt.Clear()
.Val_int_(word_id)
.Val_str_by_bry_(word)
.Exec_insert();
}
public static void Select_by_word(Cancelable cancelable, ListAdp rv, byte[] search, int results_max, Db_provider p) {
Db_qry_select qry = Db_qry_.select_()
.Cols_(Xodb_search_title_word_tbl.Fld_stw_word_id)
.From_(Xodb_search_title_word_tbl.Tbl_name, "w")
;
gplx.criterias.Criteria crt = null;
if (Bry_.Has(search, Byte_ascii.Asterisk)) {
search = Bry_.Replace(search, Byte_ascii.Asterisk, Byte_ascii.Percent);
crt = Db_crt_.like_ (Xodb_search_title_word_tbl.Fld_stw_word, String_.new_utf8_(search));
}
else
crt = Db_crt_.eq_ (Xodb_search_title_word_tbl.Fld_stw_word, String_.new_utf8_(search));
qry.Where_(crt);
DataRdr rdr = DataRdr_.Null;
ListAdp words = ListAdp_.new_();
try {
rdr = qry.Exec_qry_as_rdr(p);
while (rdr.MoveNextPeer()) {
int word_id = rdr.ReadInt(Xodb_search_title_word_tbl.Fld_stw_word_id);
words.Add(Int_obj_val.new_(word_id));
}
}
finally {rdr.Rls();}
Xodb_in_wkr_search_title_id wkr = new Xodb_in_wkr_search_title_id();
wkr.Init(words, rv);
wkr.Select_in(p, cancelable, 0, words.Count());
}
public static final String Tbl_name = "search_title_word", Fld_stw_word_id = "stw_word_id", Fld_stw_word = "stw_word";
private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS search_title_word"
, "( stw_word_id integer NOT NULL PRIMARY KEY"
, ", stw_word varchar(255) NOT NULL"
, ");"
);
private static final Db_idx_itm
Indexes_main = Db_idx_itm.sql_("CREATE UNIQUE INDEX IF NOT EXISTS search_title_word__main ON search_title_word (stw_word, stw_word_id);");
}
class Xodb_in_wkr_search_title_id extends Xodb_in_wkr_base {
private ListAdp words, pages;
@Override public int Interval() {return 990;}
public void Init(ListAdp words, ListAdp pages) {this.words = words; this.pages = pages;}
@Override public Db_qry Build_qry(int bgn, int end) {
Object[] part_ary = Xodb_in_wkr_base.In_ary(end - bgn);
String in_fld_name = Xodb_search_title_page_tbl.Fld_stp_word_id;
return Db_qry_.select_cols_
( Xodb_search_title_page_tbl.Tbl_name
, Db_crt_.in_(in_fld_name, part_ary)
)
;
}
@Override public void Fill_stmt(Db_stmt stmt, int bgn, int end) {
for (int i = bgn; i < end; i++) {
Int_obj_val word_id = (Int_obj_val)words.FetchAt(i);
stmt.Val_int_(word_id.Val());
}
}
@Override public void Eval_rslts(Cancelable cancelable, DataRdr rdr) {
while (rdr.MoveNextPeer()) {
if (cancelable.Canceled()) return;
int page_id = rdr.ReadInt(Xodb_search_title_page_tbl.Fld_stp_page_id);
Xodb_page page = new Xodb_page().Id_(page_id);
pages.Add(page);
}
}
}

49
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_site_stats_tbl.java Normal file
View File

@@ -0,0 +1,49 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*;
public class Xodb_site_stats_tbl {
public void Provider_(Db_provider provider) {this.provider = provider;} Db_provider provider;
public void Update(int num_articles, int num_pages, int num_files) {
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_update_(provider, Tbl_name, String_.Ary(Fld_ss_row_id), Fld_ss_good_articles, Fld_ss_total_pages, Fld_ss_images);
stmt.Val_int_(num_articles)
.Val_int_(num_pages)
.Val_int_(num_files)
.Val_int_(1)
.Exec_update();
;
} finally {stmt.Rls();}
}
public void Select(Xow_wiki wiki) {
Xow_wiki_stats stats = wiki.Stats();
DataRdr rdr = DataRdr_.Null;
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_select_all_(provider, Tbl_name);
rdr = stmt.Exec_select();
if (rdr.MoveNextPeer()) {
stats.NumArticles_ (rdr.ReadInt(Fld_ss_good_articles));
stats.NumPages_ (rdr.ReadInt(Fld_ss_total_pages));
stats.NumFiles_ (rdr.ReadInt(Fld_ss_images));
}
} finally {rdr.Rls(); stmt.Rls();}
}
public static final String Tbl_name = "site_stats", Fld_ss_row_id = "ss_row_id", Fld_ss_good_articles = "ss_good_articles", Fld_ss_total_pages = "ss_total_pages", Fld_ss_images = "ss_images";
}

81
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_text_tbl.java Normal file
View File

@@ -0,0 +1,81 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*; import gplx.ios.*;
public class Xodb_text_tbl {
public Xodb_text_tbl(Xodb_mgr_sql db_mgr) {this.db_mgr = db_mgr; zip_mgr = db_mgr.Wiki().App().Zip_mgr();} private Xodb_mgr_sql db_mgr; Io_stream_zip_mgr zip_mgr;
public void Delete_all(Db_provider provider) {provider.Exec_qry(Db_qry_.delete_tbl_(Tbl_name));}
public Db_stmt Insert_stmt(Db_provider prov) {return Db_stmt_.new_insert_(prov, Tbl_name, Fld_page_id, Fld_old_text);}
public void Insert(Db_stmt stmt, int page_id, byte[] text, byte storage_type) {
stmt.Clear().Val_int_(page_id).Val_bry_(text).Exec_insert();
}
public void Update(int file_id, int page_id, byte[] text) {
Db_stmt stmt = Db_stmt_.Null;
try {
Db_provider provider = db_mgr.Fsys_mgr().Get_by_db_idx(file_id).Provider();
stmt = Db_stmt_.new_update_(provider, Tbl_name, String_.Ary(Fld_page_id), Fld_old_text);
stmt.Val_bry_(text).Val_int_(page_id).Exec_update();
} finally {stmt.Rls();}
}
public byte[] Select(int file_id, int page_id) {
Db_stmt stmt = Db_stmt_.Null;
try {
Db_provider provider = db_mgr.Fsys_mgr().Get_by_db_idx(file_id).Provider();
stmt = Db_stmt_.new_select_(provider, Tbl_name, String_.Ary(Fld_page_id), Fld_old_text);
byte[] rv = (byte[])stmt.Val_int_(page_id).Exec_select_val();
rv = zip_mgr.Unzip(db_mgr.Data_storage_format(), rv);
return rv;
} finally {stmt.Rls();}
}
public void Select_in(Cancelable cancelable, Xodb_file file, OrderedHash hash) {
DataRdr rdr = DataRdr_.Null;
Db_stmt stmt = Db_stmt_.Null;
try {
int len = hash.Count();
ListAdp pages = ListAdp_.new_();
for (int i = 0; i < len; i++) {
Xodb_page page = (Xodb_page)hash.FetchAt(i);
if (page.Db_file_idx() == file.Id())
pages.Add(page);
}
len = pages.Count();
if (len == 0) return;
Object[] args_ary = new Object[len];
for (int i = 0; i < len; i++) {
if (cancelable.Canceled()) return;
args_ary[i] = 0;
}
stmt = Db_stmt_.new_select_in_(file.Provider(), Tbl_name, Fld_page_id, args_ary);
for (int i = 0; i < len; i++) {
if (cancelable.Canceled()) return;
Xodb_page page = (Xodb_page)pages.FetchAt(i);
stmt.Val_int_(page.Id());
}
rdr = stmt.Exec_select();
while (rdr.MoveNextPeer()) {
if (cancelable.Canceled()) return;
int page_id = rdr.ReadInt(Fld_page_id);
byte[] old_text = rdr.ReadBry(Fld_old_text);
old_text = zip_mgr.Unzip(db_mgr.Data_storage_format(), old_text);
Xodb_page page = (Xodb_page)hash.Fetch(Int_obj_val.new_(page_id));
page.Text_(old_text);
}
} finally {rdr.Rls(); stmt.Rls();}
}
public static final String Tbl_name = "text", Fld_page_id = "page_id", Fld_old_text = "old_text";
}

37
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_wdata_pids_tbl.java Normal file
View File

@@ -0,0 +1,37 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*; import gplx.xowa.xtns.wdatas.*;
public class Xodb_wdata_pids_tbl {
public void Purge(Db_provider p) {p.Exec_qry(Db_qry_.delete_tbl_(Tbl_name));}
public Db_stmt Insert_stmt(Db_provider p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_wp_src_lang, Fld_wp_src_ttl, Fld_wp_trg_ttl);}
public void Insert(Db_stmt stmt, byte[] src_lang, byte[] src_ttl, byte[] trg_ttl) {
stmt.Clear().Val_str_by_bry_(src_lang).Val_str_by_bry_(src_ttl).Val_str_by_bry_(trg_ttl).Exec_insert();
}
public int Select_pid(Db_provider p, byte[] src_lang, byte[] src_ttl) {
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_select_(p, Tbl_name, String_.Ary(Fld_wp_src_lang, Fld_wp_src_ttl), Fld_wp_trg_ttl);
String pid_str = (String)stmt.Val_str_by_bry_(src_lang).Val_str_by_bry_(src_ttl).Exec_select_val();
if (pid_str == null) return Wdata_wiki_mgr.Pid_null; // occurs when pid exists, but does not have entry for language; see hu.w:Marco Polo argali; DATE: 2014-02-01
byte[] pid_bry = Bry_.new_utf8_(pid_str);
return pid_bry == null ? Wdata_wiki_mgr.Pid_null : Bry_.X_to_int_or(pid_bry, 1, pid_bry.length, Wdata_wiki_mgr.Pid_null);
} finally {stmt.Rls();}
}
public static final String Tbl_name = "wdata_pids", Fld_wp_src_lang = "wp_src_lang", Fld_wp_src_ttl = "wp_src_ttl", Fld_wp_trg_ttl = "wp_trg_ttl";
}

35
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_wdata_qids_tbl.java Normal file
View File

@@ -0,0 +1,35 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*;
public class Xodb_wdata_qids_tbl {
public void Purge(Db_provider p) {p.Exec_qry(Db_qry_.delete_tbl_(Tbl_name));}
public Db_stmt Insert_stmt(Db_provider p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_wq_src_wiki, Fld_wq_src_ns, Fld_wq_src_ttl, Fld_wq_trg_ttl);}
public void Insert(Db_stmt stmt, byte[] src_wiki, int src_ns, byte[] src_ttl, byte[] trg_ttl) {
stmt.Clear().Val_str_by_bry_(src_wiki).Val_int_(src_ns).Val_str_by_bry_(src_ttl).Val_str_by_bry_(trg_ttl).Exec_insert();
}
public byte[] Select_qid(Db_provider p, byte[] src_wiki, byte[] src_ns, byte[] src_ttl) {
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_select_(p, Tbl_name, String_.Ary(Fld_wq_src_wiki, Fld_wq_src_ns, Fld_wq_src_ttl), Fld_wq_trg_ttl);
String rv = (String)stmt.Val_str_by_bry_(src_wiki).Val_int_(Bry_.X_to_int(src_ns)).Val_str_by_bry_(src_ttl).Exec_select_val();
return rv == null ? null : Bry_.new_utf8_(rv);
} finally {stmt.Rls();}
}
public static final String Tbl_name = "wdata_qids", Fld_wq_src_wiki = "wq_src_wiki", Fld_wq_src_ns = "wq_src_ns", Fld_wq_src_ttl = "wq_src_ttl", Fld_wq_trg_ttl = "wq_trg_ttl";
}

104
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_xowa_cfg_tbl.java Normal file
View File

@@ -0,0 +1,104 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*;
public class Xodb_xowa_cfg_tbl {
public Db_provider Provider() {return provider;} public Xodb_xowa_cfg_tbl Provider_(Db_provider provider) {this.provider = provider; return this;} Db_provider provider;
private DataRdr Select(String grp) {
Db_qry qry = Db_qry_.select_cols_(Tbl_name, Db_crt_.eq_(Fld_cfg_grp, grp), Fld_cfg_key, Fld_cfg_val);
return provider.Exec_qry_as_rdr(qry);
}
public int Select_val_as_int(String grp, String key) {return Int_.parse_(Select_val(grp, key));}
public String Select_val(String grp, String key) {return Select_val_or(grp, key, null);}
public String Select_val_or(String grp, String key, String or) {
Db_qry_select qry = Db_qry_.select_val_(Tbl_name, Fld_cfg_val, Where_grp_key(grp, key));
String rv = (String)qry.ExecRdr_val(provider);
return rv == null ? or : rv;
}
public String Select_val_or_make(String grp, String key, String or) {
String rv = Select_val_or(grp, key, null);
if (rv == null) {
rv = or;
Insert_str(grp, key, rv);
}
return rv;
}
public KeyVal[] Select_kvs(String grp, String match_key, String_obj_ref match_val) {
DataRdr rdr = DataRdr_.Null;
try {
rdr = this.Select(grp);
while (rdr.MoveNextPeer()) {
String key = rdr.ReadStr(Fld_cfg_key);
String val = rdr.ReadStr(Fld_cfg_val);
if (String_.Eq(key, match_key)) match_val.Val_(val);
KeyVal kv = KeyVal_.new_(key, val);
tmp_list.Add(kv);
}
return (KeyVal[])tmp_list.XtoAry(KeyVal.class);
}
finally {rdr.Rls(); tmp_list.Clear();}
} ListAdp tmp_list = ListAdp_.new_();
public void Delete(String grp, String key) {
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_delete_(provider, Tbl_name, String_.Ary(Fld_cfg_grp, Fld_cfg_key));
stmt.Val_str_(grp).Val_str_(key).Exec_delete();
} finally {stmt.Rls();}
}
public void Insert_byte(String grp, String key, byte val) {Insert_str(grp, key, Byte_.XtoStr(val));}
public void Insert_int(String grp, String key, int val) {Insert_str(grp, key, Int_.XtoStr(val));}
public void Insert_str_by_bry(String grp, String key, byte[] val) {Insert_str(grp, key, String_.new_utf8_(val));}
public void Insert_str(String grp, String key, String val) {Insert_str(provider, grp, key, val);}
public static void Insert_str(Db_provider p, String grp, String key, String val) {
Db_qry qry = Db_qry_.insert_(Tbl_name)
.Arg_(Fld_cfg_grp , grp)
.Arg_(Fld_cfg_key , key)
.Arg_(Fld_cfg_val , val)
;
p.Exec_qry(qry);
}
public Db_stmt Update_stmt() {return Db_stmt_.new_update_(provider, Tbl_name, String_.Ary(Fld_cfg_grp, Fld_cfg_key), Fld_cfg_val);}
public void Update(Db_stmt stmt, String grp, String key, long val) {Update(stmt, grp, key, Long_.XtoStr(val));}
public void Update(Db_stmt stmt, String grp, String key, int val) {Update(stmt, grp, key, Int_.XtoStr(val));}
public void Update(Db_stmt stmt, String grp, String key, String val) {
stmt.Clear()
.Val_str_(val)
.Val_str_(grp)
.Val_str_(key)
.Exec_update();
}
public void Update(String grp, String key, int val) {Update(grp, key, Int_.XtoStr(val));}
public void Update(String grp, String key, String val) {
Db_qry qry = Db_qry_.update_common_(Tbl_name, Where_grp_key(grp, key), KeyVal_.new_(Fld_cfg_val, val));
provider.Exec_qry(qry);
}
private gplx.criterias.Criteria Where_grp_key(String grp, String key) {return Db_crt_.eqMany_(KeyVal_.new_(Fld_cfg_grp, grp), KeyVal_.new_(Fld_cfg_key, key));}
public static final String Tbl_name = "xowa_cfg", Fld_cfg_grp = "cfg_grp", Fld_cfg_key = "cfg_key", Fld_cfg_val = "cfg_val";
public static void Create_table(Db_provider p) {Sqlite_engine_.Tbl_create(p, Tbl_name, Tbl_sql);}
public static void Create_index(Db_provider p) {Sqlite_engine_.Idx_create(p, Idx_select);}
private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS xowa_cfg"
, "( cfg_grp varchar(1024) NOT NULL"
, ", cfg_key varchar(1024) NOT NULL"
, ", cfg_val blob NOT NULL"
, ");"
);
private static final Db_idx_itm
Idx_select = Db_idx_itm.sql_("CREATE UNIQUE INDEX IF NOT EXISTS xowa_cfg__grp_key ON xowa_cfg (cfg_grp, cfg_key);")
;
}

88
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_xowa_db_tbl.java Normal file
View File

@@ -0,0 +1,88 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*;
public class Xodb_xowa_db_tbl {
public void Provider_(Db_provider provider) {this.provider = provider;} Db_provider provider;
public void Update_url(Db_provider provider, int db_id, String db_url) {
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_update_(provider, Tbl_name, String_.Ary(Fld_db_id), Fld_db_url);
stmt.Clear()
.Val_str_(db_url)
.Val_int_(db_id)
.Exec_update();
;
} finally {stmt.Rls();}
}
public void Commit_all(Db_provider provider, Xodb_file[] ary) {
stmt_bldr.Init(provider);
try {
int len = ary.length;
for (int i = 0; i < len; i++)
Commit_itm(ary[i]);
stmt_bldr.Commit();
} finally {stmt_bldr.Rls();}
}
private void Commit_itm(Xodb_file itm) {
Db_stmt stmt = stmt_bldr.Get(itm.Cmd_mode());
switch (itm.Cmd_mode()) {
case Db_cmd_mode.Create: stmt.Clear().Val_int_(itm.Id()) .Val_byte_(itm.Tid()).Val_str_(itm.Url_rel()).Exec_insert(); break;
case Db_cmd_mode.Update: stmt.Clear() .Val_byte_(itm.Tid()).Val_str_(itm.Url_rel()).Val_int_(itm.Id()).Exec_update(); break;
case Db_cmd_mode.Delete: stmt.Clear().Val_int_(itm.Id()).Exec_delete(); break;
case Db_cmd_mode.Ignore: break;
default: throw Err_.unhandled(itm.Cmd_mode());
}
itm.Cmd_mode_(Db_cmd_mode.Ignore);
}
public Xodb_file[] Select_all(Db_provider provider) {
DataRdr rdr = DataRdr_.Null;
ListAdp list = ListAdp_.new_();
try {
Db_qry qry = Db_qry_.select_tbl_(Tbl_name).OrderBy_asc_(Fld_db_id);
rdr = provider.Exec_qry_as_rdr(qry);
while (rdr.MoveNextPeer()) {
Xodb_file db = Xodb_file.load_(rdr.ReadInt(Fld_db_id), rdr.ReadByte(Fld_db_type), rdr.ReadStr(Fld_db_url));
list.Add(db);
}
} finally {rdr.Rls();}
Xodb_file[] rv = (Xodb_file[])list.XtoAry(Xodb_file.class);
Chk_sequential(rv);
return rv;
}
private void Chk_sequential(Xodb_file[] ary) {
int len = ary.length;
int expd_id = 0;
for (int i = 0; i < len; i++) {
Xodb_file itm = ary[i];
int actl_id = itm.Id();
if (expd_id != actl_id) throw Err_.new_fmt_("database ids are not sequential; expd={0} actl={1}", expd_id, actl_id);
++expd_id;
}
}
public static final String Tbl_name = "xowa_db", Fld_db_id = "db_id", Fld_db_type = "db_type", Fld_db_url = "db_url";
Db_stmt_bldr stmt_bldr = new Db_stmt_bldr(Tbl_name, String_.Ary(Fld_db_id), Fld_db_type, Fld_db_url);
}
class Xodb_db_sorter implements gplx.lists.ComparerAble {
public int compare(Object lhsObj, Object rhsObj) {
Xodb_file lhs = (Xodb_file)lhsObj;
Xodb_file rhs = (Xodb_file)rhsObj;
return Int_.Compare(lhs.Id(), rhs.Id());
}
public static final Xodb_db_sorter _ = new Xodb_db_sorter();
}

72
400_xowa/src/gplx/xowa/dbs/tbls/Xodb_xowa_ns_tbl.java Normal file
View File

@@ -0,0 +1,72 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.dbs.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.dbs.*;
import gplx.dbs.*;
public class Xodb_xowa_ns_tbl {
public void Provider_(Db_provider provider) {this.provider = provider;} Db_provider provider;
public int Select_ns_count(int ns_id) {
Db_qry_select qry = Db_qry_.select_val_(Tbl_name, Fld_ns_count, Db_crt_.eq_(Fld_ns_id, ns_id));
return Int_.cast_(qry.ExecRdr_val(provider));
}
public void Update_ns_count(int ns_id, int ns_count) {
provider.Exec_qry(Db_qry_.update_common_(Tbl_name
, Db_crt_.eq_(Fld_ns_id, ns_id)
, KeyVal_.Ary
( KeyVal_.new_(Fld_ns_count, ns_count)
)));
}
public void Insert(Xow_ns_mgr ns_mgr) {
Db_stmt stmt = Db_stmt_.Null;
try {
stmt = Db_stmt_.new_insert_(provider, Tbl_name, Fld_ns_id, Fld_ns_name, Fld_ns_case, Fld_ns_is_alias, Fld_ns_count);
int len = ns_mgr.Ids_len();
for (int i = 0; i < len; i++) {
Xow_ns ns = ns_mgr.Ids_get_at(i);
stmt.Clear()
.Val_int_(ns.Id())
.Val_str_(ns.Name_str())
.Val_byte_(ns.Case_match())
.Val_bool_(ns.Is_alias())
.Val_int_(ns.Count())
.Exec_insert();
;
}
} finally {stmt.Rls();}
}
public void Load(Xow_ns_mgr ns_mgr) {
ns_mgr.Clear();
DataRdr rdr = DataRdr_.Null;
try {
rdr = provider.Exec_qry_as_rdr(Db_qry_.select_tbl_(Tbl_name));
while (rdr.MoveNextPeer()) {
int ns_id = rdr.ReadInt(Fld_ns_id);
byte ns_case_match = rdr.ReadByte(Fld_ns_case);
byte[] ns_name = rdr.ReadBryByStr(Fld_ns_name);
boolean ns_is_alias = rdr.ReadBool(Fld_ns_is_alias);
ns_mgr.Add_new(ns_id, ns_name, ns_case_match, ns_is_alias);
if (ns_id < 0) continue; // don't load counts for Special / Media
int ns_count = rdr.ReadInt(Fld_ns_count);
Xow_ns ns = ns_mgr.Ids_get_or_null(ns_id);
ns.Count_(ns_count);
if (ns_count > 0) ns.Exists_(true); // ns has article; mark it as exists, else Talk tab won't show; DATE:2013-12-04
}
ns_mgr.Init();
} finally {rdr.Rls();}
}
public static final String Tbl_name = "xowa_ns", Fld_ns_id = "ns_id", Fld_ns_name = "ns_name", Fld_ns_case = "ns_case", Fld_ns_count = "ns_count", Fld_ns_is_alias = "ns_is_alias";
}