Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

uca category support and other changes

Browse Source
This commit is contained in:
gnosygnu
2016-10-12 08:57:22 -04:00
parent e3b393650d
commit 3fc2e0741f
187 changed files with 3486 additions and 2984 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
400_xowa/src/gplx/xowa/wikis/Xowv_wiki.java
View File

@@ -29,7 +29,7 @@ import gplx.xowa.parsers.*;
import gplx.xowa.apps.urls.*;
import gplx.fsdb.*; import gplx.fsdb.meta.*;
import gplx.xowa.specials.mgrs.*;
import gplx.xowa.addons.wikis.htmls.css.bldrs.*; import gplx.xowa.addons.wikis.htmls.css.mgrs.*; import gplx.xowa.addons.wikis.ctgs.htmls.pageboxs.*;
import gplx.xowa.addons.wikis.htmls.css.bldrs.*; import gplx.xowa.addons.wikis.htmls.css.mgrs.*; import gplx.xowa.addons.wikis.ctgs.htmls.pageboxs.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*;
public class Xowv_wiki implements Xow_wiki, Xow_ttl_parser, Gfo_invk {
private final Xof_fsdb_mgr__sql fsdb_mgr; private Fsdb_db_mgr db_core_mgr;
private boolean init_needed = true;
@@ -50,6 +50,7 @@ public class Xowv_wiki implements Xow_wiki, Xow_ttl_parser, Gfo_invk {
this.xwiki_mgr = new Xow_xwiki_mgr(this);
this.stats = new Xow_site_stats_mgr(this);
this.lnki_bldr = new Xoh_lnki_bldr(app, href_wtr);
this.ctg_catpage_mgr = new Xoctg_catpage_mgr(this);
}
public Xoa_app App() {return app;}
public boolean Type_is_edit() {return Bool_.N;}
@@ -73,7 +74,8 @@ public class Xowv_wiki implements Xow_wiki, Xow_ttl_parser, Gfo_invk {
public Xow_hdump_mgr Html__hdump_mgr() {return html__hdump_mgr;} private final Xow_hdump_mgr html__hdump_mgr;
public boolean Html__css_installing() {return html__css_installing;} public void Html__css_installing_(boolean v) {html__css_installing = v;} private boolean html__css_installing;
public Xoh_page_wtr_mgr Html__wtr_mgr() {return html__wtr_mgr;} private final Xoh_page_wtr_mgr html__wtr_mgr = new Xoh_page_wtr_mgr(Bool_.Y);
public Xoctg_pagebox_wtr Html__ctg_pagebox_wtr() {return ctg_pagebox_wtr;} private final Xoctg_pagebox_wtr ctg_pagebox_wtr = new Xoctg_pagebox_wtr();
public Xoctg_pagebox_wtr Ctg__pagebox_wtr() {return ctg_pagebox_wtr;} private final Xoctg_pagebox_wtr ctg_pagebox_wtr = new Xoctg_pagebox_wtr();
public Xoctg_catpage_mgr Ctg__catpage_mgr() {return ctg_catpage_mgr;} private final Xoctg_catpage_mgr ctg_catpage_mgr;
public Xow_msg_mgr Msg_mgr() {return msg_mgr;} private final Xow_msg_mgr msg_mgr;
public byte[] Wtxt__expand_tmpl(byte[] src) {return src;}
public Xow_mw_parser_mgr Mw_parser_mgr() {return mw_parser_mgr;} private final Xow_mw_parser_mgr mw_parser_mgr = new Xow_mw_parser_mgr();

2
400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java
View File

@@ -58,7 +58,7 @@ public class Xow_cache_mgr {
ifexist_cache.Clear();
wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); // moved from ctx.Clear(); DATE:2016-07-21
}
wiki.Html_mgr().Catpage_mgr().Free_mem_all();
wiki.Ctg__catpage_mgr().Free_mem_all();
tmpl_result_cache.Clear();
defn_cache.Free_mem_all();
misc_cache.Clear();

2
400_xowa/src/gplx/xowa/wikis/data/Xow_data_mgr_tst.java
View File

@@ -62,7 +62,7 @@ public class Xow_data_mgr_tst {
;
}
@Test public void Update_zip() {
// fxt.Wiki().Fsys_mgr().Dir_regy()[Xow_ns_.Tid__main].Ext_tid_(gplx.core.ios.streams.Io_stream_.Tid_zip);
// fxt.Wiki().Fsys_mgr().Dir_regy()[Xow_ns_.Tid__main].Ext_tid_(gplx.core.ios.streams.Io_stream_tid_.Tid__zip);
// fxt.Wiki().Data_mgr().Zip_mgr_(new Io_zip_mgr_mok());
// fxt .Create("A1", "A1 data")
// .Create("B12", "B12 data")

4
400_xowa/src/gplx/xowa/wikis/data/Xowd_core_db_props.java
View File

@@ -52,7 +52,7 @@ public class Xowd_core_db_props {
return cfg_tbl.Select_int_or(Cfg_grp, Cfg_key__schema_version, 1) == 1
? new Xowd_core_db_props
( 1, Xow_db_layout.Itm_lot, Xow_db_layout.Itm_lot, Xow_db_layout.Itm_lot, cfg_tbl.Select_byte_or(Xowe_wiki.Invk_db_mgr, Xodb_mgr_sql.Invk_data_storage_format
, Io_stream_.Tid_gzip), Io_stream_.Tid_gzip, Bool_.Y, Bool_.N)
, Io_stream_tid_.Tid__gzip), Io_stream_tid_.Tid__gzip, Bool_.Y, Bool_.N)
: Cfg_load(cfg_tbl);
}
private static Xowd_core_db_props Cfg_load(Db_cfg_tbl tbl) {
@@ -78,5 +78,5 @@ public class Xowd_core_db_props {
, Cfg_key__hzip_enabled = "hzip_enabled"
, Cfg_key__hzip_mode_is_b256 = "hzip_mode_is_b256"
;
public static final Xowd_core_db_props Test = new Xowd_core_db_props(2, Xow_db_layout.Itm_few, Xow_db_layout.Itm_few, Xow_db_layout.Itm_few, Io_stream_.Tid_raw, Io_stream_.Tid_raw, Bool_.Y, Bool_.Y);
public static final Xowd_core_db_props Test = new Xowd_core_db_props(2, Xow_db_layout.Itm_few, Xow_db_layout.Itm_few, Xow_db_layout.Itm_few, Io_stream_tid_.Tid__raw, Io_stream_tid_.Tid__raw, Bool_.Y, Bool_.Y);
}

2
400_xowa/src/gplx/xowa/wikis/data/tbls/Select_in_cbk.java
View File

@@ -18,9 +18,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.wikis.data.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*;
import gplx.dbs.*;
public interface Select_in_cbk {
int Hash_max();
void Write_sql(Bry_bfr bfr, int idx);
void Read_data(Db_rdr rdr);
int Hash_max();
}
class Select_in_wkr {
private final byte[] sql_bgn;

14
400_xowa/src/gplx/xowa/wikis/data/tbls/Xowd_page_tbl.java
View File

@@ -157,6 +157,20 @@ public class Xowd_page_tbl implements Rls_able {
return rdr.Move_next() ? rdr.Read_int(fld_id) : Xowd_page_itm.Id_null;
} finally {rdr.Rls();}
}
public void Select_in__id(Select_in_cbk cbk) {
int pos = 0;
Bry_bfr bfr = Bry_bfr_.New();
Select_in_wkr wkr = Select_in_wkr.New(bfr, tbl_name, Flds_select_all(), fld_id);
while (true) {
pos = wkr.Make_sql_or_null(bfr, cbk, pos);
if (pos == -1) break;
Db_rdr rdr = conn.Stmt_sql(bfr.To_str_and_clear()).Exec_select__rls_auto();
try {
while (rdr.Move_next())
cbk.Read_data(rdr);
} finally {rdr.Rls();}
}
}
public void Select_in__ttl(Cancelable cancelable, Ordered_hash rv, int ns_id, int bgn, int end) {
Xowd_page_tbl__ttl wkr = new Xowd_page_tbl__ttl();
wkr.Ctor(this, tbl_name, fld_title);

1
400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr.java
View File

@@ -27,7 +27,6 @@ public interface Xodb_load_mgr {
void Load_by_ids (Cancelable cancelable, List_adp rv, int bgn, int end);
boolean Load_by_ttl (Xowd_page_itm rv, Xow_ns ns, byte[] ttl);
void Load_by_ttls (Cancelable cancelable, Ordered_hash rv, boolean fill_idx_fields_only, int bgn, int end);
boolean Load_ctg_v1 (Xoctg_catpage_ctg rv, byte[] ttl);
void Load_ttls_for_all_pages (Cancelable cancelable, List_adp rslt_list, Xowd_page_itm rslt_nxt, Xowd_page_itm rslt_prv, Int_obj_ref rslt_count, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item);
void Load_ttls_for_search_suggest(Cancelable cancelable, List_adp rslt_list, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item);
byte[] Find_random_ttl (Xow_ns ns);

32
400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_sql.java
View File

@@ -58,11 +58,6 @@ public class Xodb_load_mgr_sql implements Xodb_load_mgr {
}
public boolean Load_by_id (Xowd_page_itm rv, int id) {return db_mgr.Core_data_mgr().Tbl__page().Select_by_id(rv, id);}
public void Load_by_ids(Cancelable cancelable, List_adp rv, int bgn, int end) {db_mgr.Core_data_mgr().Tbl__page().Select_in__id(cancelable, false, true, rv, bgn, end);}
public boolean Load_ctg_v1(Xoctg_catpage_ctg rv, byte[] ctg_bry) {
int cat_page_id = db_mgr.Core_data_mgr().Tbl__page().Select_id(Xow_ns_.Tid__category, ctg_bry); if (cat_page_id == Xowd_page_itm.Id_null) return false;
Xowd_category_itm ctg = fsys_mgr.Db__cat_core().Tbl__cat_core().Select(cat_page_id); if (ctg == Xowd_category_itm.Null) return false;
return Ctg_select_v1(db_mgr.Wiki(), db_mgr.Core_data_mgr(), rv, ctg.File_idx(), ctg);
}
public void Load_ttls_for_all_pages(Cancelable cancelable, List_adp rslt_list, Xowd_page_itm rslt_nxt, Xowd_page_itm rslt_prv, Int_obj_ref rslt_count, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item) {
db_mgr.Core_data_mgr().Tbl__page().Select_for_special_all_pages(cancelable, rslt_list, rslt_nxt, rslt_prv, rslt_count, ns, key, max_results, min_page_len, browse_len, include_redirects, fetch_prv_item);
}
@@ -74,31 +69,4 @@ public class Xodb_load_mgr_sql implements Xodb_load_mgr {
public byte[] Find_random_ttl(Xow_ns ns) {return db_mgr.Core_data_mgr().Tbl__page().Select_random(ns);}
public Xodb_page_rdr Get_page_rdr(Xowe_wiki wiki) {return new Xodb_page_rdr__sql(wiki);}
public void Clear() {}
private static boolean Ctg_select_v1(Xowe_wiki wiki, Xow_db_mgr core_data_mgr, Xoctg_catpage_ctg view_ctg, int link_db_id, Xowd_category_itm ctg) {
List_adp link_list = List_adp_.New();
core_data_mgr.Dbs__get_by_id_or_fail(link_db_id).Tbl__cat_link().Select_in(link_list, ctg.Id());
int link_list_len = link_list.Count();
link_list.Sort_by(Xowd_page_itm_sorter.IdAsc);
core_data_mgr.Tbl__page().Select_in__id(Cancelable_.Never, false, true, link_list, 0, link_list_len);
link_list.Sort_by(Xowd_page_itm_sorter.Ns_id_TtlAsc);
boolean rv = false;
for (int i = 0; i < link_list.Count(); i++) {
Xowd_page_itm page = (Xowd_page_itm)link_list.Get_at(i);
if (page.Ns_id() == Int_.Min_value) continue; // HACK: page not found; ignore
byte ctg_tid = Load_ctg_v1_tid(page.Ns_id());
Xoctg_catpage_grp ctg_grp = view_ctg.Grp_by_tid(ctg_tid);
Xoctg_catpage_itm ctg_itm = new Xoctg_catpage_itm(page.Id(), Xoa_ttl.Parse(wiki, page.Ns_id(), page.Ttl_page_db()), page.Ttl_page_db());
ctg_grp.Itms__add(ctg_itm);
rv = true;
}
view_ctg.Make_itms();
return rv;
}
public static byte Load_ctg_v1_tid(int ns_id) {
switch (ns_id) {
case Xow_ns_.Tid__category: return Xoa_ctg_mgr.Tid__subc;
case Xow_ns_.Tid__file: return Xoa_ctg_mgr.Tid__file;
default: return Xoa_ctg_mgr.Tid__page;
}
}
}

55
400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_txt.java
View File

@@ -38,7 +38,7 @@ public class Xodb_load_mgr_txt implements Xodb_load_mgr {
public void Load_page(Xowd_page_itm rv, Xow_ns ns) {Load_page(rv, rv.Text_db_id(), rv.Tdb_row_idx(), ns, false, tmp_xdat_file, tmp_xdat_itm);}
public void Load_page(Xowd_page_itm rv, int txt_fil_idx, int txt_row_idx, Xow_ns ns, boolean timestamp_enabled, Xob_xdat_file xdat_file, Xob_xdat_itm xdat_itm) {
Io_url file = fsys_mgr.Url_ns_fil(Xotdb_dir_info_.Tid_page, ns.Id(), txt_fil_idx);
byte[] bry = gplx.core.ios.streams.Io_stream_rdr_.Load_all(file); int bry_len = bry.length;
byte[] bry = Io_mgr.Instance.LoadFilBry(file); int bry_len = bry.length;
xdat_file.Clear().Parse(bry, bry_len, file).GetAt(xdat_itm, txt_row_idx);
Load_page_parse(rv, bry, bry_len, xdat_itm.Itm_bgn(), xdat_itm.Itm_end(), timestamp_enabled);
}
@@ -236,59 +236,6 @@ public class Xodb_load_mgr_txt implements Xodb_load_mgr {
count.Val_(tmp_count);
return rv;
}
public boolean Load_ctg_v1(Xoctg_catpage_ctg view_ctg, byte[] ctg_ttl) {return Load_ctg_v1_wkr(view_ctg, ctg_ttl, null);}
private boolean Load_ctg_v1_wkr(Xoctg_catpage_ctg view_ctg, byte[] ctg_ttl, Int_obj_ref count_only) {
Xowd_regy_mgr ctg_regy = Get_regy_by_site(Xotdb_dir_info_.Tid_category);
int fil_idx = ctg_regy.Files_find(ctg_ttl);
if (fil_idx == Xowd_regy_mgr.Regy_null) return false; // NOTE: must check for -1, not 0; else defect in which entries in file 0 are ignored; DATE:2013-04-11
Io_url fil = fsys_mgr.Url_site_fil(Xotdb_dir_info_.Tid_category, fil_idx);
Load_xdat_file(Cancelable_.Never, tmp_xdat_file, fil);
tmp_xdat_file.Find(tmp_xdat_itm, ctg_ttl, 0, Byte_ascii.Pipe, true);
if (tmp_xdat_itm.Missing()) return false;
List_adp ctgs = List_adp_.New();
Load_ctg_v1_parse(ctgs, wiki.Appe().Usr_dlg(), tmp_xdat_itm.Itm_bry());
ctgs.Sort_by(Xowd_page_itm_sorter.IdAsc);
this.Load_by_ids(Cancelable_.Never, ctgs, 0, ctgs.Count());
ctgs.Sort_by(Xowd_page_itm_sorter.Ns_id_TtlAsc);
int ctgs_len = ctgs.Count(); if (ctgs_len == 0) return false;
if (count_only != null) { // Ctg_count specificed
count_only.Val_(ctgs_len);
return true;
}
for (int i = 0; i < ctgs_len; i++) {
Xowd_page_itm itm = (Xowd_page_itm)ctgs.Get_at(i);
byte itm_tid = Xodb_load_mgr_sql.Load_ctg_v1_tid(itm.Ns_id());
Xoctg_catpage_itm sub = new Xoctg_catpage_itm(itm.Id(), Xoa_ttl.Parse(wiki, itm.Ns_id(), itm.Ttl_page_db()), itm.Ttl_page_db());
view_ctg.Grp_by_tid(itm_tid).Itms__add(sub);
}
view_ctg.Make_itms();
return true;
}
private static void Load_ctg_v1_parse(List_adp rv, Gfo_usr_dlg usr_dlg, byte[] ary) {
int aryLen = ary.length;
int pos = Bry_find_.Find_fwd(ary, Byte_ascii.Pipe, 0, aryLen);
int rowCount = (aryLen - pos + 1) / (Base85_.Len_int + gplx.xowa.apps.progs.Xoa_prog_mgr.Len_dlm_fld);
rv.Clear();
boolean garbage = false;
for (int i = 0; i < rowCount; i++) {
Xowd_page_itm row = new Xowd_page_itm();
rv.Add(row);
if (garbage) continue;
int bgn = pos + 1 + (i * (Base85_.Len_int + gplx.xowa.apps.progs.Xoa_prog_mgr.Len_dlm_fld));
try {
int id = Base85_.To_int_by_bry(ary, bgn, bgn + Base85_.Len_int - 1);
if (id < 0) throw Err_.new_wo_type("invalid id", "id", id);
row.Id_(id);
}
catch (Exception e) {
row.Id_(0);
usr_dlg.Warn_many("xowa.ttl_request", "parse", "failed to parse row: row=~{0} err=~{1}", String_.new_u8(ary), Err_.Message_lang(e));
garbage = true;
continue; // NOTE: must break b/c rest of ids are garbage
}
}
}
public static boolean Load_page_or_false(Xowd_page_itm page, Xob_xdat_itm xdat, int ns_id) {
byte[] src = xdat.Src(); int itm_end = xdat.Itm_end();
int bgn = xdat.Itm_bgn();

2
400_xowa/src/gplx/xowa/wikis/dbs/Xodb_mgr_sql.java
View File

@@ -45,7 +45,7 @@ public class Xodb_mgr_sql implements Xodb_mgr, Gfo_invk {
core_data_mgr.Tbl__cfg().Insert_byte(grp, key, category_version);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_data_storage_format)) return Io_stream_.Obsolete_to_str(core_data_mgr.Props().Zip_tid_text());
if (ctx.Match(k, Invk_data_storage_format)) return Io_stream_tid_.Obsolete_to_str(core_data_mgr.Props().Zip_tid_text());
else if (ctx.Match(k, Invk_data_storage_format_)) {} // SERIALIZED:000.sqlite3|xowa_cfg; ignore; read from Xow_db_props
else if (ctx.Match(k, Invk_category_version)) return category_version;
else if (ctx.Match(k, Invk_category_version_)) category_version = m.ReadByte("v");

6
400_xowa/src/gplx/xowa/wikis/dbs/Xodb_mgr_txt.java
View File

@@ -27,7 +27,7 @@ public class Xodb_mgr_txt implements Xodb_mgr {
} private Xowe_wiki wiki;
public byte Tid() {return Tid_txt;} public static final byte Tid_txt = 0;
public String Tid_name() {return "xdat";}
public byte Data_storage_format() {return data_storage_format;} public void Data_storage_format_(byte v) {data_storage_format = v;} private byte data_storage_format = gplx.core.ios.streams.Io_stream_.Tid_raw;
public byte Data_storage_format() {return data_storage_format;} public void Data_storage_format_(byte v) {data_storage_format = v;} private byte data_storage_format = gplx.core.ios.streams.Io_stream_tid_.Tid__raw;
public Xodb_load_mgr Load_mgr() {return load_mgr;} private Xodb_load_mgr_txt load_mgr;
public Xodb_save_mgr Save_mgr() {return save_mgr;} private Xodb_save_mgr_txt save_mgr;
public DateAdp Dump_date_query() {
@@ -46,8 +46,8 @@ public class Xodb_mgr_txt implements Xodb_mgr {
public byte Search_version() {return gplx.xowa.addons.wikis.searchs.specials.Srch_special_page.Version_2;}
public void Search_version_refresh() {throw Err_.new_unimplemented();}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_data_storage_format)) return Io_stream_.Obsolete_to_str(data_storage_format);
else if (ctx.Match(k, Invk_data_storage_format_)) data_storage_format = Io_stream_.Obsolete_to_tid(m.ReadStr("v"));
if (ctx.Match(k, Invk_data_storage_format)) return Io_stream_tid_.Obsolete_to_str(data_storage_format);
else if (ctx.Match(k, Invk_data_storage_format_)) data_storage_format = Io_stream_tid_.Obsolete_to_tid(m.ReadStr("v"));
else if (ctx.Match(k, Invk_category_version)) return this.Category_version();
else if (ctx.Match(k, Invk_category_version_)) category_version = m.ReadByte("v");
else if (ctx.Match(k, Invk_search_version)) return this.Search_version();

2
400_xowa/src/gplx/xowa/wikis/dbs/Xodb_page_rdr__tdb.java
View File

@@ -49,7 +49,7 @@ public class Xodb_page_rdr__tdb implements Xodb_page_rdr {
continue;
}
this.cur_file = cur_file_ary[cur_file_idx];
byte[] bry = Io_stream_rdr_.Load_all(cur_file);
byte[] bry = Io_mgr.Instance.LoadFilBry(cur_file);
cur_xdat_file.Clear().Parse(bry, bry.length, cur_file);
this.cur_xdat_len = cur_xdat_file.Count();
this.cur_xdat_idx = -1;

2
400_xowa/src/gplx/xowa/wikis/dbs/Xodb_save_mgr_txt.java
View File

@@ -43,7 +43,7 @@ public class Xodb_save_mgr_txt implements Xodb_save_mgr {
int ns_id = ttl.Ns().Id();
Xotdb_page_itm_.Txt_page_save(tmp, page_id, Datetime_now.Get(), ttl_bry, text, true);
Io_url page_rdr_url = fsys_mgr.Url_ns_fil(Xotdb_dir_info_.Tid_page, ns_id, fil_idx);
byte[] page_rdr_bry = gplx.core.ios.streams.Io_stream_rdr_.Load_all(page_rdr_url);
byte[] page_rdr_bry = Io_mgr.Instance.LoadFilBry(page_rdr_url);
Xob_xdat_file page_rdr = new Xob_xdat_file();
if (Bry_.Len_gt_0(page_rdr_bry)) page_rdr.Parse(page_rdr_bry, page_rdr_bry.length, page_rdr_url);
int row_idx = page_rdr.Count();

18
400_xowa/src/gplx/xowa/wikis/tdbs/Xotdb_dir_info.java
View File

@@ -30,24 +30,24 @@ public class Xotdb_dir_info {
ext_bry = Wtr_ext(v);
ext_str = String_.new_a7(ext_bry);
return this;
} byte ext_tid = gplx.core.ios.streams.Io_stream_.Tid_raw;
} byte ext_tid = gplx.core.ios.streams.Io_stream_tid_.Tid__raw;
public static final String Wtr_xdat_str = ".xdat", Wtr_zip_str = ".zip", Wtr_gz_str = ".gz", Wtr_bz2_str = ".bz2";
public static final byte[] Wtr_xdat_bry = Bry_.new_a7(Wtr_xdat_str), Wtr_zip_bry = Bry_.new_a7(Wtr_zip_str), Wtr_gz_bry = Bry_.new_a7(Wtr_gz_str), Wtr_bz2_bry = Bry_.new_a7(Wtr_bz2_str);
public static String Wtr_dir(byte v) {
switch (v) {
case gplx.core.ios.streams.Io_stream_.Tid_raw : return "";
case gplx.core.ios.streams.Io_stream_.Tid_zip : return "_zip";
case gplx.core.ios.streams.Io_stream_.Tid_gzip : return "_gz";
case gplx.core.ios.streams.Io_stream_.Tid_bzip2 : return "_bz2";
case gplx.core.ios.streams.Io_stream_tid_.Tid__raw : return "";
case gplx.core.ios.streams.Io_stream_tid_.Tid__zip : return "_zip";
case gplx.core.ios.streams.Io_stream_tid_.Tid__gzip : return "_gz";
case gplx.core.ios.streams.Io_stream_tid_.Tid__bzip2 : return "_bz2";
default : throw Err_.new_unhandled(v);
}
}
public static byte[] Wtr_ext(byte v) {
switch (v) {
case gplx.core.ios.streams.Io_stream_.Tid_raw : return Wtr_xdat_bry;
case gplx.core.ios.streams.Io_stream_.Tid_zip : return Wtr_zip_bry;
case gplx.core.ios.streams.Io_stream_.Tid_gzip : return Wtr_gz_bry;
case gplx.core.ios.streams.Io_stream_.Tid_bzip2 : return Wtr_bz2_bry;
case gplx.core.ios.streams.Io_stream_tid_.Tid__raw : return Wtr_xdat_bry;
case gplx.core.ios.streams.Io_stream_tid_.Tid__zip : return Wtr_zip_bry;
case gplx.core.ios.streams.Io_stream_tid_.Tid__gzip : return Wtr_gz_bry;
case gplx.core.ios.streams.Io_stream_tid_.Tid__bzip2 : return Wtr_bz2_bry;
default : throw Err_.new_unhandled(v);
}
}

10
400_xowa/src/gplx/xowa/wikis/tdbs/Xotdb_fsys_mgr.java
View File

@@ -56,14 +56,14 @@ public class Xotdb_fsys_mgr {
private static void Scan_dirs_zip(Xotdb_fsys_mgr fsys_mgr, byte id) {
Io_url[] dirs = Io_mgr.Instance.QueryDir_args(fsys_mgr.Ns_dir().GenSubDir_nest("000")).FilPath_("*page*").DirOnly_().Recur_(false).ExecAsUrlAry();
int len = dirs.length;
byte tid = gplx.core.ios.streams.Io_stream_.Tid_raw; // needed for Xoa_xowa_exec_tst
byte tid = gplx.core.ios.streams.Io_stream_tid_.Tid__raw; // needed for Xoa_xowa_exec_tst
for (int i = 0; i < len; i++) {
Io_url dir = dirs[i];
String dir_name = dir.NameOnly();
if (String_.Eq(dir_name, "page")) {tid = gplx.core.ios.streams.Io_stream_.Tid_raw; break;}
else if (String_.Eq(dir_name, "page_zip")) tid = gplx.core.ios.streams.Io_stream_.Tid_zip;
else if (String_.Eq(dir_name, "page_gz")) tid = gplx.core.ios.streams.Io_stream_.Tid_gzip;
else if (String_.Eq(dir_name, "page_bz2")) tid = gplx.core.ios.streams.Io_stream_.Tid_bzip2;
if (String_.Eq(dir_name, "page")) {tid = gplx.core.ios.streams.Io_stream_tid_.Tid__raw; break;}
else if (String_.Eq(dir_name, "page_zip")) tid = gplx.core.ios.streams.Io_stream_tid_.Tid__zip;
else if (String_.Eq(dir_name, "page_gz")) tid = gplx.core.ios.streams.Io_stream_tid_.Tid__gzip;
else if (String_.Eq(dir_name, "page_bz2")) tid = gplx.core.ios.streams.Io_stream_tid_.Tid__bzip2;
}
fsys_mgr.Tdb_dir_regy()[id].Ext_tid_(tid);
}

2
400_xowa/src/gplx/xowa/wikis/tdbs/Xotdb_fsys_mgr_tst.java
View File

@@ -31,7 +31,7 @@ class Xow_fsys_mgr_fxt {
wiki = Xoa_app_fxt.Make__wiki__edit(app);
}
Xoae_app app; Xowe_wiki wiki;
public Xow_fsys_mgr_fxt Zip_(byte tid, boolean v) {wiki.Tdb_fsys_mgr().Tdb_dir_regy()[tid].Ext_tid_(v ? gplx.core.ios.streams.Io_stream_.Tid_zip : gplx.core.ios.streams.Io_stream_.Tid_raw); return this;}
public Xow_fsys_mgr_fxt Zip_(byte tid, boolean v) {wiki.Tdb_fsys_mgr().Tdb_dir_regy()[tid].Ext_tid_(v ? gplx.core.ios.streams.Io_stream_tid_.Tid__zip : gplx.core.ios.streams.Io_stream_tid_.Tid__raw); return this;}
public void Url_ns_fil(byte tid, int ns_id, int fil_idx, String expd) {
Tfds.Eq(expd, wiki.Tdb_fsys_mgr().Url_ns_fil(tid, ns_id, fil_idx).Raw());
}

10
400_xowa/src/gplx/xowa/wikis/tdbs/hives/Xob_hive_mgr.java
View File

@@ -37,7 +37,7 @@ public class Xob_hive_mgr {
}
Xob_xdat_file xdat_load_(Xob_xdat_file xdat, byte dir_tid, Xow_ns ns, int fil_idx) {
Io_url xdat_url = fsys_mgr.Url_ns_fil(dir_tid, ns.Id(), fil_idx);
byte[] xdat_bry = gplx.core.ios.streams.Io_stream_rdr_.Load_all(xdat_url);
byte[] xdat_bry = Io_mgr.Instance.LoadFilBry(xdat_url);
xdat.Parse(xdat_bry, xdat_bry.length, xdat_url);
return xdat;
}
@@ -115,7 +115,7 @@ public class Xob_hive_mgr {
byte dir_tid = Xotdb_dir_info_.Tid_ttl;
int xdat_idx = Regy__find_file_ns(key, dir_tid, ns.Num_str());
Io_url xdat_url = fsys_mgr.Url_ns_fil(dir_tid, ns.Id(), xdat_idx);
byte[] xdat_bry = gplx.core.ios.streams.Io_stream_rdr_.Load_all(xdat_url);
byte[] xdat_bry = Io_mgr.Instance.LoadFilBry(xdat_url);
xdat.Parse(xdat_bry, xdat_bry.length, xdat_url);
xdat.Find(xdat_itm, key, Xotdb_page_itm_.Txt_ttl_pos, Byte_ascii.Tab, false);
Find_nearby_add_fwd(list, dir_tid, ns, include_redirects, count, xdat_idx, xdat_itm.Itm_idx());
@@ -132,7 +132,7 @@ public class Xob_hive_mgr {
// while (loop) {
// if (fil_idx == -1) break;
// Io_url xdat_url = fsys_mgr.Url_ns_fil(dir_tid, ns.Id(), fil_idx);
// byte[] xdat_bry = gplx.core.ios.streams.Io_stream_rdr_.Load_all(xdat_url);
// byte[] xdat_bry = Io_mgr.Instance.LoadFilBry(xdat_url);
// xdat.Parse(xdat_bry, xdat_bry.length, xdat_url);
// int row_idx = first && row_bgn != -1 ? row_bgn : xdat.Count() - 1;
// first = false;
@@ -155,7 +155,7 @@ public class Xob_hive_mgr {
while (loop) {
if (fil_idx == regy_len) break;
Io_url xdat_url = fsys_mgr.Url_ns_fil(dir_tid, ns.Id(), fil_idx);
byte[] xdat_bry = gplx.core.ios.streams.Io_stream_rdr_.Load_all(xdat_url);
byte[] xdat_bry = Io_mgr.Instance.LoadFilBry(xdat_url);
xdat.Parse(xdat_bry, xdat_bry.length, xdat_url);
int row_idx = first ? row_bgn : 0;
int rows_len = xdat.Count();
@@ -186,7 +186,7 @@ public class Xob_hive_mgr {
int Regy__find_file_by_url(byte[] key, Io_url regy_url) {regy.Init(regy_url); return regy.Files_find(key);} private Xowd_regy_mgr regy = new Xowd_regy_mgr();
private void Xdat__create_row(byte dir_tid, byte[] key, byte[] row, int xdat_idx) {
Io_url xdat_url = fsys_mgr.Url_site_fil(dir_tid, xdat_idx);
byte[] xdat_bry = gplx.core.ios.streams.Io_stream_rdr_.Load_all(xdat_url);
byte[] xdat_bry = Io_mgr.Instance.LoadFilBry(xdat_url);
Xob_xdat_file xdat_fil = new Xob_xdat_file();
if (xdat_bry.length > 0) // if file is not empty, load it and parse it
xdat_fil.Parse(xdat_bry, xdat_bry.length, xdat_url);

2
400_xowa/src/gplx/xowa/wikis/tdbs/xdats/Xob_xdat_file.java
View File

@@ -103,7 +103,7 @@ public class Xob_xdat_file {
public void Save(Io_url url) {
Bry_bfr bfr = Bry_bfr_.New();
Srl_save_bry(bfr);
Io_stream_wtr wtr = Io_stream_wtr_.new_by_url_(url);
Io_stream_wtr wtr = Io_stream_wtr_.New_by_url(url);
try {
wtr.Open();
wtr.Write(bfr.Bfr(), 0, bfr.Len());

4
400_xowa/src/gplx/xowa/wikis/tdbs/xdats/Xob_xdat_file_wtr.java
View File

@@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.wikis.tdbs.xdats; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*;
import gplx.core.ios.*; import gplx.core.ios.streams.*; import gplx.core.encoders.*; import gplx.xowa.wikis.tdbs.*;
public class Xob_xdat_file_wtr {
public static Xob_xdat_file_wtr new_file_(int fil_max, Io_url root_dir) {return new Xob_xdat_file_wtr(fil_max, root_dir, Io_stream_.Tid_raw);}
public static Xob_xdat_file_wtr new_file_(int fil_max, Io_url root_dir) {return new Xob_xdat_file_wtr(fil_max, root_dir, Io_stream_tid_.Tid__raw);}
public static Xob_xdat_file_wtr new_by_tid_(int fil_max, Io_url root_dir, byte dir_tid, byte tid) {return new Xob_xdat_file_wtr(fil_max, root_dir.GenSubDir(Xotdb_dir_info_.Tid_name(dir_tid) + Xotdb_dir_info.Wtr_dir(tid)), tid);}
Xob_xdat_file_wtr(int fil_max, Io_url root_dir, byte wtr_tid) {
this.fil_max = fil_max;
@@ -27,7 +27,7 @@ public class Xob_xdat_file_wtr {
bfr = Bry_bfr_.New_w_size(fil_max);
idx = new int[fil_max / 8]; // ASSUME: any given row must at least be 8 bytes long
Url_gen(fil_idx); // set 1st url
wtr = Io_stream_wtr_.new_by_tid_(wtr_tid);
wtr = Io_stream_wtr_.New_by_tid(wtr_tid);
} int fil_max; Io_stream_wtr wtr; byte[] fil_ext;
public int Fil_idx() {return fil_idx;} public Xob_xdat_file_wtr Fil_idx_(int v) {fil_idx = v; return this;} private int fil_idx;
public int Ns_ord_idx() {return ns_ord_idx;} public Xob_xdat_file_wtr Ns_ord_idx_(int v) {ns_ord_idx = v; return this;} private int ns_ord_idx; // NOTE: optional; needed for page cmd which will flush all wtrs, but needs ns_idx for stats

4
400_xowa/src/gplx/xowa/wikis/xwikis/Xow_xwiki_mgr.java
View File

@@ -83,8 +83,8 @@ public class Xow_xwiki_mgr {
default: return tid;
}
}
public static byte[] Get_domain_from_url(Gfo_url_parser url_parser, Gfo_url url, byte[] url_fmt) { // extract "commons.wikimedia.org" from "http://commons.wikimedia.org/wiki/Category:~{0}"
url_parser.Parse(url, url_fmt, 0, url_fmt.length);
public static byte[] Get_domain_from_url(Gfo_url_parser url_parser, byte[] url_fmt) { // extract "commons.wikimedia.org" from "http://commons.wikimedia.org/wiki/Category:~{0}"
Gfo_url url = url_parser.Parse(url_fmt, 0, url_fmt.length);
return url.Segs__get_at_1st();
}
public static byte[] Bld_url_fmt(byte[] domain_bry) {return Bry_.Add(gplx.core.net.Gfo_protocol_itm.Itm_https.Text_bry(), domain_bry, Bry__url_fmt_end);}

4
400_xowa/src/gplx/xowa/wikis/xwikis/bldrs/Xow_xwiki_itm_bldr.java
View File

@@ -22,10 +22,10 @@ import gplx.xowa.langs.*;
import gplx.xowa.wikis.domains.*;
public class Xow_xwiki_itm_bldr {
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private final Gfo_url_parser url_parser = new Gfo_url_parser(); private final Gfo_url url = new Gfo_url();
private final Gfo_url_parser url_parser = new Gfo_url_parser();
public Xow_xwiki_itm Bld_mw(Xow_domain_itm cur_domain, byte[] key, byte[] mw_url, byte[] domain_name) {return Bld_xo(cur_domain, key, Gfs_php_converter.To_gfs(tmp_bfr, mw_url), domain_name);} // EX: "//commons.wikimedia.org/wiki/Category:$1" -> "//commons.wikimedia.org/wiki/Category:~{0}"
public Xow_xwiki_itm Bld_xo(Xow_domain_itm cur_domain, byte[] key, byte[] xo_url, byte[] domain_name) {
byte[] domain_bry = Xow_xwiki_mgr.Get_domain_from_url(url_parser, url, xo_url);
byte[] domain_bry = Xow_xwiki_mgr.Get_domain_from_url(url_parser, xo_url);
Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry);
Xol_lang_stub lang_itm = Xol_lang_stub_.Get_by_key_or_null(domain_itm.Lang_actl_key());
int lang_id = lang_itm == null ? Xol_lang_stub_.Id__unknown : lang_itm.Id();

4
400_xowa/src/gplx/xowa/wikis/xwikis/parsers/Xow_xwiki_itm_parser.java
View File

@@ -23,7 +23,7 @@ import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.xwikis.bldrs.*;
public class Xow_xwiki_itm_parser extends Dsv_wkr_base {
private Xow_domain_itm owner_domain_itm;
private int cur_tid = -1; private byte[] cur_fld1, cur_fld2, cur_fld3;
private final Gfo_url_parser url_parser = new Gfo_url_parser(); private final Gfo_url url = new Gfo_url();
private final Gfo_url_parser url_parser = new Gfo_url_parser();
public Ordered_hash Xwiki_list() {return xwiki_list;} private final Ordered_hash xwiki_list = Ordered_hash_.New();
@Override public Dsv_fld_parser[] Fld_parsers() {return new Dsv_fld_parser[] {Dsv_fld_parser_.Bry_parser, Dsv_fld_parser_.Bry_parser, Dsv_fld_parser_.Bry_parser, Dsv_fld_parser_.Bry_parser};}
@Override public boolean Write_bry(Dsv_tbl_parser parser, int fld_idx, byte[] src, int bgn, int end) {
@@ -46,7 +46,7 @@ public class Xow_xwiki_itm_parser extends Dsv_wkr_base {
case Tid__manual: // EX: "0|domz|http://www.dmoz.org/~{0}|DMOZ"
xwiki_is_mw = false;
url_fmt = cur_fld2;
domain_bry = Xow_xwiki_mgr.Get_domain_from_url(url_parser, url, url_fmt);
domain_bry = Xow_xwiki_mgr.Get_domain_from_url(url_parser, url_fmt);
break;
case Tid__mw_domain: // EX: "1|w|en.wikipedia.org"
domain_bry = cur_fld2;