Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.3.4.1'

Browse Source
This commit is contained in:
gnosygnu
2016-03-27 23:44:59 -04:00
parent de67253a9c
commit baaef32df2
903 changed files with 13339 additions and 8695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
400_xowa/src/gplx/core/brys/Bit_heap_rdr_tst.java
View File

@@ -29,10 +29,8 @@ public class Bit_heap_rdr_tst {
fxt.Test__get_bool(Bool_.Y).Test__cur( 3, 6, 0);
fxt.Test__get_bool(Bool_.Y).Test__cur( 1, 7, 0);
fxt.Test__get_bool(Bool_.Y).Test__cur( 0, 0, 1);
fxt.Load(0);
fxt.Test__get_bool(Bool_.N).Test__cur( 0, 1, 0);
fxt.Load(6);
fxt.Test__get_bool(Bool_.N).Test__cur( 3, 1, 0);
fxt.Test__get_bool(Bool_.Y).Test__cur( 1, 2, 0);
@@ -40,11 +38,9 @@ public class Bit_heap_rdr_tst {
}
@Test public void Get_byte() {
fxt.Load(255).Test__get_byte(2, 3).Test__cur(63, 2, 0);
fxt.Load(255, 3);
fxt.Test__get_byte(7, 127);
fxt.Test__get_byte(3, 7);
fxt.Load(10);
fxt.Test__get_bool(false);
fxt.Test__get_byte(3, 5);

2
400_xowa/src/gplx/core/brys/Bit_heap_wtr_tst.java
View File

@@ -37,10 +37,8 @@ public class Bit_heap_wtr_tst {
@Test public void Add_byte_digits() {
fxt.Clear().Add_byte(4, 15).Test__vals(4, 15);
fxt.Clear().Add_byte(7, 127).Add_byte(2, 3).Test__vals(1, 1, 255);
fxt.Clear().Add_byte(3, 7).Add_byte(3, 7).Test__vals(6, 63);
fxt.Clear().Add_byte(6, 63).Add_byte(3, 7).Test__vals(1, 1, 255);
fxt.Clear().Add_byte(3, 6).Add_byte(3, 6).Test__vals(6, 54);
}
@Test public void Add_int_hzip() {

13
400_xowa/src/gplx/core/enums/Gfo_enum_grp.java
View File

@@ -39,16 +39,3 @@ class Gfo_enum_itm {
public int Sort() {return sort;} private int sort;
public String Xtn() {return xtn;} private String xtn;
}
/*
enum_grps
grp_guid,grp_key,grp_int,grp_name,grp_sort,grp_xtn
0-1-2-3,xowa.wiki,0,wiki,,
enum_itms
grp_int,itm_guid,itm_key,itm_int,itm_name,itm_sort,itm_xtn
1,0-1-2-3,0,en.wikipedia.org,0,enwiki,0,''
class Gfo_enum_mgr {
// public Gui
}
*/

32
400_xowa/src/gplx/core/lists/Binary_search_.java
View File

@@ -49,4 +49,36 @@ public class Binary_search_ {
}
return Int_.Min_value; // should only occur if (a) ary's 0th slot is not ""; or (b) some unknown error
}
public static int Search(List_adp__getable list, int list_len, CompareAble val) {
if (list_len == 1) return 0;
int interval = list_len / 2;
int pos = interval - List_adp_.Base1;
int pos_last = list_len - 1;
int pos_prv = -1;
int loop_count = 0;
while (loop_count++ < 32) { // 32 bit integer
CompareAble lo = (CompareAble)list.Get_at(pos);
CompareAble hi = pos + 1 == list_len ? null : (CompareAble)list.Get_at(pos + 1);
int adj = 0;
int lo_comp = val.compareTo(lo);
if (lo_comp == CompareAble_.Less) // val is < lo; search slots below
adj = -1;
else {
if (hi == null) return pos; // hi is null when at last slot in ary
int hi_comp = val.compareTo(hi);
if (hi_comp == CompareAble_.More) // val is > hi; search slots above
adj = 1;
else
return pos; // val is > lo and < hi; return slot
}
interval /= 2;
if (interval == 0) interval = 1; // do not allow 0 intervals; pos must always change;
pos += (interval * adj);
if (pos == 0 && pos_prv == 0) break; // NOTE: this will only happen when 1st member is not ""
if (pos < 0) pos = 0;
else if (pos > pos_last) pos = pos_last;
pos_prv = pos;
}
return Int_.Min_value; // should only occur if (a) ary's 0th slot is not ""; or (b) some unknown error
}
}

4
400_xowa/src/gplx/core/lists/StatRng_tst.java
View File

@@ -33,7 +33,7 @@ public class StatRng_tst {
//@Test public void Basic() {fx.Test_parse_tmpl_str_test("{{#switch:{{{1}}}|a|b|c=1|d=2}}", "{{test|a}}", "1");}
}
/*
public class Pf_func_switch_tst {
// Mwl_parser_fxt fx = new Mwl_parser_fxt(); Pf_func_lang_rsc rsc = Pf_func_lang_rsc.Instance;
public class Pf_func_switch_tst {
// Mwl_parser_fxt fx = new Mwl_parser_fxt(); Pf_func_lang_rsc rsc = Pf_func_lang_rsc.Instance;
*/

2
400_xowa/src/gplx/core/net/Gfo_url_parser.java
View File

@@ -74,7 +74,6 @@ public class Gfo_url_parser {
protocol_bry = anch = null;
path_bgn = qarg_key_bgn = qarg_val_bgn = anch_bgn = anch_nth_bgn = -1;
segs_ary.Clear(); qargs.Clear();
int pos = src_bgn;
Object protocol_obj = protocols.Match_bgn(src, src_bgn, src_end);
pos = protocols.Match_pos();
@@ -86,7 +85,6 @@ public class Gfo_url_parser {
this.protocol_tid = ((Byte_obj_val)protocol_obj).Val();
this.protocol_bry = Make_bry(src_bgn, pos);
}
area = Area__path;
path_bgn = pos;
while (true) {

6
400_xowa/src/gplx/core/tests/Tst_mgr.java
View File

@@ -20,10 +20,10 @@ import gplx.core.strings.*;
public class Tst_mgr {
public Tst_mgr ThrowError_n_() {throwError = false; return this;} private boolean throwError = true;
public List_adp Results() {return results;} List_adp results = List_adp_.new_();
public KeyValHash Vars() {return vars;} KeyValHash vars = KeyValHash.new_();
public Object Vars_get_by_key(String key) {return vars.FetchValOr(key, null);}
public Keyval_hash Vars() {return vars;} Keyval_hash vars = new Keyval_hash();
public Object Vars_get_by_key(String key) {return vars.Get_val_or(key, null);}
public String Vars_get_bry_as_str(String key, int bgn, int end) {
byte[] bry = (byte[])vars.FetchValOr(key, null); if (bry == null) return String_.Empty;
byte[] bry = (byte[])vars.Get_val_or(key, null); if (bry == null) return String_.Empty;
if (bgn < 0 || end > bry.length || end < bgn || end < 0) return "<<OUT OF BOUNDS>>";
return String_.new_u8(Bry_.Mid(bry, bgn, end));
}

4
400_xowa/src/gplx/core/threads/Gfo_thread_cmd_replace.java
View File

@@ -42,7 +42,7 @@ public class Gfo_thread_cmd_replace implements Gfo_thread_cmd {
String raw = Io_mgr.Instance.LoadFilStr(fil);
int pairs_len = pairs.Count();
for (int i = 0; i < pairs_len; i++) {
KeyVal kv = (KeyVal)pairs.Get_at(i);
Keyval kv = (Keyval)pairs.Get_at(i);
raw = String_.Replace(raw, kv.Key(), kv.Val_to_str_or_null());
}
Io_mgr.Instance.SaveFilStr(fil, raw);
@@ -53,7 +53,7 @@ public class Gfo_thread_cmd_replace implements Gfo_thread_cmd {
if (ctx.Match(k, Invk_async_bgn)) Exec_find_replace();
else if (ctx.Match(k, Invk_owner)) return owner;
else if (ctx.Match(k, Invk_fil_)) fil = Bry_fmtr_eval_mgr_.Eval_url(url_eval_mgr, m.ReadBry("v"));
else if (ctx.Match(k, Invk_add)) pairs.Add(KeyVal_.new_(m.ReadStr("find"), m.ReadStr("replace")));
else if (ctx.Match(k, Invk_add)) pairs.Add(Keyval_.new_(m.ReadStr("find"), m.ReadStr("replace")));
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_async_bgn = "async_bgn", Invk_owner = "owner", Invk_fil_ = "fil_", Invk_add = "add";

54
400_xowa/src/gplx/dbs/Db_attach_cmd.java
View File

@@ -1,54 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.dbs; import gplx.*;
import gplx.dbs.engines.sqlite.*;
public class Db_attach_cmd {
private final boolean diff_db;
private final Db_conn conn; private final String attach_name; private final Io_url attach_url;
private final List_adp sql_list = List_adp_.new_();
Db_attach_cmd(Db_conn conn, String attach_name, Io_url attach_url) {
this.conn = conn; this.attach_name = attach_name; this.attach_url = attach_url;
Sqlite_conn_info conn_info = (Sqlite_conn_info)conn.Conn_info();
this.diff_db = !String_.Eq(conn_info.Url().Raw(), attach_url.Raw());
}
public Db_attach_cmd Add_fmt(String msg, String sql_fmt, Object... sql_args) {
String sql = String_.Format(sql_fmt, sql_args);
sql = String_.Replace(sql, "<attach_db>", diff_db ? attach_name + "." : ""); // replace <attach> with either "attach_db." or "";
sql_list.Add(new Db_exec_sql_by_attach_itm(msg, sql));
return this;
}
public void Exec() {
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Instance;
if (diff_db) conn.Env_db_attach(attach_name, attach_url);
conn.Txn_bgn(attach_name); // NOTE: BEGIN TRAN must occur after ATTACH else sqlite will throw error
int len = sql_list.Count();
for (int i = 0; i < len; ++i) {
Db_exec_sql_by_attach_itm itm = (Db_exec_sql_by_attach_itm)sql_list.Get_at(i);
usr_dlg.Plog_many("", "", itm.Msg());
conn.Exec_sql(itm.Sql());
}
conn.Txn_end();
if (diff_db) conn.Env_db_detach(attach_name);
}
public static Db_attach_cmd new_(Db_conn conn, String attach_name, Io_url attach_url) {return new Db_attach_cmd(conn, attach_name, attach_url);}
}
class Db_exec_sql_by_attach_itm {
public Db_exec_sql_by_attach_itm(String msg, String sql) {this.msg = msg; this.sql = sql;}
public String Msg() {return msg;} private final String msg;
public String Sql() {return sql;} private final String sql;
}

40
400_xowa/src/gplx/dbs/Db_attach_rdr.java
View File

@@ -1,40 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.dbs; import gplx.*;
import gplx.dbs.engines.sqlite.*;
public class Db_attach_rdr {
private final boolean diff_db;
private final Db_conn conn; private final String attach_name; private final Io_url attach_url;
public Db_attach_rdr(Db_conn conn, String attach_name, Io_url attach_url) {
this.conn = conn; this.attach_name = attach_name; this.attach_url = attach_url;
Sqlite_conn_info conn_info = (Sqlite_conn_info)conn.Conn_info();
this.diff_db = !String_.Eq(conn_info.Url().Raw(), attach_url.Raw());
}
public void Attach() {
try {
if (diff_db) conn.Env_db_attach(attach_name, attach_url);
} catch (Exception e) {Err_.Noop(e); Gfo_usr_dlg_.Instance.Warn_many("", "", "db:failed to attach db; name=~{0} url=~{1}", attach_name, attach_url.Raw());}
}
public Db_rdr Exec_as_rdr(String sql) {
sql = String_.Replace(sql, "<attach_db>", diff_db ? attach_name + "." : ""); // replace <attach> with either "attach_db." or "";
return conn.Exec_sql_as_rdr_v2(sql);
}
public void Detach() {
if (diff_db) conn.Env_db_detach(attach_name);
}
}

2
400_xowa/src/gplx/dbs/cfgs/Db_cfg_itm.java
View File

@@ -36,7 +36,6 @@ public class Db_cfg_itm {
public String To_str() {Fail_if_null(); return val;}
private void Fail_if_null() {if (val == null) throw Err_.new_wo_type("cfg.val is empty", "grp", grp, "key", key); }
private Err err_parse(Exception e, String type) {return Err_.new_wo_type("cfg.val is not parseable", "grp", grp, "key", key, "val", val, "type", type).Trace_ignore_add_1_();}
private static final String Grp_none = "";
public static Db_cfg_itm new_str (String key, String val) {return new Db_cfg_itm(Grp_none , key, val);}
public static Db_cfg_itm new_str (String grp, String key, String val) {return new Db_cfg_itm(grp , key, val);}
@@ -54,6 +53,5 @@ public class Db_cfg_itm {
public static Db_cfg_itm new_DateAdp (String grp, String key, DateAdp val) {return new Db_cfg_itm(grp , key, val.XtoStr_fmt_yyyyMMdd_HHmmss());}
public static Db_cfg_itm new_guid (String key, Guid_adp val) {return new Db_cfg_itm(Grp_none , key, val.To_str());}
public static Db_cfg_itm new_guid (String grp, String key, Guid_adp val) {return new Db_cfg_itm(grp , key, val.To_str());}
public static final Db_cfg_itm Empty = new Db_cfg_itm("empty", "empty", null);
}

4
400_xowa/src/gplx/dbs/cfgs/Db_cfg_tbl.java
View File

@@ -34,7 +34,7 @@ public class Db_cfg_tbl implements Rls_able {
stmt_update = Db_stmt_.Rls(stmt_update);
stmt_select = Db_stmt_.Rls(stmt_select);
}
public void Create_tbl() {conn.Ddl_create_tbl(Dbmeta_tbl_itm.New(tbl_name, flds, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "main", fld_grp, fld_key, fld_val)));}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "main", fld_grp, fld_key, fld_val)));}
public void Delete_val(String grp, String key) {conn.Stmt_delete(tbl_name, fld_grp, fld_key).Crt_str(fld_grp, grp).Crt_str(fld_key, key).Exec_delete();}
public void Delete_grp(String grp) {conn.Stmt_delete(tbl_name, fld_grp).Crt_str(fld_grp, grp).Exec_delete();}
public void Delete_all() {conn.Stmt_delete(tbl_name, Dbmeta_fld_itm.Str_ary_empty).Exec_delete();}
@@ -49,7 +49,7 @@ public class Db_cfg_tbl implements Rls_able {
if (stmt_insert == null) stmt_insert = conn.Stmt_insert(tbl_name, flds);
try {
stmt_insert.Clear().Val_str(fld_grp, grp).Val_str(fld_key, key).Val_str(fld_val, val).Exec_insert();
} catch (Exception e) {throw Err_.new_exc(e, "db", "db_cfg.insert failed", "grp", grp, "key", key, "val", val, "db", conn.Conn_info().Xto_api());}
} catch (Exception e) {throw Err_.new_exc(e, "db", "db_cfg.insert failed", "grp", grp, "key", key, "val", val, "db", conn.Conn_info().Db_api());}
}
public void Update_yn (String grp, String key, boolean val) {Update_str(grp, key, val ? "y" : "n");}
public void Update_byte (String grp, String key, byte val) {Update_str(grp, key, Byte_.To_str(val));}

2
400_xowa/src/gplx/dbs/metas/Schema_loader_mgr_.java
View File

@@ -26,7 +26,7 @@ class Schema_loader_mgr__null implements Schema_loader_mgr {
}
class Schema_loader_mgr__sqlite implements Schema_loader_mgr {
public void Load(Schema_db_mgr db_mgr, Db_conn conn) {
Gfo_usr_dlg_.Instance.Log_many("", "", "db.schema.load.bgn: conn=~{0}", conn.Conn_info().Xto_api());
Gfo_usr_dlg_.Instance.Log_many("", "", "db.schema.load.bgn: conn=~{0}", conn.Conn_info().Db_api());
Dbmeta_tbl_mgr tbl_mgr = db_mgr.Tbl_mgr();
Db_qry__select_in_tbl qry = Db_qry__select_in_tbl.new_("sqlite_master", String_.Ary_empty, String_.Ary("type", "name", "sql"), Db_qry__select_in_tbl.Order_by_null);
Db_rdr rdr = conn.Stmt_new(qry).Exec_select__rls_auto();

74
400_xowa/src/gplx/dbs/percentiles/Log_tbl_fmtr.java Normal file
View File

@@ -0,0 +1,74 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.dbs.percentiles; import gplx.*; import gplx.dbs.*;
class Log_tbl_fmtr {
private final Bry_bfr bfr = Bry_bfr.new_();
private final List_adp itms = List_adp_.new_();
private Log_fld_itm[] ary;
public Log_tbl_fmtr Add_str(String key, int len) {ary = null; itms.Add(new Log_fld_itm__bry(Type_adp_.Tid__bry, key, len)); return this;}
public Log_tbl_fmtr Add_int(String key, int bgn, int end) {ary = null; itms.Add(new Log_fld_itm__int(Type_adp_.Tid__int, key, bgn, end)); return this;}
public void Log(Object... vals) {
if (ary == null)
ary = (Log_fld_itm[])itms.To_ary_and_clear(Log_fld_itm.class);
int len = ary.length;
for (int i = 0; i < len; ++i) {
Log_fld_itm itm = ary[i];
Object val = vals[i];
if (i != 0) bfr.Add_byte_pipe();
itm.Fmt(bfr, val);
}
bfr.Add_byte_nl();
}
public String To_str_and_clear() {return bfr.To_str_and_clear();}
}
interface Log_fld_itm {
void Fmt(Bry_bfr bfr, Object val);
}
abstract class Log_fld_itm__base implements Log_fld_itm {
public Log_fld_itm__base(int tid, String key, int len) {
this.tid = tid; this.key = key; this.len = len;
}
public int Tid() {return tid;} private final int tid;
public String Key() {return key;} private final String key;
public int Len() {return len;} protected int len;
public abstract void Fmt(Bry_bfr bfr, Object val);
}
class Log_fld_itm__bry extends Log_fld_itm__base {
public Log_fld_itm__bry(int tid, String key, int len) {super(tid, key, len);}
@Override public void Fmt(Bry_bfr bfr, Object val) {
byte[] val_bry = Bry_.cast(val);
int val_bry_len = val_bry.length;
int pad_len = this.len - val_bry_len;
bfr.Add(val_bry);
if (pad_len > 0)
bfr.Add_byte_repeat(Byte_ascii.Space, pad_len);
}
}
class Log_fld_itm__int extends Log_fld_itm__base {
public Log_fld_itm__int(int tid, String key, int bgn, int end) {super(tid, key, 0);
this.bgn = bgn; this.end = end;
this.len = Int_.DigitCount(end);
}
public int Bgn() {return bgn;} private final int bgn;
public int End() {return end;} private final int end;
@Override public void Fmt(Bry_bfr bfr, Object val) {
int val_int = Int_.cast(val);
String val_str = String_.PadBgn(Int_.To_str(val_int), this.Len(), " ");
bfr.Add_str_u8(val_str);
}
}

87
400_xowa/src/gplx/dbs/percentiles/Percentile_rng.java Normal file
View File

@@ -0,0 +1,87 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.dbs.percentiles; import gplx.*; import gplx.dbs.*;
public class Percentile_rng {
private long total_max; private int total_needed;
private int score_max, score_len_max;
private long prv_time;
public int Score_bgn() {return score_bgn;} private int score_bgn;
public int Score_end() {return score_end;} private int score_end;
public int Score_len() {return score_len;} private int score_len;
public int Found_rdr() {return found_rdr;} private int found_rdr;
public int Found_all() {return found_all;} private int found_all;
public int Elapsed() {return elapsed;} private int elapsed;
public Percentile_rng Init(long total_max, int score_max) {
this.total_max = total_max;
this.score_max = score_max;
this.score_len_max = score_max / 20; // limit to 5%
return this;
}
public void Select_init(int total_needed, int prv_score_bgn, int prv_score_len, int score_len_adj) {
this.total_needed = total_needed;
this.found_all = 0;
this.prv_time = gplx.core.envs.Env_.TickCount();
int score_unit = Calc_score_unit(total_needed, total_max, score_max);
if (prv_score_bgn == Score_null) {
score_len = score_unit + (score_unit * score_len_adj);
score_bgn = score_max;
Rng_len_(Bool_.Y);
}
else {
score_len = prv_score_len;
score_bgn = prv_score_bgn;
score_end = score_bgn + score_len;
}
}
public void Update(int found_rdr) {
this.found_rdr = found_rdr;
this.found_all += found_rdr;
// calc rng_multiplier based on found_rdr and total_needed; EX: 100=total_needed; 10=found_rdr; 40=found_all -> 6=rng_multiplier; (100 - 40 / 10)
int rng_multiplier = 1;
if (found_rdr == 0) {
rng_multiplier = 4;
} else {
int total_remaining = total_needed - found_all;
rng_multiplier = total_remaining == 0 ? 1 : Math_.Ceil_as_int(total_remaining / found_rdr);
}
// calc new score_len
int new_score_len = score_len * rng_multiplier;
if (new_score_len < 1) new_score_len = score_len;
else if (new_score_len > score_len_max) new_score_len = score_len_max;
score_len = new_score_len;
Rng_len_(Bool_.N);
// update times
long new_time = gplx.core.envs.Env_.TickCount();
this.elapsed = Int_.Subtract_long(new_time, prv_time);
prv_time = new_time;
}
private void Rng_len_(boolean first) {
score_end = score_bgn + (first ? 1 : 0); // + 1 to include rows with scores at max; EX: > 999,998 AND < 1,000,001
score_bgn = score_end - score_len;
if (score_bgn < 0) score_bgn = 0; // make sure score is not negative
}
@gplx.Internal protected static int Calc_score_unit(int total_needed, long total_max, int score_max) {// TEST:
int rv = (int)Math_.Ceil(Math_.Div_safe_as_double(total_needed, Math_.Div_safe_as_double(total_max, score_max))); // EX: 100 needed / (16 M / 1 M) -> 7 units to fill 100
if (rv > score_max) rv = score_max; // never allow score_unit to be > score_max; occurs when total_needed > total_max; EX: 50 needed; 10 available
return rv;
}
public static final int Score_null = -1;
}

43
400_xowa/src/gplx/dbs/percentiles/Percentile_rng_log.java Normal file
View File

@@ -0,0 +1,43 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.dbs.percentiles; import gplx.*; import gplx.dbs.*;
public class Percentile_rng_log {
private final Log_tbl_fmtr fmtr = new Log_tbl_fmtr();
private byte[] search; private int rslts_needed;
private int rdr_idx;
public Percentile_rng_log(int score_max) {
fmtr.Add_str("search" , 50)
.Add_int("rslts_needed" , 1, 999)
.Add_int("rdr_idx" , 0, 100) // warn if more than 100 sql queries
.Add_int("score_bgn" , 0, score_max)
.Add_int("score_end" , 0, score_max)
.Add_int("score_len" , 1, 100000)
.Add_int("rdr_found" , 0, 9999) // warn if more than 9.999 seconds
.Add_int("total_found" , 0, 999)
.Add_int("total_needed" , 1, 999)
;
}
public void Init(byte[] search, int rslts_needed) {
this.search = search; this.rslts_needed = rslts_needed;
rdr_idx = -1;
}
public void Log(int score_bgn, int score_end, int rdr_found, int total_found, int pass_time) {
fmtr.Log(search, rslts_needed, ++rdr_idx, score_bgn, score_end, score_end - score_bgn, rdr_found, total_found, pass_time);
}
public String To_str_and_clear() {return fmtr.To_str_and_clear();}
}

63
400_xowa/src/gplx/dbs/percentiles/Percentile_rng_tst.java Normal file
View File

@@ -0,0 +1,63 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.dbs.percentiles; import gplx.*; import gplx.dbs.*;
import org.junit.*;
public class Percentile_rng_tst {
private final Percentile_rng_fxt fxt = new Percentile_rng_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Found__000() {
fxt.Test__rng(999994, 1000001);
fxt.Exec__update( 0).Test__rng(999966, 999994);
fxt.Exec__update( 0).Test__rng(999854, 999966);
}
@Test public void Found__025() {
fxt.Test__rng(999994, 1000001);
fxt.Exec__update( 25).Test__rng(999973, 999994);
fxt.Exec__update( 25).Test__rng(999931, 999973);
fxt.Exec__update( 25).Test__rng(999889, 999931);
fxt.Exec__update( 25).Test__rng(999847, 999889);
}
@Test public void Calc_score_unit() {
fxt.Test__calc_score_unit(50, 16000000, 1000000, 4); // to fill 50 -> 16 pages per point -> read every 4 points to get 64 pages
fxt.Test__calc_score_unit(50, 1000, 1000000, 50000); // to fill 50 -> 1000 points per page -> read every 50k points to get 50 pages
fxt.Test__calc_score_unit(50, 25, 1000000, 1000000); // range bounds check; to fill 50, always read full amount
}
}
class Percentile_rng_fxt {
private final Percentile_rng rng = new Percentile_rng();
public void Clear() {
this.Exec__init_for_wiki(16000000, 1000000);
this.Exec__init_for_search(100, 0);
}
public Percentile_rng_fxt Exec__init_for_wiki (int pages_max, int score_max) {
rng.Init(pages_max, score_max); return this;
}
public Percentile_rng_fxt Exec__init_for_search(int request_count, int score_len_adj) {
rng.Select_init(request_count, Percentile_rng.Score_null, Percentile_rng.Score_null, score_len_adj); return this;
}
public Percentile_rng_fxt Exec__update(int rdr_found) {
rng.Update(rdr_found); return this;
}
public void Test__rng(int expd_bgn, int expd_end) {
Tfds.Eq(expd_end, rng.Score_end(), "rng_end");
Tfds.Eq(expd_bgn, rng.Score_bgn(), "rng_bgn");
}
public void Test__calc_score_unit(int request_count, long pages_max, int score_max, int expd) {
Tfds.Eq(expd, Percentile_rng.Calc_score_unit(request_count, pages_max, score_max));
}
}

67
400_xowa/src/gplx/dbs/percentiles/Percentile_select_base.java Normal file
View File

@@ -0,0 +1,67 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.dbs.percentiles; import gplx.*; import gplx.dbs.*;
public abstract class Percentile_select_base { // SELECT * FROM x ORDER BY y LIMIT 10;
protected Cancelable cxl;
protected Percentile_rng rng;
protected Percentile_rng_log rng_log;
protected void Select() {
Db_rdr rdr = null;
try {
int rdr_found = 0;
while (true) {
if (cxl.Canceled()) return;
if (rdr == null) {
rdr = Rdr__init(); // EXPENSIVE
rdr_found = 0;
if (cxl.Canceled()) return;
}
if (!Row__read(rdr)) { // EXPENSIVE
if (cxl.Canceled()) return;
rng_log.Log(rng.Score_bgn(), rng.Score_end(), rng.Found_rdr(), rng.Found_all(), rng.Elapsed());
rdr = Rdr__term(rdr);
Rng__update(rdr_found);
boolean found_enough = Found_enough();
boolean none_left = rng.Score_bgn() == 0;
Rdr__done(found_enough, none_left);
if (found_enough || none_left)
break;
else
continue; // resume from top; will create new rdrd
}
if (Row__eval()) ++rdr_found;
}
}
catch (Exception exc) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "error during percentile; err=~{0}", Err_.Message_gplx_log(exc));
}
finally {
rdr = Rdr__term(rdr);
}
}
protected abstract Db_rdr Rdr__init();
@gplx.Virtual protected void Rdr__done(boolean found_enough, boolean none_left) {}
@gplx.Virtual protected Db_rdr Rdr__term(Db_rdr rdr) {
if (rdr != null) rdr.Rls();
return null;
}
@gplx.Virtual protected void Rng__update(int rdr_found) {rng.Update(rdr_found);}
@gplx.Virtual protected boolean Row__read(Db_rdr rdr) {return true;}
@gplx.Virtual protected boolean Row__eval() {return true;} // NOTE: return true by default; DEPENDENCY: Srch_word_count_wkr
@gplx.Virtual protected boolean Found_enough() {return false;}
}

55
400_xowa/src/gplx/dbs/updates/Sql_runner.java Normal file
View File

@@ -0,0 +1,55 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.dbs.updates; import gplx.*; import gplx.dbs.*;
import gplx.dbs.stmts.*;
/*
sql =
UPDATE page
SET page_score = page_len
WHERE page_id >= ?
AND page_id < ?
state = -1|100000
*/
public class Sql_runner {
private final Db_stmt_arg_list list = new Db_stmt_arg_list();
public Db_conn Conn() {return conn;} public void Conn_(Db_conn v) {conn = v;} private Db_conn conn;
public boolean Quiet() {return quiet;} public void Quiet_(boolean v) {quiet = v;} private boolean quiet;
public String Sql_fmt() {return sql_fmt;} public void Sql_fmt_(String v) {sql_fmt = v;} private String sql_fmt;
// public Db_stmt_arg[] Sql_args() {return sql_args;} public void Sql_args_(Db_stmt_arg[] v) {sql_args = v;} private Db_stmt_arg[] sql_args;
public String Msg() {return msg;} public void Msg_(String v) {msg = v;} private String msg;
public String Fill_next(String state) {
String[] vals = String_.Split(state, "|");
int val_lo = Int_.parse(vals[0]);
int interval = Int_.parse(vals[1]);
int val_hi = val_lo + interval;
Db_stmt_arg arg = list.Get_at(0);
arg.Val = val_lo;
arg = list.Get_at(1);
arg.Val = val_hi;
return String_.Concat_with_str("|", Int_.To_str(val_hi), vals[1]);
}
public void Run() {
Db_stmt stmt = conn.Stmt_sql(sql_fmt);
// foreach (itme) Db_stmt_arg_list list = Db_stmt_arg_list
Gfo_usr_dlg_.Instance.Note_many("", "", msg);
stmt.Exec_update();
// increment ranges
}
}

2
400_xowa/src/gplx/fsdb/data/Fsd_bin_tbl.java
View File

@@ -37,7 +37,7 @@ public class Fsd_bin_tbl implements Rls_able {
stmt_insert = Db_stmt_.Rls(stmt_insert);
stmt_select = Db_stmt_.Rls(stmt_select);
}
public void Create_tbl() {conn.Ddl_create_tbl(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Insert_bgn() {conn.Txn_bgn("fsdb_bin__insert"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_commit() {conn.Txn_sav();}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}

1
400_xowa/src/gplx/fsdb/data/Fsd_dir_itm.java
View File

@@ -21,7 +21,6 @@ public class Fsd_dir_itm {
public int Dir_id() {return dir_id;} private final int dir_id;
public int Owner() {return owner;} private final int owner;
public byte[] Name() {return name;} private final byte[] name;
public static final int Owner_root = 0;
public static final Fsd_dir_itm Null = null;
}

2
400_xowa/src/gplx/fsdb/data/Fsd_dir_tbl.java
View File

@@ -34,7 +34,7 @@ public class Fsd_dir_tbl implements Rls_able {
stmt_select_by_name = Db_stmt_.Rls(stmt_select_by_name);
}
public void Create_tbl() {
conn.Ddl_create_tbl
conn.Meta_tbl_create
( Dbmeta_tbl_itm.New(tbl_name, flds
, Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "name", fld_name, fld_owner_id, fld_id)));
}

2
400_xowa/src/gplx/fsdb/data/Fsd_fil_tbl.java
View File

@@ -42,7 +42,7 @@ public class Fsd_fil_tbl implements Rls_able {
stmt_select_by_name = Db_stmt_.Rls(stmt_select_by_name);
}
public void Create_tbl() {
conn.Ddl_create_tbl(Dbmeta_tbl_itm.New(tbl_name, flds
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds
, Dbmeta_idx_itm.new_unique_by_name(tbl_name, idx_owner, fld_owner_id, fld_name, fld_id)
));
}

1
400_xowa/src/gplx/fsdb/data/Fsd_thm_itm.java
View File

@@ -37,7 +37,6 @@ public class Fsd_thm_itm {
public int Req_w() {return req_w;} private int req_w;
public double Req_time() {return req_time;} private double req_time;
public int Req_page() {return req_page;} private int req_page;
public void Init_by_req(int w, double time, int page) {this.w = w; this.time = time; this.page = page;}
public void Init_by_match(Fsd_thm_itm comp) {
this.req_w = w; this.req_time = time; this.req_page = page;

2
400_xowa/src/gplx/fsdb/data/Fsd_thm_tbl.java
View File

@@ -48,7 +48,7 @@ public class Fsd_thm_tbl implements Rls_able {
stmt_select_by_fil_near = Db_stmt_.Rls(stmt_select_by_fil_near);
}
public void Create_tbl() {
conn.Ddl_create_tbl(Dbmeta_tbl_itm.New(tbl_name, flds
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds
, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "owner", fld_owner_id, fld_id, fld_w, fld_time, fld_page)
));
}

4
400_xowa/src/gplx/fsdb/meta/Fsm_atr_tbl.java
View File

@@ -29,7 +29,7 @@ public class Fsm_atr_tbl {
this.fld_uid = flds.Add_int_pkey (fld_prefix + "uid");
this.fld_url = flds.Add_str (fld_prefix + "url", 255);
}
public void Create_tbl() {conn.Ddl_create_tbl(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public Fsm_atr_fil Select_1st_or_fail(Fsm_mnt_itm mnt_itm, Fsdb_db_mgr core_mgr, int mnt_id, boolean schema_thm_page) {
Db_rdr rdr = conn.Stmt_select(tbl_name, flds, Dbmeta_fld_itm.Str_ary_empty).Exec_select__rls_auto();
boolean schema_is_1 = core_mgr.File__schema_is_1();
@@ -47,7 +47,7 @@ public class Fsm_atr_tbl {
}
}
finally {rdr.Rls();}
throw Err_.new_wo_type("missing atr db", "conn", conn.Conn_info().Xto_api());
throw Err_.new_wo_type("missing atr db", "conn", conn.Conn_info().Db_api());
}
public void Insert(int id, String url_rel) {
conn.Stmt_insert(tbl_name, flds).Val_int(fld_uid, id).Val_str(fld_url, url_rel).Exec_insert();

2
400_xowa/src/gplx/fsdb/meta/Fsm_bin_tbl.java
View File

@@ -37,7 +37,7 @@ public class Fsm_bin_tbl {
fld_bin_max = Dbmeta_fld_itm.Key_null;
}
}
public void Create_tbl() {conn.Ddl_create_tbl(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Insert(int id, String url_rel) {
conn.Stmt_insert(tbl_name, flds).Crt_int(fld_uid, id).Val_str(fld_url, url_rel).Val_long(fld_bin_len, 0).Val_long(fld_bin_max, 0).Exec_insert();
}

2
400_xowa/src/gplx/fsdb/meta/Fsm_cfg_mgr.java
View File

@@ -24,7 +24,7 @@ public class Fsm_cfg_mgr {
}
public void Ctor_by_load() {
Db_cfg_hash hash = Grps_get_or_load(Grp_core);
this.next_id = hash.Get_by(Key_next_id).To_int_or(-1); if (next_id == -1) throw Err_.new_wo_type("next_id not found in cfg", "url", tbl.Conn().Conn_info().Xto_api());
this.next_id = hash.Get_by(Key_next_id).To_int_or(-1); if (next_id == -1) throw Err_.new_wo_type("next_id not found in cfg", "url", tbl.Conn().Conn_info().Db_api());
this.schema_thm_page = hash.Get_by(Key_schema_thm_page).To_yn_or_n();
this.patch__next_id = hash.Get_by(Key_patch__next_id).To_yn_or_n();
this.patch__page_gt_1 = hash.Get_by(Key_patch__page_gt_1).To_yn_or_n();

4
400_xowa/src/gplx/fsdb/meta/Fsm_mnt_itm.java
View File

@@ -65,8 +65,8 @@ class Fsm_mnt_itm_ {
Fsm_cfg_mgr cfg_mgr = abc_mgr.Cfg_mgr();
int last_id = -1;
Fsm_atr_fil atr_fil = atr_mgr.Db__core();
int max_fil_id = Db_conn_.Select_fld0_as_int_or(atr_fil.Conn(), "SELECT Max(fil_id) AS MaxId FROM fsdb_fil;", -1);
int max_thm_id = Db_conn_.Select_fld0_as_int_or(atr_fil.Conn(), "SELECT Max(thm_id) AS MaxId FROM fsdb_xtn_thm;", -1);
int max_fil_id = atr_fil.Conn().Exec_select_as_int("SELECT Max(fil_id) AS MaxId FROM fsdb_fil;", -1);
int max_thm_id = atr_fil.Conn().Exec_select_as_int("SELECT Max(thm_id) AS MaxId FROM fsdb_xtn_thm;", -1);
last_id = max_fil_id > max_thm_id ? max_fil_id : max_thm_id;
cfg_mgr.Patch_next_id_exec(last_id);
}

2
400_xowa/src/gplx/fsdb/meta/Fsm_mnt_tbl.java
View File

@@ -30,7 +30,7 @@ public class Fsm_mnt_tbl implements Rls_able {
}
public void Create_tbl() {
Dbmeta_tbl_itm meta = Dbmeta_tbl_itm.New(tbl_name, flds);
conn.Ddl_create_tbl(meta);
conn.Meta_tbl_create(meta);
this.Insert(Fsm_mnt_mgr.Mnt_idx_main, Mnt_name_main, Mnt_name_main);
this.Insert(Fsm_mnt_mgr.Mnt_idx_user, Mnt_name_user, Mnt_name_user);
}

6
400_xowa/src/gplx/langs/gfs/Gfs_msg_bldr_tst.java
View File

@@ -44,12 +44,12 @@ public class Gfs_msg_bldr_tst {
}
class Gfs_msg_bldr_fxt {
public void Clear() {} String_bldr sb = String_bldr_.new_(); Gfs_msg_bldr msg_bldr = Gfs_msg_bldr.Instance;
public KeyVal kv_(String key, String val) {return KeyVal_.new_(key, val);}
public GfoMsg msg_(String key, KeyVal... args) {
public Keyval kv_(String key, String val) {return Keyval_.new_(key, val);}
public GfoMsg msg_(String key, Keyval... args) {
GfoMsg rv = GfoMsg_.new_parse_(key);
int len = args.length;
for (int i = 0; i < len; i++) {
KeyVal kv = args[i];
Keyval kv = args[i];
rv.Add(kv.Key(), kv.Val());
}
return rv;

15
400_xowa/src/gplx/langs/gfs/Gfs_parser_ctx.java
View File

@@ -40,7 +40,6 @@ class Gfs_parser_ctx {
cur_idf_end = end;
} int cur_idf_bgn = -1, cur_idf_end = -1;
private void Held_word_clear() {cur_idf_bgn = -1; cur_idf_end = -1;}
public Gfs_nde Make_nde(int tkn_bgn, int tkn_end) { // "abc."; "abc("; "abc;"; "abc{"
Gfs_nde nde = new Gfs_nde().Name_rng_(cur_idf_bgn, cur_idf_end);
this.Held_word_clear();
@@ -66,17 +65,17 @@ class Gfs_parser_ctx {
}
class Gfs_err_mgr {
public void Fail_eos(Gfs_parser_ctx ctx) {Fail(ctx, Fail_msg_eos, ctx.Src_len());}
public void Fail_unknown_char(Gfs_parser_ctx ctx, int pos, byte c) {Fail(ctx, Fail_msg_unknown_char, pos, KeyVal_.new_("char", Char_.To_str((char)c)));}
public void Fail_unknown_char(Gfs_parser_ctx ctx, int pos, byte c) {Fail(ctx, Fail_msg_unknown_char, pos, Keyval_.new_("char", Char_.To_str((char)c)));}
public void Fail_nde_stack_empty(Gfs_parser_ctx ctx, int pos) {Fail(ctx, Fail_msg_nde_stack_empty, pos);}
public void Fail_invalid_lxr(Gfs_parser_ctx ctx, int pos, int lxr_tid, byte c) {
Fail(ctx, Fail_msg_invalid_lxr, pos, KeyVal_.new_("char", Char_.To_str((char)c)), KeyVal_.new_("cur_lxr", Gfs_lxr_.Tid__name(lxr_tid)), KeyVal_.new_("prv_lxr", Gfs_lxr_.Tid__name(ctx.Prv_lxr())));
Fail(ctx, Fail_msg_invalid_lxr, pos, Keyval_.new_("char", Char_.To_str((char)c)), Keyval_.new_("cur_lxr", Gfs_lxr_.Tid__name(lxr_tid)), Keyval_.new_("prv_lxr", Gfs_lxr_.Tid__name(ctx.Prv_lxr())));
}
private void Fail(Gfs_parser_ctx ctx, String msg, int pos, KeyVal... args) {
private void Fail(Gfs_parser_ctx ctx, String msg, int pos, Keyval... args) {
byte[] src = ctx.Src(); int src_len = ctx.Src_len();
Fail_args_standard(src, src_len, pos);
int len = args.length;
for (int i = 0; i < len; i++) {
KeyVal arg = args[i];
Keyval arg = args[i];
tmp_fail_args.Add(arg.Key(), arg.Val_to_str_or_empty());
}
throw Err_.new_wo_type(Fail_msg(msg, tmp_fail_args));
@@ -87,12 +86,12 @@ class Gfs_err_mgr {
tmp_fail_args.Add("pos" , pos);
}
public static final String Fail_msg_invalid_lxr = "invalid character", Fail_msg_unknown_char = "unknown char", Fail_msg_eos = "end of stream", Fail_msg_nde_stack_empty = "node stack empty";
String Fail_msg(String type, KeyValList fail_args) {
String Fail_msg(String type, Keyval_list fail_args) {
tmp_fail_bfr.Add_str_u8(type).Add_byte(Byte_ascii.Colon);
int len = fail_args.Count();
for (int i = 0; i < len; i++) {
tmp_fail_bfr.Add_byte(Byte_ascii.Space);
KeyVal kv = fail_args.GetAt(i);
Keyval kv = fail_args.Get_at(i);
tmp_fail_bfr.Add_str_u8(kv.Key());
tmp_fail_bfr.Add_byte(Byte_ascii.Eq).Add_byte(Byte_ascii.Apos);
tmp_fail_bfr.Add_str_u8(kv.Val_to_str_or_empty()).Add_byte(Byte_ascii.Apos);
@@ -100,7 +99,7 @@ class Gfs_err_mgr {
return tmp_fail_bfr.To_str_and_clear();
}
Bry_bfr tmp_fail_bfr = Bry_bfr.reset_(255);
KeyValList tmp_fail_args = new KeyValList();
Keyval_list tmp_fail_args = new Keyval_list();
private static int excerpt_len = 50;
String Fail_excerpt_bgn(byte[] src, int src_len, int pos) {
int bgn = pos - excerpt_len; if (bgn < 0) bgn = 0;

3
400_xowa/src/gplx/langs/htmls/Gfh_tag_.java
View File

@@ -105,8 +105,9 @@ public class Gfh_tag_ {
, Script_rhs = Bry_.new_a7("</script>")
, Span_lhs = Bry_.new_a7("<span")
, Span_rhs = Bry_.new_a7("</span>")
, Strong_lhs = Bry_.new_a7("<strong>")
, Strong_rhs = Bry_.new_a7("</strong>")
;
public static final String
Comm_bgn_str = "<!--"
, Comm_end_str = "-->"

1
400_xowa/src/gplx/langs/htmls/Gfh_utl.java
View File

@@ -102,7 +102,6 @@ public class Gfh_utl {
else
return dirty ? bfr.To_bry_and_clear() : bry;
}
private static final Btrie_slim_mgr unescape_trie = Btrie_slim_mgr.ci_a7()
.Add_bry_byte(Gfh_entity_.Lt_bry , Byte_ascii.Lt)
.Add_bry_byte(Gfh_entity_.Gt_bry , Byte_ascii.Gt)

2
400_xowa/src/gplx/langs/htmls/docs/Gfh_doc_wkr.java
View File

@@ -19,4 +19,4 @@ package gplx.langs.htmls.docs; import gplx.*; import gplx.langs.*; import gplx.l
public interface Gfh_doc_wkr {
byte[] Hook();
int Parse(byte[] src, int src_bgn, int src_end, int pos);
}
}

1
400_xowa/src/gplx/langs/htmls/docs/Gfh_tag_rdr.java
View File

@@ -86,7 +86,6 @@ public class Gfh_tag_rdr {
else
return false;
}
int depth = depth_obj.Val();
boolean tag_is_tail = tag.Tag_is_tail();
if (tail == tag_is_tail) {

2
400_xowa/src/gplx/langs/htmls/docs/Gfh_txt_wkr.java
View File

@@ -18,4 +18,4 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.langs.htmls.docs; import gplx.*; import gplx.langs.*; import gplx.langs.htmls.*;
public interface Gfh_txt_wkr {
void Parse(int rng_bgn, int rng_end);
}
}

4
400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder_tst.java
View File

@@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.langs.htmls.encoders; import gplx.*; import gplx.langs.*; import gplx.langs.htmls.*;
import org.junit.*;
public class Gfo_url_encoder_tst {
private final Gfo_url_encoder_fxt fxt = new Gfo_url_encoder_fxt();
private final Gfo_url_encoder_fxt fxt = new Gfo_url_encoder_fxt();
@Test public void Id__nums() {fxt.Encoder_id().Test__bicode("0123456789" , "0123456789");}
@Test public void Id__ltrs_lower() {fxt.Encoder_id().Test__bicode("abcdefghijklmnopqrstuvwxyz" , "abcdefghijklmnopqrstuvwxyz");}
@Test public void Id__ltrs_upper() {fxt.Encoder_id().Test__bicode("ABCDEFGHIJKLMNOPQRSTUVWXYZ" , "ABCDEFGHIJKLMNOPQRSTUVWXYZ");}
@@ -46,7 +46,7 @@ public class Gfo_url_encoder_tst {
fxt.Encoder_href().Test__encode("%GC", "%25GC");
}
@Test public void Fsys__wnt() {
fxt.Encoder_fsys_safe().Test__encode("Help:Options/HTML", "Help%3AOptions%2FHTML");
fxt.Encoder_fsys_safe().Test__encode("Options/HTML", "Options%2FHTML");
}
}
class Gfo_url_encoder_fxt {

2
400_xowa/src/gplx/langs/htmls/styles/Gfh_style_wkr__val_as_int.java
View File

@@ -32,4 +32,4 @@ public class Gfh_style_wkr__val_as_int implements Gfh_style_wkr {
Gfh_style_parser_.Parse(src, src_bgn, src_end, this);
return Bry_.To_int_or__lax(src, val_bgn, val_end, -1);
}
}
}

14
400_xowa/src/gplx/langs/jsons/Json_kv_ary_srl.java
View File

@@ -17,11 +17,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.langs.jsons; import gplx.*; import gplx.langs.*;
public class Json_kv_ary_srl {
public static KeyVal Kv_by_itm(Json_itm itm) {
public static Keyval Kv_by_itm(Json_itm itm) {
switch (itm.Tid()) {
case Json_itm_.Tid__kv:
Json_kv kv = (Json_kv)itm;
return KeyVal_.new_(kv.Key_as_str(), Val_by_itm(kv.Val()));
return Keyval_.new_(kv.Key_as_str(), Val_by_itm(kv.Val()));
default:
throw Err_.new_unhandled(itm.Tid());
}
@@ -39,19 +39,19 @@ public class Json_kv_ary_srl {
default: throw Err_.new_unhandled(itm.Tid());
}
}
private static KeyVal[] Val_by_itm_ary(Json_ary itm) {
private static Keyval[] Val_by_itm_ary(Json_ary itm) {
int subs_len = itm.Len();
KeyVal[] rv = new KeyVal[subs_len];
Keyval[] rv = new Keyval[subs_len];
for (int i = 0; i < subs_len; i++) {
Json_itm sub = itm.Get_at(i);
KeyVal kv = KeyVal_.new_(Int_.To_str(i + Int_.Base1), Val_by_itm(sub));
Keyval kv = Keyval_.new_(Int_.To_str(i + Int_.Base1), Val_by_itm(sub));
rv[i] = kv;
}
return rv;
}
public static KeyVal[] Val_by_itm_nde(Json_nde itm) {
public static Keyval[] Val_by_itm_nde(Json_nde itm) {
int subs_len = itm.Len();
KeyVal[] rv = new KeyVal[subs_len];
Keyval[] rv = new Keyval[subs_len];
for (int i = 0; i < subs_len; i++) {
Json_itm sub = itm.Get_at(i);
rv[i] = Kv_by_itm(sub);

18
400_xowa/src/gplx/langs/jsons/Json_kv_ary_srl_tst.java
View File

@@ -35,16 +35,16 @@ class Json_kv_ary_srl_fxt {
parser = new Json_parser();
}
} private Json_parser parser;
public void Test_parse(String raw_str, KeyVal[] expd) {
public void Test_parse(String raw_str, Keyval[] expd) {
byte[] raw_bry = Json_parser_tst.Replace_apos(Bry_.new_u8(raw_str));
Json_doc doc = parser.Parse(raw_bry);
KeyVal[] actl = Json_kv_ary_srl.Val_by_itm_nde(doc.Root_nde());
Tfds.Eq_str_lines(KeyVal_.Ary_to_str(expd), KeyVal_.Ary_to_str(actl));
Keyval[] actl = Json_kv_ary_srl.Val_by_itm_nde(doc.Root_nde());
Tfds.Eq_str_lines(Keyval_.Ary_to_str(expd), Keyval_.Ary_to_str(actl));
}
public KeyVal[] ary_(KeyVal... ary) {return ary;}
public KeyVal kv_obj_(String key, Object val) {return KeyVal_.new_(key, val);}
public KeyVal kv_str_(String key, String val) {return KeyVal_.new_(key, val);}
public KeyVal kv_int_(String key, int val) {return KeyVal_.new_(key, val);}
public KeyVal kv_bool_(String key, boolean val) {return KeyVal_.new_(key, Bool_.To_str_lower(val));}
public KeyVal kv_dec_(String key, Decimal_adp val) {return KeyVal_.new_(key, val.To_str());}
public Keyval[] ary_(Keyval... ary) {return ary;}
public Keyval kv_obj_(String key, Object val) {return Keyval_.new_(key, val);}
public Keyval kv_str_(String key, String val) {return Keyval_.new_(key, val);}
public Keyval kv_int_(String key, int val) {return Keyval_.new_(key, val);}
public Keyval kv_bool_(String key, boolean val) {return Keyval_.new_(key, Bool_.To_str_lower(val));}
public Keyval kv_dec_(String key, Decimal_adp val) {return Keyval_.new_(key, val.To_str());}
}

2
400_xowa/src/gplx/langs/jsons/Json_parser__list_nde__base.java
View File

@@ -62,7 +62,7 @@ public class Json_parser__list_nde__base extends Json_parser__itm__base {
Json_kv sub = nde.Get_at_as_kv(i);
byte[] key = sub.Key_as_bry();
byte[] val = Parse_to_list_as_kv__get_val(sub, key);
list.Add(key, KeyVal_.new_(String_.new_u8(key), String_.new_u8(val)));
list.Add(key, Keyval_.new_(String_.new_u8(key), String_.new_u8(val)));
}
}
@gplx.Virtual protected byte[] Parse_to_list_as_kv__get_val(Json_kv sub, byte[] key) {return sub.Val_as_bry();}

6
400_xowa/src/gplx/langs/jsons/Json_wtr.java
View File

@@ -211,10 +211,10 @@ public class Json_wtr {
else {
Handle_nde_as_ary_itm_0();
Write_grp_bgn(Sym_nde_bgn, Bool_.N);
KeyVal[] kvy = (KeyVal[])obj;
Keyval[] kvy = (Keyval[])obj;
int kvy_len = kvy.length;
for (int i = 0; i < kvy_len; ++i) {
KeyVal kv = kvy[i];
Keyval kv = kvy[i];
Object kv_val = kv.Val();
Kv_obj(Bry_.new_u8(kv.Key()), kv_val, Type_adp_.To_tid_obj(kv_val));
}
@@ -287,7 +287,7 @@ public class Json_wtr {
private static final int Grp_type__json_nde = 1, Grp_type__kv_ary = 2, Grp_type__json_ary = 3, Grp_type__obj_ary = 4;
private static int Grp_type__get(Object obj) {
Class<?> type = obj.getClass();
if (Type_adp_.Eq(type, KeyVal[].class)) return Grp_type__kv_ary;
if (Type_adp_.Eq(type, Keyval[].class)) return Grp_type__kv_ary;
else if (Type_adp_.Is_array(type)) return Grp_type__obj_ary;
else if (Type_adp_.Eq(type, Json_nde.class)) return Grp_type__json_nde;
else if (Type_adp_.Eq(type, Json_ary.class)) return Grp_type__json_ary;

6
400_xowa/src/gplx/langs/jsons/Json_wtr_tst.java
View File

@@ -84,9 +84,9 @@ public class Json_wtr_tst {
fxt.Wtr()
.Doc_nde_bgn()
.Ary_bgn("a0")
.Ary_itm_obj(KeyVal_.Ary
( KeyVal_.new_("k1", "v1")
, KeyVal_.new_("k2", "v2")
.Ary_itm_obj(Keyval_.Ary
( Keyval_.new_("k1", "v1")
, Keyval_.new_("k2", "v2")
))
.Ary_end()
.Doc_nde_end();

2
400_xowa/src/gplx/langs/phps/Php_srl_itm.java
View File

@@ -135,4 +135,4 @@ class Php_srl_wtr {
public static void Indent(Bry_bfr bfr, int depth) {
if (depth > 0) bfr.Add_byte_repeat(Byte_ascii.Space, depth * 2); // indent
}
}
}

10
400_xowa/src/gplx/langs/phps/Php_srl_parser.java
View File

@@ -20,18 +20,18 @@ import gplx.core.texts.*;
public class Php_srl_parser {
@gplx.Internal protected Php_srl_factory Factory() {return factory;} Php_srl_factory factory = new Php_srl_factory();
byte[] raw; int raw_len, pos;
public KeyVal[] Parse_as_kvs(byte[] raw) {
public Keyval[] Parse_as_kvs(byte[] raw) {
Php_srl_itm_ary root = Parse(raw);
return Xto_kv_ary(root);
}
KeyVal[] Xto_kv_ary(Php_srl_itm_ary ary) {
Keyval[] Xto_kv_ary(Php_srl_itm_ary ary) {
int len = ary.Subs_len();
KeyVal[] rv = new KeyVal[len];
Keyval[] rv = new Keyval[len];
for (int i = 0; i < len; i++)
rv[i] = Xto_kv(ary.Subs_get_at(i));
return rv;
}
KeyVal Xto_kv(Php_srl_itm_kv itm) {
Keyval Xto_kv(Php_srl_itm_kv itm) {
Php_srl_itm itm_key = itm.Key();
Object key = itm_key == null ? null : itm_key.Val();
Php_srl_itm itm_val = itm.Val();
@@ -48,7 +48,7 @@ public class Php_srl_parser {
val = itm_val.Val();
break;
}
return KeyVal_.obj_(key, val);
return Keyval_.obj_(key, val);
}
@gplx.Internal protected Php_srl_itm_ary Parse(byte[] raw) {
this.raw = raw; this.raw_len = raw.length; pos = 0;

12
400_xowa/src/gplx/xowa/Xoa_app_.java
View File

@@ -34,15 +34,19 @@ public class Xoa_app_ {
}
}
public static final String Name = "xowa";
public static final String Version = "3.2.1.1";
public static final String Version = "3.3.4.1";
public static String Build_date = "2012-12-30 00:00:00";
public static String Op_sys_str;
public static String User_agent = "";
public static final Gfo_msg_grp Nde = Gfo_msg_grp_.prj_(Name);
public static final Gfo_msg_grp Nde = Gfo_msg_grp_.prj_(Name);
public static Gfo_usr_dlg Usr_dlg() {return usr_dlg;} public static void Usr_dlg_(Gfo_usr_dlg v) {usr_dlg = v;} private static Gfo_usr_dlg usr_dlg;
public static Bry_bfr_mkr Utl__bfr_mkr() {return utl__bry_bfr_mkr;} private static final Bry_bfr_mkr utl__bry_bfr_mkr = new Bry_bfr_mkr();
public static Io_stream_zip_mgr Utl__zip_mgr() {return utl__zip_mgr;} private static final Io_stream_zip_mgr utl__zip_mgr = new Io_stream_zip_mgr();
public static Bry_bfr_mkr Utl__bfr_mkr() {return utl__bry_bfr_mkr;} private static final Bry_bfr_mkr utl__bry_bfr_mkr = new Bry_bfr_mkr();
public static Io_stream_zip_mgr Utl__zip_mgr() {return utl__zip_mgr;} private static final Io_stream_zip_mgr utl__zip_mgr = new Io_stream_zip_mgr();
public static Xoa_gfs_mgr Gfs_mgr() {return gfs_mgr;} public static void Gfs_mgr_(Xoa_gfs_mgr v) {gfs_mgr = v;} private static Xoa_gfs_mgr gfs_mgr;
public static void Plog_none(String mod, String fmt) {
String msg = mod + ": " + fmt;
Xoa_app_.Usr_dlg().Plog_many("", "", msg);
}
public static Gfo_usr_dlg usr_dlg_console_() {
Gfo_usr_dlg rv = new Gfo_usr_dlg_base(new Gfo_usr_dlg__log_base(), Gfo_usr_dlg__gui_.Console);
rv.Log_wkr().Queue_enabled_(true);

34
400_xowa/src/gplx/xowa/Xoa_app_fxt.java
View File

@@ -25,10 +25,20 @@ import gplx.xowa.wikis.*; import gplx.xowa.wikis.nss.*;
import gplx.xowa.apps.*; import gplx.xowa.files.exts.*;
import gplx.xowa.wikis.domains.*;
public class Xoa_app_fxt {
public static Xoae_app app_() {
public static Xoae_app Make__app__edit() {
Io_mgr.Instance.InitEngine_mem();
Db_conn_bldr.Instance.Reg_default_mem();
return app_("linux", Io_url_.mem_dir_("mem/xowa/"));
return Make__app__edit("linux", Io_url_.mem_dir_("mem/xowa/"));
}
public static Xoae_app Make__app__edit(String op_sys, Io_url root_dir) {
Io_url user_dir = root_dir.GenSubDir_nest("user", "test_user");
Gfo_usr_dlg__log_base.Instance.Log_dir_(user_dir.GenSubDir_nest("tmp", "current"));
Xoae_app app = new Xoae_app(Gfo_usr_dlg_.Test(), Xoa_app_mode.Itm_cmd, root_dir, root_dir.GenSubDir("wiki"), root_dir.GenSubDir("file"), user_dir, root_dir.GenSubDir_nest("user", "anonymous", "wiki"), op_sys);
app.Setup_mgr().Dump_mgr().Data_storage_format_(gplx.core.ios.Io_stream_.Tid_raw); // TEST: set data_storage_format to file, else bldr tests will fails (expects plain text)
GfsCore.Instance.Clear(); // NOTE: must clear
GfsCore.Instance.AddCmd(app, Xoae_app.Invk_app); // NOTE: must add app to GfsCore; app.Gfs_mgr() always adds current app to GfsCore; note this causes old test to leave behind GfsCore for new test
GfsCore.Instance.AddCmd(app, Xoae_app.Invk_xowa); // add alias for app; DATE:2014-06-09
return app;
}
public static Xoav_app Make__app__view() {
Io_mgr.Instance.InitEngine_mem();
@@ -43,16 +53,6 @@ public class Xoa_app_fxt {
rv.Wiki_mgr().Add(new Xowv_wiki(rv, Xow_domain_itm_.Bry__home, user_dir));
return rv;
}
public static Xoae_app app_(String op_sys, Io_url root_dir) {
Io_url user_dir = root_dir.GenSubDir_nest("user", "test_user");
Gfo_usr_dlg__log_base.Instance.Log_dir_(user_dir.GenSubDir_nest("tmp", "current"));
Xoae_app app = new Xoae_app(Gfo_usr_dlg_.Test(), Xoa_app_mode.Itm_cmd, root_dir, root_dir.GenSubDir("wiki"), root_dir.GenSubDir("file"), user_dir, root_dir.GenSubDir_nest("user", "anonymous", "wiki"), op_sys);
app.Setup_mgr().Dump_mgr().Data_storage_format_(gplx.core.ios.Io_stream_.Tid_raw); // TEST: set data_storage_format to file, else bldr tests will fails (expects plain text)
GfsCore.Instance.Clear(); // NOTE: must clear
GfsCore.Instance.AddCmd(app, Xoae_app.Invk_app); // NOTE: must add app to GfsCore; app.Gfs_mgr() always adds current app to GfsCore; note this causes old test to leave behind GfsCore for new test
GfsCore.Instance.AddCmd(app, Xoae_app.Invk_xowa); // add alias for app; DATE:2014-06-09
return app;
}
public static Xowv_wiki Make__wiki__view(Xoa_app app) {return Make__wiki__view(app, "en.wikipedia.org");}
public static Xowv_wiki Make__wiki__view(Xoa_app app, String domain_str) {
byte[] domain_bry = Bry_.new_u8(domain_str);
@@ -61,14 +61,14 @@ public class Xoa_app_fxt {
((Xoav_wiki_mgr)app.Wiki_mgri()).Add(rv);
return rv;
}
public static Xowe_wiki wiki_nonwmf(Xoae_app app, String key) {
public static Xowe_wiki Make__wiki__edit__nonwmf(Xoae_app app, String key) {
Xol_lang_itm lang = new Xol_lang_itm(app.Lang_mgr(), Xol_lang_itm_.Key_en).Kwd_mgr__strx_(true);
Xol_lang_itm_.Lang_init(lang);
return wiki_(app, key, lang);
return Make__wiki__edit(app, key, lang);
}
public static Xowe_wiki wiki_tst_(Xoae_app app) {return wiki_(app, "en.wikipedia.org");}
public static Xowe_wiki wiki_(Xoae_app app, String key) {return wiki_(app, key, app.Lang_mgr().Lang_en());}
public static Xowe_wiki wiki_(Xoae_app app, String key, Xol_lang_itm lang) {
public static Xowe_wiki Make__wiki__edit(Xoae_app app) {return Make__wiki__edit(app, "en.wikipedia.org");}
public static Xowe_wiki Make__wiki__edit(Xoae_app app, String key) {return Make__wiki__edit(app, key, app.Lang_mgr().Lang_en());}
public static Xowe_wiki Make__wiki__edit(Xoae_app app, String key, Xol_lang_itm lang) {
Io_url wiki_dir = app.Fsys_mgr().Wiki_dir().GenSubDir(key);
Xowe_wiki rv = new Xowe_wiki(app, lang, Xow_ns_mgr_.default_(lang.Case_mgr()), Xow_domain_itm_.parse(Bry_.new_u8(key)), wiki_dir);
rv.File_mgr().Dbmeta_mgr().Depth_(2); // TEST: written for 2 depth

16
400_xowa/src/gplx/xowa/Xoa_test_.java
View File

@@ -23,12 +23,12 @@ public class Xoa_test_ {
public static boolean Db_skip() {return Bool_.N;}
public static boolean Db_is_mem_dflt() {return Bool_.Y;}
public static void Db_init(Io_url sqlite_url) {Db_init(Db_is_mem_dflt(), sqlite_url);}
public static void Db__init__mem(Xowe_wiki wiki) {
public static void Init__db__edit(Xowe_wiki wiki) {
Db__init__mem();
wiki.Db_mgr_create_as_sql();
wiki.Data__core_mgr().Init_by_make(Xowd_core_db_props.Test, Xob_info_session.Test);
}
public static void Init__db__mem(gplx.xowa.wikis.Xowv_wiki wiki) {
public static void Init__db__view(gplx.xowa.wikis.Xowv_wiki wiki) {
Db__init__mem();
wiki.Init_by_make(Xowd_core_db_props.Test, Xob_info_session.Test);
// wiki.Data__core_mgr().Init_by_make(Xowd_core_db_props.Test, Xob_info_session.Test);
@@ -45,18 +45,6 @@ public class Xoa_test_ {
Db_conn_bldr.Instance.Reg_default_sqlite();
}
}
public static String Db__print_tbl_as_str(Bry_bfr bfr, Db_conn conn, String tbl, String... cols) {
int cols_len = cols.length;
Db_rdr rdr = conn.Stmt_select(tbl, cols).Exec_select__rls_auto();
while (rdr.Move_next()) {
for (int i = 0; i < cols_len; ++i) {
bfr.Add_obj(rdr.Read_at(i));
bfr.Add_byte(i == cols_len - 1 ? Byte_ascii.Nl : Byte_ascii.Pipe);
}
}
rdr.Rls();
return bfr.To_str_and_clear();
}
public static void Inet__init() {
Gfo_inet_conn_.new_prototype_(Gfo_inet_conn_.Tid__mem__hash);
}

15
400_xowa/src/gplx/xowa/Xoa_ttl.java
View File

@@ -29,7 +29,8 @@ public class Xoa_ttl { // PAGE:en.w:http://en.wikipedia.org/wiki/Help:Link; REF.
public byte[] Raw() {return raw;} private byte[] raw = Bry_.Empty;
public byte[] Wik_txt() {return wik_bgn == -1 ? Bry_.Empty : Bry_.Mid(full_txt, wik_bgn, ns_bgn == -1 ? page_bgn - 1 : ns_bgn - 1);}
public Xow_xwiki_itm Wik_itm() {return wik_itm;} private Xow_xwiki_itm wik_itm;
public byte[] Full_txt() {
public byte[] Full_txt_w_ttl_case() {return Xoa_ttl.Replace_unders(Full_db());}
public byte[] Full_txt_by_orig() {
int bgn = wik_bgn == -1 ? 0 : ns_bgn == -1 ? page_bgn : ns_bgn;
int end = full_txt.length;
if (anch_bgn != -1) end = anch_bgn - 1;
@@ -56,10 +57,10 @@ public class Xoa_ttl { // PAGE:en.w:http://en.wikipedia.org/wiki/Help:Link; REF.
public int Wik_bgn() {return wik_bgn;}
public int Anch_bgn() {return anch_bgn;} // NOTE: anch_bgn is not correct when page has trailing ws; EX: [[A #b]] should have anch_bgn of 3 (1st char after #), but instead it is 2
public byte[] Anch_txt() {return anch_bgn == -1 ? Bry_.Empty : Bry_.Mid(full_txt, anch_bgn, full_txt.length);}
public byte[] Talk_txt() {return ns.Id_is_talk() ? Full_txt() : Bry_.Add(tors_txt, Page_txt());}
public byte[] Subj_txt() {return ns.Id_is_subj() ? Full_txt() : Bry_.Add(tors_txt, Page_txt());}
public byte[] Talk_txt() {return ns.Id_is_talk() ? Full_txt_w_ttl_case() : Bry_.Add(tors_txt, Page_txt());}
public byte[] Subj_txt() {return ns.Id_is_subj() ? Full_txt_w_ttl_case() : Bry_.Add(tors_txt, Page_txt());}
public byte[] Full_url() {return Gfo_url_encoder_.Href.Encode(full_txt);}
public String Full_db_as_str() {return String_.new_u8(Full_db());}
public String Full_db_as_str() {return String_.new_u8(Full_db());}
public byte[] Full_db() {return ns.Gen_ttl(this.Page_db());}
public byte[] Full_db_w_anch() {return Replace_spaces(full_txt);}
public byte[] Page_url() {return Xoa_url_encoder.Instance.Encode(this.Page_txt());}
@@ -103,11 +104,11 @@ public class Xoa_ttl { // PAGE:en.w:http://en.wikipedia.org/wiki/Help:Link; REF.
return new_(wiki, wiki.Appe().Msg_log(), raw, 0, raw.length);
}
public static Xoa_ttl parse(Xowe_wiki wiki, byte[] raw) {return new_(wiki, wiki.Appe().Msg_log(), raw, 0, raw.length);}
private static final Object thread_lock = new Object();
private static final Object thread_lock = new Object();
// $dbkey = preg_replace( '/\xE2\x80[\x8E\x8F\xAA-\xAE]/S', '', $dbkey );
// $dbkey = preg_replace( '/[ _\xA0\x{1680}\x{180E}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}]+/u', '_', $dbkey );
private static final int Char__bidi = 1, Char__ws = 2;
private static final Btrie_slim_mgr char_trie = Btrie_slim_mgr.cs()
private static final Btrie_slim_mgr char_trie = Btrie_slim_mgr.cs()
.Add_many_int(Char__bidi , Bry_.New_by_ints(0xE2, 0x80, 0x8E), Bry_.New_by_ints(0xE2, 0x80, 0x8F), Bry_.New_by_ints(0xE2, 0x80, 0xAA), Bry_.New_by_ints(0xE2, 0x80, 0xAB), Bry_.New_by_ints(0xE2, 0x80, 0xAC), Bry_.New_by_ints(0xE2, 0x80, 0xAD), Bry_.New_by_ints(0xE2, 0x80, 0xAE))
.Add_many_int(Char__ws , "\u00A0", "\u1680", "\u180E", "\u2000", "\u2001", "\u2002", "\u2003", "\u2004", "\u2005", "\u2006", "\u2007", "\u2008", "\u2009", "\u200A", "\u2028", "\u2029", "\u202F", "\u205F", "\u3000")
;
@@ -386,7 +387,7 @@ public class Xoa_ttl { // PAGE:en.w:http://en.wikipedia.org/wiki/Help:Link; REF.
public static final int Anch_bgn_anchor_only = 1; // signifies lnki which is only anchor; EX: [[#anchor]]
public static final int Max_len = 2048; // ASSUME: max len of 256 * 8 bytes
public static final int Null_wik_bgn = -1;
public static final Xoa_ttl Null = null;
public static final Xoa_ttl Null = null;
}
class Xoa_ttl_trie {
public static Btrie_fast_mgr new_() {

11
400_xowa/src/gplx/xowa/Xop_fxt.java
View File

@@ -26,8 +26,8 @@ import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.tdbs.hives.*;
public class Xop_fxt {
public Xop_fxt() {
Xoae_app app = Xoa_app_fxt.app_();
wiki = Xoa_app_fxt.wiki_tst_(app);
Xoae_app app = Xoa_app_fxt.Make__app__edit();
wiki = Xoa_app_fxt.Make__wiki__edit(app);
ctor(app, wiki);
}
public Xop_fxt(Xoae_app app, Xowe_wiki wiki) {
@@ -106,7 +106,7 @@ public class Xop_fxt {
public Xop_tblw_th_tkn_chkr tkn_tblw_th_(int bgn, int end) {return (Xop_tblw_th_tkn_chkr)new Xop_tblw_th_tkn_chkr().Src_rng_(bgn, end);}
public Xop_tblw_tr_tkn_chkr tkn_tblw_tr_(int bgn, int end) {return (Xop_tblw_tr_tkn_chkr)new Xop_tblw_tr_tkn_chkr().Src_rng_(bgn, end);}
public Xop_hdr_tkn_chkr tkn_hdr_(int bgn, int end, int hdr_len) {return (Xop_hdr_tkn_chkr)new Xop_hdr_tkn_chkr().Hdr_level_(hdr_len).Src_rng_(bgn, end);}
public Xop_xnde_tkn_chkr tkn_xnde_br_(int pos) {return tkn_xnde_(pos, pos).Xnde_tagId_(Xop_xnde_tag_.Tid_br);}
public Xop_xnde_tkn_chkr tkn_xnde_br_(int pos) {return tkn_xnde_(pos, pos).Xnde_tagId_(Xop_xnde_tag_.Tid__br);}
public Xop_xnde_tkn_chkr tkn_xnde_() {return tkn_xnde_(String_.Pos_neg1, String_.Pos_neg1);}
public Xop_xnde_tkn_chkr tkn_xnde_(int bgn, int end) {return (Xop_xnde_tkn_chkr)new Xop_xnde_tkn_chkr().Src_rng_(bgn, end);}
public Xop_tkn_chkr_base tkn_curly_bgn_(int bgn) {return new Xop_tkn_chkr_base().TypeId_dynamic(Xop_tkn_itm_.Tid_tmpl_curly_bgn).Src_rng_(bgn, bgn + 2);}
@@ -370,6 +370,7 @@ public class Xop_fxt {
public void Test_html_wiki_frag(String raw, String... expd_frags) {Test_str_part_y(Exec_html_wiki(raw), expd_frags);}
public void Test_html_full_frag(String raw, String... expd_frags) {Test_str_part_y(Exec_html_full(raw), expd_frags);}
public void Test_html_full_frag_n(String raw, String... expd_frags) {Test_str_part_n(Exec_html_full(raw), expd_frags);}
public void Test__parse__tmpl_to_html(String raw, String expd) {Test_str_full(raw, gplx.langs.htmls.Gfh_utl.Replace_apos(expd), Exec_html_full(raw));}
public void Test__parse__wtxt_to_html(String raw, String expd) {
String actl = Exec_html_wiki(raw);
Tfds.Eq_str_lines(gplx.langs.htmls.Gfh_utl.Replace_apos(expd), actl, raw);
@@ -427,8 +428,8 @@ public class Xop_fxt {
}
public void Clear_ref_mgr() {this.Page().Ref_mgr().Grps_clear();} // clear to reset count
public static Xop_fxt new_nonwmf() {
Xoae_app app = Xoa_app_fxt.app_();
return new Xop_fxt(app, Xoa_app_fxt.wiki_nonwmf(app, "nethackwiki"));
Xoae_app app = Xoa_app_fxt.Make__app__edit();
return new Xop_fxt(app, Xoa_app_fxt.Make__wiki__edit__nonwmf(app, "nethackwiki"));
}
private final Bry_bfr tmp_bfr = Bry_bfr.new_(255);
public String Exec__parse_to_hdump(String src_str) {

9
400_xowa/src/gplx/xowa/Xow_wiki.java
View File

@@ -17,8 +17,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.primitives.*;
import gplx.xowa.langs.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.metas.*; import gplx.xowa.wikis.ttls.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.xwikis.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.cases.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.metas.*; import gplx.xowa.wikis.data.site_stats.*; import gplx.xowa.wikis.ttls.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.xwikis.*; import gplx.xowa.addons.*;
import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.origs.*; import gplx.xowa.files.bins.*; import gplx.fsdb.meta.*; import gplx.fsdb.*;
import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.htmls.utls.*; import gplx.xowa.htmls.core.hzips.*; import gplx.xowa.htmls.core.*; import gplx.xowa.htmls.bridges.dbuis.tbls.*;
import gplx.xowa.parsers.*;
@@ -27,6 +27,7 @@ public interface Xow_wiki extends Xow_ttl_parser, GfoInvkAble {
boolean Type_is_edit();
Xoa_app App();
Xol_lang_itm Lang();
Xol_case_mgr Case_mgr();
byte[] Domain_bry(); // EX: en.wikipedia.org
String Domain_str();
int Domain_tid(); // Xow_domain_tid_.Int__wikipedia
@@ -47,6 +48,8 @@ public interface Xow_wiki extends Xow_ttl_parser, GfoInvkAble {
Xow_mw_parser_mgr Mw_parser_mgr();
Xow_xwiki_mgr Xwiki_mgr();
Xow_wiki_props Props();
Xow_site_stats_mgr Stats();
void Init_by_wiki();
Xoa_url_parser Utl__url_parser();
Xow_url_parser Utl__url_parser();
Xoax_addon_mgr Addon_mgr();
}

47
400_xowa/src/gplx/xowa/Xowe_wiki.java
View File

@@ -18,13 +18,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa; import gplx.*;
import gplx.core.brys.*; import gplx.core.primitives.*; import gplx.core.brys.fmtrs.*;
import gplx.xowa.apps.*; import gplx.xowa.apps.fsys.*; import gplx.xowa.apps.cfgs.*; import gplx.xowa.apps.cfgs.gui.*; import gplx.xowa.apps.urls.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.msgs.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.xwikis.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.metas.*; import gplx.xowa.wikis.ttls.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.caches.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.langs.cases.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.xwikis.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.metas.*; import gplx.xowa.wikis.data.site_stats.*; import gplx.xowa.wikis.ttls.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.addons.*;
import gplx.xowa.users.*; import gplx.xowa.htmls.*; import gplx.xowa.users.history.*; import gplx.xowa.specials.*; import gplx.xowa.xtns.*; import gplx.xowa.wikis.dbs.*;
import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.origs.*; import gplx.xowa.files.bins.*; import gplx.fsdb.*; import gplx.fsdb.meta.*; import gplx.xowa.files.exts.*;
import gplx.xowa.htmls.heads.*; import gplx.xowa.htmls.core.htmls.utls.*; import gplx.xowa.htmls.core.hzips.*; import gplx.xowa.htmls.core.*; import gplx.xowa.htmls.css.*; import gplx.xowa.htmls.ns_files.*; import gplx.xowa.htmls.bridges.dbuis.tbls.*;
import gplx.xowa.bldrs.xmls.*; import gplx.xowa.bldrs.installs.*; import gplx.xowa.bldrs.setups.maints.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.utils.*;
import gplx.xowa.wikis.ctgs.*;
import gplx.xowa.guis.views.*;
import gplx.xowa.xtns.gallery.*; import gplx.xowa.xtns.pfuncs.*;
import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.tdbs.hives.*;
@@ -35,7 +36,7 @@ public class Xowe_wiki implements Xow_wiki, GfoInvkAble, GfoEvObj {
this.app = app; this.lang = lang; this.ns_mgr = ns_mgr;
this.domain_itm = domain_itm; this.domain_str = domain_itm.Domain_str(); this.domain_bry = domain_itm.Domain_bry(); this.domain_tid = domain_itm.Domain_type_id(); this.domain_abrv = Xow_abrv_wm_.To_abrv(domain_itm);
this.fsys_mgr = new Xow_fsys_mgr(wiki_dir, app.Fsys_mgr().File_dir().GenSubDir(domain_str));
this.url__parser = new Xoa_url_parser(this);
this.url__parser = new Xow_url_parser(this);
this.xwiki_mgr = new Xow_xwiki_mgr(this);
this.html__hdump_mgr = new Xow_hdump_mgr(this);
this.html_mgr = new Xow_html_mgr(this);
@@ -48,9 +49,9 @@ public class Xowe_wiki implements Xow_wiki, GfoInvkAble, GfoEvObj {
cfg_parser = new Xowc_parser(this);
props.SiteName_(domain_tid).ServerName_(domain_bry);
props.ContentLanguage_(lang.Key_bry());
stats = new Xow_site_stats_mgr(this);
Pf_func_.Reg(lang.Func_regy(), lang);
special_mgr = new Xows_mgr(this, lang);
stats = new Xow_wiki_stats(this);
sys_cfg = new Xow_sys_cfg(this);
hive_mgr = new Xob_hive_mgr(this);
util = new Xow_html_util(this);
@@ -74,20 +75,21 @@ public class Xowe_wiki implements Xow_wiki, GfoInvkAble, GfoEvObj {
cache_mgr = new Xow_cache_mgr(this);
/*if (Bry_.Eq(domain_bry, Xow_domain_itm_.Bry__home))*/ xwiki_mgr.Add_by_atrs(domain_bry, domain_bry); // add full name to xwiki_mgr; needed for lookup in home ns; EX: [[home:Help/Contents]]
}
public GfoEvMgr EvMgr() {return ev_mgr;} private final GfoEvMgr ev_mgr;
public Xow_ns_mgr Ns_mgr() {return ns_mgr;} private final Xow_ns_mgr ns_mgr;
public GfoEvMgr EvMgr() {return ev_mgr;} private final GfoEvMgr ev_mgr;
public Xow_ns_mgr Ns_mgr() {return ns_mgr;} private final Xow_ns_mgr ns_mgr;
public Xoa_ttl Ttl_parse(byte[] ttl) {return Xoa_ttl.parse(this, ttl);}
public Xoa_ttl Ttl_parse(byte[] src, int src_bgn, int src_end) {return Xoa_ttl.new_(this, app.Msg_log(), src, src_bgn, src_end);}
public Xoa_ttl Ttl_parse(int ns_id, byte[] ttl) {return Xoa_ttl.parse(this, ns_id, ttl);}
public boolean Type_is_edit() {return Bool_.Y;}
public Xoa_app App() {return app;}
public Xol_lang_itm Lang() {return lang;} private final Xol_lang_itm lang;
public byte[] Domain_bry() {return domain_bry;} private final byte[] domain_bry;
public String Domain_str() {return domain_str;} private final String domain_str;
public int Domain_tid() {return domain_tid;} private final int domain_tid;
public byte[] Domain_abrv() {return domain_abrv;} private final byte[] domain_abrv;
public Xow_domain_itm Domain_itm() {return domain_itm;} private final Xow_domain_itm domain_itm;
public Xow_fsys_mgr Fsys_mgr() {return fsys_mgr;} private final Xow_fsys_mgr fsys_mgr;
public Xol_lang_itm Lang() {return lang;} private final Xol_lang_itm lang;
public Xol_case_mgr Case_mgr() {return lang.Case_mgr();}
public byte[] Domain_bry() {return domain_bry;} private final byte[] domain_bry;
public String Domain_str() {return domain_str;} private final String domain_str;
public int Domain_tid() {return domain_tid;} private final int domain_tid;
public byte[] Domain_abrv() {return domain_abrv;} private final byte[] domain_abrv;
public Xow_domain_itm Domain_itm() {return domain_itm;} private final Xow_domain_itm domain_itm;
public Xow_fsys_mgr Fsys_mgr() {return fsys_mgr;} private final Xow_fsys_mgr fsys_mgr;
public Xowd_db_mgr Data__core_mgr() {if (db_mgr == null) return null; return db_mgr.Tid() == Xodb_mgr_txt.Tid_txt ? null : this.Db_mgr_as_sql().Core_data_mgr();} // TEST:
public Xof_fsdb_mode File__fsdb_mode() {return file_mgr.Fsdb_mode();}
public Fsdb_db_mgr File__fsdb_core() {return file_mgr.Db_core();}
@@ -98,15 +100,17 @@ public class Xowe_wiki implements Xow_wiki, GfoInvkAble, GfoEvObj {
public boolean Html__hdump_enabled() {return html_mgr__hdump_enabled;} private boolean html_mgr__hdump_enabled = Bool_.N;
public Xoh_page_wtr_mgr Html__wtr_mgr() {return html_mgr.Page_wtr_mgr();}
public boolean Html__css_installing() {return html__css_installing;} public void Html__css_installing_(boolean v) {html__css_installing = v;} private boolean html__css_installing;
public Xoa_url_parser Utl__url_parser() {return url__parser;} private final Xoa_url_parser url__parser;
public Xow_mw_parser_mgr Mw_parser_mgr() {return mw_parser_mgr;} private final Xow_mw_parser_mgr mw_parser_mgr = new Xow_mw_parser_mgr();
public Xow_xwiki_mgr Xwiki_mgr() {return xwiki_mgr;} private final Xow_xwiki_mgr xwiki_mgr;
public Xow_wiki_props Props() {return props;} private final Xow_wiki_props props = new Xow_wiki_props();
public Xow_parser_mgr Parser_mgr() {return parser_mgr;} private final Xow_parser_mgr parser_mgr;
public Xow_url_parser Utl__url_parser() {return url__parser;} private final Xow_url_parser url__parser;
public Xow_mw_parser_mgr Mw_parser_mgr() {return mw_parser_mgr;} private final Xow_mw_parser_mgr mw_parser_mgr = new Xow_mw_parser_mgr();
public Xow_xwiki_mgr Xwiki_mgr() {return xwiki_mgr;} private final Xow_xwiki_mgr xwiki_mgr;
public Xow_wiki_props Props() {return props;} private final Xow_wiki_props props = new Xow_wiki_props();
public Xow_site_stats_mgr Stats() {return stats;} private final Xow_site_stats_mgr stats;
public Xow_parser_mgr Parser_mgr() {return parser_mgr;} private final Xow_parser_mgr parser_mgr;
public Xoax_addon_mgr Addon_mgr() {return addon_mgr;} private final Xoax_addon_mgr addon_mgr = new Xoax_addon_mgr();
public Xow_hdump_mgr Html__hdump_mgr() {return html__hdump_mgr;} private final Xow_hdump_mgr html__hdump_mgr;
public Xow_hdump_mgr Html__hdump_mgr() {return html__hdump_mgr;} private final Xow_hdump_mgr html__hdump_mgr;
public Xoae_app Appe() {return app;} private Xoae_app app;
public Xow_gui_mgr Gui_mgr() {return gui_mgr;} private final Xow_gui_mgr gui_mgr = new Xow_gui_mgr();
public Xow_gui_mgr Gui_mgr() {return gui_mgr;} private final Xow_gui_mgr gui_mgr = new Xow_gui_mgr();
public Xow_user User() {return user;} private Xow_user user = new Xow_user();
public Xow_data_mgr Data_mgr() {return data_mgr;} private Xow_data_mgr data_mgr;
public Xodb_mgr Db_mgr() {return db_mgr;} private Xodb_mgr db_mgr;
@@ -137,8 +141,7 @@ public class Xowe_wiki implements Xow_wiki, GfoInvkAble, GfoEvObj {
public Xow_file_mgr File_mgr() {return file_mgr;} private Xow_file_mgr file_mgr;
public Xow_cfg_wiki_core Cfg_wiki_core() {return cfg_wiki_core;} private Xow_cfg_wiki_core cfg_wiki_core;
public Xob_import_cfg Import_cfg() {return import_cfg;} private Xob_import_cfg import_cfg;
public Xotdb_fsys_mgr Tdb_fsys_mgr() {return tdb_fsys_mgr;} private final Xotdb_fsys_mgr tdb_fsys_mgr;
public Xow_wiki_stats Stats() {return stats;} private Xow_wiki_stats stats;
public Xotdb_fsys_mgr Tdb_fsys_mgr() {return tdb_fsys_mgr;} private final Xotdb_fsys_mgr tdb_fsys_mgr;
public Xou_history_cfg Cfg_history() {return cfg_history;} private Xou_history_cfg cfg_history = new Xou_history_cfg();
public Xoh_cfg_gallery Cfg_gallery() {return cfg_gallery;} private Xoh_cfg_gallery cfg_gallery = new Xoh_cfg_gallery();
public Xoh_file_page_wtr Cfg_file_page() {return cfg_file_page;} private Xoh_file_page_wtr cfg_file_page = new Xoh_file_page_wtr();

6
400_xowa/src/gplx/xowa/specials/search/Srch_rslt_lnr.java → 400_xowa/src/gplx/xowa/addons/Xoax_addon_itm.java
View File

@@ -15,7 +15,7 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.specials.search; import gplx.*; import gplx.xowa.*; import gplx.xowa.specials.*;
public interface Srch_rslt_lnr {
void Notify_rslt_found(Srch_rslt_itm rslt);
package gplx.xowa.addons; import gplx.*; import gplx.xowa.*;
public interface Xoax_addon_itm {
byte[] Addon__key();
}

23
400_xowa/src/gplx/xowa/addons/Xoax_addon_mgr.java Normal file
View File

@@ -0,0 +1,23 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons; import gplx.*; import gplx.xowa.*;
public class Xoax_addon_mgr {
private final Ordered_hash hash = Ordered_hash_.New_bry();
public Xoax_addon_itm Itms__get_or_null(byte[] key) {return (Xoax_addon_itm)hash.Get_by(key);}
public void Itms__add(Xoax_addon_itm itm) {hash.Add(itm.Addon__key(), itm);}
}

55
400_xowa/src/gplx/xowa/addons/ctgs/Xoax_ctg_addon.java Normal file
View File

@@ -0,0 +1,55 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*;
import gplx.xowa.wikis.data.tbls.*;
public class Xoax_ctg_addon implements Xoax_addon_itm { // TODO:mem_mgr
private final Xow_wiki wiki;
private final Hash_adp_bry hash = Hash_adp_bry.cs();
public Xoax_ctg_addon(Xow_wiki wiki) {this.wiki = wiki;}
public byte[] Addon__key() {return Key_const;} public static final byte[] Key_const = Bry_.new_a7("xowa.category");
public Xoctg_ctg_itm Itms__get_or_null(byte[] key) {return (Xoctg_ctg_itm)hash.Get_by_bry(key);}
public Xoctg_ctg_itm Itms__load(byte[] key) {
Xowd_page_itm tmp_page = new Xowd_page_itm();
wiki.Data__core_mgr().Tbl__page().Select_by_ttl(tmp_page, wiki.Ns_mgr().Ns_category(), key);
gplx.xowa.wikis.data.tbls.Xowd_category_itm itm = wiki.Data__core_mgr().Db__cat_core().Tbl__cat_core().Select(tmp_page.Id());
return Itms__add(key, itm.Count_pages(), itm.Count_subcs(), itm.Count_files());
}
public Xoctg_ctg_itm Itms__add(byte[] key, int pages, int subcs, int files) { // TEST:
Xoctg_ctg_itm rv = new Xoctg_ctg_itm(key, pages, subcs, files);
hash.Add(key, rv);
return rv;
}
/*
public long Mem__size__max() {return mem__size__max;} private long mem__size__max;
public long Mem__size__reduce() {return mem__size__reduce;} private long mem__size__reduce;
public void Mem__free__all() {hash.Clear();}
public void Mem__free__reduce() {Mem_mgr_.Free__reduce(hash);}
public void Mem__free__unused() {Mem_mgr_.Free__unused(hash);}
public long Mem__stat__size() {return mem__stat__size;} private long mem__stat__size;
public long Mem__stat__last() {return mem__stat__last;} private long mem__stat__last;
public int Mem__stat__count() {return mem__stat__count;} private int mem__stat__count;
*/
public static Xoax_ctg_addon Get(Xow_wiki wiki) {
Xoax_ctg_addon rv = (Xoax_ctg_addon)wiki.Addon_mgr().Itms__get_or_null(Key_const);
if (rv == null) {
rv = new Xoax_ctg_addon(wiki);
wiki.Addon_mgr().Itms__add(rv);
}
return rv;
}
}

32
400_xowa/src/gplx/xowa/addons/ctgs/Xoctg_ctg_itm.java Normal file
View File

@@ -0,0 +1,32 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*;
public class Xoctg_ctg_itm {
public Xoctg_ctg_itm(byte[] ttl_wo_ns, int pages, int subcs, int files) {
this.Ttl_wo_ns = ttl_wo_ns;
this.Pages = pages;
this.Subcs = subcs;
this.Files = files;
this.All = pages + subcs + files;
}
public final byte[] Ttl_wo_ns;
public final int Pages;
public final int Subcs;
public final int Files;
public final int All;
}

51
400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_pagelinks_parser_cmd.java → 400_xowa/src/gplx/xowa/addons/pagelinks/bldrs/Pglnk_bldr_cmd.java
View File

@@ -15,40 +15,40 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_pagelinks_parser_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd {
private Db_conn core_conn;
private Xowd_pagelinks_temp_tbl temp_tbl;
public Xob_pagelinks_parser_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
@Override public String Sql_file_name() {return "pagelinks";}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_wiki_pagelink;}
private static final byte Fld__pl_from = 0, Fld__pl_namespace = 1, Fld__pl_title = 2;
package gplx.xowa.addons.pagelinks.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.pagelinks.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.*; import gplx.xowa.addons.pagelinks.dbs.*;
public class Pglnk_bldr_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd {
private Db_conn conn;
private Pglnk_page_link_temp_tbl temp_tbl;
private int tmp_src_id, tmp_trg_ns;
private int rows = 0;
public Pglnk_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
@Override public String Sql_file_name() {return Dump_type_key;} public static final String Dump_type_key = "pagelinks";
@Override public String Cmd_key() {return Xob_cmd_keys.Key_wiki_page_link;}
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
parser.Fld_cmd_(this).Flds_req_idx_(4, 0, 1, 2);
wiki.Init_assert();
Xowd_db_file core_db = wiki.Data__core_mgr().Db__core();
this.core_conn = core_db.Conn();
this.temp_tbl = new Xowd_pagelinks_temp_tbl(core_conn);
core_conn.Ddl_delete_tbl(temp_tbl.Tbl_name());
parser.Fld_cmd_(this).Flds_req_idx_(4, 0, 1, 2);
Xob_db_file page_link_db = Xob_db_file.New__page_link(wiki);
this.conn = page_link_db.Conn();
this.temp_tbl = new Pglnk_page_link_temp_tbl(conn);
conn.Meta_tbl_drop(temp_tbl.Tbl_name());
temp_tbl.Create_tbl();
temp_tbl.Insert_bgn();
}
@Override public void Cmd_end() {
if (fail) return;
temp_tbl.Insert_end();
temp_tbl.Create_idx();
Xowd_pagelinks_tbl actl_tbl = new Xowd_pagelinks_tbl(core_conn);
core_conn.Ddl_delete_tbl(actl_tbl.Tbl_name());
Pglnk_page_link_tbl actl_tbl = new Pglnk_page_link_tbl(conn);
conn.Meta_tbl_drop(actl_tbl.Tbl_name());
actl_tbl.Create_tbl();
core_conn.Exec_sql(Sql__pagelinks__make);
core_conn.Ddl_delete_tbl(temp_tbl.Tbl_name());
new Db_attach_mgr(conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()))
.Exec_sql_w_msg("updating page_link", Sql__page_link__make);
conn.Meta_tbl_drop(temp_tbl.Tbl_name());
actl_tbl.Create_idx__src_trg();
actl_tbl.Create_idx__trg_src();
core_conn.Env_vacuum();
conn.Env_vacuum();
}
public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
switch (fld_idx) {
@@ -61,13 +61,14 @@ public class Xob_pagelinks_parser_cmd extends Xob_sql_dump_base implements Sql_f
break;
}
}
private static final String Sql__pagelinks__make = String_.Concat_lines_nl_skip_last
( "INSERT INTO pagelinks (src_id, trg_id, trg_count)"
private static final byte Fld__pl_from = 0, Fld__pl_namespace = 1, Fld__pl_title = 2;
private static final String Sql__page_link__make = String_.Concat_lines_nl_skip_last
( "INSERT INTO page_link (src_id, trg_id, trg_count)"
, "SELECT pl.src_id"
, ", p.page_id"
, ", Count(p.page_id)"
, "FROM pagelinks_temp pl"
, " JOIN page p ON pl.trg_ns = p.page_namespace AND pl.trg_ttl = p.page_title"
, "FROM page_link_temp pl"
, " JOIN <page_db>page p ON pl.trg_ns = p.page_namespace AND pl.trg_ttl = p.page_title"
, "GROUP BY pl.src_id, p.page_id"
);
}

14
400_xowa/src/gplx/xowa/wikis/data/tbls/Xowd_pagelinks_tbl.java → 400_xowa/src/gplx/xowa/addons/pagelinks/dbs/Pglnk_page_link_tbl.java
View File

@@ -15,13 +15,13 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.wikis.data.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*;
package gplx.xowa.addons.pagelinks.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.pagelinks.*;
import gplx.core.ios.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.dbs.cfgs.*;
public class Xowd_pagelinks_tbl implements Rls_able {
private final String tbl_name = "pagelink"; private final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
public class Pglnk_page_link_tbl implements Rls_able {
private final String tbl_name = "page_link"; private final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
private final String fld_src_id, fld_trg_id;
private final Db_conn conn;
public Xowd_pagelinks_tbl(Db_conn conn) {
public Pglnk_page_link_tbl(Db_conn conn) {
this.conn = conn;
fld_src_id = flds.Add_int("src_id");
fld_trg_id = flds.Add_int("trg_id");
@@ -30,8 +30,8 @@ public class Xowd_pagelinks_tbl implements Rls_able {
}
public Db_conn Conn() {return conn;}
public String Tbl_name() {return tbl_name;}
public void Create_tbl() {conn.Ddl_create_tbl(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx__src_trg() {conn.Ddl_create_idx(Gfo_usr_dlg_.Instance, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "src_trg", fld_src_id, fld_trg_id));}
public void Create_idx__trg_src() {conn.Ddl_create_idx(Gfo_usr_dlg_.Instance, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "trg_src", fld_trg_id, fld_src_id));}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx__src_trg() {conn.Meta_idx_create(Gfo_usr_dlg_.Instance, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "src_trg", fld_src_id, fld_trg_id));}
public void Create_idx__trg_src() {conn.Meta_idx_create(Gfo_usr_dlg_.Instance, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "trg_src", fld_trg_id, fld_src_id));}
public void Rls() {}
}

18
400_xowa/src/gplx/xowa/wikis/data/tbls/Xowd_pagelinks_temp_tbl.java → 400_xowa/src/gplx/xowa/addons/pagelinks/dbs/Pglnk_page_link_temp_tbl.java
View File

@@ -15,13 +15,13 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.wikis.data.tbls; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*;
package gplx.xowa.addons.pagelinks.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.pagelinks.*;
import gplx.core.ios.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.dbs.cfgs.*;
public class Xowd_pagelinks_temp_tbl implements Rls_able {
private final String tbl_name = "pagelinks_temp"; private final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
public class Pglnk_page_link_temp_tbl implements Rls_able {
private final String tbl_name = "page_link_temp"; private final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
private final String fld_src_id, fld_trg_ns, fld_trg_ttl;
private final Db_conn conn; private Db_stmt stmt_insert;
public Xowd_pagelinks_temp_tbl(Db_conn conn) {
public Pglnk_page_link_temp_tbl(Db_conn conn) {
this.conn = conn;
flds.Add_int_pkey_autonum("uid");
fld_src_id = flds.Add_int("src_id");
@@ -30,11 +30,11 @@ public class Xowd_pagelinks_temp_tbl implements Rls_able {
conn.Rls_reg(this);
}
public Db_conn Conn() {return conn;}
public String Tbl_name() {return tbl_name;}
public void Create_tbl() {conn.Ddl_create_tbl(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx() {conn.Ddl_create_idx(Gfo_usr_dlg_.Instance, Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "main", fld_src_id, fld_trg_ns, fld_trg_ttl));}
public void Insert_bgn() {conn.Txn_bgn("schema__pagelinks__insert"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public String Tbl_name() {return tbl_name;}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx() {conn.Meta_idx_create(Gfo_usr_dlg_.Instance, Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "main", fld_src_id, fld_trg_ns, fld_trg_ttl));}
public void Insert_bgn() {conn.Txn_bgn("schema__page_link__insert"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public void Insert(int src_id, int trg_ns, byte[] trg_ttl) {
if (stmt_insert == null) stmt_insert = conn.Stmt_insert(tbl_name, flds);
stmt_insert.Clear().Val_int(fld_src_id, src_id).Val_int(fld_trg_ns, trg_ns).Val_bry_as_str(fld_trg_ttl, trg_ttl).Exec_insert();

51
400_xowa/src/gplx/xowa/addons/searchs/Srch_search_addon.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*;
import gplx.xowa.addons.searchs.dbs.*; import gplx.xowa.addons.searchs.searchers.*; import gplx.xowa.addons.searchs.parsers.*; import gplx.xowa.addons.searchs.searchers.rslts.*; import gplx.xowa.addons.searchs.searchers.cbks.*;
import gplx.xowa.langs.cases.*;
public class Srch_search_addon implements Xoax_addon_itm {
private final Srch_search_mgr search_mgr;
public Srch_search_addon(Xow_wiki wiki) {
this.wiki_domain = wiki.Domain_bry();
this.db_mgr = new Srch_db_mgr(wiki).Init();
this.ttl_parser = new Srch_text_parser().Init_for_ttl(wiki.Case_mgr());
this.search_mgr = new Srch_search_mgr(this, wiki, ttl_parser);
}
public byte[] Addon__key() {return Key_const;} public static final byte[] Key_const = Bry_.new_a7("xowa.search");
public byte[] Wiki_domain() {return wiki_domain;} private final byte[] wiki_domain;
public Srch_db_mgr Db_mgr() {return db_mgr;} private final Srch_db_mgr db_mgr;
public Srch_text_parser Ttl_parser() {return ttl_parser;} private final Srch_text_parser ttl_parser;
public void Search_cancel() {search_mgr.Search_cancel();}
public void Search(Srch_search_qry qry, Srch_rslt_cbk cbk) {search_mgr.Search(qry, cbk);}
public void Clear_rslts_cache() {search_mgr.Clear_rslts_cache();}
public static final int Score_max = 1000000;
public static final byte Wildcard__star = Byte_ascii.Star;
public static Srch_search_addon Get(Xow_wiki wiki) {
synchronized (wiki.App()) { // THREAD: must synchronized on app, else 2 wikis with simultaneous search commands will get same addon
Srch_search_addon rv = (Srch_search_addon)wiki.Addon_mgr().Itms__get_or_null(Key_const);
if (rv == null) {
Xoa_app_.Usr_dlg().Log_many("", "", "addons.init: ~{0}", Key_const);
rv = new Srch_search_addon(wiki);
wiki.Addon_mgr().Itms__add(rv);
}
return rv;
}
}
}

110
400_xowa/src/gplx/xowa/addons/searchs/Srch_search_cfg.java Normal file
View File

@@ -0,0 +1,110 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*;
import gplx.core.net.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.addons.searchs.searchers.cbks.*; import gplx.xowa.addons.searchs.searchers.*;
public class Srch_search_cfg implements GfoInvkAble {
private final Xoae_app app;
private String args_default_str = "";// default args for search
private byte search_mode = Tid_search_mode_all_pages_v2;
private int all_pages_extend = 1000; // look ahead by 1000
private int all_pages_min = 10000; // only look at pages > 10 kb
private boolean log_enabled = false;
public Srch_search_cfg(Xoae_app app) {
this.app = app;
ns_mgr.Add_main_if_empty();
}
public boolean Enabled() {return enabled;} private boolean enabled = true;
public int Rslts_max() {return results_max;} private int results_max = 25;
public boolean Auto_wildcard() {return auto_wildcard;} private boolean auto_wildcard = false; // automatically add wild-card; EX: Earth -> *Earth*
public Srch_ns_mgr Ns_mgr() {return ns_mgr;} private final Srch_ns_mgr ns_mgr = new Srch_ns_mgr();
public Gfo_qarg_itm[] Args_default() {return args_default;} private Gfo_qarg_itm[] args_default = Gfo_qarg_itm.Ary_empty;
public void Args_default_str_(String v) {
this.args_default_str = v;
byte[] bry = Bry_.new_a7("http://x.org/a?" + v);
Gfo_url tmp_url = new Gfo_url();
app.User().Wikii().Utl__url_parser().Url_parser().Parse(tmp_url, bry, 0, bry.length);
args_default = tmp_url.Qargs();
}
public void Cancel() {if (addon != null) addon.Search_cancel();}
private Srch_search_addon addon;
public void Search(Xowe_wiki wiki, byte[] search_bry, byte[] cbk_func) {
if ( !enabled
|| search_bry.length == 0
) return;
if (addon == null)
addon = Srch_search_addon.Get(wiki);
else {
if (!Bry_.Eq(wiki.Domain_bry(), addon.Wiki_domain())) // NOTE: suggest-box caches addon at wiki level; need to check if wiki has changed
addon = Srch_search_addon.Get(wiki);
}
this.Cancel();
Srch_search_qry qry = Srch_search_qry.New__suggest_box(wiki, this, search_bry);
Srch_rslt_cbk__suggest_box cbk = new Srch_rslt_cbk__suggest_box(wiki.Appe(), cbk_func, search_bry);
addon.Search(qry, cbk);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_enabled)) return Yn.To_str(enabled);
else if (ctx.Match(k, Invk_enabled_)) enabled = m.ReadYn("v");
else if (ctx.Match(k, Invk_results_max)) return results_max;
else if (ctx.Match(k, Invk_results_max_)) results_max = m.ReadInt("v");
else if (ctx.Match(k, Invk_search_mode)) return Search_mode_str(search_mode);
else if (ctx.Match(k, Invk_search_mode_)) search_mode = Search_mode_parse(m.ReadStr("v"));
else if (ctx.Match(k, Invk_search_mode_list)) return Options_search_mode_list;
else if (ctx.Match(k, Invk_all_pages_extend)) return all_pages_extend;
else if (ctx.Match(k, Invk_all_pages_extend_)) all_pages_extend = m.ReadInt("v");
else if (ctx.Match(k, Invk_all_pages_min)) return all_pages_min;
else if (ctx.Match(k, Invk_all_pages_min_)) all_pages_min = m.ReadInt("v");
else if (ctx.Match(k, Invk_auto_wildcard)) return Yn.To_str(auto_wildcard);
else if (ctx.Match(k, Invk_auto_wildcard_)) auto_wildcard = m.ReadYn("v");
else if (ctx.Match(k, Invk_log_enabled)) return Yn.To_str(log_enabled);
else if (ctx.Match(k, Invk_log_enabled_)) log_enabled = m.ReadYn("v");
else if (ctx.Match(k, Invk_args_default)) return args_default_str;
else if (ctx.Match(k, Invk_args_default_)) Args_default_str_(m.ReadStr("v"));
else return GfoInvkAble_.Rv_unhandled;
return this;
}
public static final String
Invk_enabled = "enabled", Invk_enabled_ = "enabled_", Invk_results_max = "results_max", Invk_results_max_ = "results_max_"
, Invk_search_mode = "search_mode", Invk_search_mode_ = "search_mode_", Invk_search_mode_list = "search_mode_list"
, Invk_all_pages_extend = "all_pages_extend", Invk_all_pages_extend_ = "all_pages_extend_"
, Invk_all_pages_min = "all_pages_min", Invk_all_pages_min_ = "all_pages_min_"
, Invk_auto_wildcard = "auto_wildcard", Invk_auto_wildcard_ = "auto_wildcard_"
, Invk_log_enabled = "log_enabled", Invk_log_enabled_ = "log_enabled_"
, Invk_args_default = "args_default", Invk_args_default_ = "args_default_"
;
private static final String Str_search_mode_search = "Search", Str_search_mode_all_pages_v1 = "AllPages", Str_search_mode_all_pages_v2 = "AllPages_(v2)";
public static final int[] Ns_default_main = new int[] {Xow_ns_.Tid__main};
public static final byte Tid_search_mode_all_pages_v1 = 0, Tid_search_mode_search = 1, Tid_search_mode_all_pages_v2 = 2;
private static Keyval[] Options_search_mode_list = Keyval_.Ary(Keyval_.new_(Str_search_mode_search), Keyval_.new_(Str_search_mode_all_pages_v1), Keyval_.new_(Str_search_mode_all_pages_v2));
private static byte Search_mode_parse(String v) {
if (String_.Eq(v, Str_search_mode_search)) return Tid_search_mode_search;
else if (String_.Eq(v, Str_search_mode_all_pages_v1)) return Tid_search_mode_all_pages_v1;
else if (String_.Eq(v, Str_search_mode_all_pages_v2)) return Tid_search_mode_all_pages_v2;
else throw Err_.new_unhandled(v);
}
private static String Search_mode_str(byte v) {
switch (v) {
case Tid_search_mode_search: return Str_search_mode_search;
case Tid_search_mode_all_pages_v1: return Str_search_mode_all_pages_v1;
case Tid_search_mode_all_pages_v2: return Str_search_mode_all_pages_v2;
default: throw Err_.new_unhandled(v);
}
}
}

42
400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_db_cfg.java Normal file
View File

@@ -0,0 +1,42 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
public class Srch_db_cfg {
public Srch_db_cfg(int version_id, long page_count, int word_count, int link_count_score_max, int link_count_score_cutoff, int link_score_max) {
this.version_id = version_id;
this.page_count = page_count;
this.word_count = word_count;
this.link_count_score_max = link_count_score_max;
this.link_count_score_cutoff = link_count_score_cutoff;
this.link_score_max = link_score_max;
}
public int Version_id() {return version_id;} private int version_id;
public boolean Version_id__needs_upgrade() {return version_id < Srch_db_upgrade.Version__link_score;}
public long Page_count() {return page_count;} private long page_count;
public int Word_count() {return word_count;} private int word_count;
public int Link_count_score_max() {return link_count_score_max;} private int link_count_score_max;
public int Link_count_score_cutoff() {return link_count_score_cutoff;} private int link_count_score_cutoff;
public int Link_score_max() {return link_score_max;} private int link_score_max;
public void Update_link(int link_score_max) {this.link_score_max = link_score_max;}
public void Update_word(int word_count, int link_count_score_max, int link_count_score_cutoff) {
this.version_id = Srch_db_upgrade.Version__link_score;
this.word_count = word_count;
this.link_count_score_max = link_count_score_max;
this.link_count_score_cutoff = link_count_score_cutoff;
}
}

61
400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_db_cfg_.java Normal file
View File

@@ -0,0 +1,61 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.dbs.cfgs.*;
public class Srch_db_cfg_ {
public static Srch_db_cfg New(Db_cfg_tbl cfg_tbl, long page_count, int version_id) { // NOTE: dflt values are for old search dbs
int word_count = cfg_tbl.Assert_int(Grp__search__cfg, Key__word_count, 0);
int link_count_score_max = cfg_tbl.Assert_int(Grp__search__cfg, Key__link_count_score_max, 0);
int link_count_score_cutoff = cfg_tbl.Assert_int(Grp__search__cfg, Key__link_count_score_cutoff, 0);
int link_score_max = cfg_tbl.Assert_int(Grp__search__cfg, Key__link_score_max, 0);
return new Srch_db_cfg(version_id, page_count, word_count, link_count_score_max, link_count_score_cutoff, link_score_max);
}
public static void Update__bldr__link(Db_cfg_tbl cfg_tbl, Srch_db_cfg cfg, int link_score_max) {
cfg.Update_link(link_score_max);
cfg_tbl.Upsert_int(Grp__search__cfg, Key__link_score_max, link_score_max);
}
public static void Update__bldr__word(Db_cfg_tbl cfg_tbl, Srch_db_cfg cfg, int word_count, int link_count_score_max, int link_count_score_cutoff) {
cfg.Update_word(word_count, link_count_score_max, link_count_score_cutoff);
cfg_tbl.Upsert_int(Grp__search__cfg, Key__version_id, Srch_db_upgrade.Version__link_score);
cfg_tbl.Upsert_int(Grp__search__cfg, Key__word_count, word_count);
cfg_tbl.Upsert_int(Grp__search__cfg, Key__link_count_score_max, link_count_score_max);
cfg_tbl.Upsert_int(Grp__search__cfg, Key__link_count_score_cutoff, link_count_score_cutoff);
}
public static int Select__version_id(Db_cfg_tbl cfg_tbl, Srch_word_tbl word_tbl) {
int rv = cfg_tbl.Select_int_or(Grp__search__cfg, Key__version_id, -1);
if (rv >= Srch_db_upgrade.Version__link_score) return rv; // version_id exists; return it;
if (rv == Srch_db_upgrade.Version__link_score_alpha) { // R.16.03.13 has same schema as later version
rv = Srch_db_upgrade.Version__link_score;
} else {
boolean version_is_page_count = word_tbl.conn.Meta_fld_exists(word_tbl.tbl_name, "word_page_count");
rv = version_is_page_count ? Srch_db_upgrade.Version__page_count : Srch_db_upgrade.Version__initial;
}
cfg_tbl.Upsert_int(Grp__search__cfg, Key__version_id, rv);
return rv;
}
public static final int Link_count_score_cutoff = 300;
private static final String
Grp__search__cfg = "xowa.search.cfg"
, Key__version_id = "version_id"
, Key__word_count = "word_count"
, Key__link_count_score_max = "link_count_score_max"
, Key__link_count_score_cutoff = "link_count_score_cutoff"
, Key__link_score_max = "link_score_max"
;
}

114
400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_db_mgr.java Normal file
View File

@@ -0,0 +1,114 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.dbs.cfgs.*;
import gplx.xowa.wikis.data.*;
public class Srch_db_mgr {
private final Xow_wiki wiki;
public final Srch_db_upgrade Upgrade_mgr;
public Srch_db_mgr(Xow_wiki wiki) {
this.wiki = wiki;
Upgrade_mgr = new Srch_db_upgrade(wiki, this);
}
public Srch_db_cfg Cfg() {return cfg;} private Srch_db_cfg cfg;
public Db_cfg_tbl Tbl__cfg() {return tbl__cfg;} private Db_cfg_tbl tbl__cfg;
public Srch_word_tbl Tbl__word() {return tbl__word;} private Srch_word_tbl tbl__word;
public int Tbl__link__len() {return tbl__link__ary.length;}
public Srch_link_tbl Tbl__link__get_at(int i) {return tbl__link__ary[i];}
public Srch_link_tbl[] Tbl__link__ary() {return tbl__link__ary;} private Srch_link_tbl[] tbl__link__ary = Srch_link_tbl.Ary_empty;
public Srch_db_mgr Init() {
Xowd_db_mgr db_mgr = wiki.Data__core_mgr();
Xowd_db_file word_db = null;
if (db_mgr.Db__core().Db_props().Layout_text().Tid_is_all_or_few()) {
// single_db; core_db has search_word and search_link
word_db = db_mgr.Db__core();
tbl__cfg = new Db_cfg_tbl(word_db.Conn(), "xowa_cfg");
tbl__word = new Srch_word_tbl(word_db.Conn(), word_db.Db_props().Schema_is_1());
tbl__link__ary = new Srch_link_tbl[1];
Tbl__link__ary__set(tbl__link__ary, 0, word_db);
} else {
// many_db; figure out link_dbs
word_db = db_mgr.Dbs__get_by_tid_or_null(Srch_db_mgr_.Dbtid__search_core);
if (word_db == null) return this; // HACK: called during db build; skip;
tbl__cfg = new Db_cfg_tbl(word_db.Conn(), "xowa_cfg");
tbl__word = new Srch_word_tbl(word_db.Conn(), word_db.Db_props().Schema_is_1());
Ordered_hash hash = db_mgr.Dbs__get_hash_by_tid(Srch_db_mgr_.Dbtid__search_link);
if (hash == null) { // v2 file layout where search_word and search_link is in 1 search_db
tbl__link__ary = new Srch_link_tbl[1];
Tbl__link__ary__set(tbl__link__ary, 0, word_db);
} else { // v3 file layout where search_link is in many db
int dbs_len = hash.Count();
tbl__link__ary = new Srch_link_tbl[dbs_len];
for (int i = 0; i < dbs_len; ++i) {
Xowd_db_file db_file = (Xowd_db_file)hash.Get_at(i);
Tbl__link__ary__set(tbl__link__ary, i, db_file);
}
}
}
cfg = Srch_db_cfg_.New(tbl__cfg, wiki.Stats().Num_pages(), Srch_db_cfg_.Select__version_id(tbl__cfg, tbl__word));
return this;
}
public void Delete_all(Xowd_db_mgr core_data_mgr) {
tbl__word.conn.Meta_tbl_drop(Srch_link_reg_tbl.Tbl_name);
if (wiki.Data__core_mgr().Db__core().Db_props().Layout_text().Tid_is_all_or_few()) {
// single_db; just drop tables
tbl__word.conn.Meta_tbl_drop(tbl__word.tbl_name);
Srch_link_tbl link_tbl = tbl__link__ary[0];
link_tbl.conn.Meta_tbl_drop(link_tbl.tbl_name);
} else {
// many_db; drop databases
core_data_mgr.Dbs__delete_by_tid(Xowd_db_file_.Tid_search_core);
core_data_mgr.Dbs__delete_by_tid(Xowd_db_file_.Tid_search_link);
}
}
public void Create_all() {
Xowd_db_mgr db_mgr = wiki.Data__core_mgr();
if (wiki.Data__core_mgr().Db__core().Db_props().Layout_text().Tid_is_all_or_few()) {
// single_db; put both in core_db
Xowd_db_file search_db = db_mgr.Db__core();
tbl__word = new Srch_word_tbl(search_db.Conn(), Bool_.N); tbl__word.Create_tbl();
Srch_link_tbl tbl__link = new Srch_link_tbl(search_db.Conn(), Bool_.N); tbl__link.Create_tbl();
tbl__link__ary = new Srch_link_tbl[] {tbl__link};
Srch_link_reg_tbl tbl__lreg = new Srch_link_reg_tbl(search_db.Conn()); tbl__lreg.Create_tbl();
Tbl__link__ary__new(tbl__lreg, tbl__link__ary, db_mgr, 0, Bool_.N, search_db);
} else {
// many_db: put in 3 db;
Xowd_db_file word_db = db_mgr.Dbs__make_by_tid(Srch_db_mgr_.Dbtid__search_core);
tbl__word = new Srch_word_tbl(word_db.Conn(), Bool_.N); tbl__word.Create_tbl();
tbl__link__ary = new Srch_link_tbl[2];
Srch_link_reg_tbl tbl__lreg = new Srch_link_reg_tbl(word_db.Conn()); tbl__lreg.Create_tbl();
Tbl__link__ary__new(tbl__lreg, tbl__link__ary, db_mgr, 0, Bool_.Y, null);
Tbl__link__ary__new(tbl__lreg, tbl__link__ary, db_mgr, 1, Bool_.N, null);
}
}
private static Srch_link_tbl Tbl__link__ary__set(Srch_link_tbl[] ary, int idx, Xowd_db_file db) {
Srch_link_tbl tbl = new Srch_link_tbl(db.Conn(), db.Db_props().Schema_is_1());
ary[idx] = tbl;
return tbl;
}
private static void Tbl__link__ary__new(Srch_link_reg_tbl lreg_tbl, Srch_link_tbl[] ary, Xowd_db_mgr db_mgr, int idx, boolean ns_ids_is_main, Xowd_db_file db) {
if (db == null) {
String ns_ids = (ns_ids_is_main ? "000" : "999");
String suffix = "-xtn.search.link-title-ns." + ns_ids + "-db.001.xowa"; // -xtn.search.link-title-ns.main-db.001.xowa
db = db_mgr.Dbs__make_by_tid(Srch_db_mgr_.Dbtid__search_link, ns_ids, idx, suffix);
}
Srch_link_tbl tbl = Tbl__link__ary__set(ary, idx, db);
tbl.Create_tbl();
lreg_tbl.Insert(idx, db.Id(), Srch_link_reg_tbl.Db_type__title, ns_ids_is_main ? Srch_link_reg_tbl.Ns_type__main : Srch_link_reg_tbl.Ns_type__rest, 0, -1, -1);
}
}

8
400_xowa/src/gplx/xowa/drds/Xod_search_cmd.java → 400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_db_mgr_.java
View File

@@ -15,8 +15,8 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.drds; import gplx.*; import gplx.xowa.*;
import gplx.xowa.specials.search.*;
public interface Xod_search_cmd {
void Search(Cancelable cancelable, Srch_rslt_lnr rslt_lnr, Xow_wiki wiki, String search);
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.xowa.wikis.data.*;
public class Srch_db_mgr_ {
public static final byte Dbtid__search_core = 4, Dbtid__search_link = 16; // SYNC:Xowd_db_file_
}

55
400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_db_upgrade.java Normal file
View File

@@ -0,0 +1,55 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*;
public class Srch_db_upgrade {
private final Xow_wiki wiki;
private boolean upgrade_prompted;
public Srch_db_upgrade(Xow_wiki wiki, Srch_db_mgr search_db_mgr) {
this.wiki = wiki;
}
public void Upgrade() {
if (!wiki.App().Mode().Tid_is_gui()) return; // ignore if html-server or drd-app
if (upgrade_prompted) return;
upgrade_prompted = true;
Xoae_app app = ((Xoae_app)wiki.App());
boolean ok = app.Gui_mgr().Kit().Ask_ok_cancel("", "", String_.Concat_lines_nl_skip_last
( "XOWA would like to upgrade your search database for " + wiki.Domain_str() + "."
, ""
, "* Press OK to upgrade. This may take an hour for English Wikipedia."
, "* Press Cancel to skip. You will not be able to search."
, ""
, "If you cancel, XOWA will ask you to upgrade this wiki again next time you restart the application."
, ""
, "Note that you can run this upgrade process manually by doing:"
, " Main Menu -> Tools -> Wiki Maintenance -> Search"
));
if (!ok) return;
Xowe_wiki wikie = (Xowe_wiki)wiki;
gplx.xowa.addons.searchs.dbs.bldrs.Srch_bldr_mgr_.Setup(wikie);
gplx.xowa.bldrs.Xob_bldr bldr = app.Bldr();
bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_alert_cmd(bldr, wikie).Msg_("search upgrade finished"));
gplx.core.threads.Thread_adp_.invk_("search_upgrade", app.Bldr(), gplx.xowa.bldrs.Xob_bldr.Invk_run).Start();
}
public static final int
Version__link_score_alpha = 0 // in 2016-02 android alpha
, Version__initial = 1
, Version__page_count = 2
, Version__link_score = 3
;
}

44
400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_link_reg_tbl.java Normal file
View File

@@ -0,0 +1,44 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.dbs.*;
public class Srch_link_reg_tbl implements Rls_able {
private final String tbl_name; public final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
private final String fld_id, fld_db_id, fld_db_type, fld_ns_ids, fld_sub_id, fld_score_min, fld_score_max;
private final Db_conn conn;
public Srch_link_reg_tbl(Db_conn conn) {
this.conn = conn;
tbl_name = Tbl_name;
fld_id = flds.Add_int_pkey ("reg_id"); // corresponds to link_tbl_ary_idx
fld_db_id = flds.Add_int ("reg_db_id"); // corresponds to xowa_db
fld_db_type = flds.Add_int ("reg_db_type"); // "title"; "text"
fld_ns_ids = flds.Add_int ("reg_ns_ids"); // "0"; "*"
fld_sub_id = flds.Add_int ("reg_sub_id"); // "0"; "*"
fld_score_min = flds.Add_int ("reg_score_min"); // -1
fld_score_max = flds.Add_int ("reg_score_max"); // -1
}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Rls() {}
public void Insert(int id, int db_id, String db_type, String ns_ids, int sub_id, int score_min, int score_max) {
Db_stmt stmt_insert = conn.Stmt_insert(tbl_name, flds);
stmt_insert.Clear().Val_int(fld_id, id).Val_int(fld_db_id, db_id).Val_str(fld_db_type, db_type).Val_str(fld_ns_ids, ns_ids).Val_int(fld_sub_id, sub_id).Val_int(fld_score_min, score_min).Val_int(fld_score_max, score_max).Exec_insert();
}
public static final String Tbl_name = "search_link_reg";
public static final String Db_type__title = "title" , Db_type__text = "text";
public static final String Ns_type__main = "0" , Ns_type__rest = "*";
}

41
400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_link_tbl.java Normal file
View File

@@ -0,0 +1,41 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.dbs.*;
public class Srch_link_tbl {
public final String tbl_name;
public final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
public final String fld_word_id, fld_page_id, fld_link_score;
public final boolean schema_is_1;
public final Db_conn conn;
public Srch_link_tbl(Db_conn conn, boolean schema_is_1) {
this.conn = conn; this.schema_is_1 = schema_is_1;
String fld_prefix = "";
if (schema_is_1) {tbl_name = "search_title_page"; fld_prefix = "stp_";}
else {tbl_name = "search_link";}
fld_word_id = flds.Add_int(fld_prefix + "word_id");
fld_page_id = flds.Add_int(fld_prefix + "page_id");
fld_link_score = flds.Add_int_dflt(fld_prefix + Fld_link_score, 0);
}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx__page_id() {} // TODO: conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "page_id", fld_page_id));}
public void Create_idx__link_score() {conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "word_id__link_score", fld_word_id, Fld_link_score));}
public static final Srch_link_tbl[] Ary_empty = new Srch_link_tbl[0];
public static final String Fld_link_score = "link_score";
}

45
400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_temp_tbl.java Normal file
View File

@@ -0,0 +1,45 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.dbs.*;
public class Srch_temp_tbl {
public final String tbl_name = "search_temp";
private final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
private final String fld_word_id, fld_page_id, fld_word_text;
public final Db_conn conn; private Db_stmt stmt_insert;
public Srch_temp_tbl(Db_conn conn) {
this.conn = conn;
flds.Add_int_pkey_autonum("word_uid");
fld_word_id = flds.Add_int("word_id");
fld_page_id = flds.Add_int("page_id");
fld_word_text = flds.Add_str("word_text", 255);
}
public void Insert_bgn() {
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
stmt_insert = conn.Stmt_insert(tbl_name, flds);
conn.Txn_bgn("schema__search_temp__insert");
}
public void Insert_cmd_by_batch(int word_id, int page_id, byte[] word) {
stmt_insert.Clear().Val_int(fld_word_id, word_id).Val_int(fld_page_id, page_id).Val_bry_as_str(fld_word_text, word).Exec_insert();
}
public void Insert_end() {
conn.Txn_end();
stmt_insert = Db_stmt_.Rls(stmt_insert);
conn.Meta_idx_create(Xoa_app_.Usr_dlg(), Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "main", fld_word_text, fld_page_id));
}
}

30
400_xowa/src/gplx/xowa/specials/search/crts/Srch_crt_tkn.java → 400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_word_row.java
View File

@@ -15,23 +15,17 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.specials.search.crts; import gplx.*; import gplx.xowa.*; import gplx.xowa.specials.*; import gplx.xowa.specials.search.*;
class Srch_crt_tkn {
public final byte tid;
public final byte[] val;
public Srch_crt_tkn(byte tid, byte[] val) {this.tid = tid; this.val = val;}
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
public class Srch_word_row {
public Srch_word_row(int id, byte[] text, int link_count, int link_score_min, int link_score_max) {
this.Id = id; this.Text = text; this.Link_count = link_count;
this.Link_score_min = link_score_min; this.Link_score_max = link_score_max;
}
public final int Id;
public final byte[] Text;
public final int Link_count;
public final int Link_score_min;
public final int Link_score_max;
public static final byte
Tid_root = 1
, Tid_word = 2
, Tid_word_quoted = 3
, Tid_space = 4
, Tid_quote = 5
, Tid_not = 6
, Tid_paren_bgn = 7
, Tid_paren_end = 8
, Tid_or = 9
, Tid_and = 10
, Tid_eos = 11
;
public static final Srch_word_row Empty = new Srch_word_row(-1, Bry_.Empty, 0, 0, 0);
}

66
400_xowa/src/gplx/xowa/addons/searchs/dbs/Srch_word_tbl.java Normal file
View File

@@ -0,0 +1,66 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.dbs.*;
public class Srch_word_tbl implements Rls_able {
public final String tbl_name;
public final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
public final String fld_id, fld_text, fld_link_count, fld_link_count_score, fld_link_score_min, fld_link_score_max;
public final Db_conn conn; private Db_stmt stmt_insert, stmt_select_by;
public Srch_word_tbl(Db_conn conn, boolean schema_is_1) {
this.conn = conn;
String fld_prefix = "", fld_text_name = "word_text";
if (schema_is_1) {tbl_name = "search_title_word"; fld_prefix = "stw_"; fld_text_name = "stw_word";}
else {tbl_name = "search_word";}
this.fld_id = flds.Add_int_pkey(fld_prefix + "word_id");
this.fld_text = flds.Add_str(fld_text_name, 255);
this.fld_link_count = flds.Add_int("link_count");
this.fld_link_count_score = Dbmeta_fld_itm.Make_or_null(conn, flds, tbl_name, Dbmeta_fld_tid.Tid__int, 0, "link_count_score");
this.fld_link_score_min = Dbmeta_fld_itm.Make_or_null(conn, flds, tbl_name, Dbmeta_fld_tid.Tid__int, Int_.Max_value__31, "link_score_min");
this.fld_link_score_max = Dbmeta_fld_itm.Make_or_null(conn, flds, tbl_name, Dbmeta_fld_tid.Tid__int, 0, "link_score_max");
conn.Rls_reg(this);
}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx() {
// idx for rng_bgn, rng_end
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "word_text__link_score_max__link_score_min", fld_text, Fld_link_score_max, Fld_link_score_min));
// idx for like
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "link_score_max__link_score_min", Fld_link_score_max, Fld_link_score_min));
}
public void Insert_bgn() {conn.Txn_bgn("schema__search_word__insert"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public void Insert_cmd_by_batch(int id, byte[] word, int page_count) {stmt_insert.Clear().Val_int(fld_id, id).Val_bry_as_str(fld_text, word).Val_int(fld_link_count, page_count).Exec_insert();}
public Srch_word_row Select_or_empty(byte[] word) {
if (stmt_select_by == null) stmt_select_by = conn.Stmt_select(tbl_name, flds, fld_text);
Db_rdr rdr = stmt_select_by.Clear().Crt_bry_as_str(fld_text, word).Exec_select__rls_manual();
try {return rdr.Move_next() ? New_row(rdr) : Srch_word_row.Empty;}
finally {rdr.Rls();}
}
public Srch_word_row New_row(Db_rdr rdr) {
int page_count = fld_link_count == Dbmeta_fld_itm.Key_null ? 0 : rdr.Read_int(fld_link_count);
int link_score_min = fld_link_score_min == Dbmeta_fld_itm.Key_null ? page_count : rdr.Read_int(fld_link_score_min);
int link_score_max = fld_link_score_max == Dbmeta_fld_itm.Key_null ? page_count : rdr.Read_int(fld_link_score_max);
return new Srch_word_row(rdr.Read_int(fld_id), rdr.Read_bry_by_str(fld_text), page_count, link_score_min, link_score_max);
}
public void Rls() {
stmt_insert = Db_stmt_.Rls(stmt_insert);
stmt_select_by = Db_stmt_.Rls(stmt_select_by);
}
public static final String Fld_link_score_min = "link_score_min", Fld_link_score_max = "link_score_max";
}

34
400_xowa/src/gplx/xowa/addons/searchs/dbs/bldrs/Srch_bldr_cmd.java Normal file
View File

@@ -0,0 +1,34 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.dbs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Srch_bldr_cmd extends Xob_cmd__base implements Xob_cmd {
private int commit_interval = 100000, progress_interval = 10000;
public Srch_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_text_search_cmd;}
@Override public void Cmd_run() {
if (!gplx.core.envs.Env_.Mode_testing()) wiki.Init_assert();
new Srch_temp_tbl_wkr().Exec_by_cmd(wiki, commit_interval, progress_interval);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_commit_interval_)) commit_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_progress_interval_)) progress_interval = m.ReadInt("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_";
}

38
400_xowa/src/gplx/xowa/addons/searchs/dbs/bldrs/Srch_bldr_mgr_.java Normal file
View File

@@ -0,0 +1,38 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.dbs.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.addons.searchs.dbs.bldrs.cmds.*;
public class Srch_bldr_mgr_ {
public static void Setup(Xowe_wiki wiki) {
Xoae_app app = wiki.Appe();
Xob_bldr bldr = app.Bldr();
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_text_search_cmd);
int page_rank_iterations = app.Api_root().Bldr().Wiki().Import().Page_rank().Iteration_max();
boolean page_rank_enabled = page_rank_iterations > 0;
if (page_rank_enabled) {
bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_download_cmd(bldr, wiki).Dump_type_(gplx.xowa.addons.pagelinks.bldrs.Pglnk_bldr_cmd.Dump_type_key));
bldr.Cmd_mgr().Add_many(wiki, Xob_cmd_keys.Key_wiki_page_link);
}
bldr.Cmd_mgr().Add(new Srch__page__page_score(bldr, wiki).Iteration_max_(page_rank_iterations));
bldr.Cmd_mgr().Add(new Srch__link__link_score(bldr, wiki).Page_rank_enabled_(page_rank_enabled).Delete_plink_db_(Bool_.Y));
bldr.Cmd_mgr().Add(new Srch__word__link_count(bldr, wiki));
bldr.Cmd_mgr().Add(new gplx.xowa.bldrs.cmds.utils.Xob_delete_cmd(bldr, wiki).Patterns_ary_("*pagelinks.sql", "*pagelinks.sql.gz", "*pagelinks.sqlite3"));
}
}

36
400_xowa/src/gplx/xowa/addons/searchs/dbs/bldrs/Srch_bldr_wkr.java Normal file
View File

@@ -0,0 +1,36 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.dbs.*;
import gplx.xowa.bldrs.*;
public class Srch_bldr_wkr implements Xob_page_wkr {
private final Xowe_wiki wiki;
private final Srch_temp_tbl_wkr temp_tbl_wkr = new Srch_temp_tbl_wkr();
public Srch_bldr_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.wiki = wiki;}
public String Wkr_key() {return Xob_cmd_keys.Key_text_search_wkr;}
public void Wkr_bgn(Xob_bldr bldr) {
temp_tbl_wkr.Init(Bool_.N, wiki);
}
public void Wkr_run(gplx.xowa.wikis.data.tbls.Xowd_page_itm page) {
try {temp_tbl_wkr.Exec_by_wkr(page.Id(), page.Ttl_page_db());}
catch (Exception e) {Gfo_usr_dlg_.Instance.Warn_many("", "", "search:error: page=~{0} err=~{1}", page.Ttl_page_db(), Err_.Message_gplx_full(e));}
}
public void Wkr_end() {
temp_tbl_wkr.Term();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return this;}
}

171
400_xowa/src/gplx/xowa/addons/searchs/dbs/bldrs/Srch_temp_tbl_wkr.java Normal file
View File

@@ -0,0 +1,171 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.dbs.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.addons.searchs.dbs.*; import gplx.xowa.addons.searchs.parsers.*;
class Srch_temp_tbl_wkr implements Srch_text_parser_wkr {
private Xowe_wiki wiki; private Xowd_db_mgr core_data_mgr; private Srch_search_addon search_addon;
private Srch_text_parser title_parser;
private Srch_temp_tbl search_temp_tbl; private int word_id, page_id; // needed for Parse_done
public Srch_temp_tbl_wkr Init(boolean cmd, Xowe_wiki wiki) {
// init
this.wiki = wiki;
this.core_data_mgr = wiki.Db_mgr_as_sql().Core_data_mgr();
this.search_addon = Srch_search_addon.Get(wiki);
this.title_parser = search_addon.Ttl_parser();
this.word_id = 0;
// assert search_word / search_link tables
Srch_db_mgr search_db_mgr = search_addon.Db_mgr();
if (cmd) search_db_mgr.Delete_all(core_data_mgr); // always delete if launched by cmd
search_db_mgr.Create_all();
// start processing search_temp
this.search_temp_tbl = new Srch_temp_tbl(search_db_mgr.Tbl__word().conn);
search_temp_tbl.Insert_bgn();
return this;
}
public void Term() {
// end inserts
search_temp_tbl.Insert_end();
// init
Srch_db_mgr search_db_mgr = search_addon.Db_mgr();
Db_conn word_conn = search_temp_tbl.conn;
Db_conn page_conn = wiki.Data__core_mgr().Tbl__page().conn;
// update search_word ids if they exist
Update_word_id(word_conn, wiki);
word_conn.Exec_sql("filling search_word (please wait)", Sql__create_word);
// create search_link
if (search_db_mgr.Tbl__link__len() == 1) {
// single_db; just run sql;
Srch_link_tbl link_tbl = search_db_mgr.Tbl__link__ary()[0];
new Db_attach_mgr(word_conn, new Db_attach_itm("link_db", link_tbl.conn))
.Exec_sql(Sql__create_link__one);
link_tbl.Create_idx__page_id();
} else {
// many_db; split into main links and non-main; ASSUME:link_dbs_is_2
int len = search_db_mgr.Tbl__link__len();
Db_attach_mgr attach_mgr = new Db_attach_mgr();
for (int i = 0; i < len; ++i) {
Srch_link_tbl link_tbl = search_db_mgr.Tbl__link__ary()[i];
attach_mgr.Main_conn_(word_conn).Init
( new Db_attach_itm("page_db", page_conn)
, new Db_attach_itm("link_db", link_tbl.conn));
attach_mgr.Exec_sql(Sql__create_link__many, i == 0 ? " = 0" : " != 0");
link_tbl.Create_idx__page_id();
}
}
// remove search_temp
word_conn.Meta_tbl_drop(search_temp_tbl.tbl_name);
}
public void Exec_by_wkr(int page_id, byte[] page_ttl) {
this.page_id = page_id;
title_parser.Parse(this, page_ttl);
}
public void Exec_by_cmd(Xowe_wiki wiki, int commit_interval, int progress_interval) {
this.Init(Bool_.Y, wiki);
Xowd_page_tbl page_tbl = core_data_mgr.Tbl__page(); String fld_page_id = page_tbl.Fld_page_id(); String fld_page_ttl = page_tbl.Fld_page_title();
Db_rdr page_rdr = page_tbl.Select_all__id__ttl(); int page_count = 0;
try {
while (page_rdr.Move_next()) {
this.page_id = page_rdr.Read_int(fld_page_id);
byte[] page_ttl = page_rdr.Read_bry_by_str(fld_page_ttl);
try {title_parser.Parse(this, page_ttl);}
catch (Exception e) {Xoa_app_.Usr_dlg().Warn_many("", "", "error while parsing title; id=~{0} title=~{1} err=~{2}", page_id, page_ttl, Err_.Message_gplx_log(e));}
++page_count;
if ((page_count % commit_interval) == 0) search_temp_tbl.conn.Txn_sav();
if ((page_count % progress_interval) == 0) Gfo_usr_dlg_.Instance.Prog_many("", "", "parse progress: count=~{0} last=~{1}", page_count, String_.new_u8(page_ttl));
}
} finally {page_rdr.Rls();}
this.Term();
}
public void Parse_done(Srch_word_itm word) {
search_temp_tbl.Insert_cmd_by_batch(++word_id, page_id, word.Word);
}
private void Update_word_id(Db_conn cur_conn, Xow_wiki wiki) {
// see if prv_url exists
Io_url cur_url = gplx.dbs.engines.sqlite.Sqlite_conn_info.To_url(cur_conn); // EX: /xowa/wiki/en.wikipedia.org/en.wikipedia.org-xtn.search.core.xowa
Io_url prv_url = wiki.Fsys_mgr().Root_dir().GenSubFil_nest("prv", cur_url.NameAndExt()); // EX: /xowa/wiki/en.wikipedia.org/prv/en.wikipedia.org-xtn.search.core.xowa
if (!Io_mgr.Instance.Exists(prv_url)) return;
// update ids for old_words
cur_conn.Exec_sql("UPDATE search_temp SET word_id = -1");
Db_conn prv_conn = Db_conn_bldr.Instance.Get_or_noop(prv_url);
new Db_attach_mgr(cur_conn, new Db_attach_itm("prv_db", prv_conn))
.Exec_sql_w_msg("updating old word ids", String_.Concat_lines_nl_skip_last
( "UPDATE search_temp"
, "SET word_id = Coalesce((SELECT prv.word_id FROM <prv_db>search_word prv WHERE prv.word_text = search_temp.word_text), -1)"
));
// assign incrementing int to new_words
int nxt_word_id = cur_conn.Exec_select_as_int("SELECT Max(word_id) AS word_id FROM search_temp", -1); if (nxt_word_id == -1) throw Err_.new_("dbs", "max_id not found");
int uids_max = 10000;
int[] uids_ary = new int[uids_max]; int uid_last = -1;
Db_stmt update_stmt = cur_conn.Stmt_update("search_temp", String_.Ary("word_uid"), "word_id");
while (true) {
// read 10,000 into memory
int uids_len = 0;
Db_rdr rdr = cur_conn.Exec_rdr("SELECT word_uid FROM search_temp WHERE word_id = -1 AND word_uid > " + Int_.To_str(uid_last));
try {
while (rdr.Move_next()) {
int uid = rdr.Read_int("word_uid");
uids_ary[uids_len++] = uid;
if (uids_len == uids_max) {
uid_last = uid;
break;
}
}
}
finally {rdr.Rls();}
if (uids_len == 0) break;
// update
for (int i = 0; i < uids_max; ++i) {
int uid = uids_ary[i];
update_stmt.Clear().Val_int("word_id", ++nxt_word_id).Crt_int("word_uid", uid).Exec_update();
}
}
}
private static final String
Sql__create_word = String_.Concat_lines_nl
( "INSERT INTO search_word (word_id, word_text, link_count)"
, "SELECT word_id"
, ", word_text"
, ", Count(DISTINCT page_id)"
, "FROM search_temp"
, "GROUP BY "
, " word_text"
, ";"
)
, Sql__create_link__one = String_.Concat_lines_nl
( "INSERT INTO <link_db>search_link (word_id, page_id)"
, "SELECT w.word_id"
, ", t.page_id"
, "FROM search_temp t"
, " JOIN search_word w ON t.word_text = w.word_text"
, ";"
)
, Sql__create_link__many = String_.Concat_lines_nl
( "INSERT INTO <link_db>search_link (word_id, page_id)"
, "SELECT w.word_id"
, ", t.page_id"
, "FROM search_temp t"
, " JOIN search_word w ON t.word_text = w.word_text"
, " JOIN <page_db>page p ON t.page_id = p.page_id"
, "WHERE p.page_namespace {0}"
, ";"
)
;
}

182
400_xowa/src/gplx/xowa/addons/searchs/dbs/bldrs/cmds/Srch__link__link_score.java Normal file
View File

@@ -0,0 +1,182 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.dbs.*; import gplx.xowa.addons.searchs.dbs.bldrs.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.addons.sqlite_utils.bldrs.*;
import gplx.xowa.addons.searchs.dbs.*;
public class Srch__link__link_score extends Xob_cmd__base implements Xob_cmd {
private int score_multiplier = 100000000;
private boolean page_rank_enabled = false;
private boolean delete_plink_db = false;
public Srch__link__link_score(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
public Srch__link__link_score Page_rank_enabled_(boolean v) {this.page_rank_enabled = v; return this;}
public Srch__link__link_score Delete_plink_db_(boolean v) {this.delete_plink_db = v; return this;}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_search__link__link_score;}
@Override public void Cmd_run() {
wiki.Init_assert();
Db_conn plink_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, wiki.Fsys_mgr().Root_dir().GenSubFil(Xob_db_file.Name__page_link));
String page_rank_tbl = Srch__page__page_score.Pagerank__tbl_name;
String log_module = "search.page.score";
Xoa_app_.Plog_none(log_module, "adding fields to page_rank_temp");
plink_conn.Meta_fld_assert(page_rank_tbl, "page_len" , Dbmeta_fld_tid.Itm__int, 0);
plink_conn.Meta_fld_assert(page_rank_tbl, "page_len_score" , Dbmeta_fld_tid.Itm__int, 0);
plink_conn.Meta_fld_assert(page_rank_tbl, "page_rank_score" , Dbmeta_fld_tid.Itm__double, 0);
plink_conn.Meta_fld_assert(page_rank_tbl, "page_score" , Dbmeta_fld_tid.Itm__int, 0);
int link_score_max = Srch_search_addon.Score_max;
// percentize page_len_score to 0 : 100,000,000; NOTE: 100,000,000 so that no individual score should have 2+ page; i.e.: score_range > page_count
new Db_attach_mgr(plink_conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Tbl__page().conn))
.Exec_sql_w_msg("filling page_rank_temp.page_len", String_.Concat_lines_nl_skip_last
( "UPDATE page_rank_temp"
, "SET page_len = (SELECT page_len FROM <page_db>page p WHERE p.page_id = page_rank_temp.page_id)"
));
new Sqlite_percentile_cmd(bldr, wiki).Init_by_rel_url(Xob_db_file.Name__page_link, "temp_page_len", link_score_max, String_.Concat_lines_nl_skip_last
( "SELECT p.page_id, p.page_len"
, "FROM <page_db>page p"
, "ORDER BY p.page_len" // NOTE: ORDER BY is needed b/c INSERT into AUTO INCREMENT table
)).Cmd_run();
plink_conn.Exec_sql("finalizing page_rank_temp.page_len_score", String_.Concat_lines_nl_skip_last
( "UPDATE page_rank_temp"
, "SET page_len_score = (SELECT tpl.row_score FROM temp_page_len tpl WHERE tpl.row_key = page_rank_temp.page_id)"
));
// calc page_score
if (page_rank_enabled) {
// get min / max
Xoa_app_.Plog_none(log_module, "calculating page_rank range");
double page_rank_min = plink_conn.Exec_select_as_double("SELECT Min(page_rank) FROM " + page_rank_tbl, Double_.MinValue); if (page_rank_min == Double_.MinValue) throw Err_.new_("bldr", "failed to get min; tbl=~{0}", page_rank_tbl);
double page_rank_max = plink_conn.Exec_select_as_double("SELECT Max(page_rank) FROM " + page_rank_tbl, Double_.MinValue); if (page_rank_max == Double_.MinValue) throw Err_.new_("bldr", "failed to get max; tbl=~{0}", page_rank_tbl);
double page_rank_rng = page_rank_max - page_rank_min;
if (page_rank_rng == 0) page_rank_rng = 1; // if 0, set to 1 to prevent divide by 0 below;
String score_multiplier_as_str = Dbmeta_fld_itm.To_double_str_by_int(score_multiplier);
// normalize page_rank to 0 : 100,000,000
plink_conn.Exec_sql
( "normalizing page_rank_temp.page_rank_score"
, Bry_fmt.Make_str
("UPDATE ~{tbl} SET page_rank_score = ((Coalesce(page_rank, 0)) / ~{page_score_rng}) * ~{score_multiplier}"
, page_rank_tbl, Double_.To_str(page_rank_rng), score_multiplier_as_str)
);
// adjust for page_len where pages in lower 60% will get penalized; EX: page_rank = 100; page_len = 50,000,000 -> 100 * (50 M / 100 M) -> 50
String page_len_cutoff = Dbmeta_fld_itm.To_double_str_by_int((int)(score_multiplier * .6));
plink_conn.Exec_sql
( "penalizing short pages"
, Bry_fmt.Make_str(String_.Concat_lines_nl_skip_last
( "UPDATE page_rank_temp"
, "SET page_rank_score = (page_rank_score * CASE WHEN page_len_score < ~{page_len_cutoff} THEN (page_len_score / ~{score_multiplier}) ELSE 1 END)"
), page_len_cutoff, score_multiplier_as_str));
// percentize page_rank_score to 0 : 100,000
new Sqlite_percentile_cmd(bldr, wiki).Init_by_rel_url(Xob_db_file.Name__page_link, "temp_page_score", link_score_max, String_.Concat_lines_nl_skip_last
( "SELECT prt.page_id, prt.page_rank_score"
, "FROM page_rank_temp prt"
, "ORDER BY prt.page_rank_score" // NOTE: ORDER BY is needed b/c INSERT into AUTO INCREMENT table
)).Cmd_run();
plink_conn.Exec_sql
( "finalizing page_rank_temp.page_score"
, String_.Concat_lines_nl_skip_last
( "UPDATE page_rank_temp"
, "SET page_score = Cast((SELECT tmp.row_score FROM temp_page_score tmp WHERE tmp.row_key = page_rank_temp.page_id) AS int)"
));
}
else {
plink_conn.Exec_sql
( "finalizing page_rank_temp.page_score"
, String_.Format(String_.Concat_lines_nl_skip_last
( "UPDATE page_rank_temp"
, "SET page_score = Cast(page_len_score AS int)"
)));
}
plink_conn.Meta_idx_create(Xoa_app_.Usr_dlg(), Dbmeta_idx_itm.new_normal_by_tbl("page_rank_temp", "page_score", "page_id", "page_score"));
// update page table
Xowd_page_tbl page_tbl = wiki.Data__core_mgr().Tbl__page();
Db_conn page_conn = page_tbl.conn;
if (page_tbl.Fld_page_score() == Dbmeta_fld_itm.Key_null) {
page_conn.Meta_fld_append(page_tbl.Tbl_name(), Dbmeta_fld_itm.new_int(Xowd_page_tbl.Fld__page_score__key).Default_(0));
page_tbl = wiki.Data__core_mgr().Db__core().Tbl__page__rebind();
}
new Db_attach_mgr(page_conn, new Db_attach_itm("plink_db", plink_conn))
.Exec_sql_w_msg("updating page.page_score", String_.Concat_lines_nl_skip_last
( "UPDATE page"
, "SET page_score = Coalesce((SELECT Cast(pr.page_score AS integer) FROM <plink_db>page_rank_temp pr WHERE pr.page_id = page.page_id), 0)"
));
// update link tables
Srch_db_mgr search_db_mgr = Srch_search_addon.Get(wiki).Db_mgr();
Srch_word_tbl word_tbl = search_db_mgr.Tbl__word();
if (!page_tbl.conn.Eq(word_tbl.conn)) page_tbl.conn.Env_vacuum(); // don't vacuum if single-db
int link_tbls_len = search_db_mgr.Tbl__link__len();
for (int i = 0; i < link_tbls_len; ++i) {
Srch_link_tbl link_tbl = search_db_mgr.Tbl__link__get_at(i);
// update search_link.link_score
link_tbl.conn.Meta_fld_assert(link_tbl.tbl_name, Srch_link_tbl.Fld_link_score, Dbmeta_fld_tid.Itm__int, 0);
new Db_attach_mgr(link_tbl.conn
, new Db_attach_itm("page_db", page_conn)
).Exec_sql_w_msg
( Bry_fmt.Make_str("updating search_link.link_score: ~{idx} of ~{total}", i + 1, link_tbls_len)
, String_.Concat_lines_nl_skip_last
( "UPDATE search_link"
, "SET link_score = Coalesce((SELECT page_score FROM <page_db>page p WHERE p.page_id = search_link.page_id), 0)"
));
// update word_tbl.page_score_max
new Db_attach_mgr(word_tbl.conn
, new Db_attach_itm("link_db", link_tbl.conn)
)
.Exec_sql_w_msg
( Bry_fmt.Make_str("updating link_score_min: ~{idx} of ~{total}", i + 1, link_tbls_len)
, String_.Concat_lines_nl_skip_last
( "REPLACE INTO search_word (word_id, word_text, link_count, link_score_min, link_score_max)"
, "SELECT sw.word_id, sw.word_text, sw.link_count, Min(sl.link_score), sw.link_score_max"
, "FROM search_word sw"
, " JOIN <link_db>search_link sl ON sl.word_id = sw.word_id"
, "GROUP BY sw.word_id, sw.word_text, sw.link_count, sw.link_score_max"
, "HAVING sw.link_score_min > Min(sl.link_score)"
))
.Exec_sql_w_msg
( Bry_fmt.Make_str("updating link_score_max: ~{idx} of ~{total}", i + 1, link_tbls_len)
, String_.Concat_lines_nl_skip_last
( "REPLACE INTO search_word (word_id, word_text, link_count, link_score_min, link_score_max)"
, "SELECT sw.word_id, sw.word_text, sw.link_count, sw.link_score_min, Max(sl.link_score)"
, "FROM search_word sw"
, " JOIN <link_db>search_link sl ON sl.word_id = sw.word_id"
, "GROUP BY sw.word_id, sw.word_text, sw.link_count, sw.link_score_min"
, "HAVING sw.link_score_max < Max(sl.link_score)"
))
;
link_tbl.Create_idx__link_score();
if (!link_tbl.conn.Eq(word_tbl.conn)) link_tbl.conn.Env_vacuum(); // don't vacuum if single-db
word_tbl.Create_idx();
}
Srch_db_cfg_.Update__bldr__link(search_db_mgr.Tbl__cfg(), search_db_mgr.Cfg(), link_score_max);
if (delete_plink_db) {
Xob_db_file plink_db = Xob_db_file.New__page_link(wiki);
plink_db.Conn().Rls_conn();
Io_mgr.Instance.DeleteFil(plink_db.Url());
}
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__page_rank_enabled_)) page_rank_enabled = m.ReadYn("v");
else if (ctx.Match(k, Invk__delete_plink_db_)) delete_plink_db = m.ReadYn("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk__page_rank_enabled_ = "page_rank_enabled_", Invk__delete_plink_db_ = "delete_plink_db_";
}

109
400_xowa/src/gplx/xowa/addons/searchs/dbs/bldrs/cmds/Srch__page__page_score.java Normal file
View File

@@ -0,0 +1,109 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.dbs.*; import gplx.xowa.addons.searchs.dbs.bldrs.*;
import gplx.dbs.*; import gplx.dbs.qrys.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.addons.pagelinks.dbs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Srch__page__page_score extends Xob_cmd__base implements Xob_cmd { // create page_rank in page_db; drop and vaccuum later; avoid cross db for now
private double damping_factor = .85;
private double error_margin = .0001d;
private int iteration_max = 1000;
private Db_conn page_conn, plink_conn;
private int page_count = 0;
public Srch__page__page_score(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
public Srch__page__page_score Iteration_max_(int iteration_max) {this.iteration_max = iteration_max; return this;}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_search__page__page_score;}
@Override public void Cmd_run() {
Init__tbl();
Calc__main();
}
private void Init__tbl() {
wiki.Init_assert();
this.page_conn = wiki.Data__core_mgr().Db__core().Conn();
page_count = page_conn.Exec_select_as_int("SELECT Count(page_id) FROM page", -1);
Xob_db_file page_link_db = Xob_db_file.New__page_link(wiki);
this.plink_conn = page_link_db.Conn();
if (plink_conn.Meta_tbl_exists(Pagerank__tbl_name)) plink_conn.Meta_tbl_drop(Pagerank__tbl_name);
Pglnk_page_link_tbl plink_tbl = new Pglnk_page_link_tbl(plink_conn);
if (!plink_conn.Meta_tbl_exists(plink_tbl.Tbl_name())) plink_tbl.Create_tbl();// create page_link if it doesn't exist; occurs when page_rank_enabled == false;
plink_conn.Meta_tbl_create(Dbmeta_tbl_itm.New
( Pagerank__tbl_name
, Dbmeta_fld_itm.new_int (Pagerank__fld_page_id).Primary_y_()
, Dbmeta_fld_itm.new_int (Pagerank__fld_link_count)
, Dbmeta_fld_itm.new_int (Pagerank__fld_has_converged).Default_(0)
, Dbmeta_fld_itm.new_double (Pagerank__fld_page_rank).Default_(1)
));
new Db_attach_mgr(plink_conn, new Db_attach_itm("page_db", page_conn))
.Exec_sql_w_msg("generating list of pages", String_.Concat_lines_nl_skip_last
( "INSERT INTO page_rank_temp (page_id, link_count)"
, "SELECT page_id"
, ", Coalesce(Count(pl.trg_id), {0}) AS link_count"
, "FROM <page_db>page p" // NOTE: JOIN needed to filter out [[User:]] pages which are in pagelinks.sql, but not in pages-articles.xml
, " LEFT JOIN page_link pl ON p.page_id = pl.src_id"
// , "WHERE p.page_namespace = 0"
, "GROUP BY p.page_id"
), page_count);
}
private void Calc__main() {
int iteration_idx = 0;
double damping_factor_inverse = 1 - damping_factor;
while (true) {
if (iteration_idx == iteration_max) break;
int converged_count = plink_conn.Exec_select_as_int("SELECT Count(page_id) FROM page_rank_temp WHERE has_converged = 0;", 0);
if (converged_count == 0) break;
new Db_attach_mgr(plink_conn, new Db_attach_itm("page_db", page_conn))
.Exec_sql_w_msg(String_.Format("calculating page_rank; iteration={0} unconverged={1}", iteration_idx, converged_count)
, String_.Concat_lines_nl_skip_last
( "REPLACE INTO page_rank_temp (page_id, page_rank, link_count, has_converged)"
, "SELECT pr.page_id"
, ", {1} + ({0} * Coalesce(w.page_rank, 0)) AS page_rank"
, ", pr.link_count"
, ", CASE WHEN Abs(pr.page_rank - ({1} + ({0} * Coalesce(w.page_rank, 0)))) < {2} THEN 1 ELSE 0 END AS has_converged"
, "FROM page_rank_temp pr"
, " LEFT JOIN"
, " (SELECT lnk.trg_id"
, " , Sum((Cast(pr2.page_rank AS double) / Cast(pr2.page_rank AS double))) * {0} AS page_rank"
, " FROM page_rank_temp pr2"
, " JOIN page_link lnk ON pr2.page_id = lnk.src_id"
, " JOIN <page_db>page p ON lnk.src_id = p.page_id"
// , " WHERE p.page_namespace = 0"
, " GROUP BY lnk.trg_id"
, " ) AS w ON w.trg_id = pr.page_id"
, "WHERE pr.has_converged = 0"
), damping_factor, damping_factor_inverse, error_margin);
++iteration_idx;
}
Gfo_usr_dlg_.Instance.Note_many("", "", "page_rank done; iteration=~{0}", iteration_idx);
}
public static final String
Pagerank__tbl_name = "page_rank_temp"
, Pagerank__fld_page_id = "page_id"
, Pagerank__fld_page_rank = "page_rank"
, Pagerank__fld_link_count = "link_count"
, Pagerank__fld_has_converged = "has_converged"
;
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__damping_factor_)) damping_factor = m.ReadDouble("v");
else if (ctx.Match(k, Invk__error_margin_)) error_margin = m.ReadDouble("v");
else if (ctx.Match(k, Invk__iteration_max_)) iteration_max = m.ReadInt("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk__damping_factor_ = "damping_factor_", Invk__error_margin_ = "error_margin_", Invk__iteration_max_ = "iteration_max_";
}

52
400_xowa/src/gplx/xowa/addons/searchs/dbs/bldrs/cmds/Srch__word__link_count.java Normal file
View File

@@ -0,0 +1,52 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.dbs.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.dbs.*; import gplx.xowa.addons.searchs.dbs.bldrs.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.addons.sqlite_utils.bldrs.*;
import gplx.xowa.addons.searchs.dbs.*;
public class Srch__word__link_count extends Xob_cmd__base implements Xob_cmd {
private int score_multiplier = Srch_search_addon.Score_max;
public Srch__word__link_count(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_search__word__link_count;}
@Override public void Cmd_run() {
wiki.Init_assert();
Srch_db_mgr search_db_mgr = Srch_search_addon.Get(wiki).Db_mgr();
Srch_word_tbl word_tbl = search_db_mgr.Tbl__word();
Db_conn word_conn = word_tbl.conn;
String percentile_tbl = "search_word__link_count";
Sqlite_percentile_cmd percentile_cmd = new Sqlite_percentile_cmd(bldr, wiki).Init_by_conn(word_conn, percentile_tbl, score_multiplier, String_.Concat_lines_nl_skip_last
( "SELECT sw.word_id, sw.link_count"
, "FROM search_word sw"
, "ORDER BY sw.link_count" // NOTE: ORDER BY is needed b/c INSERT into AUTO INCREMENT table
));
percentile_cmd.Cmd_run();
word_conn.Exec_sql("finalizing search_word.link_count_score", String_.Concat_lines_nl_skip_last
( "UPDATE search_word"
, "SET link_count_score = (SELECT tpl.row_score FROM search_word__link_count tpl WHERE tpl.row_key = search_word.word_id)"
));
word_conn.Meta_tbl_drop(percentile_tbl);
word_conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(word_tbl.tbl_name, "link_count_score__word_text", word_tbl.fld_link_count_score, word_tbl.fld_text));
int link_count_score_cutoff = word_conn.Exec_select_as_int("SELECT Cast(Min(link_count_score) AS int) AS val FROM search_word WHERE link_count > " + Int_.To_str(Srch_db_cfg_.Link_count_score_cutoff), 0);
Srch_db_cfg_.Update__bldr__word(search_db_mgr.Tbl__cfg(), search_db_mgr.Cfg(), percentile_cmd.count, score_multiplier, link_count_score_cutoff);
word_conn.Env_vacuum();
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return this;}
}

117
400_xowa/src/gplx/xowa/addons/searchs/parsers/Srch_highlight_mgr.java Normal file
View File

@@ -0,0 +1,117 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.core.btries.*;
import gplx.xowa.langs.cases.*;
public class Srch_highlight_mgr {
private final Xol_case_mgr case_mgr;
private final Bry_bfr tmp_bfr = Bry_bfr.new_(32);
private Srch_highlight_itm[] srch_lc_itms;
private int srch_words_len;
public Srch_highlight_mgr(Xol_case_mgr case_mgr) {this.case_mgr = case_mgr;}
public Srch_highlight_mgr Search_(byte[] srch_mc_bry) {
synchronized (tmp_bfr) {
// build array of search_words
byte[] srch_lc_bry = case_mgr.Case_build_lower(srch_mc_bry);
byte[][] srch_lc_ary = Bry_split_.Split(srch_lc_bry, Byte_ascii.Space, Bool_.Y);
this.srch_words_len = srch_lc_ary.length;
this.srch_lc_itms = new Srch_highlight_itm[srch_words_len];
for (int i = 0; i < srch_words_len; ++i) {
byte[] srch_lc_itm = srch_lc_ary[i];
srch_lc_itms[i] = new Srch_highlight_itm(i, srch_lc_itm);
}
// sort to search first by longest search_word; needed for searches like "A Abc" and titles like "Abc A", else "A" will match "Abc" and "Abc" will match nothing
Array_.Sort(srch_lc_itms, Srch_highlight_bry_sorter.Instance);
return this;
}
}
public byte[] Highlight(byte[] page_mc_bry) {
synchronized (tmp_bfr) {
byte[][] page_mc_words = Bry_split_.Split(page_mc_bry, Byte_ascii.Space, Bool_.Y);
byte[][] page_lc_words = Bry_split_.Split(case_mgr.Case_build_lower(page_mc_bry), Byte_ascii.Space, Bool_.Y);
int page_words_len = page_lc_words.length;
boolean[] page_words_done = new boolean[page_words_len];
// loop over search_words; EX: search_raw="a b" -> "a", "b"
for (int i = 0; i < srch_words_len; ++i) {
Srch_highlight_itm srch_lc_itm = srch_lc_itms[i];
byte[] srch_lc_word = srch_lc_itm.Word;
// loop over page_title words; EX: page_raw="A B C" -> "a", "b", "c"
for (int j = 0; j < page_words_len; ++j) {
byte[] page_lc_word = page_lc_words[j];
int find_pos = Bry_find_.Find_fwd(page_lc_word, srch_lc_word);
// skip: search is not in page; EX: page_raw="D e"; should not happen, but exit now else Array out of bounds exception below
if (find_pos == Bry_find_.Not_found) continue;
// skip: find_pos is not BOS and prv byte is not dash or paren; EX: "Za" should be skipped; "-a" and "(a" should not
if (find_pos > 0) {
byte prv_byte = page_lc_word[find_pos - 1];
if ( prv_byte != Byte_ascii.Dash
&& prv_byte != Byte_ascii.Paren_bgn
)
continue;
}
// skip: item already matched; EX: "a"; "a a"
if (page_words_done[j]) continue;
page_words_done[j] = true;
// NOTE: lc_idx is guaranteed to equal mc_idx
byte[] page_mc_word = page_mc_words[j];
int srch_lc_word_len = srch_lc_word.length;
// build new word; EX: "<strong>A</strong>"
tmp_bfr.Clear();
tmp_bfr.Add_mid(page_mc_word, 0, find_pos);
tmp_bfr.Add(gplx.langs.htmls.Gfh_tag_.Strong_lhs);
tmp_bfr.Add_mid(page_mc_word, find_pos, find_pos + srch_lc_word_len);
tmp_bfr.Add(gplx.langs.htmls.Gfh_tag_.Strong_rhs);
tmp_bfr.Add_mid(page_mc_word, find_pos + srch_lc_word_len, page_mc_word.length);
byte[] repl = tmp_bfr.To_bry_and_clear();
// overwrite page_title word
page_mc_words[j] = repl;
}
}
// rebuild page_words ary
for (int i = 0; i < page_words_len; ++i) {
if (i != 0) tmp_bfr.Add_byte_space(); // NOTE: this assumes one space separating titles which is true for all MW titles
tmp_bfr.Add(page_mc_words[i]);
}
return tmp_bfr.To_bry_and_clear();
}
}
}
class Srch_highlight_itm {
public Srch_highlight_itm(int idx, byte[] word) {this.Idx = idx; this.Word = word; this.Word_len = word.length;}
public final int Idx;
public final byte[] Word;
public final int Word_len;
}
class Srch_highlight_bry_sorter implements gplx.core.lists.ComparerAble {
public int compare(Object lhsObj, Object rhsObj) {
Srch_highlight_itm lhs = (Srch_highlight_itm)lhsObj;
Srch_highlight_itm rhs = (Srch_highlight_itm)rhsObj;
return -Int_.Compare(lhs.Word_len, rhs.Word_len); // - for descending
}
public static final Srch_highlight_bry_sorter Instance = new Srch_highlight_bry_sorter(); Srch_highlight_bry_sorter() {}
}

47
400_xowa/src/gplx/xowa/addons/searchs/parsers/Srch_highlight_mgr_tst.java Normal file
View File

@@ -0,0 +1,47 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import org.junit.*; import gplx.xowa.langs.cases.*;
public class Srch_highlight_mgr_tst {
private final Srch_highlight_mgr_tstr tstr = new Srch_highlight_mgr_tstr();
@Test public void Full__one() {tstr.Test("a" , "A" , "<strong>A</strong>");}
@Test public void Full__many() {tstr.Test("a b" , "A B" , "<strong>A</strong> <strong>B</strong>");}
@Test public void Part__one() {tstr.Test("a" , "A1" , "<strong>A</strong>1");}
@Test public void Part__many() {tstr.Test("a b" , "A1 B1" , "<strong>A</strong>1 <strong>B</strong>1");}
@Test public void Unordered() {tstr.Test("b a" , "A1 B1" , "<strong>A</strong>1 <strong>B</strong>1");}
@Test public void Repeat__part() {tstr.Test("a ab" , "Ab A" , "<strong>Ab</strong> <strong>A</strong>");}
@Test public void Repeat__full() {tstr.Test("a" , "A A" , "<strong>A</strong> <strong>A</strong>");}
@Test public void Close() {tstr.Test("a" , "Ba Aa" , "Ba <strong>A</strong>a");}
@Test public void Comma() {tstr.Test("a" , "A, b" , "<strong>A</strong>, b");}
@Test public void Dash() {tstr.Test("b" , "A-B c" , "A-<strong>B</strong> c");}
@Test public void Parens() {tstr.Test("a" , "(A)" , "(<strong>A</strong>)");}
@Test public void Strong() {tstr.Test("strong" , "strong strong" , "<strong>strong</strong> <strong>strong</strong>");}
@Test public void Dash_w_mixed_cases() {tstr.Test("b" , "A-a B" , "A-a <strong>B</strong>");} // search_parser treats A-a separately from a-a
@Test public void Acronymn() {tstr.Test("ab" , "A.B." , "A.B.");}
// @Test public void Slash() {tstr.Test("b" , "A/B/C" , "A/<strong>B</strong>/C");}
}
class Srch_highlight_mgr_tstr {
private final Srch_highlight_mgr mgr;
private final Xol_case_mgr case_mgr = Xol_case_mgr_.A7();
public Srch_highlight_mgr_tstr() {
mgr = new Srch_highlight_mgr(case_mgr);
}
public void Test(String search, String title, String expd) {
Tfds.Eq(expd, String_.new_u8(mgr.Search_(Bry_.new_u8(search)).Highlight(Bry_.new_u8(title))));
}
}

262
400_xowa/src/gplx/xowa/addons/searchs/parsers/Srch_sym_parser.java Normal file
View File

@@ -0,0 +1,262 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.core.btries.*; import gplx.core.primitives.*;
interface Srch_sym_parser {
int Tid();
byte[][] Hooks_ary();
int Parse(Srch_text_parser mgr, byte[] src, int src_end, int hook_bgn, int hook_end);
}
class Srch_sym_parser_ {
public static final int Tid__terminal = 1, Tid__split = 2, Tid__enclosure = 3, Tid__dot = 4, Tid__ellipsis = 5, Tid__apos = 6, Tid__dash = 7;
}
class Srch_sym_parser__terminal implements Srch_sym_parser {
public Srch_sym_parser__terminal(byte[] hook_bry) {this.hooks_ary = Bry_.Ary(hook_bry);}
public int Tid() {return Srch_sym_parser_.Tid__terminal;}
public byte[][] Hooks_ary() {return hooks_ary;} private final byte[][] hooks_ary;
public int Parse(Srch_text_parser mgr, byte[] src, int src_end, int hook_bgn, int hook_end) {
if (mgr.Cur__end__chk(hook_end)) { // hook at word_end; EX: "a, b" -> "a", "b"
int word_bgn = mgr.Cur__bgn();
if (word_bgn == Srch_text_parser.None) return hook_end;
int word_end = hook_bgn;
for (int i = hook_bgn - 1; i >= word_bgn; --i) { // loop bwd to gobble up streams of terminals; EX: "?!"
byte b = src[i];
if (mgr.word_end_trie.Match_bgn_w_byte(b, src, i, hook_bgn) == null) break;
word_end = i;
}
mgr.Words__add_if_pending_and_clear(word_end); // make word; note that " , " will not make word b/c word_bgn == -1
return hook_end;
}
else {}// hook at word_mid or word_bgn; noop; EX: "1,000" -> "1,000"; note that " ,abc" will ignore ","
// if (mgr.Cur__bgn() == - 1) {
// mgr.Cur__bgn__set(hook_bgn);
// }
// }
return hook_end;
}
}
class Srch_sym_parser__split implements Srch_sym_parser {
private final Btrie_slim_mgr trie = Btrie_slim_mgr.cs();
private final boolean handle_eos;
public Srch_sym_parser__split(boolean handle_eos, String... hooks_ary_as_str) {
this.handle_eos = handle_eos;
this.hooks_ary = Bry_.Ary(hooks_ary_as_str);
int hooks_len = hooks_ary.length;
for (int i = 0; i < hooks_len; ++i) {
byte[] hook = hooks_ary[i];
trie.Add_obj(hook, Int_obj_val.new_(hook.length));
}
}
public int Tid() {return Srch_sym_parser_.Tid__split;}
public byte[][] Hooks_ary() {return hooks_ary;} private final byte[][] hooks_ary;
public int Parse(Srch_text_parser mgr, byte[] src, int src_end, int hook_bgn, int hook_end) {
mgr.Words__add_if_pending_and_clear(hook_bgn);
int rv = hook_end;
while (rv < src_end) { // loop to skip multiple syms in same group; EX: "a \n\t\r b"
byte b = src[rv];
Object o = trie.Match_bgn_w_byte(b, src, rv, src_end);
if (o == null) break; // current sequence is not in grp; stop
Int_obj_val itm_len = (Int_obj_val)o;
rv += itm_len.Val(); // add len of sym to pos
}
return rv;
}
public int Find_fwd(byte[] src, int src_bgn, int src_end) {
for (int i = src_bgn; i < src_end; ++i) {
byte b = src[i];
Object o = trie.Match_bgn_w_byte(b, src, i, src_end);
if (o != null) return i;
}
return src_end;
}
public boolean Is_next(byte[] src, int pos, int end) {
if (pos >= end) return handle_eos;
byte b = src[pos];
Object o = trie.Match_bgn_w_byte(b, src, pos, end);
return o != null;
}
}
class Srch_sym_parser__paren_bgn implements Srch_sym_parser {
private final byte bgn_byte, end_byte;
public Srch_sym_parser__paren_bgn(byte bgn_byte, byte end_byte) {
this.bgn_byte = bgn_byte; this.end_byte = end_byte;
this.hooks_ary = Bry_.Ary(Bry_.New_by_byte(bgn_byte));
}
public int Tid() {return Srch_sym_parser_.Tid__enclosure;}
public byte[][] Hooks_ary() {return hooks_ary;} private final byte[][] hooks_ary;
public int Parse(Srch_text_parser mgr, byte[] src, int src_end, int paren_lhs_bgn, int paren_lhs_end) {
int word_bgn = mgr.Cur__bgn();
paren_lhs_end = Bry_find_.Find_fwd_while(src, paren_lhs_end, src_end, bgn_byte);
int paren_rhs_bgn = Bry_find_.Find_fwd(src, end_byte, paren_lhs_end, src_end);
if (paren_rhs_bgn == Bry_find_.Not_found) return paren_lhs_end; // paren_rhs missing; noop; NOTE: handles both "a(b" -> "a(b" and "a (b" -> "a", "b"
int paren_rhs_end = Bry_find_.Find_fwd_while(src, paren_rhs_bgn, src_end, end_byte);
int word_end = mgr.Cur__end__find__text_only(paren_lhs_end + 1);
if (word_end < paren_rhs_bgn)
word_end = mgr.Cur__end__find__text_only(paren_rhs_end);
boolean recurse = false;
if (word_bgn == -1) { // paren_lhs at word_bgn; EX: "a (b)"
if (word_end == paren_rhs_end) // paren_rhs at word_end; EX: "a (b) c"
recurse = true;
else { // paren_rhs at word_mid; EX: "a (b)c"
mgr.Words__add_direct(paren_lhs_bgn, word_end);
mgr.Words__add_direct(paren_rhs_end, word_end);
}
}
else { // paren_lhs at word_mid; EX: "a(b)"
mgr.Words__add_direct(word_bgn, word_end);
if (word_end == paren_rhs_end) // paren_rhs at word_end; EX: "a(b) c"
mgr.Words__add_direct(word_bgn, paren_lhs_bgn);
else {} // paren_rhs at word_mid; EX: "a (b)c"
mgr.Cur__bgn__reset();
}
if (recurse)
mgr.Make_copy().Parse(mgr.lcase, src, paren_lhs_end, paren_rhs_bgn);
return word_end;
}
}
class Srch_sym_parser__dot implements Srch_sym_parser { // handle periods which will add two entries; EX: "H. G. Wells" -> "H.", "G.", "H", "G", "Wells"
private final byte[] sym; private final int sym_len;
public Srch_sym_parser__dot(String sym_str) {this.sym = Bry_.new_u8(sym_str); this.sym_len = sym.length;}
public int Tid() {return Srch_sym_parser_.Tid__dot;}
public byte[][] Hooks_ary() {return Hooks_const;} private static final byte[][] Hooks_const = Bry_.Ary(".");
public int Parse(Srch_text_parser mgr, byte[] src, int src_end, int hook_bgn, int hook_end) {
int word_bgn = mgr.Cur__bgn();
if (word_bgn == -1) { // dot at start of word; EX: ".NET", "Colt .45"
int word_end = mgr.Cur__end__find(hook_bgn + 1);
if (word_end - hook_bgn == 1) return hook_end; // ignore stand-alone sym; EX: "a . . . b"
mgr.Words__add__chk_dash(hook_bgn, word_end); // make word; EX: ".45"
if (Bry_find_.Find_fwd(src, sym, hook_bgn + 1, word_end) == -1) // only add "root" word, if sym is not in middle of String; EX: ".int" -> "int"; ".A.B" x> "A.B"
mgr.Words__add__chk_dash(hook_bgn + 1, word_end); // make word; EX: "45"
mgr.Cur__bgn__reset();
return word_end;
}
else {
if (mgr.Cur__end__chk(hook_end)) { // at end of word; EX: "U.S.A. b" vs. "H. G. b"
mgr.Words__add__chk_dash(word_bgn, hook_end); // make word; EX: "vs."
if (Bry_find_.Find_fwd(src, sym, word_bgn, hook_bgn) == -1) // no dots in word; add "root" word; "vs." -> "vs.", "vs"; "U.S.A." -> "U.S.A." x> "U.S.A"
mgr.Words__add__chk_dash(word_bgn, hook_bgn); // make word; EX: "vs"
else {
for (int i = word_bgn; i < hook_end; ++i) { // dots in middle of String; extract word; EX: "U.S.A." -> USA
if (Bry_.Eq(src, i, i + sym_len, sym)) continue;// skip syms
mgr.Tmp_bfr.Add_byte(src[i]);
}
mgr.Words__add_direct(mgr.Tmp_bfr.To_bry_and_clear());
}
mgr.Cur__bgn__reset();
}
}
return hook_end;
}
}
class Srch_sym_parser__ellipsis implements Srch_sym_parser { // ellipsis which have variable length; EX: "..", "...", ".... "
private final byte ellipsis_byte;
public Srch_sym_parser__ellipsis(byte ellipsis_byte, String hook) {
this.ellipsis_byte = ellipsis_byte;
this.hooks_ary = Bry_.Ary(hook);
}
public int Tid() {return Srch_sym_parser_.Tid__ellipsis;}
public byte[][] Hooks_ary() {return hooks_ary;} private final byte[][] hooks_ary;
public int Parse(Srch_text_parser mgr, byte[] src, int src_end, int hook_bgn, int hook_end) {
mgr.Words__add_if_pending_and_clear(hook_bgn); // add word; EX: "Dreams" in "Dreams..."
int rv = Bry_find_.Find_fwd_while(src, hook_end, src_end, ellipsis_byte); // skip multiple ellipsis; EX: ......
mgr.Words__add__chk_dash(hook_bgn, rv); // add ellipsis
return rv;
}
}
class Srch_sym_parser__apos implements Srch_sym_parser { // apos which can be contraction ("I'm"), possessive ("today's") and plural ("plans'")
private final byte[] hook_bry;
public Srch_sym_parser__apos(String hook) {
this.hook_bry = Bry_.new_u8(hook);
this.hooks_ary = Bry_.Ary(hook_bry);
}
public int Tid() {return Srch_sym_parser_.Tid__apos;}
public byte[][] Hooks_ary() {return hooks_ary;} private final byte[][] hooks_ary;
public int Parse(Srch_text_parser mgr, byte[] src, int src_end, int hook_bgn, int hook_end) {
int word_bgn = mgr.Cur__bgn();
int word_end = -1;
if (word_bgn == -1) { // sym at word_bgn; EX: "'n'"
word_bgn = hook_bgn;
word_end = mgr.Cur__end__find(hook_bgn + 1);
mgr.Words__add__chk_dash(word_bgn, word_end);
// "'n'" -> "n"
int alt_word_bgn = word_bgn + 1;
int alt_word_end = Bry_find_.Find_bwd__skip(src, word_end, alt_word_bgn, hook_bry);
if (alt_word_end - alt_word_bgn > 0) // do not add if 0-len; EX: "' (disambiguation)"
mgr.Words__add__chk_dash(alt_word_bgn, alt_word_end);
mgr.Cur__bgn__reset();
return word_end;
}
word_end = mgr.Cur__end__find(hook_end);
if (hook_end == word_end) { // sym at word_end; EX: "A' "
mgr.Words__add_if_pending_and_clear(hook_bgn); // add root only; EX: "a'" -> "a" x> "a", "a'"
return hook_end;
}
else {
byte[] root_word = null;
if (hook_bgn - word_bgn == 1)
root_word = mgr.Tmp_bfr.Add_byte(src[word_bgn]).Add_mid(src, hook_bgn + 1, word_end).To_bry_and_clear();
else if (word_end - hook_end == 1)
root_word = mgr.Tmp_bfr.Add_mid(src, word_bgn, hook_bgn).Add_byte(src[hook_end]).To_bry_and_clear();
// Tfds.Write(word_bgn, hook_bgn, hook_end, word_end);
if (root_word != null) {
mgr.Words__add__chk_dash(word_bgn, word_end); // add full; EX: "o'clock"
mgr.Words__add_direct(root_word); // add root; EX: "oclock"
mgr.Cur__bgn__reset();
}
return word_end;
}
}
}
class Srch_sym_parser__dash implements Srch_sym_parser {
public Srch_sym_parser__dash(String hook) {this.hooks_ary = Bry_.Ary(hook);}
public int Tid() {return Srch_sym_parser_.Tid__dash;}
public byte[][] Hooks_ary() {return hooks_ary;} private final byte[][] hooks_ary;
public int Parse(Srch_text_parser mgr, byte[] src, int src_end, int hook_bgn, int hook_end) {
int cur_bgn = mgr.Cur__bgn(); // get word_bgn
if (cur_bgn == -1) { // no word_bgn; "-" is 1st char
if (mgr.Cur__end__chk(hook_end)) { // next char is space
mgr.Words__add_direct(hook_bgn, hook_end); // add dash; EX: "a - b" -> "a", "-", "b"
}
else { // next char is something else
mgr.Cur__bgn__set(hook_bgn); // update word_bgn to dash
mgr.Dash__bgn_(hook_end);
}
return hook_end;
}
else { // word_bgn exists
int dash_bgn = mgr.Dash__bgn();
if (dash_bgn == Srch_text_parser.None) { // 1st dash
mgr.Words__add__chk_dash(cur_bgn, hook_bgn);// add word; EX: "a" in "a-b"
} else { // 2nd or more; add stub; EX: "a-b-c"; 2nd "-" should add "b"
if (hook_bgn > dash_bgn) // only add if len > 0; handles multiple dashes; EX: "---"
mgr.Words__add_direct(dash_bgn, hook_bgn);
}
}
mgr.Dash__bgn_(hook_end);
return hook_end;
}
public void Add_pending_word(Srch_text_parser mgr, byte[] src, int word_end) {
int dash_bgn = mgr.Dash__bgn();
if (dash_bgn == Srch_text_parser.None) return;
if (word_end - dash_bgn > 0) // only add if there is word to right of dash; EX: "a-"
mgr.Words__add_direct(dash_bgn, word_end);
mgr.Dash__bgn_(Srch_text_parser.None); // clear the dash
}
}

173
400_xowa/src/gplx/xowa/addons/searchs/parsers/Srch_text_parser.java Normal file
View File

@@ -0,0 +1,173 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.core.btries.*; import gplx.xowa.langs.cases.*;
public class Srch_text_parser {
private Btrie_slim_mgr parser_trie = Btrie_slim_mgr.cs(); public Btrie_slim_mgr word_end_trie = Btrie_slim_mgr.cs(); private Btrie_slim_mgr word_bgn_trie = Btrie_slim_mgr.cs();
private Xol_case_mgr case_mgr;
public final Bry_bfr Tmp_bfr = Bry_bfr.new_(32);
private byte[] src; private int end;
private Srch_sym_parser__split parser__ws; private Srch_sym_parser__dash parser__dash;
public Srch_word_hash word_hash = new Srch_word_hash();
public boolean lcase = true;
public Srch_text_parser Init_for_ttl(Xol_case_mgr case_mgr) {
this.case_mgr = case_mgr;
parser_trie.Clear(); word_end_trie.Clear();
parser__ws = new Srch_sym_parser__split(Bool_.Y, " ", "\t", "\n", "\r", "_");
parser__dash = new Srch_sym_parser__dash("-");
Parsers__reg(Parsers__make__word_end
( "!", "?", ",", ":", ";", "\"", "~"
//, "@", "&", "*", "`", "+" // should add for symmetry of word_bgn trie but strips "@Home" to "Home" only; also, several have many "*", "`", "+"
));
Parsers__reg(new Srch_sym_parser__split(Bool_.N, "/"));
Parsers__reg(parser__ws, parser__dash
, new Srch_sym_parser__paren_bgn(Byte_ascii.Paren_bgn, Byte_ascii.Paren_end)
, new Srch_sym_parser__dot("."), new Srch_sym_parser__ellipsis(Byte_ascii.Dot, ".."), new Srch_sym_parser__apos("'")); // NOTE: [ ] { } do not exist in titles
word_bgn_trie.Add_many_int(1
// , "!", "?", ",", ":", ";" // low #; should add for symmetry of word_end trie;
, "\"" // adding for symmetry of word_end trie;
, "@", "&", "*", "`", "~", "+" // "@Home", "&c.", "&NSYNC", "Muhammad_ibn_`Ali_at-Tirmidhi"; "Phantom_~Requiem_for_the_Phantom~"; "+pool"
);
return this;
}
public Srch_text_parser Make_copy() {
Srch_text_parser rv = new Srch_text_parser();
rv.parser_trie = parser_trie; rv.word_end_trie = word_end_trie; rv.word_bgn_trie = word_bgn_trie;
rv.parser__ws = parser__ws; rv.parser__dash = parser__dash;
rv.word_hash = word_hash; rv.case_mgr = case_mgr;
rv.lcase = lcase;
return rv;
}
public byte[][] Parse_to_bry_ary(boolean lcase, byte[] src_orig) {
word_hash.Clear();
this.lcase = lcase;
Parse(lcase, src_orig, 0, src_orig.length);
int hash_len = word_hash.Len();
byte[][] rv = new byte[hash_len][];
for (int i = 0; i < hash_len; ++i) {
Srch_word_itm itm = (Srch_word_itm)word_hash.Get_at(i);
rv[i] = itm.Word;
}
return rv;
}
public void Parse(Srch_text_parser_wkr wkr, byte[] src_orig) {
word_hash.Clear();
Parse(Bool_.Y, src_orig, 0, src_orig.length);
int hash_len = word_hash.Len();
for (int i = 0; i < hash_len; ++i) {
Srch_word_itm itm = (Srch_word_itm)word_hash.Get_at(i);
wkr.Parse_done(itm);
}
}
public void Parse(boolean lcase, byte[] src_orig, int bgn, int end_orig) {
this.src = lcase ? case_mgr.Case_build_lower(src_orig) : src_orig;
this.end = end_orig + (src.length - src_orig.length);
this.cur_bgn = dash_bgn = Srch_text_parser.None;
int pos = bgn;
while (true) {
boolean pos_is_last = pos == end;
if (pos_is_last) { // EOS and pending word; add it
if (cur_bgn != -1)
Words__add__chk_dash(cur_bgn, end);
break;
}
byte b = src[pos];
Object o = parser_trie.Match_bgn_w_byte(b, src, pos, end);
if (o == null) { // unknown sequence; word-char
if (cur_bgn == -1) cur_bgn = pos; // set 1st char for word
++pos;
}
else {
Srch_sym_parser parser = (Srch_sym_parser)o;
pos = parser.Parse(this, src, end, pos, parser_trie.Match_pos());
}
}
}
public int Cur__bgn() {return cur_bgn;} private int cur_bgn; public void Cur__bgn__reset() {this.cur_bgn = -1;}
public void Cur__bgn__set(int v) {this.cur_bgn = v;} // called from dash parser
public int Dash__bgn() {return dash_bgn;} private int dash_bgn; public void Dash__bgn_(int v) {this.dash_bgn = v;}
public int Cur__end__find(int pos) {return parser__ws.Find_fwd(src, pos, end);}
public boolean Cur__end__chk(int pos) {return parser__ws.Is_next(src, pos, end);}
public int Cur__end__find__text_only(int pos) { // primarily for parens and finding word_end after ")"; EX: "(city), " vs "(a)b "
while (pos < end) {
byte b = src[pos];
Object parser_obj = parser_trie.Match_bgn_w_byte(b, src, pos, end);
if (parser_obj == null) // b is text; increment by 1 and continue searching
++pos;
else // b is some sort of symbol; end word here; EX: "a," should produce "a", not "a,"
return pos;
}
return end;
}
public void Words__add_if_pending_and_clear(int word_end) {
if (cur_bgn == -1) return; // exit; no pending words
this.Words__add__chk_dash(cur_bgn, word_end);
cur_bgn = -1;
}
public void Words__add__chk_dash(int word_bgn, int word_end) {
parser__dash.Add_pending_word(this, src, word_end);
byte[] word = Bry_.Mid(src, word_bgn, word_end);
Words__add_direct(word);
}
public void Words__add_direct(int bgn, int end) {Words__add_direct(Bry_.Mid(src, bgn, end));}
public void Words__add_direct(byte[] bry) {
word_hash.Add(bry);
// remove punctuation at bgn of word; EX: "@Home" -> "Home"
boolean dirty = false;
int pos = 0; int len = bry.length;
while (pos < len) {
byte b = bry[pos];
if (word_bgn_trie.Match_bgn_w_byte(b, bry, pos, len) != null) { // b is symbol;
dirty = true;
pos = word_bgn_trie.Match_pos();
}
else {
break;
}
}
if (dirty && pos < len) {
byte[] trunc = Bry_.Mid(bry, pos, len);
// if (!word_hash.Has(trunc)) // don't add if it exists; EX: "'tis"
word_hash.Add(trunc);
}
}
private void Parsers__reg(Srch_sym_parser... parsers_ary) {
int parsers_len = parsers_ary.length;
for (int i = 0; i < parsers_len; ++i) {
Srch_sym_parser parser = parsers_ary[i];
byte[][] hooks_ary = parser.Hooks_ary();
int hooks_len = hooks_ary.length;
for (int j = 0; j < hooks_len; ++j) {
byte[] hook = hooks_ary[j];
parser_trie.Add_obj(hook, parser);
if (parser.Tid() == Srch_sym_parser_.Tid__terminal)
word_end_trie.Add_obj(hook, parser);
}
}
}
private static Srch_sym_parser[] Parsers__make__word_end(String... hooks) {
int rv_len = hooks.length;
Srch_sym_parser[] rv = new Srch_sym_parser[rv_len];
for (int i = 0; i < rv_len; ++i)
rv[i] = new Srch_sym_parser__terminal(Bry_.new_u8(hooks[i]));
return rv;
}
public static final int None = -1;
}

147
400_xowa/src/gplx/xowa/addons/searchs/parsers/Srch_text_parser_tst.java Normal file
View File

@@ -0,0 +1,147 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import org.junit.*; import gplx.xowa.langs.cases.*;
public class Srch_text_parser_tst {
private final Srch_text_parser_fxt fxt = new Srch_text_parser_fxt();
@Before public void init() {fxt.Init();}
@Test public void Word__one() {fxt.Clear().Test__split("abcd" , "abcd");}
@Test public void Word__many() {fxt.Clear().Test__split("abc d ef" , "abc", "d", "ef");}
@Test public void Ws__many() {fxt.Clear().Test__split("a b" , "a", "b");}
@Test public void Ws__bgn() {fxt.Clear().Test__split(" a" , "a");}
@Test public void Ws__end() {fxt.Clear().Test__split("a " , "a");}
@Test public void Under() {fxt.Clear().Test__split("a_b" , "a", "b");} // NOTE: same as space
@Test public void Lowercase() {fxt.Clear().Test__split("A B C" , "a", "b", "c");}
@Test public void Dupe() {fxt.Clear().Test__split("a a a" , fxt.Make_word("a", 3));}
@Test public void Dupe__lowercase() {fxt.Clear().Test__split("a A" , fxt.Make_word("a", 2));}
@Test public void Comma__end() {fxt.Clear().Test__split("a, b" , "a", "b");} // EX: "Henry VI, Part 3"; "Bergen County, New Jersey"
@Test public void Comma__mid() {fxt.Clear().Test__split("a,b" , "a,b");} // EX: "20,000 Leagues Under the Sea"
@Test public void Comma__bgn() {fxt.Clear().Test__split(",a b" , "a", "b");} // EX: skip bad usages; EX: "Little Harbour,Pictou ,Nova Scotia"; "The Hindu Succession Act ,1956"
@Test public void Colon__end() {fxt.Clear().Test__split("a: b" , "a", "b");}
@Test public void Colon__mid() {fxt.Clear().Test__split("a:b" , "a:b");} // EX: "3:10 to Yuma (2007 film)"; "6:02 AM EST"; "24:7 Theatre Festival"; "Library of Congress Classification:Class P -- Language and Literature"
@Test public void Colon__bgn() {fxt.Clear().Test__split(":a b" , "a", "b");}
@Test public void Semic__end() {fxt.Clear().Test__split("a; b" , "a", "b");}
@Test public void Semic__mid() {fxt.Clear().Test__split("a;b" , "a;b");}
@Test public void Semic__bgn() {fxt.Clear().Test__split(";a b" , "a", "b");}
@Test public void Bang__end() {fxt.Clear().Test__split("a! b" , "a", "b");}
@Test public void Bang__mid() {fxt.Clear().Test__split("a!b" , "a!b");}
@Test public void Bang__bgn() {fxt.Clear().Test__split("!a b" , "a", "b");}
@Test public void Question__end() {fxt.Clear().Test__split("a? b" , "a", "b");}
@Test public void Question__mid() {fxt.Clear().Test__split("a?b" , "a?b");}
@Test public void Question__bgn() {fxt.Clear().Test__split("?a b" , "a", "b");}
@Test public void Question__sentence() {fxt.Clear().Test__split("a?" , "a");}
@Test public void Multiple__1() {fxt.Clear().Test__split("a?!" , "a");}
@Test public void Multiple__2() {fxt.Clear().Test__split("a!?" , "a");}
@Test public void Dot__word() {fxt.Clear().Test__split("a.org" , "a.org");} // EX: "en.wikipedia.org"; "Earth.png"; "IEEE_802.15"
@Test public void Dot__abrv() {fxt.Clear().Test__split("a vs. b" , "a", "vs.", "vs", "b");} // EX: "vs.", "no.", "dr.", "st.", "inc."
@Test public void Dot__initials() {fxt.Clear().Test__split("a. b. cde" , "a.", "a", "b.", "b", "cde");} // EX: "H. G. Wells"
@Test public void Dot__acronym() {fxt.Clear().Test__split("abc D.E.F. ghi" , "abc", "d.e.f.", "def", "ghi");} // EX: "History of U.S.A. Science", "G.I. Bill", "Washington, D.C."; "Barcelona F.C."; "The Office (U.S.)"; "Agents of S.H.I.E.L.D."; "Gunfight at the O.K. Corral"; "H.M.S. Pinafore"; "R.E.M. discography"
@Test public void Dot__bgn() {fxt.Clear().Test__split("a .bcd e" , "a", ".bcd", "bcd", "e");} // EX: "Colt .45", "List of organizations with .int domain names"
@Test public void Dot__bgn__end() {fxt.Clear().Test__split("a .b. c" , "a", ".b.", "c");}
@Test public void Dot__ellipsis_like() {fxt.Clear().Test__split("a . . . b" , "a", "b");} // EX: "Did you know . . ."
@Test public void Ellipsis__len_3() {fxt.Clear().Test__split("a... bc d" , "a", "...", "bc", "d");} // EX: "Nights into Dreams..."
@Test public void Ellipsis__len_3__bgn() {fxt.Clear().Test__split("a ...b" , "a", "...", "b"); ;} // NOTE: make sure "dot_bgn" code doesn't break this
@Test public void Ellipsis__len_2() {fxt.Clear().Test__split("a.. b" , "a", "..", "b");} // EX: "3.. 6.. 9 Seconds of Light"
@Test public void Ellipsis__bgn() {fxt.Clear().Test__split("...a" , "...", "a");}
@Test public void Ellipsis__end() {fxt.Clear().Test__split("a..." , "a", "...");}
@Test public void Ellipsis__no_ws() {fxt.Clear().Test__split("a...b" , "a", "...", "b");}
@Test public void Ellipsis__term() {fxt.Clear().Test__split("a...?!" , "a", "...");} // EX: "Wetten, dass..?"
@Test public void Apos__merge__end__eos() {fxt.Clear().Test__split("ab's" , "ab's", "abs");} // EX: "A Midsummer Night's Dream"; "Director's cut"
@Test public void Apos__merge__end__word() {fxt.Clear().Test__split("ab's c" , "ab's", "abs", "c");} // EX: "Director's cut"; "Cap'n Crunch";
@Test public void Apos__merge__bgn() {fxt.Clear().Test__split("a o'bc" , "a", "o'bc", "obc");} // EX: "Twelve O'Clock High"; "Shaqille O'Neal"; "Banca d'Italia"
@Test public void Apos__merge__mid() {fxt.Clear().Test__split("i'm" , "i'm", "im");}
@Test public void Apos__bgn__long() {fxt.Clear().Test__split("a 'tis b" , "a", "'tis", "tis", "b");} // EX: "My Country, 'Tis of Thee"; "Omaha hold 'em"; "Slash'EM"; "Expo '92"
@Test public void Apos__end__eos() {fxt.Clear().Test__split("a'" , "a");}
@Test public void Apos__end__short() {fxt.Clear().Test__split("a' b" , "a", "b");} // EX: "Will-o'-the-wisp"; "Portuguese man o' war";
@Test public void Apos__end__long() {fxt.Clear().Test__split("ab' c" , "ab", "c");} // EX: "Dunkin' Donuts"; "'Allo 'Allo!"; "Catherine de' Medici"
@Test public void Apos__both__n() {fxt.Clear().Test__split("a 'n' b" , "a", "'n'", "n", "b");} // EX: "Rock 'n' Roll"; "Town 'n' Country, Florida"; "Hill 'n Dale, Florida"; "Chip 'n Dale Rescue Rangers"
@Test public void Apos__multiple() {fxt.Clear().Test__split("ab''cd" , "ab''cd");}
@Test public void Apos__lone() {fxt.Clear().Test__split("' a" , "'", "a");} // EX: "' (disambiguation)"
@Test public void Dash__one() {fxt.Clear().Test__split("a-b" , "a", "b", "a-b");} // EX: "The Amazing Spider-Man"; "On-super percentage"; "Basic Role-Playing"; "Context-sensitive"; "Cross-country skiing"; "Double-barreled shotgun"; "Dot-com bubble"; "Many-worlds interpretation"; "Faster-than-light"; "Gram-positive bacteria"; "Half-life", "Jean-Paul Sartre"; "Austria-Hungary"
@Test public void Dash__many() {fxt.Clear().Test__split("a-b-c" , "a", "b", "c", "a-b-c");}
@Test public void Dash__ws() {fxt.Clear().Test__split("a - b" , "a", "-", "b");}
@Test public void Dash__eos() {fxt.Clear().Test__split("a-" , "a", "a-");}
@Test public void Dash__bos() {fxt.Clear().Test__split("-a" , "a", "-a");}
@Test public void Dash__mult__2() {fxt.Clear().Test__split("--" , "--");}
@Test public void Dash__mult__3() {fxt.Clear().Test__split("---" , "---");}
@Test public void Dash__mult__2__words() {fxt.Clear().Test__split("a--b" , "a", "b", "a--b");}
@Test public void Dash__w_comma() {fxt.Clear().Test__split("a-, b" , "a", "a-", "b");}
@Test public void Slash__one() {fxt.Clear().Test__split("a/b" , "a", "b");} // EX: "Good cop/bad cop"; "Snooker world rankings 2004/2005"; "Debian GNU/Hurd"; "HIV/AIDS in the United States"; "List of minor planets/1<>100"
@Test public void Slash__many() {fxt.Clear().Test__split("a/b/c" , "a", "b", "c");} // EX: "Age/sex/location";
@Test public void Slash__ws() {fxt.Clear().Test__split("a / b" , "a", "b");}
@Test public void Dash__slash() {fxt.Clear().Test__split("a-b/c-d-e/f-g" , "a", "b", "a-b", "c", "d", "e", "c-d-e", "f", "g", "f-g");}
@Test public void Paren__both__one() {fxt.Clear().Test__split("a (b) c" , "a", "b", "c");} // EX: "A (letter)"
@Test public void Paren__both__many() {fxt.Clear().Test__split("a (b c) d" , "a", "b", "c", "d");} // EX: "A (2016 film)"
@Test public void Paren__bgn__multiple() {fxt.Clear().Test__split("a (((b)))" , "a", "b");}
@Test public void Paren__unmatched() {fxt.Clear().Test__split("a(b" , "a(b");}
@Test public void Paren__unmatched__bgn() {fxt.Clear().Test__split("a (b" , "a", "b");}
@Test public void Paren__mid() {fxt.Clear().Test__split("a(b)c" , "a(b)c");} // EX: "Chloro(pyridine)cobaloxime"; "Exi(s)t"
@Test public void Paren__end() {fxt.Clear().Test__split("a(b)" , "a(b)", "a");} // EX: "Come What(ever) May"; "501(c) organization"; "Reindeer(s) Are Better Than People"; "(Miss)understood"; "Chromium(III) picolinate"
@Test public void Paren__bgn() {fxt.Clear().Test__split("(a)b" , "(a)b", "b");} // EX: "International Student Congress of (bio)Medical Sciences"
@Test public void Paren__end__dash() {fxt.Clear().Test__split("a(b-c) d" , "a(b-c)", "a", "d");} // EX: "Bis(2-ethylhexyl) phthalate"
@Test public void Paren__end__comma() {fxt.Clear().Test__split("a(b,c) d" , "a(b,c)", "a", "d");} // EX: "Iron(II,III) oxide"
@Test public void Paren__comma() {fxt.Clear().Test__split("a (b), c" , "a", "b", "c");} // EX: "Corning (city), New York"
@Test public void Paren__multiple() {fxt.Clear().Test__split("(a) (b)" , "a", "b");}
@Test public void Quote__both() {fxt.Clear().Test__split("a \"b\" c" , "a", "b", "c");}
@Test public void Word_bgn__at() {fxt.Clear().Test__split("@a" , "@a", "a");}
@Test public void Word_bgn__tilde() {fxt.Clear().Test__split("~a~" , "a");} // EX: "Phantom ~Requiem for the Phantom~"
}
class Srch_text_parser_fxt {
private final Srch_text_parser word_parser = new Srch_text_parser();
private final Bry_bfr tmp_bfr = Bry_bfr.new_(32);
private Xol_case_mgr case_mgr;
public void Init() {
case_mgr = Xol_case_mgr_.A7();
word_parser.Init_for_ttl(case_mgr);
}
public Srch_text_parser_fxt Clear() {
word_parser.word_hash.Clear();
return this;
}
public Srch_word_itm Make_word(String raw, int count) {return new Srch_word_itm(Bry_.new_u8(raw)).Count_(count);}
public void Test__split(String src, String... expd_words) {
int len = expd_words.length;
Srch_word_itm[] ary = new Srch_word_itm[len];
for (int i = 0; i < len; ++i) {
ary[i] = Make_word(expd_words[i], 1);
}
Test__split(src, ary);
}
public void Test__split(String src, Srch_word_itm... expd_words) {
byte[] src_bry = Bry_.new_u8(src);
word_parser.Parse(Bool_.Y, src_bry, 0, src_bry.length);
Tfds.Eq_str_lines(To_str(expd_words), To_str(word_parser.word_hash));
}
private String To_str(Srch_word_itm[] word_ary) {
int len = word_ary.length;
for (int i = 0; i < len; ++i) {
if (i != 0) tmp_bfr.Add_byte_nl();
Srch_word_itm word = word_ary[i];
tmp_bfr.Add(word.Word).Add_byte_pipe();
tmp_bfr.Add_int_variable(word.Count());
}
return tmp_bfr.To_str_and_clear();
}
private String To_str(Srch_word_hash word_mgr) {
int len = word_mgr.Len();
Srch_word_itm[] ary = new Srch_word_itm[len];
for (int i = 0; i < len; ++i)
ary[i] = word_mgr.Get_at(i);
return To_str(ary);
}
}

25
400_xowa/src/gplx/xowa/addons/searchs/parsers/Srch_text_parser_wkr.java Normal file
View File

@@ -0,0 +1,25 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
public interface Srch_text_parser_wkr {
void Parse_done(Srch_word_itm word);
}
class Srch_text_parser_wkr__noop implements Srch_text_parser_wkr {
public void Parse_done(Srch_word_itm word) {}
public static final Srch_text_parser_wkr__noop Instance = new Srch_text_parser_wkr__noop(); Srch_text_parser_wkr__noop() {}
}

33
400_xowa/src/gplx/xowa/addons/searchs/parsers/Srch_word_hash.java Normal file
View File

@@ -0,0 +1,33 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
public class Srch_word_hash {
private final Ordered_hash hash = Ordered_hash_.New_bry();
public void Clear() {hash.Clear();}
public int Len() {return hash.Count();}
public boolean Has(byte[] word) {return hash.Has(word);}
public Srch_word_itm Get_at(int i) {return (Srch_word_itm)hash.Get_at(i);}
public void Add(byte[] word) {
Srch_word_itm itm = (Srch_word_itm)hash.Get_by(word);
if (itm == null) {
itm = new Srch_word_itm(word);
hash.Add(word, itm);
}
itm.Count_add_1_();
}
}

16
400_xowa/src/gplx/xowa/specials/search/Srch_rslt_list.java → 400_xowa/src/gplx/xowa/addons/searchs/parsers/Srch_word_itm.java
View File

@@ -15,10 +15,14 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.specials.search; import gplx.*; import gplx.xowa.*; import gplx.xowa.specials.*;
class Srch_rslt_list {
private final List_adp list = List_adp_.new_();
public int Len() {return list.Count();}
public void Add(Srch_rslt_itm rslt) {list.Add(rslt);}
public Srch_rslt_itm Get_at(int i) {return (Srch_rslt_itm)list.Get_at(i);}
package gplx.xowa.addons.searchs.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
public class Srch_word_itm {
public Srch_word_itm(byte[] word) {
this.Word = word;
this.count = 0;
}
public final byte[] Word;
public int Count() {return count;} private int count;
public void Count_add_1_() {++count;}
@gplx.Internal protected Srch_word_itm Count_(int v) {this.count = v; return this;}
}

63
400_xowa/src/gplx/xowa/specials/search/Xows_ns_mgr.java → 400_xowa/src/gplx/xowa/addons/searchs/searchers/Srch_ns_mgr.java
View File

@@ -15,25 +15,27 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.specials.search; import gplx.*; import gplx.xowa.*; import gplx.xowa.specials.*;
package gplx.xowa.addons.searchs.searchers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.core.primitives.*;
import gplx.xowa.wikis.nss.*;
public class Xows_ns_mgr {
private final Ordered_hash ns_hash = Ordered_hash_.New(); private final Int_obj_ref tmp_ns_id = Int_obj_ref.neg1_(); private final Bry_bfr tmp_bfr = Bry_bfr.reset_(32);
public class Srch_ns_mgr {
private final Ordered_hash ns_hash = Ordered_hash_.New(); private final Int_obj_ref tmp_ns_id = Int_obj_ref.neg1_();
private final Bry_bfr tmp_bfr = Bry_bfr.reset_(32);
private boolean ns_all, ns_main;
public void Clear() {
ns_hash.Clear();
ns_all = ns_main = false;
}
public boolean Ns_main_only() {return ns_main;}
public boolean Has(int ns_id) {
return ns_all // all flag set
|| ns_main && ns_id == Xow_ns_.Tid__main // main flag set
|| ns_hash.Has(tmp_ns_id.Val_(ns_id)) // check against ns_hash
;
return ns_all // ns_all always returns true
|| ns_main && ns_id == Xow_ns_.Tid__main // ns_main returns true if main_ns
|| ns_hash.Has(tmp_ns_id.Val_(ns_id)); // ns_hash returns true if has ns_id
}
public void Add_by_id(int ns_id) {
if (ns_hash.Has(tmp_ns_id.Val_(ns_id)))
ns_hash.Del(tmp_ns_id);
public void Add_all() {ns_all = true;}
public Srch_ns_mgr Add_main_if_empty() {if (ns_hash.Count() == 0) ns_main = true; return this;}
public void Add_by_id(int ns_id) {
if (ns_hash.Has(tmp_ns_id.Val_(ns_id))) ns_hash.Del(tmp_ns_id);
ns_hash.Add_as_key_and_val(Int_obj_ref.new_(ns_id));
}
public void Add_by_name(byte[] ns_name) {
@@ -41,14 +43,11 @@ public class Xows_ns_mgr {
if (id != Xow_ns_.Tid__null)
Add_by_id(id);
}
public void Add_all() {
ns_all = true;
}
public void Add_by_parse(byte[] key, byte[] val) {
int ns_enabled = Bry_.To_int_or_neg1(val);
if (ns_enabled == 1) { // make sure set to 1; EX: ignore &ns0=0
int key_len = key.length;
if (key_len == 3 && key[2] == Byte_ascii.Star) // translate ns* as ns_all
if (key_len == 3 && key[2] == Srch_search_addon.Wildcard__star) // key=ns* sets ns_all to true
ns_all = true;
else {
int ns_id = Bry_.To_int_or(key, 2, key_len, Int_.Min_value);
@@ -59,15 +58,9 @@ public class Xows_ns_mgr {
}
}
}
public void Add_main_if_empty() {
if (ns_hash.Count() == 0)
ns_main = true;
}
public byte[] To_hash_key() {
if (ns_all)
return Hash_key_all;
else if (ns_main)
return Hash_key_main;
if (ns_all) return Hash_key_all;
else if (ns_main) return Hash_key_main;
else {
int ns_hash_len = ns_hash.Count();
for (int i = 0; i < ns_hash_len; i++) {
@@ -78,5 +71,29 @@ public class Xows_ns_mgr {
return tmp_bfr.To_bry_and_clear();
}
}
private static final byte[] Hash_key_all = new byte[] {Byte_ascii.Star}, Hash_key_main = new byte[] {Byte_ascii.Num_0};
public void Add_by_int_ids(int[] ns_ids) {
this.Clear();
if (ns_ids.length == 0) {
this.Add_all();
} else if (ns_ids.length == 1 && ns_ids[0] == Xow_ns_.Tid__main) {
this.Add_main_if_empty();
} else {
for (int ns_id : ns_ids)
this.Add_by_id(ns_id);
}
}
public int[] To_int_ary() {
if (ns_all) return Int_.Ary_empty;
else if (ns_main) return Int_.Ary(Xow_ns_.Tid__main);
else {
int len = ns_hash.Count();
int[] rv = new int[len];
for (int i = 0; i < len; i++) {
Int_obj_ref ns_id_ref = (Int_obj_ref)ns_hash.Get_at(i);
rv[i] = ns_id_ref.Val();
}
return rv;
}
}
private static final byte[] Hash_key_all = new byte[] {Srch_search_addon.Wildcard__star}, Hash_key_main = new byte[] {Byte_ascii.Num_0};
}

55
400_xowa/src/gplx/xowa/addons/searchs/searchers/Srch_search_cmd.java Normal file
View File

@@ -0,0 +1,55 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.xowa.addons.searchs.searchers.rslts.*;
import gplx.xowa.addons.searchs.searchers.crts.*;
public class Srch_search_cmd implements Cancelable, GfoInvkAble {
private final Srch_search_mgr search_mgr;
public final Srch_search_qry qry;
public final Srch_crt_mgr crt_mgr;
private final Srch_rslt_cbk rslt_cbk;
private final Srch_rslt_list rslts_list;
public Srch_search_cmd(Srch_search_mgr search_mgr, Srch_search_qry qry, Srch_crt_mgr crt_mgr, Srch_rslt_cbk rslt_cbk, Srch_rslt_list rslts_list) {
this.search_mgr = search_mgr; this.qry = qry; this.crt_mgr = crt_mgr; this.rslt_cbk = rslt_cbk; this.rslts_list = rslts_list;
}
public boolean Canceled() {return canceled;} private boolean canceled;
public void Cancel() {
canceled = true;
rslt_cbk.On_cancel();
}
private void Search() {
try {
search_mgr.Search_async(this, qry, crt_mgr, rslt_cbk, rslts_list); // NOTE: must handle any errors in async mode
}
catch(Exception e) {
Xoa_app_.Usr_dlg().Prog_many("", "", "error during search: err=~{0}", Err_.Message_gplx_log(e));
}
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__search)) Search();
else return GfoInvkAble_.Rv_unhandled;
return this;
}
public static final String Invk__search = "search";
public static Srch_search_cmd Noop() {
if (noop == null) {
noop = new Srch_search_cmd(null, null, null, Srch_rslt_cbk_.Noop, null);
}
return noop;
} private static Srch_search_cmd noop;
}

69
400_xowa/src/gplx/xowa/addons/searchs/searchers/Srch_search_ctx.java Normal file
View File

@@ -0,0 +1,69 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.xowa.langs.cases.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.addons.searchs.dbs.*;
import gplx.xowa.addons.searchs.searchers.crts.*; import gplx.xowa.addons.searchs.searchers.rslts.*; import gplx.dbs.percentiles.*; import gplx.xowa.addons.searchs.searchers.wkrs.*;
import gplx.xowa.addons.searchs.parsers.*;
public class Srch_search_ctx {
public Srch_search_ctx(Cancelable cxl, Xow_wiki wiki, Srch_search_addon addon
, Srch_rslt_list cache__page, Hash_adp_bry cache__word_counts
, Srch_search_qry qry, Srch_crt_scanner_syms scanner_syms, Srch_crt_mgr crt_mgr, Srch_rslt_list rslts_list) {
this.Cxl = cxl;
this.Wiki = wiki;
this.Wiki_domain = wiki.Domain_bry();
this.Case_mgr = wiki.Case_mgr();
this.Addon = addon;
this.Cache__page = cache__page;
this.Cache__word_counts = cache__word_counts;
this.Qry = qry;
this.Scanner_syms = scanner_syms;
this.Crt_mgr = crt_mgr;
this.Crt_mgr__root = crt_mgr.Root;
this.Rslts_list = rslts_list;
this.Db__core = wiki.Data__core_mgr().Db__core();
this.Tbl__page = Db__core.Tbl__page();
this.Tbl__word = addon.Db_mgr().Tbl__word();
this.Tbl__link__ary = addon.Db_mgr().Tbl__link__ary();
long page_count = wiki.Stats().Num_pages();
this.Score_rng.Init(page_count, addon.Db_mgr().Cfg().Link_score_max());
int rslts_needed = qry.Slab_end - rslts_list.Len();
if (rslts_needed < 0) rslts_needed = 0;
this.Rslts_needed = rslts_needed;
this.Highlight_mgr = new Srch_highlight_mgr(this.Case_mgr).Search_(qry.Phrase.Orig);
}
public final Cancelable Cxl;
public final Xow_wiki Wiki;
public final byte[] Wiki_domain;
public final Srch_search_addon Addon;
public final Xol_case_mgr Case_mgr;
public final Srch_rslt_list Cache__page;
public final Hash_adp_bry Cache__word_counts;
public final Xowd_db_file Db__core;
public final Xowd_page_tbl Tbl__page;
public final Srch_word_tbl Tbl__word;
public final Srch_link_tbl[] Tbl__link__ary;
public final Srch_search_qry Qry;
public final Srch_crt_scanner_syms Scanner_syms;
public final Srch_rslt_list Rslts_list;
public final int Rslts_needed;
public final Percentile_rng Score_rng = new Percentile_rng();
public final Srch_crt_mgr Crt_mgr;
public final Srch_crt_itm Crt_mgr__root;
public final Srch_highlight_mgr Highlight_mgr;
}

85
400_xowa/src/gplx/xowa/addons/searchs/searchers/Srch_search_mgr.java Normal file
View File

@@ -0,0 +1,85 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.xowa.addons.searchs.searchers.rslts.*; import gplx.xowa.addons.searchs.searchers.wkrs.*; import gplx.xowa.addons.searchs.parsers.*; import gplx.xowa.addons.searchs.searchers.crts.*;
import gplx.xowa.addons.searchs.searchers.crts.visitors.*;
public class Srch_search_mgr {
private final Srch_search_addon addon;
private final Xow_wiki wiki;
private final Srch_rslt_list cache__page = new Srch_rslt_list();
private final Hash_adp_bry cache__word_counts = Hash_adp_bry.cs();
private final Srch_rslt_cache cache__rslts = new Srch_rslt_cache();
private final Srch_page_tbl_wkr page_tbl_searcher = new Srch_page_tbl_wkr();
private final Srch_crt_parser crt_parser;
private final Srch_search_cmd[] cur_cmds;
private final Object mutex = new Object();
private int search_count;
public Srch_search_mgr(Srch_search_addon addon, Xow_wiki wiki, Srch_text_parser parser) {
this.addon = addon; this.wiki = wiki;
crt_parser = new Srch_crt_parser(Scanner_syms);
// init cur_cmds with Noop cmd to make cancel logic below easier
int len = Srch_search_qry.Tid_len;
this.cur_cmds = new Srch_search_cmd[Srch_search_qry.Tid_len];
for (int i = 0; i < len; ++i)
cur_cmds[i] = Srch_search_cmd.Noop();
}
public void Clear_rslts_cache() {cache__rslts.Clear();}
public void Search_cancel() {
cur_cmds[Srch_search_qry.Tid__suggest_box].Cancel();
}
public void Search(Srch_search_qry qry, Srch_rslt_cbk cbk) {
if (qry.Phrase.Orig.length == 0) return;
// handle obsolete search dbs;
if (addon.Db_mgr().Cfg().Version_id__needs_upgrade()) {
addon.Db_mgr().Upgrade_mgr.Upgrade();
return;
}
// cancel existing cmd
Srch_search_cmd cur_cmd = cur_cmds[qry.Tid];
cur_cmd.Cancel();
// create new one; run it;
Srch_crt_mgr crt_mgr = crt_parser.Parse_or_invalid(qry.Phrase.Compiled, qry.Phrase.Wildcard);
if (crt_mgr == Srch_crt_mgr.Invalid) return; // handle "\\" which is invalid or other fatal errors
Srch_rslt_list rslts_list = cache__rslts.Get_or_new(crt_mgr.Key);
cur_cmd = new Srch_search_cmd(this, qry, crt_mgr, cbk, rslts_list);
cur_cmds[qry.Tid] = cur_cmd;
gplx.core.threads.Thread_adp_.invk_(gplx.xowa.apps.Xoa_thread_.Key_special_suggest, cur_cmd, Srch_search_cmd.Invk__search).Start();
}
public void Search_async(Cancelable cxl, Srch_search_qry qry, Srch_crt_mgr crt_mgr, Srch_rslt_cbk rslt_cbk, Srch_rslt_list rslts_list) {
synchronized (mutex) { // force only one search at a time; do not (a) place around Thread_sleep; (b) reuse for any other locks
if (++search_count > 64) this.Clear(); // lazy way of clearing memory
Srch_search_ctx ctx = new Srch_search_ctx(cxl, wiki, addon, cache__page, cache__word_counts, qry, Scanner_syms, crt_mgr, rslts_list);
ctx.Score_rng.Select_init(ctx.Rslts_needed, rslts_list.Score_bgn, rslts_list.Score_len, Srch_link_wkr.Percentile_rng__calc_adj(crt_mgr.Words_nth__len()));
page_tbl_searcher.Search(ctx, rslt_cbk);
if (cxl.Canceled()) return;
Srch_link_wkr link_wkr = new Srch_link_wkr();
link_wkr.Search(rslts_list, rslt_cbk, ctx);
}
}
private void Clear() {
search_count = 0;
cache__page.Clear();
cache__word_counts.Clear();
cache__rslts.Clear();
}
public static final Srch_crt_scanner_syms Scanner_syms = Srch_crt_scanner_syms.Dflt;
}

133
400_xowa/src/gplx/xowa/addons/searchs/searchers/Srch_search_phrase.java Normal file
View File

@@ -0,0 +1,133 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.core.btries.*;
import gplx.xowa.addons.searchs.searchers.crts.*;
public class Srch_search_phrase {
public Srch_search_phrase(boolean wildcard, byte[] orig, byte[] compiled) {
this.Orig = orig;
this.Compiled = compiled;
this.Wildcard = wildcard;
}
public final boolean Wildcard;
public final byte[] Orig; // EX: "Earth"
public final byte[] Compiled; // EX: "earth*"
public static Srch_search_phrase New(gplx.xowa.langs.cases.Xol_case_mgr case_mgr, byte[] orig, boolean wildcard) {
int orig_len = orig.length;
if ( orig_len > 0 // if "*" at end, remove and change to wildcard; needed for Special:Search which will send in "earth*" but "earth" needed for highlighting
&& orig[orig_len - 1] == Srch_crt_scanner_syms.Dflt.Wild()) {
orig = Bry_.Mid(orig, 0, orig_len - 1);
wildcard = true;
}
byte[] lcase = case_mgr.Case_build_lower(orig);
lcase = Auto_wildcard(lcase, Srch_crt_scanner_syms.Dflt);
return new Srch_search_phrase(wildcard, orig, lcase);
}
public static byte[] Auto_wildcard(byte[] raw, Srch_crt_scanner_syms syms) {
Btrie_slim_mgr trie = syms.Trie();
int raw_len = raw.length;
int insert_pos = -1;
int fail_pos = -1;
for (int i = raw_len - 1; i > -1; --i) {
byte b = raw[i];
byte tid = trie.Match_byte_or(b, raw, i, i + 1, Byte_.Max_value_127);
if (tid == Byte_.Max_value_127) { // unknown sym
if (b == syms.Wild()) { // wildcard is not tokenized
fail_pos = i;
break;
}
else { // alphanum-char
insert_pos = i;
break;
}
}
else {
switch (tid) {
case Srch_crt_tkn.Tid__quote:
case Srch_crt_tkn.Tid__space:
case Srch_crt_tkn.Tid__not:
case Srch_crt_tkn.Tid__and:
case Srch_crt_tkn.Tid__or:
case Srch_crt_tkn.Tid__paren_bgn:
fail_pos = i; // these symbols will not auto-wildcard, unless they are escaped
i = -1;
break;
case Srch_crt_tkn.Tid__escape:
if (i > 0) {
int prv_pos = i -1;
if (raw[prv_pos] == syms.Escape()) { // an escaped escape can be wildcarded; EX: "\\" -> "\\*"
insert_pos = i;
i = -1;
}
else
fail_pos = i;
}
else
fail_pos = i;
i = -1;
break;
case Srch_crt_tkn.Tid__paren_end:
break;
}
}
}
// check if preceded by escape
if (insert_pos == -1) {
if ( fail_pos > 0
&& raw[fail_pos - 1] == syms.Escape()) {
insert_pos = fail_pos;
}
else
return raw;
}
// check if word already has wildcard; EX: "a*b" x> "a*b*"
for (int i = insert_pos - 1; i > -1; --i) {
byte b = raw[i];
if (b == syms.Wild()) {
int prv_pos = i - 1;
if (prv_pos > -1) {
if (raw[prv_pos] == syms.Escape()) { // ignore escaped wildcard
i = prv_pos;
continue;
}
}
return raw; // existing wildcard cancels auto-wildcard
}
else if (b == syms.Space()) { // stop looking when word ends
break;
}
else {} // alphanum; keep going
}
// add wildcard
if (insert_pos == raw_len - 1) return Bry_.Add(raw, syms.Wild());
else {
byte[] rv = new byte[raw_len + 1];
int wildcard_pos = insert_pos + 1;
for (int i = 0; i < wildcard_pos; ++i)
rv[i] = raw[i];
rv[wildcard_pos] = syms.Wild();
for (int i = wildcard_pos; i < raw_len; ++i)
rv[i + 1] = raw[i];
return rv;
}
}
}

46
400_xowa/src/gplx/xowa/addons/searchs/searchers/Srch_search_phrase_tst.java Normal file
View File

@@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import org.junit.*; import gplx.xowa.addons.searchs.parsers.*; import gplx.xowa.addons.searchs.searchers.crts.*;
public class Srch_search_phrase_tst {
private final Srch_search_phrase_fxt fxt = new Srch_search_phrase_fxt();
@Test public void Word() {fxt.Test__auto_wildcard("a" , "a*");}
@Test public void Paren_end() {fxt.Test__auto_wildcard("(a)" , "(a*)");}
@Test public void Quoted() {fxt.Test__auto_wildcard("\"a\"" , "\"a\"");}
@Test public void Space() {fxt.Test__auto_wildcard(" " , " ");}
@Test public void Not() {fxt.Test__auto_wildcard("-" , "-");}
@Test public void And() {fxt.Test__auto_wildcard("+" , "+");}
@Test public void Or() {fxt.Test__auto_wildcard("," , ",");}
@Test public void Paren_bgn() {fxt.Test__auto_wildcard("(" , "(");}
@Test public void Star() {fxt.Test__auto_wildcard("*" , "*");}
@Test public void Wildcard__exists__y() {fxt.Test__auto_wildcard("a*b" , "a*b");}
@Test public void Wildcard__exists__escaped() {fxt.Test__auto_wildcard("a\\*b" , "a\\*b*");}
@Test public void Wildcard__exists__n() {fxt.Test__auto_wildcard("a* bc" , "a* bc*");}
@Test public void Escape() {fxt.Test__auto_wildcard("\\*" , "\\**");}
@Test public void Escape__incomplete() {fxt.Test__auto_wildcard("a\\" , "a\\");}
@Test public void Escape__escaped() {fxt.Test__auto_wildcard("a\\\\" , "a\\\\*");}
}
class Srch_search_phrase_fxt {
private final Srch_crt_scanner_syms syms = Srch_crt_scanner_syms.Dflt;
public Srch_search_phrase_fxt() {}
public void Test__auto_wildcard(String src_str, String expd) {
byte[] src_raw = Bry_.new_u8(src_str);
byte[] actl = Srch_search_phrase.Auto_wildcard(src_raw, syms);
Tfds.Eq(expd, String_.new_u8(actl));
}
}

50
400_xowa/src/gplx/xowa/addons/searchs/searchers/Srch_search_qry.java Normal file
View File

@@ -0,0 +1,50 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*;
import gplx.xowa.wikis.domains.*;
public class Srch_search_qry {
public Srch_search_qry(byte tid, Srch_ns_mgr ns_mgr, Srch_search_phrase phrase, int slab_bgn, int slab_end) {
this.Tid = tid;
this.Ns_mgr = ns_mgr;
this.Phrase = phrase;
this.Slab_bgn = slab_bgn;
this.Slab_end = slab_end;
}
public final byte Tid;
public final Srch_ns_mgr Ns_mgr;
public final Srch_search_phrase Phrase;
public final int Slab_bgn; // EX: 0
public final int Slab_end; // EX: 20
public static final byte Tid_len = 4, Tid__url_bar = 0, Tid__suggest_box = 1, Tid__search_page = 2, Tid__android = 3;
public static Srch_search_qry New__url_bar(Xow_wiki wiki, gplx.xowa.apps.apis.xowa.addons.searchs.Xoapi_url_bar cfg, byte[] search_orig) {
return new Srch_search_qry(Tid__url_bar, cfg.Ns_mgr(), Srch_search_phrase.New(wiki.Case_mgr(), search_orig, cfg.Auto_wildcard()), 0, cfg.Max_results());
}
public static Srch_search_qry New__suggest_box(Xow_wiki wiki, Srch_search_cfg cfg, byte[] search_orig) {
return new Srch_search_qry(Tid__suggest_box, cfg.Ns_mgr(), Srch_search_phrase.New(wiki.Case_mgr(), search_orig, Bool_.Y), 0, cfg.Rslts_max());
}
public static Srch_search_qry New__search_page(Xow_domain_itm[] domains, Xow_wiki wiki, Srch_search_cfg cfg, boolean simple_search, byte[] search_orig, int slab_idx, int slab_len) {
int slab_bgn = slab_idx * slab_len;
int slab_end = slab_bgn + slab_len;
boolean wildcard = simple_search;
return new Srch_search_qry(Tid__search_page, cfg.Ns_mgr(), Srch_search_phrase.New(wiki.Case_mgr(), search_orig, wildcard), slab_bgn, slab_end);
}
public static Srch_search_qry New__drd(Xow_wiki wiki, Srch_ns_mgr ns_mgr, byte[] search_orig, int slab_bgn, int slab_end) {
return new Srch_search_qry(Tid__android, ns_mgr, Srch_search_phrase.New(wiki.Case_mgr(), search_orig, Bool_.Y), slab_bgn, slab_end);
}
}

57
400_xowa/src/gplx/xowa/addons/searchs/searchers/cbks/Srch_rslt_cbk__suggest_box.java Normal file
View File

@@ -0,0 +1,57 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers.cbks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.searchers.*;
import gplx.core.js.*;
import gplx.xowa.addons.searchs.searchers.*; import gplx.xowa.addons.searchs.searchers.rslts.*;
public class Srch_rslt_cbk__suggest_box implements Srch_rslt_cbk, GfoInvkAble {
private final Js_wtr js_wtr = new Js_wtr();
private final Xoae_app app;
private final byte[] cbk_func;
private final byte[] search_raw;
public Srch_rslt_cbk__suggest_box(Xoae_app app, byte[] cbk_func, byte[] search_raw) {
this.app = app; this.cbk_func = cbk_func;
this.search_raw = search_raw;
}
public void On_cancel() {}
public void On_rslts_found(Srch_search_qry qry, Srch_rslt_list rslts_list, int rslts_bgn, int rslts_end) {
if (!rslts_list.Rslts_are_enough && !rslts_list.Rslts_are_done) return;
js_wtr.Func_init(cbk_func);
js_wtr.Prm_bry(search_raw);
js_wtr.Prm_spr();
js_wtr.Ary_init();
int rslts_len = rslts_list.Len();
for (int i = 0; i < qry.Slab_end; i++) {
if (i >= rslts_len) break; // rslts_end will overshoot actual rslts_len; check for out of bounds and exit; EX: default suggest will have rslts_end of 25, but "earth time" will retrieve 15 results
Srch_rslt_row row = rslts_list.Get_at(i);
js_wtr.Ary_bry(row.Page_ttl.Full_txt_w_ttl_case());
js_wtr.Ary_bry(row.Page_ttl_display(Bool_.Y));
}
js_wtr.Ary_term();
js_wtr.Func_term();
GfoInvkAble_.InvkCmd(app.Gui_mgr().Kit().New_cmd_sync(this), Srch_rslt_cbk__suggest_box.Invk__notify);
}
private void Notify() {
app.Gui_mgr().Browser_win().Active_html_box().Html_js_eval_script(js_wtr.To_str_and_clear());
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__notify)) Notify();
else return GfoInvkAble_.Rv_unhandled;
return this;
}
public static final String Invk__notify = "notify";
}

86
400_xowa/src/gplx/xowa/addons/searchs/searchers/cbks/Srch_rslt_cbk__url_bar.java Normal file
View File

@@ -0,0 +1,86 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers.cbks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.searchers.*;
import gplx.gfui.*;
import gplx.xowa.addons.searchs.searchers.rslts.*;
import gplx.xowa.apps.apis.xowa.addons.searchs.*;
public class Srch_rslt_cbk__url_bar implements Srch_rslt_cbk, GfoInvkAble {
private final Xoae_app app;
private final GfuiComboBox url_bar;
private final Xoapi_url_bar url_bar_api;
private String[] cbo_ary;
private boolean rslts_finished;
private int rslts_in_this_pass;
private boolean rslts_shown = false;
public Srch_rslt_cbk__url_bar(Xoae_app app, GfuiComboBox url_bar, Xoapi_url_bar url_bar_api) {
this.app = app; this.url_bar = url_bar; this.url_bar_api = url_bar_api;
}
public void On_cancel() {}
public void On_rslts_found(Srch_search_qry qry, Srch_rslt_list rslts_list, int rslts_bgn, int rslts_end) {
int rslts_len = rslts_list.Len();
this.rslts_finished = rslts_list.Rslts_are_enough || rslts_list.Rslts_are_done;
// get # of items for drop-down; note special logic to reduce blinking
rslts_in_this_pass = rslts_end - rslts_bgn;
if ( rslts_in_this_pass == 0 // no new results;
&& rslts_bgn != 0 // if first one, still update; blanks out results from previous try;
&& !rslts_finished) // if rslts_finished, still update to force cbo to "shrink"
return; // exit now else will "blink" when refreshing;
int max_rslts = url_bar_api.Max_results();
int cbo_len = max_rslts; // force cbo_len to be max_rslts; reduces "blinking" when typing by keeping visible area to same size
if (rslts_list.Rslts_are_done) { // "shrink" cbo_len to rslts_len; EX: 10 wanted; 2 returned; shrink to 2 rows;
cbo_len = rslts_len;
}
// fill cbo_ary with rslts from search, while "blanking" out rest
this.cbo_ary = new String[cbo_len];
for (int i = 0; i < cbo_len; ++i) {
String cbo_itm = "";
if (i >= max_rslts) break;
if (i < rslts_len) {
Srch_rslt_row rslt = rslts_list.Get_at(i);
cbo_itm = String_.new_u8(rslt.Page_ttl_display(Bool_.N));
}
cbo_ary[i] = cbo_itm;
}
GfoInvkAble_.InvkCmd(app.Gui_mgr().Kit().New_cmd_sync(this), Srch_rslt_cbk__url_bar.Invk__items__update); // NOTE: needs to be sync, b/c page_wkr and link_wkr must execute in order; EX:"Portal:Science" does not show; DATE:2016-03-24
}
private void Items__update() {
url_bar.Items__update(cbo_ary);
if (!url_bar.List_visible() // rslt_list not visible
&& !rslts_shown // auto-dropdown hasn't happened yet
&& (rslts_in_this_pass > 0 || rslts_finished) // at least 1 rslt, or search done
) {
rslts_shown = true; // only auto-show dropdown once; allows user to close drop-down and not have it continually flashing
url_bar.List_visible_(Bool_.Y);
}
Xoa_app_.Usr_dlg().Prog_none("", "", "");
if (rslts_finished) {
if (cbo_ary.length == 0)
url_bar.List_visible_(Bool_.N);
else
url_bar.Items__size_to_fit(cbo_ary.length);
}
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__items__update)) Items__update();
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk__items__update = "items__update";
}

51
400_xowa/src/gplx/xowa/addons/searchs/searchers/cbks/Srch_suggest_cmd__page_tbl.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
//namespace gplx.xowa.addons.searchs.searchers.cbks {
// class Srch_suggest_cmd__page_tbl {
// private boolean Search_by_all_pages_v2() {
// rslts_2.Clear();
// Xoa_ttl search_ttl = Xoa_ttl.parse(wiki, search_bry); if (search_ttl == null) return false;
// byte[] search_ttl_bry = search_ttl.Page_db();
// wiki.Db_mgr().Load_mgr().Load_ttls_for_search_suggest(this, rslts_2, search_ttl.Ns(), search_ttl_bry, max_results, all_pages_min, all_pages_extend, true, false);
// return true;
// }
// private boolean Search_by_all_pages_v1() {
// rslts_2.Clear();
// Xowd_page_itm rslt_nxt = new Xowd_page_itm();
// Xowd_page_itm rslt_prv = new Xowd_page_itm();
// Xoa_ttl search_ttl = Xoa_ttl.parse(wiki, search_bry); if (search_ttl == null) return false;
// byte[] search_ttl_bry = search_ttl.Page_db();
// List_adp page_list = List_adp_.new_();
// wiki.Db_mgr().Load_mgr().Load_ttls_for_all_pages(this, page_list, rslt_nxt, rslt_prv, Int_obj_ref.zero_(), wiki.Ns_mgr().Ns_main(), search_ttl_bry, max_results, all_pages_min, all_pages_extend, true, false);
// Xowd_page_itm[] page_ary = (Xowd_page_itm[])page_list.To_ary_and_clear(typeof(Xowd_page_itm));
// int idx = 0, page_ary_len = page_ary.length;
// for (int i = 0; i < page_ary_len; i++) {
// Xowd_page_itm page = page_ary[i];
// if (page != null) {
// if (!Bry_.Has_at_bgn(page.Ttl_page_db(), search_ttl_bry)) continue; // look-ahead may return other titles that don't begin with search; ignore
// if (page.Text_len() > all_pages_min) {
// rslts_2.Add(page);
// idx++;
// }
// }
// if (idx == max_results) break;
// }
// return true;
// }
// }
//}

48
400_xowa/src/gplx/xowa/addons/searchs/searchers/crts/Srch_crt_itm.java Normal file
View File

@@ -0,0 +1,48 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers.crts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.searchers.*;
import gplx.langs.regxs.*;
public class Srch_crt_itm {
public Srch_crt_itm(int idx, int tid, Srch_crt_itm[] subs, byte[] raw, Srch_crt_sql raw_data) {
this.Idx = idx; this.Tid = tid; this.Subs = subs;
this.Raw = raw;
this.Sql_data = raw_data;
}
public final int Idx; // itm index; EX: "a b" -> a:0 b:1
public final int Tid;
public final byte[] Raw;
public final Srch_crt_itm[] Subs;
public final Srch_crt_sql Sql_data;
public void Accept_visitor(Srch_crt_visitor visitor) {visitor.Visit(this);}
public static final int
Tid__word = 0 // EX: 'A'
, Tid__and = 1 // EX: 'A B'
, Tid__or = 2 // EX: 'A OR B'
, Tid__not = 3 // EX: '-A'
, Tid__word_quote = 4 // EX: '"A B"'
, Tid__invalid = 5 // EX: 'A OR'; incomplete or otherwise invalid
;
public static Srch_crt_itm[] Ary_empty = new Srch_crt_itm[0];
public static final Srch_crt_itm Invalid = new Srch_crt_itm(-1, Srch_crt_itm.Tid__invalid, Srch_crt_itm.Ary_empty, null, null);
public static Srch_crt_itm New_join(int tid, int idx, Srch_crt_itm... ary) {return new Srch_crt_itm(idx, tid, ary, null, Srch_crt_sql.New_or_null(null, Byte_ascii.Null));}
public static Srch_crt_itm New_word(byte wildcard_byte, Srch_crt_tkn tkn, int idx, byte[] src) {
int tid = tkn.Tid == Srch_crt_tkn.Tid__word_w_quote ? Srch_crt_itm.Tid__word_quote : Srch_crt_itm.Tid__word;
return new Srch_crt_itm(idx, tid, Srch_crt_itm.Ary_empty, tkn.Val, Srch_crt_sql.New_or_null(src, wildcard_byte));
}
}

44
400_xowa/src/gplx/xowa/addons/searchs/searchers/crts/Srch_crt_mgr.java Normal file
View File

@@ -0,0 +1,44 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers.crts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.searchers.*;
public class Srch_crt_mgr {
public Srch_crt_mgr(byte[] key, Srch_crt_tkn[] tkns, Srch_crt_itm root, byte words_tid, Srch_crt_itm[] words_ary, Srch_crt_itm words_nth) {
this.Key = key;
this.Tkns = tkns;
this.Root = root;
this.Words_tid = words_tid;
this.Words_ary = words_ary;
this.Words_nth = words_nth;
}
public final byte[] Key;
public final Srch_crt_tkn[] Tkns;
public final Srch_crt_itm Root;
public final byte Words_tid;
public final Srch_crt_itm[] Words_ary;
public final Srch_crt_itm Words_nth;
public int Words_nth__len() {
return Words_nth == null ? 0 : Words_nth.Raw.length;
}
public static final byte
Tid__one = 0
, Tid__ands = 1
, Tid__mixed = 2
;
public static Srch_crt_mgr Invalid = new Srch_crt_mgr(Bry_.Empty, Srch_crt_tkn.Ary_empty, Srch_crt_itm.Invalid, Tid__one, Srch_crt_itm.Ary_empty, Srch_crt_itm.Invalid);
}

146
400_xowa/src/gplx/xowa/addons/searchs/searchers/crts/Srch_crt_parser.java Normal file
View File

@@ -0,0 +1,146 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.searchs.searchers.crts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.searchs.*; import gplx.xowa.addons.searchs.searchers.*;
import gplx.xowa.addons.searchs.searchers.crts.visitors.*;
public class Srch_crt_parser {
private final Srch_crt_scanner scanner;
private final Srch_crt_visitor__words words_visitor = new Srch_crt_visitor__words();
private final Srch_crt_visitor__print print_visitor = new Srch_crt_visitor__print();
private final byte wildcard_byte;
private int uid_next;
public Srch_crt_parser(Srch_crt_scanner_syms trie_bldr) {
this.wildcard_byte = trie_bldr.Wild();
this.scanner = new Srch_crt_scanner(trie_bldr);
}
public int Next_uid() {return ++uid_next;}
public Srch_crt_mgr Parse_or_invalid(byte[] src, boolean auto_wildcard) {
this.uid_next = -1;
Srch_crt_tkn[] tkns_ary = scanner.Scan(src);
Srch_crt_parser_frame root_frame = new Srch_crt_parser_frame(this);
Parse_tkns(root_frame, tkns_ary, 0, tkns_ary.length);
Srch_crt_itm root_itm = root_frame.Produce_or_null();
if (root_itm == null) return Srch_crt_mgr.Invalid;
byte[] key = print_visitor.Print(root_itm);
words_visitor.Gather(root_itm);
return new Srch_crt_mgr(key, tkns_ary, root_itm, words_visitor.Words_tid(), words_visitor.Words_ary(), words_visitor.Words_nth());
}
private int Parse_tkns(Srch_crt_parser_frame frame, Srch_crt_tkn[] tkns_ary, int tkns_bgn, int tkns_end) {
int tkns_cur = tkns_bgn;
while (tkns_cur < tkns_end) {
Srch_crt_tkn cur_tkn = tkns_ary[tkns_cur];
int new_tkns_cur = Process_tkn(frame, tkns_ary, tkns_cur, tkns_end, cur_tkn);
if (new_tkns_cur < 0) {
tkns_cur = -new_tkns_cur;
break;
}
else
tkns_cur = new_tkns_cur;
}
return tkns_cur;
}
private int Process_tkn(Srch_crt_parser_frame frame, Srch_crt_tkn[] tkns_ary, int tkns_cur, int tkns_end, Srch_crt_tkn cur_tkn) {
byte cur_tid = cur_tkn.Tid;
switch (cur_tid) {
case Srch_crt_tkn.Tid__word:
case Srch_crt_tkn.Tid__word_w_quote:
frame.Subs__add(Srch_crt_itm.New_word(wildcard_byte, cur_tkn, frame.Next_uid(), cur_tkn.Val));
break;
case Srch_crt_tkn.Tid__and:
frame.Eval_join(Srch_crt_itm.Tid__and);
break;
case Srch_crt_tkn.Tid__or:
frame.Eval_join(Srch_crt_itm.Tid__or);
break;
case Srch_crt_tkn.Tid__paren_bgn: {
Srch_crt_parser_frame paren_frame = new Srch_crt_parser_frame(this);
int new_tkns_cur = Parse_tkns(paren_frame, tkns_ary, tkns_cur + 1, tkns_end);
Srch_crt_itm paren_itm = paren_frame.Produce_or_null();
if (paren_itm != null) {
frame.Subs__add(paren_itm);
}
return new_tkns_cur;
}
case Srch_crt_tkn.Tid__paren_end:
return -(tkns_cur + 1);
case Srch_crt_tkn.Tid__not:
frame.Notted_y_();
break;
}
return tkns_cur + 1;
}
}
class Srch_crt_parser_frame {
public Srch_crt_parser_frame(Srch_crt_parser parser) {
this.parser = parser;
}
private int join_tid = Srch_crt_tkn.Tid__null;
private boolean notted = false;
private final List_adp subs = List_adp_.new_();
private final Srch_crt_parser parser;
public int Next_uid() {return parser.Next_uid();}
public void Notted_y_() {
if (notted) // already notted; disable; EX: "--a"
notted = false;
else
notted = true;
}
public void Subs__add(Srch_crt_itm itm) {
// if notted, wrap itm in not; EX: "-a"; "-(a & b)"
if (notted) {
itm = Srch_crt_itm.New_join(Srch_crt_itm.Tid__not, this.Next_uid(), itm);
notted = false;
}
subs.Add(itm);
// auto-and behavior; EX: "a b" -> "a & b"; EX: "a (b | c)" -> "a & (b | c)"
if ( join_tid == Srch_crt_tkn.Tid__null // if currently null
&& subs.Len() > 1 // but 2 items in list
)
join_tid = Srch_crt_itm.Tid__and; // default to AND; EX: "a (b) c"
}
public void Eval_join(int tid) {
if (join_tid == Srch_crt_tkn.Tid__null) join_tid = tid;
else if (join_tid == tid) {} // tid is same; ignore; note that this handles dupes; EX: "a & & b"
else { // tid changed; EX: a & b | c
Merge_and_add();
join_tid = tid;
}
}
public Srch_crt_itm Produce_or_null() {
int subs_len = subs.Len();
switch (subs_len) {
case 0:
return null;
case 1:
join_tid = Srch_crt_tkn.Tid__null;
return (Srch_crt_itm)subs.Get_at(0);
default:
Srch_crt_itm[] subs_ary = (Srch_crt_itm[])subs.To_ary_and_clear(Srch_crt_itm.class);
Srch_crt_itm rv = Srch_crt_itm.New_join(join_tid, parser.Next_uid(), subs_ary);
join_tid = Srch_crt_tkn.Tid__null;
return rv;
}
}
private void Merge_and_add() {
int subs_len = subs.Len();
if (subs_len > 1)
subs.Add(Produce_or_null());
}
}

Some files were not shown because too many files have changed in this diff Show More