Full-text search: Automatically execute search if search_text is available; also, add limited caching

pull/620/head
gnosygnu 7 years ago
parent e486d139e0
commit 7924e26522

@ -113,6 +113,7 @@ public class Xocfg_mgr implements Gfo_invk {
public void Set_bool_app(String key, boolean val) {Set_str(Xocfg_mgr.Ctx__app, key, Yn.To_str(val));}
public void Set_float_app(String key, float val) {Set_str(Xocfg_mgr.Ctx__app, key, Float_.To_str(val));}
public void Set_str_app(String key, String val) {Set_str(Xocfg_mgr.Ctx__app, key, val);}
public void Set_bry_app(String key, byte[] val) {Set_str(Xocfg_mgr.Ctx__app, key, String_.new_u8(val));}
public void Set_date_app(String key, DateAdp val) {Set_str(Xocfg_mgr.Ctx__app, key, val.XtoUtc().XtoStr_fmt(Fmt__time));}
public void Set_int_app(String key, int val) {Set_str(Xocfg_mgr.Ctx__app, key, Int_.To_str(val));}
public void Set_bry_wiki(Xowe_wiki wiki, String key, byte[] val) {Set_str(wiki.Domain_itm().Abrv_xo_str(), key, String_.new_u8(val));}

@ -284,7 +284,9 @@ public class Xobldr__fsdb_db__create_data extends Xob_cmd__base implements Xob_c
trg_cfg_mgr.Tbl().Conn().Txn_end();
trg_cfg_mgr.Tbl().Conn().Rls_conn();
if (!trg_mnt_itm.Db_mgr().File__solo_file()) {
trg_bin_fil.Conn().Txn_end(); trg_bin_fil.Conn().Rls_conn();
if (trg_bin_fil != null) { // NOTE: trg_bin_fil is null when there are no images in the wiki; EX: bo.b; DATE:2017-03-19
trg_bin_fil.Conn().Txn_end(); trg_bin_fil.Conn().Rls_conn();
}
}
if (exec_done) {
bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make); // delete bmks for future reruns; DATE:2014-08-20

@ -16,7 +16,9 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
package gplx.xowa.addons.wikis.fulltexts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.specials.*; import gplx.xowa.htmls.bridges.*;
import gplx.xowa.addons.wikis.fulltexts.searchers.caches.*;
public class Xosearch_fulltext_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__json, Xoax_addon_itm__bldr {
public String Addon__key() {return ADDON__KEY;} private static final String ADDON__KEY = "xowa.wiki.fulltext";
public Xob_cmd[] Bldr_cmds() {
return new Xob_cmd[]
{ gplx.xowa.addons.wikis.fulltexts.indexers.bldrs.Xofulltext_indexer_cmd.Prototype
@ -34,8 +36,10 @@ public class Xosearch_fulltext_addon implements Xoax_addon_itm, Xoax_addon_itm__
, gplx.xowa.addons.wikis.fulltexts.indexers.svcs.Xofulltext_indexer_bridge.Prototype
};
}
public String Addon__key() {return ADDON__KEY;} private static final String ADDON__KEY = "xowa.wiki.fulltext";
public Xofulltext_cache_mgr Cache_mgr() {return cache_mgr;} private final Xofulltext_cache_mgr cache_mgr = new Xofulltext_cache_mgr();
public static Xosearch_fulltext_addon Get_by_app(Xoa_app app) {
return (Xosearch_fulltext_addon)app.Addon_mgr().Itms__get_or_null(ADDON__KEY);
}
public static Io_url Get_index_dir(Xow_wiki wiki) {return Get_index_dir(wiki.Fsys_mgr().Root_dir());}
public static Io_url Get_index_dir(Io_url wiki_dir) {

@ -14,52 +14,64 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.fulltexts.searchers.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*;
import gplx.core.primitives.*;
public class Xofulltext_cache_mgr {
private final Ordered_hash qry_hash = Ordered_hash_.New();
private final Hash_adp_bry id_hash = Hash_adp_bry.cs();
public int Next_qry_id() {return next_qry_id++;} private int next_qry_id;
public void Clear() {
qry_hash.Clear();
}
public void Add(int query_id, byte[] query) {
Xofulltext_cache_qry qry = (Xofulltext_cache_qry)qry_hash.Get_by(query_id);
public int Ids__get_or_neg1(byte[] qry_key) {
Int_obj_val id = (Int_obj_val)id_hash.Get_by(qry_key);
return id == null ? -1 : id.Val();
}
public int Ids__next() {return next_qry_id++;}
public void Add(int id, byte[] key) {
id_hash.Add(key, new Int_obj_val(id));
Xofulltext_cache_qry qry = (Xofulltext_cache_qry)qry_hash.Get_by(id);
if (qry == null) {
qry = new Xofulltext_cache_qry(query_id, query);
qry_hash.Add(query_id, qry);
qry = new Xofulltext_cache_qry(id, key);
qry_hash.Add(id, qry);
}
}
public void Add(int query_id, byte[] wiki_bry, int page_id, int line_seq, byte[] line_html) {
public void Add_page(int query_id, byte[] wiki_bry, int page_id, byte[] page_ttl) {
// get qry
Xofulltext_cache_qry qry = (Xofulltext_cache_qry)qry_hash.Get_by(query_id);
if (qry == null) {
throw Err_.new_wo_type("query not found; query_id=~{0}", query_id);
}
// get wiki
Xofulltext_cache_wiki wiki = (Xofulltext_cache_wiki)qry.Wikis().Get_by(wiki_bry);
if (wiki == null) {
wiki = new Xofulltext_cache_wiki(wiki_bry);
qry.Wikis().Add(wiki_bry, wiki);
}
Xofulltext_cache_qry qry = (Xofulltext_cache_qry)qry_hash.Get_by_or_fail(query_id);
// get page
Xofulltext_cache_page page = (Xofulltext_cache_page)wiki.Pages().Get_by(page_id);
Xofulltext_cache_page page = (Xofulltext_cache_page)qry.Pages().Get_by(page_id);
if (page == null) {
page = new Xofulltext_cache_page(page_id, wiki.Pages().Count());
wiki.Pages().Add(page_id, page);
page = new Xofulltext_cache_page(page_id, qry.Pages().Count(), page_ttl);
qry.Pages().Add(page_id, page);
}
}
public void Add_line(int query_id, byte[] wiki_bry, int page_id, int line_seq, byte[] line_html) {
// get qry
Xofulltext_cache_qry qry = (Xofulltext_cache_qry)qry_hash.Get_by_or_fail(query_id);
Xofulltext_cache_page page = (Xofulltext_cache_page)qry.Pages().Get_by_or_fail(page_id);
// add line
Xofulltext_cache_line line = new Xofulltext_cache_line(line_seq, line_html);
page.Lines().Add(line);
}
public Object Get_pages_rng(int qry_id, int page_seq_bgn, int page_seq_end) {
return null;
public Xofulltext_cache_page[] Get_pages_rng(int qry_id, int page_seq_bgn, int page_seq_end) {
Xofulltext_cache_qry qry = (Xofulltext_cache_qry)qry_hash.Get_by(qry_id);
if (qry == null) return null;
List_adp list = List_adp_.New();
int len = qry.Pages().Len();
for (int i = page_seq_bgn; i < page_seq_end; i++) {
if (i >= len) break;
Xofulltext_cache_page page = (Xofulltext_cache_page)qry.Pages().Get_at(i);
list.Add(page);
}
return (Xofulltext_cache_page[])list.To_ary_and_clear(Xofulltext_cache_page.class);
}
public Xofulltext_cache_line[] Get_lines_rest(int qry_id, byte[] wiki_bry, int page_id) {
// get page
Xofulltext_cache_qry qry = (Xofulltext_cache_qry)qry_hash.Get_by(qry_id);
Xofulltext_cache_wiki wiki = (Xofulltext_cache_wiki)qry.Wikis().Get_by(wiki_bry);
Xofulltext_cache_page page = (Xofulltext_cache_page)wiki.Pages().Get_by(page_id);
Xofulltext_cache_page page = (Xofulltext_cache_page)qry.Pages().Get_by(page_id);
// loop lines from 1 to n; note "1" b/c results will always show at least 1st line
List_adp list = List_adp_.New();

@ -15,11 +15,13 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.fulltexts.searchers.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*;
public class Xofulltext_cache_page {
public Xofulltext_cache_page(int page_id, int page_seq) {
public Xofulltext_cache_page(int page_id, int page_seq, byte[] page_ttl) {
this.page_id = page_id;
this.page_seq = page_seq;
this.page_ttl = page_ttl;
}
public int Page_id() {return page_id;} private final int page_id;
public int Page_seq() {return page_seq;} private final int page_seq;
public byte[] Page_ttl() {return page_ttl;} private final byte[] page_ttl;
public List_adp Lines() {return lines;} private final List_adp lines = List_adp_.New();
}

@ -15,11 +15,11 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.fulltexts.searchers.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*;
public class Xofulltext_cache_qry {
public Xofulltext_cache_qry(int id, byte[] qry) {
public Xofulltext_cache_qry(int id, byte[] text) {
this.id = id;
this.qry = qry;
this.text = text;
}
public int Id() {return id;} private final int id;
public byte[] Qry() {return qry;} private final byte[] qry;
public Hash_adp_bry Wikis() {return wikis;} private final Hash_adp_bry wikis = Hash_adp_bry.cs();
public int Id() {return id;} private final int id;
public byte[] Text() {return text;} private final byte[] text;
public Ordered_hash Pages() {return pages;} private final Ordered_hash pages = Ordered_hash_.New();
}

@ -0,0 +1,71 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.fulltexts.searchers.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*;
import gplx.langs.jsons.*;
import gplx.xowa.addons.wikis.fulltexts.searchers.caches.*;
public class Xofulltext_args_qry {
public int qry_id;
public String page_guid;
public byte[] search_text;
public Xofulltext_cache_mgr cache_mgr;
public Xofulltext_args_wiki[] wikis_ary;
public boolean case_match;
public boolean auto_wildcard_bgn;
public boolean auto_wildcard_end;
public boolean expand_matches_section;
public boolean show_all_matches;
public byte[] Qry_key(byte[] wiki, byte[] ns_ids) {
return Bry_.Add_w_dlm(Byte_ascii.Nl, wiki, ns_ids, search_text); // EX: "en.wikipedia.org\n0|4\nearth"
}
public static Xofulltext_args_qry New_by_json(Json_nde args) {
Xofulltext_args_qry rv = new Xofulltext_args_qry();
rv.search_text = args.Get_as_bry("search");
// create wikis
byte[] wikis_bry = args.Get_as_bry("qarg_wikis");
byte[][] wikis_ary = Bry_split_.Split(wikis_bry, Byte_ascii.Pipe, true);
int wikis_len = wikis_ary.length;
Xofulltext_args_wiki[] wiki_args = new Xofulltext_args_wiki[wikis_len];
rv.wikis_ary = wiki_args;
for (int i = 0; i < wikis_len; i++) {
wiki_args[i] = new Xofulltext_args_wiki(wikis_ary[i]);
}
Set_prop(wiki_args, wikis_len, args, "ns_ids");
Set_prop(wiki_args, wikis_len, args, "offsets");
Set_prop(wiki_args, wikis_len, args, "limits");
rv.page_guid = args.Get_as_str("page_guid");
rv.case_match = args.Get_as_bool_or("case_match", false);
rv.auto_wildcard_bgn = args.Get_as_bool_or("auto_wildcard_bgn", false);
rv.auto_wildcard_end = args.Get_as_bool_or("auto_wildcard_end", false);
rv.expand_matches_section = args.Get_as_bool_or("expand_matches_section", false);
rv.show_all_matches = args.Get_as_bool_or("show_all_matches", false);
return rv;
}
private static void Set_prop(Xofulltext_args_wiki[] wikis, int wikis_len, Json_nde args, String key) {
// set ns_ids
byte[] json_val = args.Get_as_bry("qarg_" + key);
byte[][] ary = Bry_split_.Split(json_val, Byte_ascii.Pipe, true);
int ary_len = ary.length;
for (int i = 0; i < wikis_len; i++) {
byte[] val = i < ary_len ? ary[i] : ary[0];
wikis[i].Init_by_json(key, val);
}
}
}

@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.fulltexts.searchers.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*;
public class Xofulltext_args_wiki {
public byte[] wiki;
public byte[] ns_ids;
public int offset;
public int limit;
public Xofulltext_args_wiki(byte[] wiki) {
this.wiki = wiki;
}
public void Init_by_json(String key, byte[] val) {
if (String_.Eq(key, "ns_ids")) this.ns_ids = val;
else if (String_.Eq(key, "offsets")) this.offset = Bry_.To_int(val);
else if (String_.Eq(key, "limits")) this.limit = Bry_.To_int(val);
}
}

@ -16,5 +16,5 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
package gplx.xowa.addons.wikis.fulltexts.searchers.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*;
import gplx.xowa.addons.wikis.fulltexts.searchers.mgrs.uis.*;
public interface Xofulltext_searcher {
void Search(Xofulltext_searcher_ui ui, Xow_wiki wiki, Xofulltext_searcher_args args);
void Search(Xofulltext_searcher_ui ui, Xow_wiki wiki, Xofulltext_args_qry qry_args, Xofulltext_args_wiki wiki_args);
}

@ -1,44 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.fulltexts.searchers.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*;
import gplx.langs.jsons.*;
public class Xofulltext_searcher_args {
public boolean case_match;
public boolean auto_wildcard_bgn;
public boolean auto_wildcard_end;
public boolean expand_matches_section;
public boolean show_all_matches;
public int max_pages_per_wiki;
public byte[] wikis;
public byte[] query;
public String namespaces;
public int query_id;
public String page_guid;
public static Xofulltext_searcher_args New_by_json(Json_nde args) {
Xofulltext_searcher_args rv = new Xofulltext_searcher_args();
rv.case_match = args.Get_as_bool_or("case_match", false);
rv.auto_wildcard_bgn = args.Get_as_bool_or("auto_wildcard_bgn", false);
rv.auto_wildcard_end = args.Get_as_bool_or("auto_wildcard_end", false);
rv.expand_matches_section = args.Get_as_bool_or("expand_matches_section", false);
rv.show_all_matches = args.Get_as_bool_or("show_all_matches", false);
rv.max_pages_per_wiki = args.Get_as_int_or("max_pages_per_wiki", 25);
rv.wikis = args.Get_as_bry("wikis");
rv.query = args.Get_as_bry("query");
rv.namespaces = args.Get_as_str("namespaces");
rv.page_guid = args.Get_as_str("page_guid");
return rv;
}
}

@ -23,13 +23,13 @@ public class Xofulltext_searcher__brute implements Xofulltext_searcher {
private final Xofulltext_finder_mgr finder = new Xofulltext_finder_mgr();
private final Xofulltext_finder_cbk__eval cbk_eval = new Xofulltext_finder_cbk__eval();
private final Xofulltext_finder_cbk__highlight cbk_highlight = new Xofulltext_finder_cbk__highlight();
public void Search(Xofulltext_searcher_ui ui, Xow_wiki wiki, Xofulltext_searcher_args args) {
public void Search(Xofulltext_searcher_ui ui, Xow_wiki wiki, Xofulltext_args_qry args, Xofulltext_args_wiki wiki_args) {
// get pages from db
Db_conn page_conn = wiki.Data__core_mgr().Tbl__page().Conn();
Db_rdr page_rdr = page_conn.Stmt_sql("SELECT * FROM page WHERE page_namespace IN (0) ORDER BY page_score DESC").Exec_select__rls_auto();
// init finder
finder.Init(args.query, args.case_match, args.auto_wildcard_bgn, args.auto_wildcard_end, Byte_ascii.Star, Byte_ascii.Dash);
finder.Init(args.search_text, args.case_match, args.auto_wildcard_bgn, args.auto_wildcard_end, Byte_ascii.Star, Byte_ascii.Dash);
// loop
byte[] wiki_domain = wiki.Domain_bry();
@ -58,13 +58,13 @@ public class Xofulltext_searcher__brute implements Xofulltext_searcher {
ui.Send_wiki_update(wiki_domain, found, searched);
// do highlight
if (found <= args.max_pages_per_wiki) {
cbk_highlight.Init(ui, args.query_id, wiki, page_id, ttl.Full_db(), args.show_all_matches);
if (found <= wiki_args.limit) {
cbk_highlight.Init(ui, args.qry_id, wiki, page_id, ttl.Full_db(), args.show_all_matches);
ui.Send_page_add(new Xofulltext_searcher_page
( args.query_id
, String_.new_u8(wiki_domain)
( args.qry_id
, wiki_domain
, page_id
, String_.new_u8(ttl.Full_db())
, ttl.Full_db()
, args.expand_matches_section
));
finder.Match(text_mcase, 0, text_mcase.length, cbk_highlight);

@ -24,14 +24,14 @@ import gplx.xowa.addons.wikis.fulltexts.core.*;
class Xofulltext_highlighter_mgr implements Gfo_invk {
private final Xofulltext_searcher_ui ui;
private final Xow_wiki wiki;
private final Xofulltext_searcher_args searcher_args;
private final Xofulltext_args_qry searcher_args;
private final Gflucene_analyzer_data analyzer_data;
private final Gflucene_searcher_qry searcher_data;
private final Gflucene_highlighter_mgr highlighter_mgr = new Gflucene_highlighter_mgr();
private final Xoh_page hpg = new Xoh_page();
private final Ordered_hash list;
private final Xofulltext_extractor extractor = new Xofulltext_extractor();
public Xofulltext_highlighter_mgr(Xofulltext_searcher_ui ui, Xow_wiki wiki, Xofulltext_searcher_args searcher_args, Gflucene_analyzer_data analyzer_data, Gflucene_searcher_qry searcher_data, Ordered_hash list) {
public Xofulltext_highlighter_mgr(Xofulltext_searcher_ui ui, Xow_wiki wiki, Xofulltext_args_qry searcher_args, Gflucene_analyzer_data analyzer_data, Gflucene_searcher_qry searcher_data, Ordered_hash list) {
this.ui = ui;
this.wiki = wiki;
this.searcher_args = searcher_args;
@ -73,7 +73,7 @@ class Xofulltext_highlighter_mgr implements Gfo_invk {
int page_id = item.page_id;
Gflucene_highlighter_item[] lines = highlighter_mgr.Exec(searcher_data, item);
for (Gflucene_highlighter_item line : lines) {
ui.Send_line_add(searcher_args.show_all_matches, searcher_args.query_id, wiki.Domain_bry(), page_id, line.num, Bry_.new_u8(line.text));
ui.Send_line_add(searcher_args.show_all_matches, searcher_args.qry_id, wiki.Domain_bry(), page_id, line.num, Bry_.new_u8(line.text));
}
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {

@ -23,7 +23,7 @@ import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.addons.wikis.fulltexts.searchers.mgrs.uis.*;
public class Xofulltext_searcher__lucene implements Xofulltext_searcher {
private final Gflucene_searcher_mgr searcher = new Gflucene_searcher_mgr();
public void Search(Xofulltext_searcher_ui ui, Xow_wiki wiki, Xofulltext_searcher_args args) {
public void Search(Xofulltext_searcher_ui ui, Xow_wiki wiki, Xofulltext_args_qry args, Xofulltext_args_wiki wiki_args) {
// create list
Ordered_hash list = Ordered_hash_.New();
@ -34,7 +34,7 @@ public class Xofulltext_searcher__lucene implements Xofulltext_searcher {
, Xosearch_fulltext_addon.Get_index_dir(wiki).Xto_api()));
// exec search
Gflucene_searcher_qry searcher_data = new Gflucene_searcher_qry(String_.new_u8(args.query), args.max_pages_per_wiki);
Gflucene_searcher_qry searcher_data = new Gflucene_searcher_qry(String_.new_u8(args.search_text), wiki_args.limit);
searcher.Exec(list, searcher_data);
// term
@ -61,7 +61,7 @@ public class Xofulltext_searcher__lucene implements Xofulltext_searcher {
doc_data.page_full_db = page_ttl.Full_db();
// call page doc_data
Xofulltext_searcher_page page = new Xofulltext_searcher_page(args.query_id, wiki.Domain_str(), doc_data.page_id, String_.new_u8(doc_data.page_full_db), args.expand_matches_section);
Xofulltext_searcher_page page = new Xofulltext_searcher_page(args.qry_id, wiki.Domain_bry(), doc_data.page_id, doc_data.page_full_db, args.expand_matches_section);
ui.Send_page_add(page);
}
ui.Send_wiki_update(wiki.Domain_bry(), len + List_adp_.Base1, -1);

@ -15,16 +15,16 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.fulltexts.searchers.mgrs.uis; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*; import gplx.xowa.addons.wikis.fulltexts.searchers.mgrs.*;
public class Xofulltext_searcher_page {
public Xofulltext_searcher_page(int query_id, String wiki_domain, int page_id, String page_title, boolean expand_matches_section) {
public Xofulltext_searcher_page(int query_id, byte[] wiki_domain, int page_id, byte[] page_ttl, boolean expand_matches_section) {
this.query_id = query_id;
this.wiki_domain = wiki_domain;
this.page_id = page_id;
this.page_title = page_title;
this.page_ttl = page_ttl;
this.expand_matches_section = expand_matches_section;
}
public int Query_id() {return query_id;} private final int query_id;
public String Wiki_domain() {return wiki_domain;} private final String wiki_domain;
public byte[] Wiki_domain() {return wiki_domain;} private final byte[] wiki_domain;
public int Page_id() {return page_id;} private final int page_id;
public String Page_title() {return page_title;} private final String page_title;
public byte[] Page_ttl() {return page_ttl;} private final byte[] page_ttl;
public boolean Expand_matches_section() {return expand_matches_section;} private final boolean expand_matches_section;
}

@ -25,9 +25,9 @@ public class Xofulltext_searcher_ui {
this.cbk_mgr = cbk_mgr;
this.cbk_trg = cbk_trg;
}
public void Send_wiki_add(byte[] wiki_domain) {
public void Send_wiki_add(byte[] wiki) {
cbk_mgr.Send_json(cbk_trg, "xo.fulltext_searcher.results__wiki__add__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_bry("wiki", wiki_domain)
.Add_bry("wiki", wiki)
);
}
public void Send_wiki_update(byte[] wiki, int found, int searched) {
@ -40,14 +40,15 @@ public class Xofulltext_searcher_ui {
public void Send_page_add(Xofulltext_searcher_page page) {
cbk_mgr.Send_json(cbk_trg, "xo.fulltext_searcher.results__page__add__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_int("query_id", page.Query_id())
.Add_str("wiki", page.Wiki_domain())
.Add_bry("wiki", page.Wiki_domain())
.Add_int("page_id", page.Page_id())
.Add_str("page_ttl", page.Page_title())
.Add_bry("page_ttl", page.Page_ttl())
.Add_bool("expand_matches_section", page.Expand_matches_section())
);
cache_mgr.Add_page(page.Query_id(), page.Wiki_domain(), page.Page_id(), page.Page_ttl());
}
public void Send_line_add(boolean show_all_matches, int qry_id, byte[] wiki_domain, int page_id, int line_sort_order, byte[] line_html) {
cache_mgr.Add(qry_id, wiki_domain, page_id, line_sort_order, line_html);
cache_mgr.Add_line(qry_id, wiki_domain, page_id, line_sort_order, line_html);
line_sort_order += List_adp_.Base1; // NOTE: increment after cache_mgr
if (line_sort_order == 1 || show_all_matches) {

@ -1,64 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.fulltexts.searchers.specials; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*;
import gplx.langs.mustaches.*;
public class Xofulltext_searcher_doc implements Mustache_doc_itm {
private final byte[] query;
private final boolean case_match, auto_wildcard_bgn, auto_wildcard_end, expand_matches_section, show_all_matches;
private final int max_pages_per_wiki;
private final String wikis, namespaces;
public Xofulltext_searcher_doc
( byte[] query, boolean case_match, boolean auto_wildcard_bgn, boolean auto_wildcard_end
, boolean expand_matches_section, boolean show_all_matches
, int max_pages_per_wiki
, String wikis, String namespaces) {
this.query = query;
this.case_match = case_match;
this.auto_wildcard_bgn = auto_wildcard_bgn;
this.auto_wildcard_end = auto_wildcard_end;
this.expand_matches_section = expand_matches_section;
this.show_all_matches = show_all_matches;
this.max_pages_per_wiki = max_pages_per_wiki;
this.wikis = wikis;
this.namespaces = namespaces;
}
public boolean Mustache__write(String key, Mustache_bfr bfr) {
if (String_.Eq(key, "wikis"))
bfr.Add_str_u8(wikis);
else if (String_.Eq(key, "namespaces"))
bfr.Add_str_u8(namespaces);
else if (String_.Eq(key, "max_pages_per_wiki"))
bfr.Add_int(max_pages_per_wiki);
else if (String_.Eq(key, "query"))
bfr.Add_bry(query);
else
return false;
return true;
}
public Mustache_doc_itm[] Mustache__subs(String key) {
if (String_.Eq(key, "case_match"))
return Mustache_doc_itm_.Ary__bool(case_match);
else if (String_.Eq(key, "auto_wildcard_bgn"))
return Mustache_doc_itm_.Ary__bool(auto_wildcard_bgn);
else if (String_.Eq(key, "auto_wildcard_end"))
return Mustache_doc_itm_.Ary__bool(auto_wildcard_end);
else if (String_.Eq(key, "expand_matches_section"))
return Mustache_doc_itm_.Ary__bool(expand_matches_section);
else if (String_.Eq(key, "show_all_matches"))
return Mustache_doc_itm_.Ary__bool(show_all_matches);
return Mustache_doc_itm_.Ary__empty;
}
}

@ -14,36 +14,36 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.fulltexts.searchers.specials; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.fulltexts.*; import gplx.xowa.addons.wikis.fulltexts.searchers.*;
import gplx.xowa.specials.*; import gplx.langs.mustaches.*; import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.pages.tags.*;
import gplx.core.net.qargs.*;
import gplx.dbs.*;
class Xofulltext_searcher_html extends Xow_special_wtr__base {
private final byte[] query;
import gplx.xowa.specials.*; import gplx.langs.mustaches.*; import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.pages.tags.*;
import gplx.xowa.addons.apps.cfgs.*;
class Xofulltext_searcher_html extends Xow_special_wtr__base implements Mustache_doc_itm {
private final boolean case_match, auto_wildcard_bgn, auto_wildcard_end, expand_matches_section, show_all_matches;
private final int max_pages_per_wiki;
private final String wikis, namespaces;
public Xofulltext_searcher_html
( byte[] query, boolean case_match, boolean auto_wildcard_bgn, boolean auto_wildcard_end
, boolean expand_matches_section, boolean show_all_matches
, int max_pages_per_wiki
, String wikis, String namespaces) {
this.query = query;
this.case_match = case_match;
this.auto_wildcard_bgn = auto_wildcard_bgn;
this.auto_wildcard_end = auto_wildcard_end;
this.expand_matches_section = expand_matches_section;
this.show_all_matches = show_all_matches;
this.max_pages_per_wiki = max_pages_per_wiki;
this.wikis = wikis;
this.namespaces = namespaces;
private final Hash_adp props = Hash_adp_.New();
public Xofulltext_searcher_html(Xocfg_mgr cfg_mgr, Gfo_qarg_mgr url_args, Xow_wiki wiki) {
props.Add("cur_wiki", wiki.Domain_str());
props.Add("search", url_args.Read_str_or("search", ""));
props_Add(cfg_mgr, url_args, "wikis" , wiki.Domain_str());
props_Add(cfg_mgr, url_args, "ns_ids", "0");
props_Add(cfg_mgr, url_args, "limits", "100");
props_Add(cfg_mgr, url_args, "offsets", "0");
this.case_match = cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.case_match", false);
this.auto_wildcard_bgn = cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.auto_wildcard_bgn", false);
this.auto_wildcard_end = cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.auto_wildcard_end", false);
this.expand_matches_section = cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.expand_matches_section", false);
this.show_all_matches = cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.show_all_matches", false);
}
private void props_Add(Xocfg_mgr cfg_mgr, Gfo_qarg_mgr url_args, String key, String dflt_val) {
String cfg_key = "xowa.addon.search.fulltext.special.dflt_" + key;
String cfg_val = cfg_mgr.Get_str_app_or(cfg_key, dflt_val);
props.Add("dflt_" + key, cfg_val);
props.Add("qarg_" + key, url_args.Read_str_or(key, cfg_val));
}
@Override protected Io_url Get_addon_dir(Xoa_app app) {return Addon_dir(app);}
@Override protected Io_url Get_mustache_fil(Io_url addon_dir) {return addon_dir.GenSubFil_nest("bin", "xofulltext_searcher.main.template.html");}
@Override protected Mustache_doc_itm Bld_mustache_root(Xoa_app app) {
return new Xofulltext_searcher_doc
( query, case_match, auto_wildcard_bgn, auto_wildcard_end
, expand_matches_section, show_all_matches
, max_pages_per_wiki, wikis, namespaces);
}
@Override protected Mustache_doc_itm Bld_mustache_root(Xoa_app app) {return this;}
@Override protected void Bld_tags(Xoa_app app, Io_url addon_dir, Xopage_html_data page_data) {
Xopg_tag_mgr head_tags = page_data.Head_tags();
Xopg_tag_wtr_.Add__xoelem (head_tags, app.Fsys_mgr().Http_root());
@ -65,6 +65,24 @@ class Xofulltext_searcher_html extends Xow_special_wtr__base {
page_data.Js_enabled_y_();
}
public boolean Mustache__write(String key, Mustache_bfr bfr) {
String val = (String)props.Get_by(key);
if (val == null)
return false;
else {
bfr.Add_str_u8(val);
return true;
}
}
public Mustache_doc_itm[] Mustache__subs(String key) {
if (String_.Eq(key, "case_match")) return Mustache_doc_itm_.Ary__bool(case_match);
else if (String_.Eq(key, "auto_wildcard_bgn")) return Mustache_doc_itm_.Ary__bool(auto_wildcard_bgn);
else if (String_.Eq(key, "auto_wildcard_end")) return Mustache_doc_itm_.Ary__bool(auto_wildcard_end);
else if (String_.Eq(key, "expand_matches_section")) return Mustache_doc_itm_.Ary__bool(expand_matches_section);
else if (String_.Eq(key, "show_all_matches")) return Mustache_doc_itm_.Ary__bool(show_all_matches);
return Mustache_doc_itm_.Ary__empty;
}
public static Io_url Addon_dir(Xoa_app app) {
return app.Fsys_mgr().Http_root().GenSubDir_nest("bin", "any", "xowa", "addon", "wiki", "fulltext", "searcher");
}

@ -18,23 +18,13 @@ import gplx.xowa.specials.*; import gplx.core.net.qargs.*;
import gplx.xowa.addons.apps.cfgs.*;
public class Xofulltext_searcher_special implements Xow_special_page {
public void Special__gen(Xow_wiki wiki, Xoa_page page, Xoa_url url, Xoa_ttl ttl) {
// get qry if any
// get url_args, cfg_mgr
Gfo_qarg_mgr url_args = new Gfo_qarg_mgr().Init(url.Qargs_ary());
byte[] query = url_args.Read_bry_or("query", Bry_.Empty);
// get options and create page
Xocfg_mgr cfg_mgr = wiki.App().Cfg();
new Xofulltext_searcher_html
( query
, cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.case_match", false)
, cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.auto_wildcard_bgn", false)
, cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.auto_wildcard_end", false)
, cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.expand_matches_section", false)
, cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.special.show_all_matches", false)
, cfg_mgr.Get_int_app_or ("xowa.addon.search.fulltext.special.max_pages_per_wiki", 100)
, wiki.Domain_str()
, cfg_mgr.Get_str_app_or ("xowa.addon.search.fulltext.special.namespaces", "0|4")
).Bld_page_by_mustache(wiki.App(), page, this);
// create page
Xofulltext_searcher_html html = new Xofulltext_searcher_html(cfg_mgr, url_args, wiki);
html.Bld_page_by_mustache(wiki.App(), page, this);
}
Xofulltext_searcher_special(Xow_special_meta special__meta) {this.special__meta = special__meta;}
public Xow_special_meta Special__meta() {return special__meta;} private final Xow_special_meta special__meta;

@ -29,18 +29,13 @@ import gplx.xowa.addons.wikis.fulltexts.searchers.mgrs.gflucenes.*;
import gplx.xowa.addons.wikis.fulltexts.searchers.mgrs.brutes.*;
class Xofulltext_searcher_svc implements Gfo_invk {
private final Xoa_app app;
private final Xofulltext_cache_mgr cache_mgr = new Xofulltext_cache_mgr();
public Xofulltext_searcher_svc(Xoa_app app) {
this.app = app;
}
public void Search(Json_nde args) {
// for now, always clear cache; "get_lines_rest" will only work for latest search
cache_mgr.Clear();
// get search_args
Xofulltext_searcher_args search_args = Xofulltext_searcher_args.New_by_json(args);
search_args.query_id = cache_mgr.Next_qry_id();
cache_mgr.Add(search_args.query_id, search_args.query);
Xofulltext_args_qry search_args = Xofulltext_args_qry.New_by_json(args);
search_args.cache_mgr = this.Cache_mgr();
// autosave any changes if enabled
Xocfg_mgr cfg_mgr = app.Cfg();
@ -50,37 +45,72 @@ class Xofulltext_searcher_svc implements Gfo_invk {
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.auto_wildcard_end", search_args.auto_wildcard_end);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.expand_matches_section", search_args.expand_matches_section);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.show_all_matches", search_args.show_all_matches);
cfg_mgr.Set_int_app ("xowa.addon.search.fulltext.special.max_pages_per_wiki", search_args.max_pages_per_wiki);
cfg_mgr.Set_str_app ("xowa.addon.search.fulltext.special.namespaces", search_args.namespaces);
// cfg_mgr.Set_int_app ("xowa.addon.search.fulltext.special.max_pages_per_wiki", search_args.max_pages_per_wiki);
// cfg_mgr.Set_bry_app ("xowa.addon.search.fulltext.special.namespaces", search_args.namespaces);
}
// launch thread
gplx.core.threads.Thread_adp_.Start_by_val("search", Cancelable_.Never, this, Invk__search, search_args);
}
private void Search(Xofulltext_searcher_args args) {
Xofulltext_searcher_ui searcher_ui = new Xofulltext_searcher_ui(cache_mgr, app.Gui__cbk_mgr(), new Xog_cbk_trg(args.page_guid));
private void Search(Xofulltext_args_qry args) {
// create ui
Xofulltext_cache_mgr cache_mgr = args.cache_mgr;
Xofulltext_searcher_ui ui = new Xofulltext_searcher_ui(cache_mgr, app.Gui__cbk_mgr(), new Xog_cbk_trg(args.page_guid));
try {
// loop wikis
byte[][] wiki_domains = Bry_split_.Split(args.wikis, Byte_ascii.Pipe_bry);
for (byte[] wiki_domain : wiki_domains) {
// get wiki and notify
Xow_wiki wiki = app.Wiki_mgri().Get_by_or_make_init_y(wiki_domain);
searcher_ui.Send_wiki_add(wiki_domain);
// get searcher and search
Xofulltext_searcher searcher = Get_searcher(wiki);
searcher.Search(searcher_ui, wiki, args);
for (Xofulltext_args_wiki wiki_args : args.wikis_ary) {
Search_wiki(args, cache_mgr, ui, wiki_args);
}
} catch (Exception exc) {
if (app.Tid_is_edit())
((Xoae_app)app).Gui_mgr().Kit().Ask_ok("", "", Err_.Message_gplx_full(exc));
}
}
}
private void Search_wiki(Xofulltext_args_qry args, Xofulltext_cache_mgr cache_mgr, Xofulltext_searcher_ui ui, Xofulltext_args_wiki wiki_args) {
try {
// get wiki and notify
byte[] wiki_domain = wiki_args.wiki;
Xow_wiki wiki = app.Wiki_mgri().Get_by_or_make_init_y(wiki_domain);
ui.Send_wiki_add(wiki_domain);
// try to get from cache
byte[] qry_key = args.Qry_key(wiki_domain, wiki_args.ns_ids);
int qry_id = cache_mgr.Ids__get_or_neg1(qry_key);
if (qry_id == -1) {
qry_id = cache_mgr.Ids__next();
cache_mgr.Add(qry_id, qry_key);
}
else {
Xofulltext_cache_page[] cached_pages = cache_mgr.Get_pages_rng(qry_id, wiki_args.offset, wiki_args.limit);
if (cached_pages != null) {
for (Xofulltext_cache_page page : cached_pages) {
ui.Send_page_add(new Xofulltext_searcher_page(qry_id, wiki.Domain_bry(), page.Page_id(), page.Page_ttl(), args.expand_matches_section));
int len = page.Lines().Len();
for (int i = 0; i < len; i++) {
Xofulltext_cache_line line = (Xofulltext_cache_line)page.Lines().Get_at(i);
ui.Send_line_add(args.show_all_matches, qry_id, wiki.Domain_bry(), page.Page_id(), line.Line_seq(), line.Line_html());
}
}
return;
}
}
args.qry_id = qry_id;
// do search
Xofulltext_searcher searcher = Get_searcher(wiki);
searcher.Search(ui, wiki, args, wiki_args);
}
catch (Exception exc) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "failed to search_wiki; err=~{0}", Err_.Message_gplx_log(exc));
}
}
public void Get_lines_rest(Json_nde args) {
Get_lines_rest(args.Get_as_int("qry_id"), args.Get_as_bry("wiki"), args.Get_as_int("page_id"), args.Get_as_str("page_guid"));
}
private void Get_lines_rest(int qry_id, byte[] wiki_bry, int page_id, String page_guid) {
Xofulltext_cache_mgr cache_mgr = this.Cache_mgr();
Xofulltext_searcher_ui searcher_ui = new Xofulltext_searcher_ui(cache_mgr, app.Gui__cbk_mgr(), new Xog_cbk_trg(page_guid));
Xofulltext_cache_line[] lines = cache_mgr.Get_lines_rest(qry_id, wiki_bry, page_id);
@ -96,9 +126,12 @@ class Xofulltext_searcher_svc implements Gfo_invk {
return new Xofulltext_searcher__brute();
}
}
private Xofulltext_cache_mgr Cache_mgr() {
return Xosearch_fulltext_addon.Get_by_app(app).Cache_mgr();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__search)) this.Search((Xofulltext_searcher_args)m.ReadObj("v"));
if (ctx.Match(k, Invk__search)) this.Search((Xofulltext_args_qry)m.ReadObj("v"));
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk__search = "search";

@ -1 +1 @@
add sponsors
add hindi and tibetan wikis

Loading…
Cancel
Save