Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2024-10-27 20:34:16 +00:00
Code Issues Projects Releases Wiki Activity

Full-text search: Add lucene searcher

Browse Source
This commit is contained in:
gnosygnu 2017-03-12 14:45:54 -04:00
parent c91416801b
commit ae9d0fccd3
26 changed files with 552 additions and 196 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/Xosearch_fulltext_addon.java
View File

@ -14,8 +14,14 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; package gplx.xowa.addons.wikis.searchs.fulltexts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.specials.*; import gplx.xowa.htmls.bridges.*; import gplx.xowa.specials.*; import gplx.xowa.htmls.bridges.*;
public class Xosearch_fulltext_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__json { public class Xosearch_fulltext_addon implements Xoax_addon_itm, Xoax_addon_itm__special, Xoax_addon_itm__json, Xoax_addon_itm__bldr {
public Xob_cmd[] Bldr_cmds() {
return new Xob_cmd[]
{ gplx.xowa.addons.wikis.searchs.fulltexts.indexers.Xosearch_indexer_cmd.Prototype
};
}
public Xow_special_page[] Special_pages() { public Xow_special_page[] Special_pages() {
return new Xow_special_page[] return new Xow_special_page[]
{ gplx.xowa.addons.wikis.searchs.fulltexts.specials.Xosearch_fulltext_special.Prototype { gplx.xowa.addons.wikis.searchs.fulltexts.specials.Xosearch_fulltext_special.Prototype
@ -23,7 +29,7 @@ public class Xosearch_fulltext_addon implements Xoax_addon_itm, Xoax_addon_itm__
} }
public Bridge_cmd_itm[] Json_cmds() { public Bridge_cmd_itm[] Json_cmds() {
return new Bridge_cmd_itm[] return new Bridge_cmd_itm[]
{ gplx.xowa.addons.wikis.searchs.fulltexts.cbks.Xosearch_fulltext_bridge.Prototype { gplx.xowa.addons.wikis.searchs.fulltexts.svcs.Xosearch_fulltext_bridge.Prototype
}; };
} }

8
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/caches/Xosearch_cache_mgr.java
View File

@ -47,12 +47,6 @@ public class Xosearch_cache_mgr {
page.Lines().Add(line); page.Lines().Add(line);
} }
public Object Get_pages_rng(int qry_id, byte[] wiki, int page_seq_bgn, int page_seq_end) { public Object Get_pages_rng(int qry_id, byte[] wiki, int page_seq_bgn, int page_seq_end) {
// List_adp list = List_adp_.New();
// for (int i = page_seq_bgn; i < page_seq_end; i++) {
// Xosearch_cache_qry page = (Xosearch_cache_qry)qry_hash.Get_at(i);
// list.Add(page);
// }
// return list.To_ary_and_clear(typeof(Xosearch_cache_itm));
return null; return null;
} }
public Xosearch_cache_line[] Get_lines_rest(int qry_id, byte[] wiki_bry, int page_id) { public Xosearch_cache_line[] Get_lines_rest(int qry_id, byte[] wiki_bry, int page_id) {
@ -61,7 +55,7 @@ public class Xosearch_cache_mgr {
Xosearch_cache_wiki wiki = (Xosearch_cache_wiki)qry.Wikis().Get_by(wiki_bry); Xosearch_cache_wiki wiki = (Xosearch_cache_wiki)qry.Wikis().Get_by(wiki_bry);
Xosearch_cache_page page = (Xosearch_cache_page)wiki.Pages().Get_by(page_id); Xosearch_cache_page page = (Xosearch_cache_page)wiki.Pages().Get_by(page_id);
// loop lines from 1 to n // loop lines from 1 to n; note "1" b/c results will always show at least 1st line
List_adp list = List_adp_.New(); List_adp list = List_adp_.New();
int lines_len = page.Lines().Len(); int lines_len = page.Lines().Len();
for (int i = 1; i < lines_len; i++) { for (int i = 1; i < lines_len; i++) {

173
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/cbks/Xosearch_fulltext_svc.java
View File

@ -1,173 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.cbks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*;
import gplx.core.btries.*;
import gplx.langs.jsons.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.specials.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.finders.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.caches.*;
import gplx.xowa.addons.wikis.searchs.searchers.crts.*;
import gplx.xowa.addons.wikis.searchs.searchers.crts.visitors.*;
import gplx.xowa.addons.apps.cfgs.*;
class Xosearch_fulltext_svc implements Gfo_invk {
private final Xoa_app app;
private final gplx.xowa.guis.cbks.Xog_cbk_trg cbk_trg = gplx.xowa.guis.cbks.Xog_cbk_trg.New(Xosearch_fulltext_special.Prototype.Special__meta().Ttl_bry());
private final Xosearch_finder_mgr finder = new Xosearch_finder_mgr();
private final Xosearch_finder_cbk__eval cbk_eval = new Xosearch_finder_cbk__eval();
private final Xosearch_finder_cbk__highlight cbk_highlight;
private final Xosearch_cache_mgr cache_mgr = new Xosearch_cache_mgr();
public Xosearch_fulltext_svc(Xoa_app app) {
this.app = app;
cbk_highlight = new Xosearch_finder_cbk__highlight(app, cbk_trg, cache_mgr);
}
public void Search(Json_nde args) {
cache_mgr.Clear();
Xocfg_mgr cfg_mgr = app.Cfg();
Xosearch_search_args thread_args = Xosearch_search_args.New_by_json(args);
if (cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.options.autosave_enabled", true)) {
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.case_match", thread_args.case_match);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.auto_wildcard_bgn", thread_args.auto_wildcard_bgn);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.auto_wildcard_end", thread_args.auto_wildcard_end);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.expand_matches_section", thread_args.expand_matches_section);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.show_all_matches", thread_args.show_all_matches);
cfg_mgr.Get_int_app_or ("xowa.addon.search.fulltext.special.max_pages_per_wiki", thread_args.max_pages_per_wiki);
cfg_mgr.Get_str_app_or ("xowa.addon.search.fulltext.special.namespaces", thread_args.namespaces);
}
gplx.core.threads.Thread_adp_.Start_by_val("search", Cancelable_.Never, this, Invk__search, thread_args);
}
private void Search(Xosearch_search_args args) {
try {
byte[][] wiki_domains = Bry_split_.Split(args.wikis, Byte_ascii.Pipe_bry);
for (byte[] wiki_domain : wiki_domains) {
Xow_wiki wiki = app.Wiki_mgri().Get_by_or_make_init_y(wiki_domain);
Search_wiki(wiki, args);
}
} catch (Exception exc) {
if (app.Tid_is_edit())
((Xoae_app)app).Gui_mgr().Kit().Ask_ok("", "", Err_.Message_gplx_full(exc));
}
}
public void Get_lines_rest(Json_nde args) {
Get_lines_rest(args.Get_as_int("qry_id"), args.Get_as_bry("wiki"), args.Get_as_int("page_id"));
}
private void Get_lines_rest(int qry_id, byte[] wiki_bry, int page_id) {
Xosearch_cache_line[] lines = cache_mgr.Get_lines_rest(qry_id, wiki_bry, page_id);
for (Xosearch_cache_line line : lines) {
app.Gui__cbk_mgr().Send_json(cbk_trg, "xo.search_fulltext.results__line__add__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_bry("wiki", wiki_bry)
.Add_int("page_id", page_id)
.Add_int("line", line.Line_seq() + 1)
.Add_bry("html", line.Line_html())
);
}
}
private void Search_wiki(Xow_wiki wiki, Xosearch_search_args args) {
byte[] wiki_domain = wiki.Domain_bry();
Db_conn page_conn = wiki.Data__core_mgr().Tbl__page().Conn();
Db_rdr page_rdr = page_conn.Stmt_sql("SELECT * FROM page WHERE page_namespace IN (0) ORDER BY page_score DESC").Exec_select__rls_auto();
app.Gui__cbk_mgr().Send_json(cbk_trg, "xo.search_fulltext.results__wiki__add__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_bry("wiki", wiki_domain)
.Add_long("page_count", 0)
);
finder.Init(args.query, args.case_match, args.auto_wildcard_bgn, args.auto_wildcard_end, Byte_ascii.Star, Byte_ascii.Dash);
int query_id = cache_mgr.Next_qry_id();
try {
int found = 0;
int searched = 0;
while (page_rdr.Move_next()) {
int page_id = page_rdr.Read_int("page_id");
int text_db_id = page_rdr.Read_int("page_text_db_id");
byte[] text_mcase = wiki.Data__core_mgr().Dbs__get_by_id_or_fail(text_db_id).Tbl__text().Select(page_id);
int ns_id = page_rdr.Read_int("page_namespace");
byte[] ttl_bry = page_rdr.Read_bry_by_str("page_title");
Xoa_ttl ttl = wiki.Ttl_parse(ns_id, ttl_bry);
// do eval
cbk_eval.Init(ttl.Full_db());
finder.Match(text_mcase, 0, text_mcase.length, cbk_eval);
searched++;
if (cbk_eval.found) {
++found;
Notify_pages_found_and_searched(wiki_domain, found, searched);
// do highlight
if (found <= args.max_pages_per_wiki) {
cbk_highlight.Init(args.query, query_id, wiki, page_id, ttl.Full_db(), args.show_all_matches);
app.Gui__cbk_mgr().Send_json(cbk_trg, "xo.search_fulltext.results__page__add__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_int("query_id", query_id)
.Add_bry("wiki", wiki_domain)
.Add_int("page_id", page_id)
.Add_bry("page_ttl", ttl.Full_db())
.Add_bool("expand_matches_section", args.expand_matches_section)
);
finder.Match(text_mcase, 0, text_mcase.length, cbk_highlight);
}
}
if (searched % 100 == 0) {
Notify_pages_found_and_searched(wiki_domain, found, searched);
}
}
Notify_pages_found_and_searched(wiki_domain, found, searched);
} finally {
page_rdr.Rls();
}
}
private void Notify_pages_found_and_searched(byte[] wiki, int found, int searched) {
app.Gui__cbk_mgr().Send_json(cbk_trg, "xo.search_fulltext.results__wiki__update__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_bry("wiki", wiki)
.Add_int("found", found)
.Add_int("searched", searched)
);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__search)) this.Search((Xosearch_search_args)m.ReadObj("v"));
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk__search = "search";
}
class Xosearch_search_args {
public boolean case_match;
public boolean auto_wildcard_bgn;
public boolean auto_wildcard_end;
public boolean expand_matches_section;
public boolean show_all_matches;
public int max_pages_per_wiki;
public byte[] wikis;
public byte[] query;
public String namespaces;
public static Xosearch_search_args New_by_json(Json_nde args) {
Xosearch_search_args rv = new Xosearch_search_args();
rv.case_match = args.Get_as_bool_or("case_match", false);
rv.auto_wildcard_bgn = args.Get_as_bool_or("auto_wildcard_bgn", false);
rv.auto_wildcard_end = args.Get_as_bool_or("auto_wildcard_end", false);
rv.expand_matches_section = args.Get_as_bool_or("expand_matches_section", false);
rv.show_all_matches = args.Get_as_bool_or("show_all_matches", false);
rv.max_pages_per_wiki = args.Get_as_int_or("max_pages_per_wiki", 25);
rv.wikis = args.Get_as_bry("wikis");
rv.query = args.Get_as_bry("query");
rv.namespaces = args.Get_as_str("namespaces");
return rv;
}
}

8
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/indexers/Xosearch_indexer.java
View File

@ -24,9 +24,11 @@ public class Xosearch_indexer {
} }
public void Index(Xoae_page wpg) { public void Index(Xoae_page wpg) {
// TODO: skip if not main_ns // TODO: skip if not main_ns
byte[] html = wpg.Db().Html().Html_bry(); Index(wpg.Db().Page().Id(), wpg.Db().Page().Score(), wpg.Ttl().Page_txt(), wpg.Db().Html().Html_bry());
}
index_wtr.Exec(String_.new_u8(wpg.Ttl().Page_db()), String_.new_u8(html)); public void Index(int page_id, int score, byte[] ttl, byte[] html) {
Gflucene_index_data data = new Gflucene_index_data(page_id, score, String_.new_u8(ttl), String_.new_u8(html));
index_wtr.Exec(data);
} }
public void Term() { public void Term() {
index_wtr.Term(); index_wtr.Term();

28
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/indexers/Xosearch_indexer_cmd.java Normal file
View File

@ -0,0 +1,28 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.indexers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xosearch_indexer_cmd extends Xob_cmd__base {
public Xosearch_indexer_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
new Xosearch_indexer_mgr().Exec(wiki);
}
@Override public String Cmd_key() {return "search.index";}
public static final Xob_cmd Prototype = new Xosearch_indexer_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xosearch_indexer_cmd(bldr, wiki);}
}

65
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/indexers/Xosearch_indexer_mgr.java Normal file
View File

@ -0,0 +1,65 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.indexers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*;
import gplx.dbs.*;
import gplx.xowa.htmls.*;
import gplx.xowa.wikis.data.*;
import gplx.xowa.htmls.core.dbs.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.indexers.*;
class Xosearch_indexer_mgr {
public void Exec(Xowe_wiki wiki) {
Xow_db_file core_db = wiki.Data__core_mgr().Db__core();
gplx.xowa.wikis.data.tbls.Xowd_page_tbl page_tbl = core_db.Tbl__page();
Xoh_page hpg = new Xoh_page();
Xosearch_indexer indexer = new Xosearch_indexer();
indexer.Init(wiki);
Db_conn conn = page_tbl.Conn();
Db_rdr rdr = conn.Exec_rdr("SELECT page_id, page_score, page_namespace, page_title, page_html_db_id FROM page WHERE page_namespace = 0;");
int count = 0;
while (rdr.Move_next()) {
int page_namespace = rdr.Read_int("page_namespace");
byte[] page_ttl_bry = rdr.Read_bry_by_str("page_title");
int page_id = rdr.Read_int("page_id");
int page_score = rdr.Read_int("page_score");
int html_db_id = rdr.Read_int("page_html_db_id");
// ignore redirects
if (html_db_id == -1) continue;
try {
// load page
Xoa_ttl page_ttl = wiki.Ttl_parse(page_namespace, page_ttl_bry);
if (page_ttl == null)
continue;
Xow_db_file html_db = html_db_id == -1 ? core_db : wiki.Data__core_mgr().Dbs__get_by_id_or_fail(html_db_id);
hpg.Ctor_by_hview(wiki, wiki.Utl__url_parser().Parse(page_ttl.Full_db()), page_ttl, page_id);
if (!html_db.Tbl__html().Select_by_page(hpg))
continue;
byte[] html_text = wiki.Html__hdump_mgr().Load_mgr().Parse(hpg, hpg.Db().Html().Zip_tid(), hpg.Db().Html().Hzip_tid(), hpg.Db().Html().Html_bry());
indexer.Index(page_id, page_score, page_ttl.Page_txt(), html_text);
if ((++count % 10000) == 0)
Gfo_usr_dlg_.Instance.Prog_many("", "", "indexing page: ~{0}", count);
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "err: ~{0}", Err_.Message_gplx_log(e));
}
}
indexer.Term();
}
}

20
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/Xosearch_searcher.java Normal file
View File

@ -0,0 +1,20 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.searchers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.uis.*;
public interface Xosearch_searcher {
void Search(Xosearch_searcher_ui ui, Xow_wiki wiki, Xosearch_searcher_args args);
}

42
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/Xosearch_searcher_args.java Normal file
View File

@ -0,0 +1,42 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.searchers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*;
import gplx.langs.jsons.*;
public class Xosearch_searcher_args {
public boolean case_match;
public boolean auto_wildcard_bgn;
public boolean auto_wildcard_end;
public boolean expand_matches_section;
public boolean show_all_matches;
public int max_pages_per_wiki;
public byte[] wikis;
public byte[] query;
public String namespaces;
public int query_id;
public static Xosearch_searcher_args New_by_json(Json_nde args) {
Xosearch_searcher_args rv = new Xosearch_searcher_args();
rv.case_match = args.Get_as_bool_or("case_match", false);
rv.auto_wildcard_bgn = args.Get_as_bool_or("auto_wildcard_bgn", false);
rv.auto_wildcard_end = args.Get_as_bool_or("auto_wildcard_end", false);
rv.expand_matches_section = args.Get_as_bool_or("expand_matches_section", false);
rv.show_all_matches = args.Get_as_bool_or("show_all_matches", false);
rv.max_pages_per_wiki = args.Get_as_int_or("max_pages_per_wiki", 25);
rv.wikis = args.Get_as_bry("wikis");
rv.query = args.Get_as_bry("query");
rv.namespaces = args.Get_as_str("namespaces");
return rv;
}
}

90
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/Xosearch_searcher__brute.java Normal file
View File

@ -0,0 +1,90 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*;
import gplx.dbs.*;
import gplx.xowa.guis.cbks.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.uis.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.caches.*;
public class Xosearch_searcher__brute implements Xosearch_searcher {
private final Xosearch_finder_mgr finder = new Xosearch_finder_mgr();
private final Xosearch_finder_cbk__eval cbk_eval = new Xosearch_finder_cbk__eval();
private final Xosearch_finder_cbk__highlight cbk_highlight;
public Xosearch_searcher__brute(Xoa_app app, Xog_cbk_trg cbk_trg, Xosearch_cache_mgr cache_mgr) {
this.cbk_highlight = new Xosearch_finder_cbk__highlight(app, cbk_trg, cache_mgr);
}
public void Search(Xosearch_searcher_ui ui, Xow_wiki wiki, Xosearch_searcher_args args) {
// get pages from db
Db_conn page_conn = wiki.Data__core_mgr().Tbl__page().Conn();
Db_rdr page_rdr = page_conn.Stmt_sql("SELECT * FROM page WHERE page_namespace IN (0) ORDER BY page_score DESC").Exec_select__rls_auto();
// init finder
finder.Init(args.query, args.case_match, args.auto_wildcard_bgn, args.auto_wildcard_end, Byte_ascii.Star, Byte_ascii.Dash);
// loop
byte[] wiki_domain = wiki.Domain_bry();
int found = 0;
int searched = 0;
try {
while (page_rdr.Move_next()) {
// read data from reader
int page_id = page_rdr.Read_int("page_id");
int text_db_id = page_rdr.Read_int("page_text_db_id");
byte[] text_mcase = wiki.Data__core_mgr().Dbs__get_by_id_or_fail(text_db_id).Tbl__text().Select(page_id);
int ns_id = page_rdr.Read_int("page_namespace");
byte[] ttl_bry = page_rdr.Read_bry_by_str("page_title");
Xoa_ttl ttl = wiki.Ttl_parse(ns_id, ttl_bry);
// eval query
cbk_eval.Init(ttl.Full_db());
finder.Match(text_mcase, 0, text_mcase.length, cbk_eval);
searched++;
// check if page matches query
if (cbk_eval.found) {
++found;
// update pages found
ui.Send_wiki_update(wiki_domain, found, searched);
// do highlight
if (found <= args.max_pages_per_wiki) {
cbk_highlight.Init(args.query, args.query_id, wiki, page_id, ttl.Full_db(), args.show_all_matches);
ui.Send_page_add(new Xosearch_searcher_page
( args.query_id
, String_.new_u8(wiki_domain)
, page_id
, String_.new_u8(ttl.Full_db())
, args.expand_matches_section
));
finder.Match(text_mcase, 0, text_mcase.length, cbk_highlight);
}
}
// update update pages found every 100 pages
if (searched % 100 == 0) {
ui.Send_wiki_update(wiki_domain, found, searched);
}
}
}
finally {
page_rdr.Rls();
}
// update one last time for final searched
ui.Send_wiki_update(wiki_domain, found, searched);
}
}

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/finders/Xosearch_finder_cbk.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/finders/Xosearch_finder_cbk.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
import gplx.xowa.guis.cbks.*; import gplx.xowa.guis.cbks.*;
public interface Xosearch_finder_cbk { public interface Xosearch_finder_cbk {
byte[] Page_ttl(); byte[] Page_ttl();

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/finders/Xosearch_finder_cbk__eval.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/finders/Xosearch_finder_cbk__eval.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
public class Xosearch_finder_cbk__eval implements Xosearch_finder_cbk { public class Xosearch_finder_cbk__eval implements Xosearch_finder_cbk {
public boolean found; public boolean found;
public byte[] Page_ttl() {return page_ttl;} private byte[] page_ttl; public byte[] Page_ttl() {return page_ttl;} private byte[] page_ttl;

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/finders/Xosearch_finder_cbk__eval__tst.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/finders/Xosearch_finder_cbk__eval__tst.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
import org.junit.*; import gplx.core.tests.*; import org.junit.*; import gplx.core.tests.*;
public class Xosearch_finder_cbk__eval__tst { public class Xosearch_finder_cbk__eval__tst {
private final Xosearch_finder_cbk__eval__fxt fxt = new Xosearch_finder_cbk__eval__fxt(); private final Xosearch_finder_cbk__eval__fxt fxt = new Xosearch_finder_cbk__eval__fxt();

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/finders/Xosearch_finder_cbk__highlight.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/finders/Xosearch_finder_cbk__highlight.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
import gplx.xowa.guis.cbks.*; import gplx.xowa.guis.cbks.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.caches.*; import gplx.xowa.addons.wikis.searchs.fulltexts.caches.*;
public class Xosearch_finder_cbk__highlight implements Xosearch_finder_cbk { public class Xosearch_finder_cbk__highlight implements Xosearch_finder_cbk {

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/finders/Xosearch_finder_mgr.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/finders/Xosearch_finder_mgr.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
import gplx.xowa.guis.cbks.*; import gplx.xowa.guis.cbks.*;
import gplx.core.btries.*; import gplx.core.btries.*;
import gplx.xowa.addons.wikis.searchs.searchers.crts.*; import gplx.xowa.addons.wikis.searchs.searchers.crts.*;

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/finders/Xosearch_word_bounds.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/finders/Xosearch_word_bounds.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
public class Xosearch_word_bounds { public class Xosearch_word_bounds {
public int word_bgn; public int word_bgn;
public int word_end; public int word_end;

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/finders/Xosearch_word_lang.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/finders/Xosearch_word_lang.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
import gplx.core.btries.*; import gplx.core.btries.*;
import gplx.core.intls.*; import gplx.core.intls.*;
public class Xosearch_word_lang { public class Xosearch_word_lang {

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/finders/Xosearch_word_node.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/finders/Xosearch_word_node.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
import gplx.core.btries.*; import gplx.core.btries.*;
import gplx.xowa.addons.wikis.searchs.searchers.crts.*; import gplx.xowa.addons.wikis.searchs.searchers.crts.*;
public class Xosearch_word_node { public class Xosearch_word_node {

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/finders/Xosearch_word_node_.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/brutes/finders/Xosearch_word_node_.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.finders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
import gplx.core.btries.*; import gplx.core.btries.*;
import gplx.xowa.addons.wikis.searchs.searchers.crts.*; import gplx.xowa.addons.wikis.searchs.searchers.crts.*;
public class Xosearch_word_node_ { public class Xosearch_word_node_ {

44
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/gflucenes/Xosearch_searcher__lucene.java Normal file
View File

@ -0,0 +1,44 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.gflucenes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*;
import gplx.gflucene.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.uis.*;
public class Xosearch_searcher__lucene implements Xosearch_searcher {
private final Gflucene_searcher searcher = new Gflucene_searcher();
public void Search(Xosearch_searcher_ui cbk, Xow_wiki wiki, Xosearch_searcher_args args) {
// create list
List_adp list = List_adp_.New();
// init searcher with wiki
searcher.Init(wiki.Fsys_mgr().Root_dir().GenSubDir_nest("data", "search").Xto_api());
// exec search
searcher.Exec(list, new Gflucene_searcher_data(String_.new_u8(args.query), args.max_pages_per_wiki));
// term
searcher.Term();
// loop list
int len = list.Len();
for (int i = 0; i < len; i++) {
Gflucene_index_data found = (Gflucene_index_data)list.Get_at(i);
// call page found
Xosearch_searcher_page page = new Xosearch_searcher_page(args.query_id, wiki.Domain_str(), found.page_id, found.title, false);
cbk.Send_page_add(page);
}
}
}

28
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/uis/Xosearch_searcher_line.java Normal file
View File

@ -0,0 +1,28 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.uis; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*;
public class Xosearch_searcher_line {
public Xosearch_searcher_line(String wiki_domain, int page_id, int found_idx, String excerpt) {
this.wiki_domain = wiki_domain;
this.page_id = page_id;
this.found_idx = found_idx;
this.excerpt = excerpt;
}
public String Wiki_domain() {return wiki_domain;} private final String wiki_domain;
public int Page_id() {return page_id;} private final int page_id;
public int Found_idx() {return found_idx;} private final int found_idx;
public String Excerpt() {return excerpt;} private final String excerpt;
}

30
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/uis/Xosearch_searcher_page.java Normal file
View File

@ -0,0 +1,30 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.uis; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*;
public class Xosearch_searcher_page {
public Xosearch_searcher_page(int query_id, String wiki_domain, int page_id, String page_title, boolean expand_matches_section) {
this.query_id = query_id;
this.wiki_domain = wiki_domain;
this.page_id = page_id;
this.page_title = page_title;
this.expand_matches_section = expand_matches_section;
}
public int Query_id() {return query_id;} private final int query_id;
public String Wiki_domain() {return wiki_domain;} private final String wiki_domain;
public int Page_id() {return page_id;} private final int page_id;
public String Page_title() {return page_title;} private final String page_title;
public boolean Expand_matches_section() {return expand_matches_section;} private final boolean expand_matches_section;
}

22
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/uis/Xosearch_searcher_ui.java Normal file
View File

@ -0,0 +1,22 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.uis; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*;
public interface Xosearch_searcher_ui {
void Send_wiki_add(byte[] wiki_domain);
void Send_wiki_update(byte[] wiki, int found, int searched);
void Send_page_add(Xosearch_searcher_page page);
void Send_line_add(Xosearch_searcher_line line);
}

54
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/searchers/uis/Xosearch_searcher_ui__gui.java Normal file
View File

@ -0,0 +1,54 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.searchers.uis; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*;
import gplx.xowa.guis.cbks.*;
public class Xosearch_searcher_ui__gui implements Xosearch_searcher_ui {
private final Xog_cbk_mgr cbk_mgr;
private final Xog_cbk_trg cbk_trg;
public Xosearch_searcher_ui__gui(Xog_cbk_mgr cbk_mgr, Xog_cbk_trg cbk_trg) {
this.cbk_mgr = cbk_mgr;
this.cbk_trg = cbk_trg;
}
public void Send_wiki_add(byte[] wiki_domain) {
cbk_mgr.Send_json(cbk_trg, "xo.search_fulltext.results__wiki__add__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_bry("wiki", wiki_domain)
);
}
public void Send_wiki_update(byte[] wiki, int found, int searched) {
cbk_mgr.Send_json(cbk_trg, "xo.search_fulltext.results__wiki__update__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_bry("wiki", wiki)
.Add_int("found", found)
.Add_int("searched", searched)
);
}
public void Send_page_add(Xosearch_searcher_page page) {
cbk_mgr.Send_json(cbk_trg, "xo.search_fulltext.results__page__add__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_int("query_id", page.Query_id())
.Add_str("wiki", page.Wiki_domain())
.Add_int("page_id", page.Page_id())
.Add_str("page_ttl", page.Page_title())
.Add_bool("expand_matches_section", page.Expand_matches_section())
);
}
public void Send_line_add(Xosearch_searcher_line match) {
cbk_mgr.Send_json(cbk_trg, "xo.search_fulltext.results__line__add__recv", gplx.core.gfobjs.Gfobj_nde.New()
.Add_str("wiki", match.Wiki_domain())
.Add_int("page_id", match.Page_id())
.Add_int("line", match.Found_idx())
.Add_str("html", match.Excerpt())
);
}
}

2
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/cbks/Xosearch_fulltext_bridge.java → 400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/svcs/Xosearch_fulltext_bridge.java
View File

@ -13,7 +13,7 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/ */
package gplx.xowa.addons.wikis.searchs.fulltexts.cbks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*; package gplx.xowa.addons.wikis.searchs.fulltexts.svcs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*;
import gplx.langs.jsons.*; import gplx.langs.jsons.*;
import gplx.xowa.htmls.bridges.*; import gplx.xowa.htmls.bridges.*;
public class Xosearch_fulltext_bridge implements Bridge_cmd_itm { public class Xosearch_fulltext_bridge implements Bridge_cmd_itm {

104
400_xowa/src/gplx/xowa/addons/wikis/searchs/fulltexts/svcs/Xosearch_fulltext_svc.java Normal file
View File

@ -0,0 +1,104 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.wikis.searchs.fulltexts.svcs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.fulltexts.*;
import gplx.core.btries.*;
import gplx.langs.jsons.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.guis.cbks.*;
import gplx.xowa.addons.apps.cfgs.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.specials.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.caches.*;
import gplx.xowa.addons.wikis.searchs.searchers.crts.*;
import gplx.xowa.addons.wikis.searchs.searchers.crts.visitors.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.uis.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.gflucenes.*;
import gplx.xowa.addons.wikis.searchs.fulltexts.searchers.brutes.*;
class Xosearch_fulltext_svc implements Gfo_invk {
private final Xoa_app app;
private final Xog_cbk_trg cbk_trg = Xog_cbk_trg.New(Xosearch_fulltext_special.Prototype.Special__meta().Ttl_bry());
private final Xosearch_cache_mgr cache_mgr = new Xosearch_cache_mgr();
private final Xosearch_searcher_ui searcher_cbk;
public Xosearch_fulltext_svc(Xoa_app app) {
this.app = app;
this.searcher_cbk = new Xosearch_searcher_ui__gui(app.Gui__cbk_mgr(), cbk_trg);
}
public void Search(Json_nde args) {
// for now, always clear cache; "get_lines_rest" will only work for latest search
cache_mgr.Clear();
// get search_args
Xosearch_searcher_args search_args = Xosearch_searcher_args.New_by_json(args);
search_args.query_id = cache_mgr.Next_qry_id();
// autosave any changes if enabled
Xocfg_mgr cfg_mgr = app.Cfg();
if (cfg_mgr.Get_bool_app_or("xowa.addon.search.fulltext.options.autosave_enabled", true)) {
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.case_match", search_args.case_match);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.auto_wildcard_bgn", search_args.auto_wildcard_bgn);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.auto_wildcard_end", search_args.auto_wildcard_end);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.expand_matches_section", search_args.expand_matches_section);
cfg_mgr.Set_bool_app("xowa.addon.search.fulltext.special.show_all_matches", search_args.show_all_matches);
cfg_mgr.Get_int_app_or ("xowa.addon.search.fulltext.special.max_pages_per_wiki", search_args.max_pages_per_wiki);
cfg_mgr.Get_str_app_or ("xowa.addon.search.fulltext.special.namespaces", search_args.namespaces);
}
// launch thread
gplx.core.threads.Thread_adp_.Start_by_val("search", Cancelable_.Never, this, Invk__search, search_args);
}
private void Search(Xosearch_searcher_args args) {
try {
// loop wikis
byte[][] wiki_domains = Bry_split_.Split(args.wikis, Byte_ascii.Pipe_bry);
for (byte[] wiki_domain : wiki_domains) {
// get wiki and notify
Xow_wiki wiki = app.Wiki_mgri().Get_by_or_make_init_y(wiki_domain);
searcher_cbk.Send_wiki_add(wiki_domain);
// get searcher and search
Xosearch_searcher searcher = Get_searcher(wiki);
searcher.Search(searcher_cbk, wiki, args);
}
} catch (Exception exc) {
if (app.Tid_is_edit())
((Xoae_app)app).Gui_mgr().Kit().Ask_ok("", "", Err_.Message_gplx_full(exc));
}
}
public void Get_lines_rest(Json_nde args) {
Get_lines_rest(args.Get_as_int("qry_id"), args.Get_as_bry("wiki"), args.Get_as_int("page_id"));
}
private void Get_lines_rest(int qry_id, byte[] wiki_bry, int page_id) {
Xosearch_cache_line[] lines = cache_mgr.Get_lines_rest(qry_id, wiki_bry, page_id);
for (Xosearch_cache_line line : lines) {
Xosearch_searcher_line match = new Xosearch_searcher_line(String_.new_u8(wiki_bry), page_id, line.Line_seq() + 1, String_.new_u8(line.Line_html()));
searcher_cbk.Send_line_add(match);
}
}
private Xosearch_searcher Get_searcher(Xow_wiki wiki) {
if (Io_mgr.Instance.ExistsDir(wiki.Fsys_mgr().Root_dir().GenSubDir_nest("data", "search"))) {
return new Xosearch_searcher__lucene();
}
else {
return new Xosearch_searcher__brute(app, cbk_trg, cache_mgr);
}
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__search)) this.Search((Xosearch_searcher_args)m.ReadObj("v"));
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk__search = "search";
}

2
xowa.home.version
View File

@ -1 +1 @@
v4.4.0.1