Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Cldr: Add initial support for cldr

Browse Source
This commit is contained in:
gnosygnu
2018-08-05 21:21:40 -04:00
parent be3979c5af
commit e78382a8ac
27 changed files with 1052 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_lang_tbl.java
View File

@@ -1,45 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
import gplx.dbs.*;
class Cldr_lang_tbl implements Rls_able {
private final String tbl_name = "cldr_lang"; private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private final String fld_cldr_code, fld_lang_code, fld_lang_name;
private final Db_conn conn; private Db_stmt stmt_select, stmt_insert;
public Cldr_lang_tbl(Db_conn conn) {
this.conn = conn;
this.fld_cldr_code = flds.Add_str("cldr_code", 32);
this.fld_lang_code = flds.Add_str("lang_code", 32);
this.fld_lang_name = flds.Add_str("lang_name", 2048);
conn.Rls_reg(this);
}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "main", fld_cldr_code, fld_lang_code)));}
public void Insert_bgn() {conn.Txn_bgn("cldr_lang__inser"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public void Insert_cmd_by_batch(byte[] cldr_code, byte[] lang_code, byte[] lang_name) {
stmt_insert.Clear().Val_bry_as_str(fld_cldr_code, cldr_code).Val_bry_as_str(fld_lang_code, lang_code).Val_bry_as_str(fld_lang_name, lang_name).Exec_insert();
}
public byte[] Select(byte[] cldr_code, byte[] lang_code) {
if (stmt_select == null) stmt_select = conn.Stmt_select(tbl_name, flds, fld_cldr_code, fld_lang_code);
Db_rdr rdr = stmt_select.Clear().Val_bry_as_str(fld_cldr_code, cldr_code).Val_bry_as_str(fld_lang_code, lang_code).Exec_select__rls_manual();
try {return (byte[])rdr.Read_bry(fld_lang_name);}
finally {rdr.Rls();}
}
public void Rls() {
stmt_select = Db_stmt_.Rls(stmt_select);
stmt_insert = Db_stmt_.Rls(stmt_insert);
}
}

113
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_converter.java Normal file
View File

@@ -0,0 +1,113 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
import gplx.core.primitives.*;
import gplx.langs.phps.*;
import gplx.langs.jsons.*;
class Cldr_name_converter {
private final Php_parser parser = new Php_parser();
private final Php_evaluator eval = new Php_evaluator(new gplx.core.log_msgs.Gfo_msg_log("test"));
private final Php_text_itm_parser text_itm_parser = new Php_text_itm_parser().Quote_is_single_(true);
private final Json_doc_wtr doc_wtr = new Json_doc_wtr();
private final List_adp tmp_list = List_adp_.New();
private final Byte_obj_ref tmp_result = Byte_obj_ref.zero_();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
public void Convert(Io_url src, Io_url trg) {
Cldr_name_file[] fils = Parse_dir(src);
for (Cldr_name_file fil : fils) {
Io_mgr.Instance.SaveFilStr(trg.GenSubDir(File_CldrNames).GenSubFil_ary(File_CldrNames, fil.Key(), ".json"), To_json(fil));
Gfo_usr_dlg_.Instance.Note_many("", "", "converted file; key={0}", fil.Key());
}
}
public Cldr_name_file[] Parse_dir(Io_url dir) {
List_adp rv = List_adp_.New();
Io_url[] fils = Io_mgr.Instance.QueryDir_fils(dir);
for (Io_url fil : fils) {
String key = Extract_key_or_fail(fil.NameAndExt());
byte[] src = Io_mgr.Instance.LoadFilBry(fil);
rv.Add(Parse_fil(key, src));
}
return (Cldr_name_file[])rv.To_ary_and_clear(Cldr_name_file.class);
}
public String Extract_key_or_fail(String fil_name) {
if (!String_.Has_at_bgn(fil_name, File_CldrNames) || !String_.Has_at_end(fil_name, ".php")) throw Err_.new_wo_type("file name must have a format of CldrNamesLANG.php", "fil_name", fil_name);
return String_.Mid(fil_name, String_.Len(File_CldrNames), String_.Len(fil_name) - String_.Len(".php"));
}
public Cldr_name_file Parse_fil(String key, byte[] src) {
Cldr_name_file file = new Cldr_name_file(key);
parser.Parse_tkns(src, eval);
Php_line[] lines = (Php_line[])eval.List().To_ary(Php_line.class);
int lines_len = lines.length;
for (int i = 0; i < lines_len; i++) {
Php_line line = lines[i];
Php_line_assign assign_line = (Php_line_assign)line;
String assign_key = String_.new_u8(assign_line.Key().Val_obj_bry());
Ordered_hash hash = null;
if (String_.Eq(assign_key, Node_languageNames)) hash = file.Language_names();
else if (String_.Eq(assign_key, Node_currencyNames)) hash = file.Currency_names();
else if (String_.Eq(assign_key, Node_currencySymbols)) hash = file.Currency_symbols();
else if (String_.Eq(assign_key, Node_countryNames)) hash = file.Country_names();
else if (String_.Eq(assign_key, Node_timeUnits)) hash = file.Time_units();
else throw Err_.new_unhandled_default(assign_key);
Parse_assign_line(key, assign_key, assign_line, hash);
}
eval.Clear();
return file;
}
private void Parse_assign_line(String lang_key, String assign_key, Php_line_assign assign, Ordered_hash hash) {
Php_itm_ary ary = (Php_itm_ary)assign.Val();
int ary_len = ary.Subs_len();
for (int i = 0; i < ary_len; i++) {
Php_itm_kv kv = (Php_itm_kv)ary.Subs_get(i);
String key_str = String_.new_u8(kv.Key().Val_obj_bry());
String val_str = String_.new_u8(text_itm_parser.Parse_as_bry(tmp_list, kv.Val().Val_obj_bry(), tmp_result, tmp_bfr));
if (hash.Has(key_str)) throw Err_.new_wo_type("key already exists", "lang", lang_key, "type", assign_key, "key", key_str);
hash.Add(key_str, Keyval_.new_(key_str, val_str));
}
}
public String To_json(Cldr_name_file name_file) {
Int_obj_ref written = Int_obj_ref.New_zero();
doc_wtr.Nde_bgn();
To_json(written, Node_languageNames , name_file.Language_names());
To_json(written, Node_currencyNames , name_file.Currency_names());
To_json(written, Node_currencySymbols , name_file.Currency_symbols());
To_json(written, Node_countryNames , name_file.Country_names());
To_json(written, Node_timeUnits , name_file.Time_units());
doc_wtr.Nde_end();
return doc_wtr.Bld_as_str();
}
private void To_json(Int_obj_ref written, String node_name, Ordered_hash hash) {
int len = hash.Count();
if (len == 0) return;
doc_wtr.Key(written.Val() != 0, node_name);
doc_wtr.Nde_bgn();
for (int i = 0; i < len; i++) {
Keyval kv = (Keyval)hash.Get_at(i);
doc_wtr.Kv(i != 0, Bry_.new_u8(kv.Key()), Bry_.new_u8(kv.Val_to_str_or_null()));
}
doc_wtr.Nde_end();
written.Val_add_post();
}
public static final String
File_CldrNames = "CldrNames"
, Node_languageNames = "languageNames"
, Node_currencyNames = "currencyNames"
, Node_currencySymbols = "currencySymbols"
, Node_countryNames = "countryNames"
, Node_timeUnits = "timeUnits"
;
}

166
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_converter_tst.java Normal file
View File

@@ -0,0 +1,166 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
import org.junit.*; import gplx.core.tests.*;
import gplx.langs.phps.*;
public class Cldr_name_converter_tst {
private final Cldr_name_converter_fxt fxt = new Cldr_name_converter_fxt();
@Test public void Extract_key_or_fail() {
fxt.Test__Extract_key_or_fail("CldrNamesEn.php" , "En");
fxt.Test__Extract_key_or_fail("CldrNameEn.php" , null);
fxt.Test__Extract_key_or_fail("CldrNamesEn.txt" , null);
}
@Test public void Parse_fil() {
Cldr_name_file file = fxt.Exec__Parse_fil("En", String_.Concat_lines_nl
( "$languageNames = ["
, " 'aa' => 'Afar',"
, " 'mic' => 'Mi\\'kmaq',"
, " 'zza' => 'Zaza',"
, "];"
, ""
, "$currencyNames = ["
, " 'ADP' => 'Andorran Peseta',"
, " 'ZWR' => 'Zimbabwean Dollar (2008)',"
, "];"
, ""
, "$currencySymbols = ["
, " 'JPY' => '¥',"
, " 'USD' => '$',"
, "];"
, ""
, "$countryNames = ["
, " 'AC' => 'Ascension Island',"
, " 'ZW' => 'Zimbabwe',"
, "];"
, ""
, "$timeUnits = ["
, " 'century-one' => '{0} century',"
, " 'year-short-past-other' => '{0} yr. ago',"
, "];"
));
Assert__parse_fil(file);
String expd = String_.Concat_lines_nl
( "{"
, " \"languageNames\":"
, " {"
, " \"aa\":\"Afar\""
, " , \"mic\":\"Mi'kmaq\""
, " , \"zza\":\"Zaza\""
, " }"
, ", \"currencyNames\":"
, " {"
, " \"ADP\":\"Andorran Peseta\""
, " , \"ZWR\":\"Zimbabwean Dollar (2008)\""
, " }"
, ", \"currencySymbols\":"
, " {"
, " \"JPY\":\"¥\""
, " , \"USD\":\"$\""
, " }"
, ", \"countryNames\":"
, " {"
, " \"AC\":\"Ascension Island\""
, " , \"ZW\":\"Zimbabwe\""
, " }"
, ", \"timeUnits\":"
, " {"
, " \"century-one\":\"{0} century\""
, " , \"year-short-past-other\":\"{0} yr. ago\""
, " }"
, "}"
);
fxt.Test__To_json(file, expd);
file = fxt.Exec__To_file("En", expd);
Assert__parse_fil(file);
}
private void Assert__parse_fil(Cldr_name_file file) {
fxt.Test__node(file.Language_names()
, Keyval_.new_("aa", "Afar")
, Keyval_.new_("mic", "Mi'kmaq")
, Keyval_.new_("zza", "Zaza")
);
fxt.Test__node(file.Currency_names()
, Keyval_.new_("ADP", "Andorran Peseta")
, Keyval_.new_("ZWR", "Zimbabwean Dollar (2008)")
);
fxt.Test__node(file.Currency_symbols()
, Keyval_.new_("JPY", "¥")
, Keyval_.new_("USD", "$")
);
fxt.Test__node(file.Country_names()
, Keyval_.new_("AC", "Ascension Island")
, Keyval_.new_("ZW", "Zimbabwe")
);
fxt.Test__node(file.Time_units()
, Keyval_.new_("century-one", "{0} century")
, Keyval_.new_("year-short-past-other", "{0} yr. ago")
);
}
@Test public void Smoke() {
Cldr_name_converter bldr = new Cldr_name_converter();
bldr.Convert(Io_url_.new_dir_("C:\\000\\100_bin\\200_server\\200_http\\100_apache\\100_v2.4\\htdocs\\mediawiki\\v1.29.1\\extensions\\cldr\\CldrNames\\"), Io_url_.new_dir_("C:\\xowa\\bin\\any\\xowa\\xtns\\cldr\\"));
}
}
class Cldr_name_converter_fxt {
private final Cldr_name_converter bldr = new Cldr_name_converter();
private final String dir_name = "mem/CldrNames/";
public void Init__file(String fil_name, String txt) {
Io_mgr.Instance.SaveFilStr(Io_url_.new_fil_(dir_name + fil_name), txt);
}
public void Test__Extract_key_or_fail(String fil_name, String expd) {
String actl = null;
try {
actl = bldr.Extract_key_or_fail(fil_name);
}
catch (Exception exc) {
Err_.Noop(exc);
actl = null;
}
if (expd == null && actl == null) {
}
else if (expd == null) {
Tfds.Fail("expecting null; got " + actl);
}
else if (actl == null) {
Tfds.Fail("got null; expected " + expd);
}
else {
Gftest.Eq__str(expd, actl);
}
}
public Cldr_name_file[] Exec__Parse_dir() {
return bldr.Parse_dir(Io_url_.new_fil_(dir_name));
}
public Cldr_name_file Exec__Parse_fil(String key, String src) {
return bldr.Parse_fil(key, Bry_.new_u8(src));
}
public Cldr_name_file Exec__To_file(String key, String json) {
Cldr_name_loader json_parser = new Cldr_name_loader(Io_url_.mem_dir_("mem/Cldr"));
return json_parser.Load(key, Bry_.new_u8(json));
}
public void Test__node(Ordered_hash hash, Keyval... expd) {
Keyval[] actl = (Keyval[])hash.To_ary(Keyval.class);
Gftest.Eq__ary__lines(Keyval_.Ary_to_str(expd), Keyval_.Ary_to_str(actl), "cldr_names_comp_failed");
}
public void Test__To_json(Cldr_name_file file, String expd) {
String actl = bldr.To_json(file);
Gftest.Eq__ary__lines(expd, actl, "json_failed");
}
}

27
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_file.java Normal file
View File

@@ -0,0 +1,27 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
public class Cldr_name_file {
public Cldr_name_file(String key) {
this.key = key;
}
public String Key() {return key;} private final String key;
public Ordered_hash Language_names() {return language_names;} private final Ordered_hash language_names = Ordered_hash_.New();
public Ordered_hash Currency_names() {return currency_names;} private final Ordered_hash currency_names = Ordered_hash_.New();
public Ordered_hash Currency_symbols() {return currency_symbols;} private final Ordered_hash currency_symbols = Ordered_hash_.New();
public Ordered_hash Country_names() {return country_names;} private final Ordered_hash country_names = Ordered_hash_.New();
public Ordered_hash Time_units() {return time_units;} private final Ordered_hash time_units = Ordered_hash_.New();
}

70
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_loader.java Normal file
View File

@@ -0,0 +1,70 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
import gplx.langs.jsons.*;
public class Cldr_name_loader {
private final Json_parser parser = new Json_parser();
private final Io_url cldr_dir;
private final Hash_adp hash = Hash_adp_.New();
public Cldr_name_loader(Io_url xowa_xtn_dir) {
cldr_dir = xowa_xtn_dir.GenSubDir_nest("cldr", "CldrNames");
}
public Cldr_name_file Load(String lang_key) {
Cldr_name_file file = (Cldr_name_file)hash.Get_by(lang_key);
if (file != null) return file;
byte[] json = Io_mgr.Instance.LoadFilBry(cldr_dir.GenSubFil_ary("CldrNames", lang_key, ".json"));
if (json == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "no cldrName file exists for lang; lang=~{lang}", lang_key);
return null;
}
file = Load(lang_key, json);
hash.Add(lang_key, file);
return file;
}
public Cldr_name_file Load(String lang_key, byte[] json) {
Cldr_name_file file = new Cldr_name_file(lang_key);
Json_doc jdoc = parser.Parse(json);
Json_nde root = jdoc.Root_nde();
int nodes_len = root.Len();
for (int i = 0; i < nodes_len; i++) {
Json_kv node = root.Get_at_as_kv(i);
String key = node.Key_as_str();
Json_nde val = node.Val_as_nde();
Ordered_hash hash = null;
if (String_.Eq(key, Cldr_name_converter.Node_languageNames)) hash = file.Language_names();
else if (String_.Eq(key, Cldr_name_converter.Node_currencyNames)) hash = file.Currency_names();
else if (String_.Eq(key, Cldr_name_converter.Node_currencySymbols)) hash = file.Currency_symbols();
else if (String_.Eq(key, Cldr_name_converter.Node_countryNames)) hash = file.Country_names();
else if (String_.Eq(key, Cldr_name_converter.Node_timeUnits)) hash = file.Time_units();
else throw Err_.new_unhandled_default(key);
Load_ary(file, hash, val);
}
return file;
}
private void Load_ary(Cldr_name_file file, Ordered_hash hash, Json_nde nde) {
int len = nde.Len();
for (int i = 0; i < len; i++) {
Json_kv kv = (Json_kv)nde.Get_at(i);
String key = kv.Key_as_str();
hash.Add(key, Keyval_.new_(key, String_.new_u8(kv.Val_as_bry())));
}
}
}

1
400_xowa/src/gplx/xowa/xtns/wbases/Wdata_doc.java
View File

@@ -26,6 +26,7 @@ public class Wdata_doc {
this.slink_list = slink_list; this.label_list = label_list; this.descr_list = descr_list; this.alias_list = alias_list; this.claim_list = claim_list;
}
public Json_doc Jdoc() {return jdoc;} private Json_doc jdoc;
public int Jdoc_size() {return jdoc == null ? 1 : jdoc.Src().length;}
public byte[] Qid() {return qid;} private byte[] qid;
public byte[][] Sort_langs() {return sort_langs;} public void Sort_langs_(byte[][] v) {sort_langs = v;} private byte[][] sort_langs = Bry_.Ary_empty;
public Ordered_hash Slink_list() {if (slink_list == null) slink_list = mgr.Wdoc_parser(jdoc).Parse_sitelinks(qid, jdoc); return slink_list;} private Ordered_hash slink_list;

4
400_xowa/src/gplx/xowa/xtns/wbases/Wdata_wiki_mgr.java
View File

@@ -31,8 +31,8 @@ public class Wdata_wiki_mgr implements Gfo_evt_itm, Gfo_invk {
this.app = app;
this.evt_mgr = new Gfo_evt_mgr(this);
this.Qid_mgr = new Wbase_qid_mgr(this);
this.Pid_mgr = new Wbase_pid_mgr(this);
this.Doc_mgr = new Wbase_doc_mgr(this, this.Qid_mgr, app.Cache_mgr().Doc_cache());
this.Pid_mgr = new Wbase_pid_mgr(this);
this.Doc_mgr = new Wbase_doc_mgr(this, this.Qid_mgr);
this.prop_mgr = new Wbase_prop_mgr(Wbase_prop_mgr_loader_.New_db(this));
this.prop_val_visitor = new Wdata_prop_val_visitor(app, this);
this.Enabled_(true);

50
400_xowa/src/gplx/xowa/xtns/wbases/stores/Wbase_doc_cache.java Normal file
View File

@@ -0,0 +1,50 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.stores; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.core.lists.caches.*;
import gplx.core.logs.*;
public interface Wbase_doc_cache {
void Add(byte[] qid, Wdata_doc doc);
Wdata_doc Get_or_null(byte[] qid);
void Clear();
void Term();
}
class Wbase_doc_cache__null implements Wbase_doc_cache {
public void Add(byte[] qid, Wdata_doc doc) {}
public Wdata_doc Get_or_null(byte[] qid) {return null;}
public void Clear() {}
public void Term() {}
}
class Wbase_doc_cache__hash implements Wbase_doc_cache {
private final Hash_adp_bry hash = Hash_adp_bry.cs();
public void Add(byte[] qid, Wdata_doc doc) {hash.Add(qid, doc);}
public Wdata_doc Get_or_null(byte[] qid) {return (Wdata_doc)hash.Get_by_bry(qid);}
public void Clear() {hash.Clear();}
public void Term() {hash.Clear();}
}
class Wbase_doc_cache__mru implements Wbase_doc_cache {
private final Mru_cache_mgr cache;
public Wbase_doc_cache__mru(long cache_max, long compress_size, long used_weight) {
this.cache = Mru_cache_mgr.New_by_mb_secs(Gfo_log_wtr.New_dflt("wbase", "cache_mru_{0}.csv"), cache_max, compress_size, used_weight);
}
public void Add(byte[] qid, Wdata_doc doc) {cache.Add(String_.new_a7(qid), doc, doc.Jdoc_size());}
public Wdata_doc Get_or_null(byte[] qid) {return (Wdata_doc)cache.Get_or_null(String_.new_a7(qid));}
public void Clear() {}
public void Term() {
cache.Flush();
cache.Print();
}
}

85
400_xowa/src/gplx/xowa/xtns/wbases/stores/Wbase_doc_mgr.java
View File

@@ -14,22 +14,49 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.stores; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.core.logs.*; import gplx.core.primitives.*;
import gplx.langs.jsons.*;
import gplx.xowa.wikis.pages.*;
import gplx.xowa.apps.caches.*;
import gplx.xowa.xtns.wbases.core.*;
public class Wbase_doc_mgr {
private final Wdata_wiki_mgr wbase_mgr;
private final Wbase_qid_mgr qid_mgr;
private final Wbase_doc_cache doc_cache;
public Wbase_doc_mgr(Wdata_wiki_mgr wbase_mgr, Wbase_qid_mgr qid_mgr, Wbase_doc_cache doc_cache) {
private Wbase_doc_cache doc_cache;
private final Object thread_lock = new Object();
private final Ordered_hash wbase_db_hash = Ordered_hash_.New_bry();
private final Gfo_log_wtr wbase_db_log;
public Wbase_doc_mgr(Wdata_wiki_mgr wbase_mgr, Wbase_qid_mgr qid_mgr) {
this.wbase_mgr = wbase_mgr;
this.qid_mgr = qid_mgr;
this.doc_cache = doc_cache;
this.doc_cache = new Wbase_doc_cache__hash();
this.wbase_db_log = Gfo_log_wtr.New_dflt("wbase", "db_log_{0}.csv");
}
public void Enabled_(boolean v) {this.enabled = v;} private boolean enabled;
public void Cache__init(String cache_type, long cache_max, long compress_size, long used_weight) {
if (String_.Eq(cache_type, "null")) doc_cache = new Wbase_doc_cache__null();
else if (String_.Eq(cache_type, "hash")) doc_cache = new Wbase_doc_cache__hash();
else if (String_.Eq(cache_type, "mru" )) doc_cache = new Wbase_doc_cache__mru(cache_max, compress_size, used_weight);
else throw Err_.new_unhandled_default(cache_type);
}
public void Cleanup() {
doc_cache.Term();
wbase_db_log__flush();
}
private void wbase_db_log__flush() {
int len = wbase_db_hash.Len();
Bry_bfr tmp_bfr = Bry_bfr_.New();
for (int i = 0; i < len; i++) {
Wbase_db_log_itm itm = (Wbase_db_log_itm)wbase_db_hash.Get_at(i);
tmp_bfr.Add(itm.Ttl());
tmp_bfr.Add_byte_pipe().Add_int_variable(itm.Count());
tmp_bfr.Add_byte_pipe().Add_int_variable(itm.Elapsed());
tmp_bfr.Add_byte_nl();
wbase_db_log.Write(tmp_bfr);
}
wbase_db_log.Flush();
}
public void Clear() {
synchronized (doc_cache) { // LOCK:app-level
synchronized (thread_lock) { // LOCK:app-level
doc_cache.Clear();
}
}
@@ -43,7 +70,10 @@ public class Wbase_doc_mgr {
}
public Wdata_doc Get_by_exact_id_or_null(byte[] ttl_bry) {// must correct case and ns; EX:"Q2" or "Property:P1"; not "q2" or "P2"
// load from cache
Wdata_doc rv = doc_cache.Get_or_null(ttl_bry);
Wdata_doc rv = null;
synchronized (thread_lock) {
rv = doc_cache.Get_or_null(ttl_bry);
}
if (rv == null) {
// load from db
rv = Load_wdoc_or_null(ttl_bry);
@@ -54,7 +84,17 @@ public class Wbase_doc_mgr {
}
private Wdata_doc Load_wdoc_or_null(byte[] ttl_bry) { // EX:"Q2" or "Property:P1"
if (!enabled) return null;
synchronized (doc_cache) { // LOCK:app-level; jdoc_parser; moved synchronized higher up; DATE:2016-09-03
// loggging
Wbase_db_log_itm wbase_db_itm = (Wbase_db_log_itm)wbase_db_hash.Get_by(ttl_bry);
if (wbase_db_itm == null) {
wbase_db_itm = new Wbase_db_log_itm(ttl_bry);
wbase_db_hash.Add(ttl_bry, wbase_db_itm);
}
long time_bgn = gplx.core.envs.System_.Ticks();
Wdata_doc rv = null;
synchronized (thread_lock) { // LOCK:app-level; jdoc_parser; moved synchronized higher up; DATE:2016-09-03
byte[] cur_ttl_bry = ttl_bry;
int load_count = -1;
while (load_count < 2) { // limit to 2 tries (i.e.: 1 redirect)
@@ -62,18 +102,18 @@ public class Wbase_doc_mgr {
Xoa_ttl cur_ttl = wbase_mgr.Wdata_wiki().Ttl_parse(cur_ttl_bry);
if (cur_ttl == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "invalid wbase ttl: orig=~{0} cur=~{1}", ttl_bry, cur_ttl_bry);
return null;
break;
}
// get page
Xoae_page page = wbase_mgr.Wdata_wiki().Data_mgr().Load_page_by_ttl(cur_ttl);
if (!page.Db().Page().Exists()) return null;
if (!page.Db().Page().Exists()) break;
// parse jdoc
Json_doc jdoc = wbase_mgr.Jdoc_parser().Parse(page.Db().Text().Text_bry());
if (jdoc == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "invalid jdoc for ttl: orig=~{0} cur=~{1}", ttl_bry, cur_ttl_bry);
return null;
break;
}
// check for redirect; EX: {"entity":"Q22350516","redirect":"Q21006972"}; PAGE:fr.w:Tour_du_Táchira_2016; DATE:2016-08-13
@@ -81,22 +121,39 @@ public class Wbase_doc_mgr {
byte[] redirect_ttl = jdoc_root.Get_as_bry_or(Bry__redirect, null);
if (redirect_ttl != null) {
cur_ttl_bry = redirect_ttl;
load_count++;
continue;
}
// is json doc, and not a redirect; return
return new Wdata_doc(cur_ttl_bry, wbase_mgr, jdoc);
rv = new Wdata_doc(cur_ttl_bry, wbase_mgr, jdoc);
break;
}
Gfo_usr_dlg_.Instance.Warn_many("", "", "too many redirects for ttl: orig=~{0} cur=~{1}", ttl_bry, cur_ttl_bry);
if (rv == null && load_count >= 2)
Gfo_usr_dlg_.Instance.Warn_many("", "", "too many redirects for ttl: orig=~{0} cur=~{1}", ttl_bry, cur_ttl_bry);
}
return null;
wbase_db_itm.Update(gplx.core.envs.System_.Ticks__elapsed_in_frac(time_bgn));
return rv;
}
private static final byte[] Bry__redirect = Bry_.new_a7("redirect");
public void Add(byte[] full_db, Wdata_doc page) { // TEST:
synchronized (doc_cache) { // LOCK:app-level
synchronized (thread_lock) { // LOCK:app-level
if (doc_cache.Get_or_null(full_db) == null)
doc_cache.Add(full_db, page);
}
}
}
class Wbase_db_log_itm {
public Wbase_db_log_itm(byte[] ttl) {
this.ttl = ttl;
}
public byte[] Ttl() {return ttl;} private final byte[] ttl;
public int Count() {return count;} private int count;
public int Elapsed() {return elapsed;} private int elapsed;
public void Update(int elapsed_diff) {
count++;
this.elapsed += elapsed_diff;
}
}

2
400_xowa/src/gplx/xowa/xtns/wbases/stores/Wbase_pid_mgr.java
View File

@@ -42,7 +42,7 @@ public class Wbase_pid_mgr { // EX: "en|road_map" -> 15 ("Property:P15")
if (rv == -1) {
// get from db
rv = wbase_mgr.Wdata_wiki().Db_mgr().Load_mgr().Load_pid(lang_key, pid_name);
if (rv == Wbase_pid.Id_null) return Wbase_pid.Id_null;
if (rv == Wbase_pid.Id_null) rv = Wbase_pid.Id_null;
Add(pid_key, rv);
}
return rv;