Mass_parse: Do not reuse json_parser else threading issues with itwiki [#818]

master
gnosygnu 4 years ago
parent 541f5f6524
commit 845253af79

@ -106,7 +106,6 @@ public class Wdata_wiki_mgr implements Gfo_evt_itm, Gfo_invk {
}
return wdata_wiki;
} private Xowe_wiki wdata_wiki;
public Json_parser Jdoc_parser() {return jdoc_parser;} private Json_parser jdoc_parser = new Json_parser();
public void Init_by_app() {}
public Wdata_doc_parser Wdoc_parser(Json_doc jdoc) {
Json_kv itm_0 = Json_kv.Cast(jdoc.Root_nde().Get_at(0)); // get 1st node

@ -29,6 +29,7 @@ import gplx.Tfds;
import gplx.core.primitives.Gfo_number_parser;
import gplx.core.primitives.Int_obj_ref;
import gplx.langs.jsons.Json_doc;
import gplx.langs.jsons.Json_parser;
import gplx.xowa.Xoa_ttl;
import gplx.xowa.Xoae_app;
import gplx.xowa.Xoae_page;
@ -62,6 +63,7 @@ public class Wdata_wiki_mgr_fxt {
private Xoae_app app; private Xowe_wiki wiki; private Wdata_doc_bldr wdoc_bldr;
private final Wdata_xwiki_link_wtr wdata_lang_wtr = new Wdata_xwiki_link_wtr();
private final Bry_bfr tmp_time_bfr = Bry_bfr_.New();
private final Json_parser jsonParser = new Json_parser();
public Xowe_wiki Wiki() {return parser_fxt.Wiki();}
public Wdata_wiki_mgr_fxt Init() {return Init(new Xop_fxt(), true);}
public Wdata_wiki_mgr_fxt Init(Xop_fxt parser_fxt, boolean reset) {
@ -224,7 +226,7 @@ public class Wdata_wiki_mgr_fxt {
byte[] raw_bry = Bry_.new_a7(raw_str);
raw_bry = Bry_.new_u8(Json_doc.Make_str_by_apos(raw_str));
Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b512();
Wdata_wiki_mgr.Write_json_as_html(wdata_mgr.Jdoc_parser(), bfr, raw_bry);
Wdata_wiki_mgr.Write_json_as_html(jsonParser, bfr, raw_bry);
Tfds.Eq(expd, bfr.To_str_and_rls());
}
public static String New_json(String entity_id, String grp_key, String[] grp_vals) {

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,18 +13,56 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.bldrs.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.files.origs.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.langs.jsons.*;
import gplx.xowa.langs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.bldrs.cmds.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.xtns.wbases.*; import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.claims.*; import gplx.xowa.xtns.wbases.claims.itms.*;
package gplx.xowa.xtns.wbases.imports;
import gplx.Bool_;
import gplx.Bry_;
import gplx.Byte_ascii;
import gplx.Gfo_usr_dlg;
import gplx.Int_ary_;
import gplx.Ordered_hash;
import gplx.String_;
import gplx.dbs.Db_conn;
import gplx.dbs.Db_idx_itm;
import gplx.dbs.Db_stmt;
import gplx.dbs.Db_stmt_;
import gplx.dbs.cfgs.Db_cfg_tbl;
import gplx.dbs.engines.sqlite.Sqlite_engine_;
import gplx.langs.jsons.Json_doc;
import gplx.langs.jsons.Json_grp;
import gplx.langs.jsons.Json_itm;
import gplx.langs.jsons.Json_itm_;
import gplx.langs.jsons.Json_kv;
import gplx.langs.jsons.Json_nde;
import gplx.langs.jsons.Json_parser;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.bldrs.Xob_bldr;
import gplx.xowa.bldrs.Xob_cmd_keys;
import gplx.xowa.bldrs.Xob_db_file;
import gplx.xowa.bldrs.cmds.Xob_dump_mgr_base;
import gplx.xowa.bldrs.wkrs.Xob_cmd;
import gplx.xowa.langs.Xol_lang_itm_;
import gplx.xowa.wikis.data.tbls.Xowd_page_itm;
import gplx.xowa.wikis.nss.Xow_ns;
import gplx.xowa.wikis.nss.Xow_ns_;
import gplx.xowa.xtns.wbases.Wdata_doc;
import gplx.xowa.xtns.wbases.Wdata_wiki_mgr;
import gplx.xowa.xtns.wbases.claims.Wbase_claim_grp;
import gplx.xowa.xtns.wbases.claims.Wbase_claim_visitor;
import gplx.xowa.xtns.wbases.claims.itms.Wbase_claim_base;
import gplx.xowa.xtns.wbases.claims.itms.Wbase_claim_entity;
import gplx.xowa.xtns.wbases.claims.itms.Wbase_claim_globecoordinate;
import gplx.xowa.xtns.wbases.claims.itms.Wbase_claim_monolingualtext;
import gplx.xowa.xtns.wbases.claims.itms.Wbase_claim_quantity;
import gplx.xowa.xtns.wbases.claims.itms.Wbase_claim_string;
import gplx.xowa.xtns.wbases.claims.itms.Wbase_claim_time;
import gplx.xowa.xtns.wbases.claims.itms.Wbase_claim_value;
public class Xob_wdata_db_cmd extends Xob_dump_mgr_base implements Xob_cmd {
private Wdata_tbl_mgr tbl_mgr = new Wdata_tbl_mgr();
private Wdata_wiki_mgr wdata_mgr; private Json_parser json_parser;
private Wdata_wiki_mgr wdata_mgr;
private byte[] lang_key = Xol_lang_itm_.Key_en;
private final Json_parser json_parser = new Json_parser();
public Xob_wdata_db_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_wbase_db;}
@Override public byte Init_redirect() {return Bool_.N_byte;} // json will never be found in a redirect
@ -41,7 +79,6 @@ public class Xob_wdata_db_cmd extends Xob_dump_mgr_base implements Xob_cmd {
}
@Override protected void Cmd_bgn_end() {
wdata_mgr = bldr.App().Wiki_mgr().Wdata_mgr();
json_parser = wdata_mgr.Jdoc_parser();
tbl_mgr.Conn().Txn_bgn("bldr__wdata_db");
}
@Override public void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm page, byte[] page_src) {

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,23 +13,43 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.claims.enums.*; import gplx.xowa.xtns.wbases.parsers.*; import gplx.xowa.xtns.wbases.dbs.*;
import gplx.langs.jsons.*;
package gplx.xowa.xtns.wbases.imports;
import gplx.Bool_;
import gplx.Bry_;
import gplx.Gfo_invk;
import gplx.Ordered_hash;
import gplx.Ordered_hash_;
import gplx.String_;
import gplx.dbs.Db_conn;
import gplx.langs.jsons.Json_doc;
import gplx.langs.jsons.Json_parser;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.bldrs.Xob_bldr;
import gplx.xowa.bldrs.wkrs.Xob_itm_dump_base;
import gplx.xowa.bldrs.wkrs.Xob_page_wkr;
import gplx.xowa.wikis.data.tbls.Xowd_page_itm;
import gplx.xowa.xtns.wbases.Wdata_wiki_mgr;
import gplx.xowa.xtns.wbases.claims.enums.Wbase_claim_type;
import gplx.xowa.xtns.wbases.claims.enums.Wbase_claim_type_;
import gplx.xowa.xtns.wbases.claims.enums.Wbase_enum_hash;
import gplx.xowa.xtns.wbases.core.Wdata_dict_mainsnak;
import gplx.xowa.xtns.wbases.core.Wdata_langtext_itm;
import gplx.xowa.xtns.wbases.dbs.Wbase_pid_tbl;
import gplx.xowa.xtns.wbases.dbs.Xowb_prop_tbl;
import gplx.xowa.xtns.wbases.parsers.Wdata_doc_parser;
public class Xob_wdata_pid extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
private Db_conn conn;
private Wbase_pid_tbl tbl__pid;
private Xowb_prop_tbl tbl__prop;
private Json_parser jdoc_parser;
private final Ordered_hash datatype_hash = Ordered_hash_.New_bry();
private final Json_parser jdoc_parser = new Json_parser();
private final Ordered_hash datatype_hash = Ordered_hash_.New_bry();
public Xob_wdata_pid(Db_conn conn) {
this.conn = conn;
}
public Xob_wdata_pid Ctor(Xob_bldr bldr, Xowe_wiki wiki) {
this.Cmd_ctor(bldr, wiki);
this.jdoc_parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
return this;
}
public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_pid;}

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,24 +13,43 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.langs.jsons.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.dbs.*; import gplx.xowa.xtns.wbases.parsers.*;
package gplx.xowa.xtns.wbases.imports;
import gplx.Bry_;
import gplx.Bry_bfr;
import gplx.Bry_bfr_;
import gplx.Gfo_invk;
import gplx.Ordered_hash;
import gplx.String_;
import gplx.dbs.Db_conn;
import gplx.langs.jsons.Json_doc;
import gplx.langs.jsons.Json_parser;
import gplx.xowa.Xoa_ttl;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.bldrs.Xob_bldr;
import gplx.xowa.bldrs.wkrs.Xob_itm_dump_base;
import gplx.xowa.bldrs.wkrs.Xob_page_wkr;
import gplx.xowa.wikis.data.Xow_db_file;
import gplx.xowa.wikis.data.Xow_db_file_;
import gplx.xowa.wikis.data.Xow_db_mgr;
import gplx.xowa.wikis.data.tbls.Xowd_page_itm;
import gplx.xowa.wikis.nss.Xow_ns_;
import gplx.xowa.xtns.wbases.core.Wdata_sitelink_itm;
import gplx.xowa.xtns.wbases.dbs.Wbase_qid_tbl;
import gplx.xowa.xtns.wbases.parsers.Wdata_doc_parser;
public class Xob_wdata_qid extends Xob_itm_dump_base implements Xob_page_wkr, Gfo_invk {
private Db_conn conn;
private Wbase_qid_tbl tbl;
private final Object thread_lock = new Object();
private Json_parser parser; private Xob_wbase_ns_parser ns_parser; private final Xob_wbase_ns_parser_rslt ns_parser_rslt = new Xob_wbase_ns_parser_rslt();
private final Object thread_lock = new Object();
private final Json_parser parser = new Json_parser();
private Xob_wbase_ns_parser ns_parser; private final Xob_wbase_ns_parser_rslt ns_parser_rslt = new Xob_wbase_ns_parser_rslt();
public Xob_wdata_qid(Db_conn conn) {
this.conn = conn;
}
public String Page_wkr__key() {return gplx.xowa.bldrs.Xob_cmd_keys.Key_wbase_qid;}
public Xob_wdata_qid Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); return this;}
public void Page_wkr__bgn() {
this.parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
this.ns_parser = new Xob_wbase_ns_parser(bldr.App().Fsys_mgr().Cfg_site_meta_fil());
this.Qid__bgn();
}

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,16 +13,41 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.imports.json; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*; import gplx.xowa.xtns.wbases.imports.*;
import gplx.core.ios.*;
import gplx.langs.jsons.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.sqls.*; import gplx.xowa.apps.apis.xowa.bldrs.imports.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.parsers.*;
package gplx.xowa.xtns.wbases.imports.json;
import gplx.Bool_;
import gplx.Bry_;
import gplx.Byte_ascii;
import gplx.DateAdp;
import gplx.DateAdp_;
import gplx.Datetime_now;
import gplx.Gfo_usr_dlg;
import gplx.core.ios.Io_stream_zip_mgr;
import gplx.langs.jsons.Json_doc;
import gplx.langs.jsons.Json_parser;
import gplx.xowa.Xoae_app;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.Xowe_wiki_;
import gplx.xowa.bldrs.Xob_bldr;
import gplx.xowa.bldrs.Xob_ns_to_db_mgr;
import gplx.xowa.bldrs.Xobldr_cfg;
import gplx.xowa.bldrs.cmds.Xob_ns_file_itm;
import gplx.xowa.bldrs.cmds.texts.sqls.Xob_ns_to_db_wkr__text;
import gplx.xowa.wikis.data.Xow_db_file;
import gplx.xowa.wikis.data.Xow_db_file_;
import gplx.xowa.wikis.data.Xow_db_mgr;
import gplx.xowa.wikis.data.Xowd_cfg_key_;
import gplx.xowa.wikis.data.tbls.Xowd_page_tbl;
import gplx.xowa.wikis.nss.Xow_ns;
import gplx.xowa.wikis.nss.Xow_ns_mgr;
import gplx.xowa.xtns.wbases.Wdata_wiki_mgr;
import gplx.xowa.xtns.wbases.imports.Xob_wdata_pid;
import gplx.xowa.xtns.wbases.imports.Xob_wdata_qid;
class Xowb_json_dump_db {
private final Xoae_app app; private final Gfo_usr_dlg usr_dlg; private final Xowe_wiki wiki; private final Xob_bldr bldr;
private final Json_parser json_parser;
private final Xob_wdata_pid pid_cmd; private final Xob_wdata_qid qid_cmd;
private final Xoae_app app; private final Gfo_usr_dlg usr_dlg; private final Xowe_wiki wiki; private final Xob_bldr bldr;
private final Json_parser json_parser = new Json_parser();
private final Xob_wdata_pid pid_cmd; private final Xob_wdata_qid qid_cmd;
private Xow_ns_mgr ns_mgr; private Xow_db_mgr db_mgr;
private Xowd_page_tbl page_tbl; private Xob_ns_to_db_mgr ns_to_db_mgr;
private Io_stream_zip_mgr text_zip_mgr; private byte text_zip_tid;
@ -30,7 +55,6 @@ class Xowb_json_dump_db {
private int page_id = 0, page_count_main = 0;
public Xowb_json_dump_db(Xob_bldr bldr, Xowe_wiki wiki) {
this.app = bldr.App(); this.usr_dlg = app.Usr_dlg(); this.wiki = wiki; this.bldr = bldr;
this.json_parser = bldr.App().Wiki_mgr().Wdata_mgr().Jdoc_parser();
this.ns_mgr = wiki.Ns_mgr();
this.pid_cmd = new Xob_wdata_pid(wiki.Data__core_mgr().Db__wbase().Conn());
this.qid_cmd = new Xob_wdata_qid(wiki.Data__core_mgr().Db__wbase().Conn());
@ -98,5 +122,5 @@ class Xowb_json_dump_db {
db_core.Tbl__ns().Insert(ns_mgr); // save ns
db_mgr.Tbl__cfg().Insert_str(Xowd_cfg_key_.Grp__wiki_init, Xowd_cfg_key_.Key__init__modified_latest, page_modified_on.XtoStr_fmt(DateAdp_.Fmt_iso8561_date_time));
}
private static final byte[] Bry__id_key = Bry_.new_a7("id");
private static final byte[] Bry__id_key = Bry_.new_a7("id");
}

@ -1,6 +1,6 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
Copyright (C) 2012-2020 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
@ -13,18 +13,35 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.stores; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.core.logs.*; import gplx.core.primitives.*;
import gplx.langs.jsons.*;
import gplx.xowa.wikis.pages.*;
import gplx.xowa.xtns.wbases.core.*;
package gplx.xowa.xtns.wbases.stores;
import gplx.Bry_;
import gplx.Bry_bfr;
import gplx.Bry_bfr_;
import gplx.Err_;
import gplx.Gfo_usr_dlg_;
import gplx.Ordered_hash;
import gplx.Ordered_hash_;
import gplx.String_;
import gplx.core.logs.Gfo_log_wtr;
import gplx.langs.jsons.Json_doc;
import gplx.langs.jsons.Json_nde;
import gplx.langs.jsons.Json_parser;
import gplx.xowa.Xoa_ttl;
import gplx.xowa.Xoae_page;
import gplx.xowa.Xowe_wiki;
import gplx.xowa.xtns.wbases.Wdata_doc;
import gplx.xowa.xtns.wbases.Wdata_wiki_mgr;
import gplx.xowa.xtns.wbases.core.Wbase_pid;
public class Wbase_doc_mgr {
private final Wdata_wiki_mgr wbase_mgr;
private final Wbase_qid_mgr qid_mgr;
private final Wdata_wiki_mgr wbase_mgr;
private final Wbase_qid_mgr qid_mgr;
private Wbase_doc_cache doc_cache;
private final Object thread_lock = new Object();
private final Ordered_hash wbase_db_hash = Ordered_hash_.New_bry();
private final Gfo_log_wtr wbase_db_log;
private final Object thread_lock = new Object();
private final Ordered_hash wbase_db_hash = Ordered_hash_.New_bry();
private final Gfo_log_wtr wbase_db_log;
private final Json_parser jsonParser = new Json_parser();
public Wbase_doc_mgr(Wdata_wiki_mgr wbase_mgr, Wbase_qid_mgr qid_mgr) {
this.wbase_mgr = wbase_mgr;
this.qid_mgr = qid_mgr;
@ -110,7 +127,7 @@ public class Wbase_doc_mgr {
if (!page.Db().Page().Exists()) break;
// parse jdoc
Json_doc jdoc = wbase_mgr.Jdoc_parser().Parse(page.Db().Text().Text_bry());
Json_doc jdoc = jsonParser.Parse(page.Db().Text().Text_bry());
if (jdoc == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "invalid jdoc for ttl: orig=~{0} cur=~{1}", ttl_bry, cur_ttl_bry);
break;
@ -136,7 +153,7 @@ public class Wbase_doc_mgr {
wbase_db_itm.Update(gplx.core.envs.System_.Ticks__elapsed_in_frac(time_bgn));
return rv;
}
private static final byte[] Bry__redirect = Bry_.new_a7("redirect");
private static final byte[] Bry__redirect = Bry_.new_a7("redirect");
public void Add(byte[] full_db, Wdata_doc page) { // TEST:
synchronized (thread_lock) { // LOCK:app-level
@ -149,7 +166,7 @@ class Wbase_db_log_itm {
public Wbase_db_log_itm(byte[] ttl) {
this.ttl = ttl;
}
public byte[] Ttl() {return ttl;} private final byte[] ttl;
public byte[] Ttl() {return ttl;} private final byte[] ttl;
public int Count() {return count;} private int count;
public int Elapsed() {return elapsed;} private int elapsed;
public void Update(int elapsed_diff) {

Loading…
Cancel
Save