1
0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
This commit is contained in:
gnosygnu
2015-08-17 02:09:16 -04:00
parent 34c34f227c
commit df10db140c
421 changed files with 4867 additions and 2429 deletions

View File

@@ -52,7 +52,7 @@ public class Xob_cmd_keys {
, Key_wbase_qid = "wbase.qid" // "text.wdata.qid"
, Key_wbase_pid = "wbase.pid" // "text.wdata.pid"
, Key_wbase_db = "wbase.db" // "wiki.wdata_db"
, Key_wbase_ns = "wbase.ns"
, Key_site_meta = "util.site_meta"
, Key_tdb_text_init = "tdb.text.init" // "core.init"
, Key_tdb_make_page = "tdb.text.page" // "core.make_page"
, Key_tdb_make_id = "core.make_id"

View File

@@ -60,7 +60,7 @@ public class Xob_cmd_mgr implements GfoInvkAble {
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_qid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_qid_sql().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_sql().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_db)) return Add(new Xob_wdata_db_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_ns)) return Add(new Xob_site_ns_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_site_meta)) return Add(new Xob_site_meta_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_init)) return Add(new Xob_init_tdb(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_page)) return Xml_rdr_direct_add(wiki, new Xob_page_txt(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_id)) return Xml_rdr_direct_add(wiki, new Xob_make_id_wkr(bldr, wiki));

View File

@@ -94,7 +94,7 @@ public abstract class Xob_ctg_v1_base extends Xob_itm_dump_base implements Xobd_
if (delete_temp) Io_mgr.I.DeleteDirDeep(temp_dir);
}
private Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
Btrie_fast_mgr trie = Btrie_fast_mgr.cs_().Add_stub(Tid_brack_end, "]]").Add_stub(Tid_pipe, "|").Add_stub(Tid_nl, "\n").Add_stub(Tid_brack_bgn, "[[");
Btrie_fast_mgr trie = Btrie_fast_mgr.cs().Add_stub(Tid_brack_end, "]]").Add_stub(Tid_pipe, "|").Add_stub(Tid_nl, "\n").Add_stub(Tid_brack_bgn, "[[");
static final int row_fixed_len = 5 + 1 + 1; // 5=rowId; 1=|; 1=\n
List_adp category_list = List_adp_.new_(); Int_obj_ref cur_pos = Int_obj_ref.zero_();
static final byte Tid_eos = 0, Tid_brack_end = 1, Tid_pipe = 2, Tid_nl = 3, Tid_brack_bgn = 4;

View File

@@ -73,14 +73,14 @@ public class Xob_fsdb_make_cmd extends Xob_itm_basic_base implements Xob_cmd {
this.trg_atr_fil = trg_mnt_itm.Atr_mgr().Db__core();
this.trg_cfg_mgr = trg_mnt_itm.Cfg_mgr();
bin_db_mgr.Init_by_mnt_mgr(trg_mnt_mgr);
trg_atr_fil.Conn().Txn_bgn();
trg_atr_fil.Conn().Txn_bgn("bldr__fsdb_make__trg_atr_fil");
if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn())) // need to create txn for v1; DATE:2015-07-04
trg_cfg_mgr.Tbl().Conn().Txn_bgn();
trg_cfg_mgr.Tbl().Conn().Txn_bgn("bldr__fsdb_make__trg_cfg_fil");
// bldr_db
Xob_db_file bldr_db = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir());
this.bldr_conn = bldr_db.Conn();
this.bldr_cfg_tbl = bldr_db.Tbl__cfg(); // NOTE: cfg and atr is in same db; use it
bldr_cfg_tbl.Conn().Txn_bgn();
bldr_cfg_tbl.Conn().Txn_bgn("bldr__fsdb_make__bldr_cfg_tbl");
}
public void Cmd_run() {
Init_bldr_bmks();
@@ -129,7 +129,7 @@ public class Xob_fsdb_make_cmd extends Xob_itm_basic_base implements Xob_cmd {
Db_cfg_hash bmk_hash = bldr_cfg_tbl.Select_as_hash(Cfg_fsdb_make);
String tier_id_str = bmk_hash.Get(Cfg_tier_id_bmk).To_str_or(null);
if (tier_id_str == null) { // bmks not found; new db;
bldr_conn.Txn_bgn();
bldr_conn.Txn_bgn("bldr__fsdb_make__bldr_conn");
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_tier_id_bmk , tier_id_bmk);
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_page_id_bmk , page_id_bmk);
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_lnki_id_bmk , lnki_id_bmk);
@@ -227,21 +227,21 @@ public class Xob_fsdb_make_cmd extends Xob_itm_basic_base implements Xob_cmd {
}
private void Make_trg_bin_file(boolean try_nth, Xodb_tbl_oimg_xfer_itm fsdb, long src_rdr_len) {
boolean make = true, txn_bgn = true;
if (trg_bin_fil != null) { // pre-existing bin_file;
if (trg_mnt_itm.Db_mgr().File__solo_file())
txn_bgn = false; // solo file; do nothing
else
trg_bin_fil.Conn().Txn_end(); // close txn before making new db
}
int tier_id = fsdb.Lnki_tier_id();
Xob_bin_db_itm nth_bin_db = bin_db_mgr.Get_nth_by_tier(tier_id);
if (try_nth) { // try_nth is true; occurs for new runs or changed tier
if ( nth_bin_db.Id() != -1 // nth exists;
if (try_nth) { // try_nth is true; occurs for new runs or changed tier
if ( nth_bin_db.Id() != -1 // nth exists;
&& nth_bin_db.Db_len() + src_rdr_len < trg_bin_db_max) { // if src_rdr_len exceeds
make = false; // do not make; use existing
txn_bgn = false;
make = false; // do not make; use existing
}
}
if (make) { // no nth; make it;
if (trg_bin_fil != null) { // pre-existing bin_file;
trg_bin_fil.Conn().Txn_end(); // close txn before making new db
if (trg_mnt_itm.Db_mgr().File__solo_file())
txn_bgn = false;
}
if (make) { // no nth; make it;
int ns_id = bin_db_mgr.Get_ns_id(tier_id);
int pt_id = bin_db_mgr.Increment_pt_id(nth_bin_db);
String new_bin_db_name = bin_db_mgr.Gen_name(wiki.Domain_str(), ns_id, pt_id);
@@ -252,12 +252,12 @@ public class Xob_fsdb_make_cmd extends Xob_itm_basic_base implements Xob_cmd {
Fsdb_db_mgr__v2_bldr.Make_cfg_data(wiki, trg_atr_fil.Url_rel(), trg_bin_db, Xowd_db_file_.Tid_file_data, trg_bin_fil.Id() + List_adp_.Base1);
}
}
else { // nth available; use it
else { // nth available; use it
this.trg_bin_fil = trg_mnt_itm.Bin_mgr().Dbs__get_at(nth_bin_db.Id());
trg_bin_fil.Bin_len_(nth_bin_db.Db_len());
}
if (txn_bgn)
trg_bin_fil.Conn().Txn_bgn();
trg_bin_fil.Conn().Txn_bgn("bldr__fsdb_make__trg_bin_fil");
}
private void Txn_sav() {
usr_dlg.Prog_many("", "", "committing data: count=~{0} failed=~{1}", exec_count, exec_fail);

View File

@@ -40,7 +40,7 @@ class Xob_lnki_temp_tbl {
public Xob_lnki_temp_tbl(Db_conn conn) {this.conn = conn;}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public void Create_tbl() {conn.Ddl_create_tbl(Db_meta_tbl.new_(Tbl_name, flds));}
public void Insert_bgn() {conn.Txn_bgn(); stmt_insert = conn.Stmt_insert(Tbl_name, flds);}
public void Insert_bgn() {conn.Txn_bgn("bldr__lnki_temp"); stmt_insert = conn.Stmt_insert(Tbl_name, flds);}
public void Insert_commit() {conn.Txn_sav();}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public void Insert_cmd_by_batch(int tier_id, int page_id, byte[] ttl, byte[] ttl_commons, byte ext_id, byte img_type, byte lnki_src_tid, int w, int h, double upright, double time, int page) {

View File

@@ -24,7 +24,7 @@ public class Xob_xfer_regy_cmd extends Xob_itm_basic_base implements Xob_cmd {
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_run() {
Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
conn.Txn_bgn();
conn.Txn_bgn("bldr__xfer_regy");
Xob_xfer_regy_tbl.Create_table(conn);
Xob_xfer_regy_tbl.Create_data(usr_dlg, conn);
Xob_xfer_regy_tbl.Create_index(usr_dlg, conn);

View File

@@ -37,7 +37,7 @@ public class Xob_xfer_regy_update_cmd extends Xob_itm_basic_base implements Xob_
Io_url fsdb_atr_url = ((gplx.dbs.engines.sqlite.Sqlite_conn_info)conn.Conn_info()).Url();
Sqlite_engine_.Tbl_create_and_delete(make_db_provider, Xob_fsdb_regy_tbl.Tbl_name, Xob_fsdb_regy_tbl.Tbl_sql);
Sqlite_engine_.Db_attach(make_db_provider, "fsdb_db", fsdb_atr_url.Raw());
make_db_provider.Txn_bgn();
make_db_provider.Txn_bgn("bldr__xfer_regy_update");
make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Update_regy_nil);
make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Insert_fsdb_fil);
String fsdb_thm_tbl = fsdb_abc_mgr.Db_mgr().File__schema_is_1() ? "fsdb_xtn_thm" : "fsdb_thm";
@@ -51,7 +51,7 @@ public class Xob_xfer_regy_update_cmd extends Xob_itm_basic_base implements Xob_
Sqlite_engine_.Db_detach(make_db_provider, "fsdb_db");
}
private void Update_status(Db_conn make_db_provider) {
make_db_provider.Txn_bgn();
make_db_provider.Txn_bgn("bldr__xfer_regy_update_status");
make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Update_regy_fil);
make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Update_regy_thm);
make_db_provider.Txn_end();

View File

@@ -26,7 +26,7 @@ public class Xob_xfer_temp_cmd_orig extends Xob_itm_basic_base implements Xob_cm
Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
Xob_xfer_temp_tbl.Create_table(conn);
Db_stmt trg_stmt = Xob_xfer_temp_tbl.Insert_stmt(conn);
conn.Txn_bgn();
conn.Txn_bgn("bldr__xfer_temp");
DataRdr rdr = conn.Exec_sql_as_rdr(Sql_select);
long[] ext_maxs = Calc_ext_max();
while (rdr.MoveNextPeer()) {

View File

@@ -25,7 +25,7 @@ public class Xob_xfer_temp_cmd_thumb extends Xob_itm_basic_base implements Xob_c
Db_conn conn = Xob_db_file.new__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
Xob_xfer_temp_tbl.Create_table(conn);
Db_stmt trg_stmt = Xob_xfer_temp_tbl.Insert_stmt(conn);
conn.Txn_bgn();
conn.Txn_bgn("bldr__xfer_temp_thumb");
DataRdr rdr = conn.Exec_sql_as_rdr(Sql_select);
Xob_xfer_temp_itm temp_itm = new Xob_xfer_temp_itm();
Xof_img_size img_size = new Xof_img_size();

View File

@@ -32,7 +32,7 @@ public class Xob_css_cmd implements Xob_cmd {
usr_dlg.Plog_many("", "", Cmd_key() + ":css_dir; dir=~{0}", css_dir.Raw());
wiki.Init_db_mgr(); // NOTE: must follow Install b/c Init_assert also calls Install; else will download any css from db
Xowd_db_file core_db = wiki.Db_mgr_as_sql().Core_data_mgr().Db__core();
core_db.Conn().Txn_bgn();
core_db.Conn().Txn_bgn("bldr__css");
core_db.Tbl__css_core().Create_tbl();
core_db.Tbl__css_file().Create_tbl();
gplx.xowa.html.css.Xowd_css_core_mgr.Set(core_db.Tbl__css_core(), core_db.Tbl__css_file(), css_dir, css_key);

View File

@@ -36,7 +36,7 @@ public class Xob_page_cmd extends Xob_itm_basic_base implements Xobd_wkr, GfoInv
this.ns_to_db_mgr = new Xob_ns_to_db_mgr(new Xob_ns_to_db_wkr__text(), db_mgr, import_cfg.Text_db_max());
if (redirect_id_enabled) {
this.redirect_tbl = new Xob_redirect_tbl(wiki.Fsys_mgr().Root_dir(), Xoa_app_.Utl__encoder_mgr().Http_url_ttl()).Create_table();
redirect_tbl.Conn().Txn_bgn();
redirect_tbl.Conn().Txn_bgn("bldr__page__redirect");
}
this.dg_match_mgr = app.Api_root().Bldr().Wiki().Filter().Dansguardian().New_mgr(wiki.Domain_str(), wiki.Fsys_mgr().Root_dir());
app.Bldr().Dump_parser().Trie_tab_del_(); // disable swapping &#09; for \t

View File

@@ -0,0 +1,34 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.xowa.bldrs.*;
import gplx.xowa.wmfs.*; import gplx.xowa.wmfs.data.*;
import gplx.xowa.wikis.domains.*;
public class Xob_site_meta_cmd implements Xob_cmd {
private final Xob_bldr bldr;
public Xob_site_meta_cmd(Xob_bldr bldr, Xow_wiki wiki) {this.bldr = bldr;}
public String Cmd_key() {return Xob_cmd_keys.Key_site_meta;}
public void Cmd_run() {
Site_meta_itm.Build_site_meta(bldr.App().Wmf_mgr(), bldr.App().Fsys_mgr().Cfg_site_meta_fil(), Xow_wmf_api_mgr.Wikis, DateAdp_.Now());
}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_end() {}
public void Cmd_term() {}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return GfoInvkAble_.Rv_unhandled;}
}

View File

@@ -37,7 +37,7 @@ public class Xob_image_cmd extends Xob_itm_dump_base implements Xob_cmd, GfoInvk
parser.Src_fil_(src_fil).Trg_fil_gen_(dump_url_gen).Fld_cmd_(this).Flds_req_idx_(20, Fld_img_name, Fld_img_size, Fld_img_width, Fld_img_height, Fld_img_bits, Fld_img_media_type, Fld_img_minor_mime, Fld_img_timestamp);
this.conn = Xob_db_file.new__wiki_image(wiki.Fsys_mgr().Root_dir()).Conn();
conn.Txn_bgn();
conn.Txn_bgn("bldr__image");
this.tbl_image = new Xob_image_tbl();
tbl_image.Create_table(conn);
this.stmt = tbl_image.Insert_stmt(conn);

View File

@@ -30,6 +30,7 @@ class Xobd_page_dump_tbl {
}
public void Create_data(Io_url page_db_url, int text_db_id) {
conn.Ddl_create_tbl(Db_meta_tbl.new_(Tbl_name, flds));
conn.Stmt_delete(Tbl_name).Exec_delete(); // always clear tables again; allows commands to be rerun; DATE:2015-08-04
Db_attach_cmd.new_(conn, "page_db", page_db_url)
.Add_fmt("text_db_prep.clone_page", Sql_insert_data, text_db_id)
.Exec();

View File

@@ -35,7 +35,7 @@ public class Xob_redirect_cmd extends Xob_dump_mgr_base {
encoder = Xoa_app_.Utl__encoder_mgr().Http_url_ttl();
redirect_tbl = new Xob_redirect_tbl(wiki.Fsys_mgr().Root_dir(), Xoa_app_.Utl__encoder_mgr().Http_url_ttl()).Create_table();
conn = redirect_tbl.Conn();
conn.Txn_bgn();
conn.Txn_bgn("bldr__redirect");
return conn;
}
@Override protected void Cmd_bgn_end() {}

View File

@@ -40,7 +40,7 @@ public class Xoa_css_extractor {
Xof_download_wkr download_wkr = app.Wmf_mgr().Download_wkr();
this.download_xrg = download_wkr.Download_xrg();
css_img_downloader = new Xoa_css_img_downloader().Ctor(usr_dlg, download_wkr, Bry_.new_u8(protocol_prefix));
failover_dir = app.Fsys_mgr().Bin_any_dir().GenSubDir_nest("html", "xowa", "import");
failover_dir = app.Fsys_mgr().Bin_xowa_dir().GenSubDir_nest("html", "css", "failover");
url_encoder = Xoa_app_.Utl__encoder_mgr().Http_url();
}
public void Install(Xow_wiki wiki, String css_key) {

View File

@@ -42,6 +42,6 @@ class Crt__match_exact implements Criteria {
}
public void Val_as_obj_(Object v) {Val_as_bry_ary_((byte[][])v);}
public void Val_from_args(Hash_adp args) {throw Err_.new_unimplemented();}
public String XtoStr() {return String_.Concat_any(this.To_str_name(), " ", String_.Ary(ary));}
public String To_str() {return String_.Concat_any(this.To_str_name(), " ", String_.Ary(ary));}
public byte Tid_match_exact = 12;
}

View File

@@ -31,7 +31,7 @@ class Dg_log_mgr {
tbl_rule.Conn_(conn, created);
tbl_page_score.Conn_(conn, created);
tbl_page_rule.Conn_(conn, created);
conn.Txn_bgn();
conn.Txn_bgn("dansguardian");
}
public void Insert_file(Dg_file file) {tbl_file.Insert(file.Id(), file.Rel_path(), file.Lines().length);}
public void Insert_rule(Dg_rule rule) {tbl_rule.Insert(rule.File_id(), rule.Id(), rule.Idx(), rule.Score(), Dg_word.Ary_concat(rule.Words(), tmp_bfr, Byte_ascii.Tilde));}

View File

@@ -31,7 +31,7 @@ public class Xob_info_file {
public String Core_file_name() {return core_file_name;} private final String core_file_name;
public String Orig_file_name() {return orig_file_name;} private final String orig_file_name;
public void Save(Db_cfg_tbl tbl) {
tbl.Conn().Txn_bgn();
tbl.Conn().Txn_bgn("make__info__file");
tbl.Insert_int (Cfg_grp, Cfg_key__id , id);
tbl.Insert_str (Cfg_grp, Cfg_key__type , type);
tbl.Insert_str (Cfg_grp, Cfg_key__ns_ids , ns_ids);

View File

@@ -28,7 +28,7 @@ public class Xob_info_session {
public DateAdp Time() {return time;} private final DateAdp time;
public Guid_adp Uuid() {return guid;} private final Guid_adp guid;
public void Save(Db_cfg_tbl tbl) {
tbl.Conn().Txn_bgn();
tbl.Conn().Txn_bgn("make__info__session");
tbl.Insert_str (Cfg_grp, Cfg_key__user , user);
tbl.Insert_str (Cfg_grp, Cfg_key__version , version);
tbl.Insert_str (Cfg_grp, Cfg_key__wiki_domain , wiki_domain);

View File

@@ -19,7 +19,7 @@ package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xow
import gplx.core.btries.*;
public class Xob_xml_parser_ {
public static Btrie_fast_mgr trie_() {
Btrie_fast_mgr rv = Btrie_fast_mgr.cs_();
Btrie_fast_mgr rv = Btrie_fast_mgr.cs();
trie_add(rv, Bry_page_bgn, Id_page_bgn); trie_add(rv, Bry_page_bgn_frag, Id_page_bgn_frag); trie_add(rv, Bry_page_end, Id_page_end);
trie_add(rv, Bry_id_bgn, Id_id_bgn); trie_add(rv, Bry_id_bgn_frag, Id_id_bgn_frag); trie_add(rv, Bry_id_end, Id_id_end);
trie_add(rv, Bry_title_bgn, Id_title_bgn); trie_add(rv, Bry_title_bgn_frag, Id_title_bgn_frag); trie_add(rv, Bry_title_end, Id_title_end);