Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.4.1.1'

Browse Source
This commit is contained in:
gnosygnu
2016-04-04 21:26:29 -04:00
parent baaef32df2
commit b2a06b8125
252 changed files with 2882 additions and 2118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
400_xowa/src/gplx/xowa/bldrs/Db_mgr_fxt.java
View File

@@ -19,7 +19,8 @@ package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.primitives.*; import gplx.core.strings.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.ctgs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.infos.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.infos.*;
public class Db_mgr_fxt {
public Db_mgr_fxt Ctor_fsys() {bldr_fxt = new Xob_fxt().Ctor(Xoa_test_.Url_root().GenSubDir("root")); return this;}
public Db_mgr_fxt Ctor_mem() {bldr_fxt = new Xob_fxt().Ctor_mem(); return this;} private Xob_fxt bldr_fxt;
@@ -120,7 +121,7 @@ public class Db_mgr_fxt {
}
public void Init_db_sqlite() {
Xowe_wiki wiki = this.Wiki();
Db_conn_pool.Instance.Clear();
Db_conn_pool.Instance.Rls_all();
Db_conn_bldr.Instance.Reg_default_sqlite();
Io_mgr.Instance.DeleteDir_cmd(wiki.Fsys_mgr().Root_dir()).MissingIgnored_().Exec();
wiki.Db_mgr_create_as_sql().Core_data_mgr().Init_by_make(Xowd_core_db_props.Test, Xob_info_session.Test);

3
400_xowa/src/gplx/xowa/bldrs/Xob_base_fxt.java
View File

@@ -16,7 +16,8 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.ios.*; import gplx.xowa.bldrs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.core.ios.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_base_fxt {
public Xob_base_fxt Clear() {
if (app == null) {

57
400_xowa/src/gplx/xowa/bldrs/Xob_bldr.java
View File

@@ -18,15 +18,18 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.consoles.*; import gplx.core.envs.*;
import gplx.xowa.apps.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.xmls.*; import gplx.xowa.bldrs.cfgs.*; import gplx.xowa.langs.bldrs.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.langs.jsons.*;
public class Xob_bldr implements GfoInvkAble {
private boolean pause_at_end = false; private long prv_prog_time; private Xob_xml_parser dump_parser;
public Xob_bldr(Xoae_app app) {
this.app = app;
this.cmd_mgr = new Xob_cmd_mgr(this);
this.cmd_mgr = new Xob_cmd_mgr(this, cmd_regy);
this.import_marker = new Xob_import_marker();
this.wiki_cfg_bldr = new Xob_wiki_cfg_bldr(this);
}
public Xoae_app App() {return app;} private final Xoae_app app;
public Xob_cmd_regy Cmd_regy() {return cmd_regy;} private final Xob_cmd_regy cmd_regy = new Xob_cmd_regy();
public Xob_cmd_mgr Cmd_mgr() {return cmd_mgr;} private final Xob_cmd_mgr cmd_mgr;
public Gfo_usr_dlg Usr_dlg() {return app.Usr_dlg();}
public int Sort_mem_len() {return sort_mem_len;} public Xob_bldr Sort_mem_len_(int v) {sort_mem_len = v; return this;} private int sort_mem_len = 16 * Io_mgr.Len_mb;
@@ -42,6 +45,46 @@ public class Xob_bldr implements GfoInvkAble {
if (pct_idx > -1) ary[pct_idx] = Decimal_adp_.CalcPctStr(cur, end, "00.00");
app.Usr_dlg().Prog_many("", "", fmt, ary);
}
public Xob_bldr Exec_json(String script) {
try {
this.cmd_mgr.Clear();
Json_parser jdoc_parser = new Json_parser();
Json_doc jdoc = jdoc_parser.Parse(script);
Json_ary cmds = jdoc.Root_ary();
int cmds_len = cmds.Len();
for (int i = 0; i < cmds_len; ++i) {
Json_nde cmd = cmds.Get_at_as_nde(i);
byte[] key = cmd.Get_bry_or_null("key");
Xob_cmd prime = cmd_regy.Get_or_null(String_.new_u8(key));
if (prime == null) throw Err_.new_("bldr", "bldr.cmd does not exists: cmd={0}", key);
byte[] wiki_key = cmd.Get_bry_or_null("wiki");
Xowe_wiki wiki = wiki_key == null ? app.Usere().Wiki() : app.Wiki_mgr().Get_by_or_make(wiki_key);
Xob_cmd clone = prime.Cmd_new(this, wiki);
int atrs_len = cmd.Len();
for (int j = 0; j < atrs_len; ++j) {
Json_kv atr_kv = cmd.Get_at_as_kv(j);
String atr_key = atr_kv.Key_as_str();
if ( String_.Eq(atr_key, "key")
|| String_.Eq(atr_key, "wiki")) continue;
byte[] atr_val = atr_kv.Val_as_bry();
GfoInvkAble_.InvkCmd_val(clone, atr_key + GfoInvkAble_.Mutator_suffix, String_.new_u8(atr_val));
}
cmd_mgr.Add(clone);
}
gplx.core.threads.Thread_adp_.invk_("bldr_by_json", this, Invk_run_by_kit).Start();
} catch (Exception e) {
app.Gui_mgr().Kit().Ask_ok("", "", "error: ~{0}", Err_.Message_gplx_log(e));
}
return this;
}
private void Run_by_kit() { // same as Run, but shows exception; don't want to change backward compatibility on Run
try {this.Run();}
catch (Exception e) {
String log_msg = Err_.Message_gplx_log(e);
Xoa_app_.Usr_dlg().Log_many("", "", log_msg);
app.Gui_mgr().Kit().Ask_ok("", "", "error: ~{0}", Err_.Message_gplx_full(e));
}
}
public void Run() {
try {
app.Bldr__running_(true);
@@ -57,9 +100,13 @@ public class Xob_bldr implements GfoInvkAble {
Xob_cmd cmd = cmd_mgr.Get_at(i);
app.Usr_dlg().Note_many("", "", "cmd bgn: ~{0}", cmd.Cmd_key());
long time_cur = Env_.TickCount();
cmd.Cmd_bgn(this);
cmd.Cmd_run();
cmd.Cmd_end();
try {
cmd.Cmd_bgn(this);
cmd.Cmd_run();
cmd.Cmd_end();
} catch (Exception e) {
throw Err_.new_exc(e, "bldr", "unknown error", "key", cmd.Cmd_key());
}
Env_.GarbageCollect();
app.Usr_dlg().Note_many("", "", "cmd end: ~{0} ~{1}", cmd.Cmd_key(), TimeSpanAdp_.from_(time_cur).XtoStrUiAbbrv());
}
@@ -91,6 +138,7 @@ public class Xob_bldr implements GfoInvkAble {
else if (ctx.Match(k, Invk_dump_fil_len_)) dump_fil_len = gplx.core.ios.Io_size_.Load_int_(m);
else if (ctx.Match(k, Invk_make_fil_len_)) make_fil_len = gplx.core.ios.Io_size_.Load_int_(m);
else if (ctx.Match(k, Invk_run)) Run();
else if (ctx.Match(k, Invk_run_by_kit)) Run_by_kit();
else if (ctx.Match(k, Invk_cancel)) Cancel();
else return GfoInvkAble_.Rv_unhandled;
return this;
@@ -99,6 +147,7 @@ public class Xob_bldr implements GfoInvkAble {
Invk_cmds = "cmds", Invk_wiki_cfg_bldr = "wiki_cfg_bldr"
, Invk_pause_at_end_ = "pause_at_end_", Invk_sort_mem_len_ = "sort_mem_len_", Invk_dump_fil_len_ = "dump_fil_len_", Invk_make_fil_len_ = "make_fil_len_"
, Invk_cancel = "cancel"
, Invk_run_by_kit = "run_by_kit"
;
public static final String Invk_run = "run";
}

24
400_xowa/src/gplx/xowa/bldrs/Xob_cmd_keys.java
View File

@@ -28,30 +28,11 @@ public class Xob_cmd_keys {
, Key_text_cat_link = "text.cat.link" // "import.sql.categorylinks"
, Key_text_cat_hidden = "text.cat.hidden" // "import.sql.hiddencat"
, Key_text_term = "text.term" // "import.sql.term"
, Key_wiki_redirect = "wiki.redirect" // "wiki.redirect"
, Key_wiki_image = "wiki.image" // "wiki.image"
, Key_wiki_page_dump_make = "wiki.page_dump.make" // "file.page_dump"
, Key_wiki_page_dump_drop = "wiki.page_dump.drop"
, Key_wiki_page_link = "wiki.page_link"
, Key_file_lnki_temp = "file.lnki_temp"
, Key_file_lnki_regy = "file.lnki_regy"
, Key_file_page_regy = "file.page_regy"
, Key_file_orig_regy = "file.orig_regy"
, Key_file_xfer_temp_thumb = "file.xfer_temp.thumb"
, Key_file_xfer_temp_orig = "file.xfer_temp.orig"
, Key_file_xfer_regy = "file.xfer_regy"
, Key_file_xfer_regy_update = "file.xfer_regy_update"
, Key_file_fsdb_make = "file.fsdb_make"
, Key_file_fsdb_reduce = "file.fsdb_reduce"
, Key_file_orig_reg = "file.orig_reg"
, Key_file_xfer_update = "file.xfer_update"
, Key_html_redlinks = "html.redlinks"
, Key_util_cleanup = "util.cleanup" // "core.cleanup"
, Key_util_download = "util.download" // "file.download"
, Key_util_xml_dump = "util.xml_dump"
, Key_util_random = "util.random"
, Key_util_sqlite_normalize = "util.sqlite.normalize"
, Key_util_alert = "util.alert"
, Key_util_delete = "util.delete"
, Key_wbase_json_dump = "wbase.json_dump"
, Key_wbase_qid = "wbase.qid" // "text.wdata.qid"
@@ -61,9 +42,6 @@ public class Xob_cmd_keys {
, Key_diff_build = "diff.build"
, Key_diff_merge = "diff.merge"
, Key_text_delete_page = "text.delete_page"
, Key_search__page__page_score = "search.page__page_score"
, Key_search__link__link_score = "search.link__link_score"
, Key_search__word__link_count = "search.word__link_count"
, Key_tdb_text_init = "tdb.text.init" // "core.init"
, Key_tdb_make_page = "tdb.text.page" // "core.make_page"
@@ -78,8 +56,6 @@ public class Xob_cmd_keys {
, Key_tdb_cat_hidden_ttl = "ctg.hiddencat_ttl"
, Key_tdb_text_wdata_qid = "tdb.text.wdata.qid"
, Key_tdb_text_wdata_pid = "tdb.text.wdata.pid"
, Key_diff_regy_exec = "file.diff_regy.exec"
, Key_diff_regy_make = "file.diff_regy.make"
, Key_exec_sql = "import.sql.exec_sql"
, Key_deploy_zip = "deploy.zip"
, Key_deploy_copy = "deploy.copy"

40
400_xowa/src/gplx/xowa/bldrs/Xob_cmd_mgr.java
View File

@@ -18,21 +18,29 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.primitives.*;
import gplx.xowa.wikis.*; import gplx.xowa.xtns.wdatas.imports.*;
import gplx.xowa.bldrs.cmds.texts.*; import gplx.xowa.bldrs.cmds.texts.sqls.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*; import gplx.xowa.bldrs.cmds.files.*; import gplx.xowa.bldrs.cmds.ctgs.*; import gplx.xowa.bldrs.cmds.utils.*; import gplx.xowa.bldrs.cmds.wikis.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.cmds.texts.*; import gplx.xowa.bldrs.cmds.texts.sqls.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*; import gplx.xowa.addons.builds.files.*; import gplx.xowa.bldrs.cmds.ctgs.*; import gplx.xowa.bldrs.cmds.utils.*;
import gplx.xowa.bldrs.cmds.diffs.*;
import gplx.xowa.files.origs.*; import gplx.xowa.htmls.core.bldrs.*;
import gplx.xowa.addons.searchs.dbs.bldrs.*;
import gplx.xowa.addons.apps.searchs.bldrs.*; import gplx.xowa.addons.builds.files.cmds.*;
public class Xob_cmd_mgr implements GfoInvkAble {
public Xob_cmd_mgr(Xob_bldr bldr) {this.bldr = bldr;} private Xob_bldr bldr;
private final Xob_bldr bldr;
public final Xob_cmd_regy cmd_regy;
public Xob_cmd_mgr(Xob_bldr bldr, Xob_cmd_regy cmd_regy) {this.bldr = bldr; this.cmd_regy = cmd_regy;}
public void Clear() {list.Clear(); dump_rdrs.Clear();}
public int Len() {return list.Count();} List_adp list = List_adp_.new_();
public int Len() {return list.Count();} private final List_adp list = List_adp_.new_();
public Xob_cmd Get_at(int i) {return (Xob_cmd)list.Get_at(i);}
public Xob_cmd Add(Xob_cmd cmd) {list.Add(cmd); return cmd;}
public GfoInvkAble Add_cmd(Xowe_wiki wiki, String cmd_key) {
Xob_cmd prototype = cmd_regy.Get_or_null(cmd_key);
if (prototype != null) {
Xob_cmd clone = prototype.Cmd_new(bldr, wiki);
Add(clone);
return clone;
}
if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_init)) return Add(new Xob_init_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_page)) return Xml_rdr_direct_add(wiki, new Xob_page_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_css)) return Add(new Xob_css_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_search_wkr)) return Xml_rdr_direct_add(wiki, new gplx.xowa.addons.searchs.dbs.bldrs.Srch_bldr_wkr(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_search_wkr)) return Xml_rdr_direct_add(wiki, new gplx.xowa.addons.apps.searchs.bldrs.Srch_bldr_wkr(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_search_cmd)) return Add(new Srch_bldr_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_core_v1)) return Xml_rdr_parser_add(wiki, new Xob_ctg_v1_sql().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_core)) return Add(new Xob_category_registry_sql(bldr, wiki));
@@ -40,22 +48,6 @@ public class Xob_cmd_mgr implements GfoInvkAble {
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_cat_hidden)) return Add(new Xoctg_hiddencat_parser_sql(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_term)) return Add(new Xob_term_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_text_delete_page)) return Add(new Xob_page_delete_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wiki_page_dump_make)) return Add(new Xob_page_dump_cmd_make(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wiki_page_dump_drop)) return Add(new Xob_page_dump_cmd_drop(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wiki_redirect)) return Add(new Xob_redirect_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wiki_image)) return Add(new Xob_image_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wiki_page_link)) return Add(new gplx.xowa.addons.pagelinks.bldrs.Pglnk_bldr_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_lnki_temp)) return Add(new Xob_lnki_temp_wkr(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_lnki_regy)) return Add(new Xob_lnki_regy_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_page_regy)) return Add(new Xob_page_regy_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_orig_regy)) return Add(new Xob_orig_regy_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_temp_thumb)) return Add(new Xob_xfer_temp_cmd_thumb(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_temp_orig)) return Add(new Xob_xfer_temp_cmd_orig(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_regy)) return Add(new Xob_xfer_regy_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_regy_update)) return Add(new Xob_xfer_regy_update_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_fsdb_make)) return Add(new Xob_fsdb_make_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_orig_reg)) return Add(new Xob_orig_tbl_bldr(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_file_xfer_update)) return Add(new Xob_xfer_update_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_html_redlinks)) return Add(new Xob_redlink_mkr_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_util_cleanup)) return Add(new Xob_cleanup_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_util_delete)) return Add(new Xob_delete_cmd(bldr, wiki));
@@ -66,10 +58,6 @@ public class Xob_cmd_mgr implements GfoInvkAble {
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_sql().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_wbase_db)) return Add(new Xob_wdata_db_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_site_meta)) return Add(new Xob_site_meta_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_search__page__page_score)) return Add(new gplx.xowa.addons.searchs.dbs.bldrs.cmds.Srch__page__page_score(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_search__link__link_score)) return Add(new gplx.xowa.addons.searchs.dbs.bldrs.cmds.Srch__link__link_score(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_search__word__link_count)) return Add(new gplx.xowa.addons.searchs.dbs.bldrs.cmds.Srch__word__link_count(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_util_sqlite_normalize)) return Add(new gplx.xowa.addons.sqlite_utils.bldrs.Sqlite_percentile_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_init)) return Add(new Xob_init_tdb(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_make_page)) return Xml_rdr_direct_add(wiki, new Xob_page_txt(bldr, wiki));
@@ -85,8 +73,6 @@ public class Xob_cmd_mgr implements GfoInvkAble {
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_qid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_qid_txt().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_tdb_text_wdata_pid)) return Xml_rdr_direct_add(wiki, new Xob_wdata_pid_txt().Ctor(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_diff_build)) return Add(new Xob_diff_build_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_diff_regy_exec)) return Add(new Xob_diff_regy_exec_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_diff_regy_make)) return Add(new Xob_diff_regy_make_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_exec_sql)) return Add(new Xob_exec_sql_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_decompress_bz2)) return Add(new Xob_decompress_bz2_cmd(bldr, wiki));
else if (String_.Eq(cmd_key, Xob_cmd_keys.Key_deploy_zip)) return Add(new Xob_deploy_zip_cmd(bldr, wiki));

30
400_xowa/src/gplx/xowa/bldrs/Xob_cmd_regy.java Normal file
View File

@@ -0,0 +1,30 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_cmd_regy {
private final Ordered_hash regy = Ordered_hash_.New();
public Xob_cmd Get_or_null(String key) {return (Xob_cmd)regy.Get_by(key);}
public void Add_many(Xob_cmd... ary) {
int len = ary.length;
for (int i = 0; i < len; ++i) {
Xob_cmd cmd = ary[i];
regy.Add(cmd.Cmd_key(), cmd);
}
}
}

4
400_xowa/src/gplx/xowa/bldrs/Xob_db_file.java
View File

@@ -32,6 +32,10 @@ public class Xob_db_file {
public static Xob_db_file New__temp_log(Io_url dir) {return New(dir, Name__temp_log);}
public static Xob_db_file New__redlink(Io_url dir) {return New(dir, Name__redlink);}
public static Xob_db_file New__page_link(Xow_wiki wiki) {return New(wiki.Fsys_mgr().Root_dir(), Name__page_link);}
public static Xob_db_file New__deletion_db(Xow_wiki wiki) {
String name = String_.Format("{0}-file-core-deletion_db-{1}.xowa", wiki.Domain_str(), DateAdp_.Now().XtoStr_fmt("yyyy.MM"));
return New(wiki.Fsys_mgr().Root_dir(), name);
}
public static Xob_db_file New(Io_url dir, String name) {
Io_url url = dir.GenSubFil(name);
Db_conn_bldr_data conn_data = Db_conn_bldr.Instance.Get_or_new(url);

1
400_xowa/src/gplx/xowa/bldrs/Xob_fxt.java
View File

@@ -18,6 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.tests.*; import gplx.core.ios.*; import gplx.core.times.*;
import gplx.dbs.*; import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.cmds.texts.tdbs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_fxt {
public Xob_fxt Ctor_mem() {
Io_mgr.Instance.InitEngine_mem();

3
400_xowa/src/gplx/xowa/bldrs/Xob_page_wkr_cmd.java
View File

@@ -18,11 +18,12 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
import gplx.core.consoles.*; import gplx.core.ios.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.parsers.tmpls.*;
import gplx.xowa.bldrs.xmls.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.xmls.*;
public class Xob_page_wkr_cmd implements Xob_cmd {
private final Xob_bldr bldr; private final Xowe_wiki wiki;
public Xob_page_wkr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki;}
public String Cmd_key() {return KEY;} public static final String KEY = "dump_mgr";
public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public void Cmd_run() {
Xob_page_wkr[] wkr_ary = (Xob_page_wkr[])wkrs.To_ary(Xob_page_wkr.class); int wkr_ary_len = wkr_ary.length;
for (int i = 0; i < wkr_ary_len; i++)

5
400_xowa/src/gplx/xowa/bldrs/cmds/Xob_dump_mgr_base.java
View File

@@ -17,10 +17,11 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.envs.*;
import gplx.dbs.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.bldrs.cmds.files.*; import gplx.xowa.files.origs.*;
import gplx.dbs.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.addons.builds.files.*; import gplx.xowa.files.origs.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.addons.builds.files.utls.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*;
public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xob_cmd, GfoInvkAble {
private Xob_dump_src_id page_src;
@@ -209,7 +210,7 @@ public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xo
private void Notify_restoring(String itm, int val) {
usr_dlg.Note_many("", "", "restoring: itm=~{0} val=~{1}", itm, val);
}
public static final String
public static final String
Invk_progress_interval_ = "progress_interval_", Invk_commit_interval_ = "commit_interval_", Invk_cleanup_interval_ = "cleanup_interval_", Invk_rate_interval_ = "rate_interval_"
, Invk_select_size_ = "select_size_"
, Invk_ns_bgn_ = "ns_bgn_", Invk_db_bgn_ = "db_bgn_", Invk_pg_bgn_ = "pg_bgn_"

4
400_xowa/src/gplx/xowa/bldrs/cmds/ctgs/Xob_category_registry_sql.java
View File

@@ -16,12 +16,14 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.ctgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.ios.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.bldrs.*;
import gplx.core.ios.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.dbs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.bldrs.wtrs.*;
public class Xob_category_registry_sql implements Xob_cmd {
public Xob_category_registry_sql(Xob_bldr bldr, Xowe_wiki wiki) {this.wiki = wiki;} private Xowe_wiki wiki;
public String Cmd_key() {return Xob_cmd_keys.Key_text_cat_core;}
public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public void Cmd_end() { // NOTE: placing in end, b/c must run *after* page_sql
// setup
wiki.Html_mgr().Importing_ctgs_(Bool_.Y);

4
400_xowa/src/gplx/xowa/bldrs/cmds/diffs/Xob_diff_build_cmd.java
View File

@@ -16,12 +16,14 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.diffs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_diff_build_cmd implements Xob_cmd {
private final Xob_bldr bldr; private final Xowe_wiki wiki;
private final Xob_bldr bldr; private final Xowe_wiki wiki;
private String prev_url, curr_url, diff_url; private int commit_interval;
private int[] db_ids = Int_.Ary_empty; private String bld_name = "all";
public Xob_diff_build_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki;}
public String Cmd_key() {return Xob_cmd_keys.Key_diff_build;}
public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public void Cmd_run() {
new Xob_diff_build_wkr(bldr, wiki, prev_url, curr_url, diff_url, commit_interval, new Xowd_tbl_mapr(bld_name, db_ids)).Exec();
}

10
400_xowa/src/gplx/xowa/bldrs/cmds/diffs/Xob_diff_build_wkr.java
View File

@@ -20,10 +20,10 @@ import gplx.core.brys.*; import gplx.core.brys.fmts.*;
import gplx.dbs.*; import gplx.dbs.metas.*; import gplx.dbs.diffs.*; import gplx.dbs.diffs.builds.*; import gplx.dbs.diffs.itms.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
class Xob_diff_build_wkr {
private final Gfdb_diff_bldr dif_bldr = new Gfdb_diff_bldr();
private final Xowe_wiki wiki;
private final Gfdb_diff_bldr dif_bldr = new Gfdb_diff_bldr();
private final Xowe_wiki wiki;
private Db_conn old_conn, new_conn, dif_conn;
private final Xowd_tbl_mapr tbl_mapr;
private final Xowd_tbl_mapr tbl_mapr;
public Xob_diff_build_wkr(Xob_bldr bldr, Xowe_wiki wiki, String old_url, String new_url, String dif_url, int commit_interval, Xowd_tbl_mapr tbl_mapr) {
this.wiki = wiki;
wiki.Init_by_wiki();
@@ -62,8 +62,8 @@ class Xob_diff_build_wkr {
// }
}
private void Compare(Gdif_bldr_ctx ctx) {
Dbmeta_tbl_mgr old_tbl_mgr = old_conn.Meta_load_all();
Dbmeta_tbl_mgr new_tbl_mgr = old_conn.Meta_load_all();
Dbmeta_tbl_mgr old_tbl_mgr = old_conn.Meta_mgr();
Dbmeta_tbl_mgr new_tbl_mgr = old_conn.Meta_mgr();
int new_tbl_len = new_tbl_mgr.Len();
for (int i = 0; i < new_tbl_len; ++i) {
Dbmeta_tbl_itm new_tbl = new_tbl_mgr.Get_at(i);

57
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_bin_db_itm.java
View File

@@ -1,57 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.fsdb.meta.*;
class Xob_bin_db_itm {
public Xob_bin_db_itm(int id, Io_url db_url, int ns_id, int pt_id) {this.id = id; this.db_url = db_url; this.ns_id = ns_id; this.pt_id = pt_id;}
public int Id() {return id;} private int id;
public int Ns_id() {return ns_id;} private final int ns_id;
public int Pt_id() {return pt_id;} private int pt_id;
public long Db_len() {return db_len;} public void Db_len_(long v) {this.db_len = v;} private long db_len;
public Io_url Db_url() {return db_url;} public void Db_url_(Io_url v) {db_url = v;} private Io_url db_url;
public void Set(int id, int pt_id, Io_url db_url) {
this.id = id; this.pt_id = pt_id; this.db_url = db_url;
}
public static String Gen_name_v1(int pt_id) {
return String_.Format("fsdb.bin.{0}.sqlite3", Int_.To_str_pad_bgn_zero(pt_id, 4));
}
public static String Gen_name_v2(String domain_str, int ns_id, int pt_id) {
String ns_id_str = Int_.To_str_pad_bgn_zero(ns_id, 3);
String pt_id_str = Int_.To_str_pad_bgn_zero(pt_id, 3);
return String_.Format("{0}-file-ns.{1}-db.{2}.xowa", domain_str, ns_id_str, pt_id_str);
}
public static Xob_bin_db_itm new_v1(Fsm_bin_fil fil) {
byte[] name = Bry_.new_u8(fil.Url_rel()); // EX: "fsdb.bin.0000.sqlite3"
int ns_id = 0; // assume v1 dbs are all in main ns
int pt_id = Bry_.To_int_or(name, 9 , 13, Int_.Min_value); if (pt_id == Int_.Min_value) throw Err_.new_wo_type("bin_db_itm.parse: invalid pt_id", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Raw());
return new Xob_bin_db_itm(fil.Id(), fil.Url(), ns_id, pt_id);
}
public static Xob_bin_db_itm new_v2(Fsm_bin_fil fil) {
byte[] ns_bgn_tkn = Bry_.new_a7("file-ns."), ns_end_tkn = Bry_.new_a7("-db."), pt_end_tkn = Bry_.new_a7(".xowa");
int ns_bgn_tkn_len = ns_bgn_tkn.length, ns_end_tkn_len = ns_end_tkn.length;
byte[] name = Bry_.new_u8(fil.Url_rel()); // EX: en.wikipedia.org-file-ns.000-db.001.xowa
int ns_bgn = Bry_find_.Find_fwd(name, ns_bgn_tkn, 0); if (ns_bgn == Bry_find_.Not_found) throw Err_.new_wo_type("bin_db_itm.parse: invalid ns_bgn", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Raw());
ns_bgn += ns_bgn_tkn_len;
int ns_end = Bry_find_.Find_fwd(name, ns_end_tkn, ns_bgn); if (ns_end == Bry_find_.Not_found) throw Err_.new_wo_type("bin_db_itm.parse: invalid ns_end", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Raw());
int pt_bgn = ns_end + ns_end_tkn_len;
int pt_end = Bry_find_.Find_fwd(name, pt_end_tkn, pt_bgn); if (pt_end == Bry_find_.Not_found) throw Err_.new_wo_type("bin_db_itm.parse: invalid pt_end", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Raw());
int ns_id = Bry_.To_int_or(name, ns_bgn, ns_end, Int_.Min_value); if (ns_id == Int_.Min_value) throw Err_.new_wo_type("bin_db_itm.parse: invalid ns_id", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Raw());
int pt_id = Bry_.To_int_or(name, pt_bgn, pt_end, Int_.Min_value); if (pt_id == Int_.Min_value) throw Err_.new_wo_type("bin_db_itm.parse: invalid pt_id", "name", fil.Url_rel(), "conn", fil.Conn().Conn_info().Raw());
return new Xob_bin_db_itm(fil.Id(), fil.Url(), ns_id, pt_id);
}
}

71
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_bin_db_mgr.java
View File

@@ -1,71 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.primitives.*; import gplx.core.ios.*;
import gplx.fsdb.meta.*;
class Xob_bin_db_mgr {
private final int[] ns_ids; private final int ns_ids_len;
private final Ordered_hash nth_hash = Ordered_hash_.New(); private final Int_obj_ref tier_key = Int_obj_ref.neg1_();
public Xob_bin_db_mgr(int[] ns_ids) {
this.ns_ids = ns_ids; this.ns_ids_len = ns_ids.length;
}
public boolean Schema_is_1() {return schema_is_1;} private boolean schema_is_1;
public void Init_by_mnt_mgr(Fsm_mnt_mgr trg_mnt_mgr) {
Fsm_mnt_itm trg_mnt_itm = trg_mnt_mgr.Mnts__get_main();
this.schema_is_1 = trg_mnt_itm.Db_mgr().File__schema_is_1();
Fsm_bin_mgr bin_db_mgr = trg_mnt_itm.Bin_mgr();
int len = ns_ids_len;
for (int i = 0; i < len; ++i) { // iterate ns_ids and add default nth
int ns_id = ns_ids[i];
Xob_bin_db_itm nth = new Xob_bin_db_itm(-1, null, ns_id, 0);
nth_hash.Add(Int_obj_ref.new_(ns_ids[i]), nth);
}
len = bin_db_mgr.Dbs__len();
for (int i = 0; i < len; ++i) { // iterate bin_dbs to find max pt_id for each ns
Fsm_bin_fil fil = bin_db_mgr.Dbs__get_at(i);
Xob_bin_db_itm itm = schema_is_1 ? Xob_bin_db_itm.new_v1(fil) : Xob_bin_db_itm.new_v2(fil);
int ns_id = itm.Ns_id();
Xob_bin_db_itm nth = (Xob_bin_db_itm)nth_hash.Get_by(tier_key.Val_(ns_id));
if (itm.Pt_id() > nth.Pt_id()) // update max pt_id
nth.Set(itm.Id(), itm.Pt_id(), itm.Db_url()); // note that ns_id is same
}
len = nth_hash.Count();
for (int i = 0; i < len; ++i) { // iterated tiers to calculate max_size
Xob_bin_db_itm nth = (Xob_bin_db_itm)nth_hash.Get_at(i);
if (nth.Id() == -1) continue; // ignore default nth
IoItmFil nth_itm = Io_mgr.Instance.QueryFil(nth.Db_url());
nth.Db_len_(nth_itm.Size());
}
}
public boolean Tier_id_is_last(int tier_id) {return tier_id >= ns_ids_len;} // assumes tier_id is 0 based; EX: 0,1,2 for
public int Get_ns_id(int tier_id) {return ns_ids[tier_id];}
public int Increment_pt_id(Xob_bin_db_itm itm) {
itm.Set(-1, itm.Pt_id() + 1, null);
itm.Db_len_(0);
return itm.Pt_id();
}
public String Gen_name(String domain_str, int ns_id, int pt_id) {
return schema_is_1 ? Xob_bin_db_itm.Gen_name_v1(pt_id) : Xob_bin_db_itm.Gen_name_v2(domain_str, ns_id, pt_id);
}
public Xob_bin_db_itm Get_nth_by_tier(int tier_id) {
if (schema_is_1) return (Xob_bin_db_itm)nth_hash.Get_by(tier_key.Val_(0)); // v1 is always in ns_0
if (tier_id >= ns_ids_len) throw Err_.new_wo_type("tier out of range", "tier_id", tier_id, "len", ns_ids_len);
int ns_id = ns_ids[tier_id];
return (Xob_bin_db_itm)nth_hash.Get_by(tier_key.Val_(ns_id));
}
}

101
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_diff_regy_exec_cmd.java
View File

@@ -1,101 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.fsdb.*; import gplx.fsdb.meta.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_diff_regy_exec_cmd extends Xob_itm_basic_base implements Xob_cmd {
private Io_url sql_dir;
public Xob_diff_regy_exec_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_diff_regy_exec;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_run() {Exec_main();}
public void Cmd_end() {}
public void Cmd_term() {}
private void Exec_main() {
if (sql_dir == null)
sql_dir = wiki.Parser_mgr().Ctx().App().Fsys_mgr().File_dir().GenSubDir_nest(wiki.Domain_str(), "tmp_sql");
Xob_diff_regy_sql_runner runner = new Xob_diff_regy_sql_runner();
Io_url[] urls = Io_mgr.Instance.QueryDir_fils(sql_dir);
int urls_len = urls.length;
for (int i = 0; i < urls_len; ++i)
runner.Exec(app, urls[i]);
Xob_diff_regy_sql_runner.Get_conn(wiki, 0, Fsdb_db_tid_.Tid_atr).Exec_sql("VACUUM;");
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_sql_dir_)) sql_dir = m.ReadIoUrl("v");
else return super.Invk(ctx, ikey, k, m);
return this;
} private static final String Invk_sql_dir_ = "sql_dir_";
}
class Xob_diff_regy_sql_runner {
public Io_url Url() {return url;} private Io_url url;
public String Wiki_domain() {return wiki_domain;} private String wiki_domain;
public int Fsdb_db_id() {return fsdb_db_id;} private int fsdb_db_id;
public byte Fsdb_db_tid() {return fsdb_db_tid;} private byte fsdb_db_tid;
public void Exec(Xoae_app app, Io_url url) {
Parse_url(url);
Run_sql(app);
}
public void Parse_url(Io_url url) {
this.url = url;
String[] parts = String_.Split(url.NameOnly(), "-");
wiki_domain = parts[0];
fsdb_db_id = Int_.parse(parts[1]);
fsdb_db_tid = Fsdb_db_tid_.Xto_tid(parts[2]);
}
public void Run_sql(Xoae_app app) {
Xowe_wiki wiki = (Xowe_wiki)app.Wiki_mgr().Get_by_or_null(Bry_.new_u8(wiki_domain));
app.Usr_dlg().Prog_many("", "", "running sql: url=~{0}", url.NameAndExt());
Db_conn conn = Get_conn(wiki, fsdb_db_id, fsdb_db_tid);
conn.Exec_sql(Io_mgr.Instance.LoadFilStr(url));
if (fsdb_db_tid == Fsdb_db_tid_.Tid_bin)
conn.Exec_sql("VACUUM;");
}
public static Db_conn Get_conn(Xowe_wiki wiki, int fsdb_db_id, byte fsdb_db_tid) {
wiki.File_mgr().Init_file_mgr_by_load(wiki);
Fsm_mnt_itm abc_mgr = wiki.File_mgr().Fsdb_mgr().Mnt_mgr().Mnts__get_main();
if (fsdb_db_tid == Fsdb_db_tid_.Tid_bin)
return abc_mgr.Bin_mgr().Dbs__get_at(fsdb_db_id).Conn();
else if (fsdb_db_tid == Fsdb_db_tid_.Tid_atr)
return abc_mgr.Atr_mgr().Db__core().Conn();
else
throw Err_.new_unhandled(fsdb_db_tid);
}
public static String Build_url(String wiki_domain, int fsdb_db_id, String fsdb_db_type) {
return String_.Format("{0}-{1}-{2}.sql", wiki_domain, Int_.To_str_pad_bgn_zero(fsdb_db_id, 3), fsdb_db_type);
}
}
class Fsdb_db_tid_ {
public static final byte Tid_cfg = 0, Tid_atr = 1, Tid_bin = 2;
public static final String Key_cfg = "cfg", Key_atr = "atr", Key_bin = "bin";
public static byte Xto_tid(String s) {
if (String_.Eq(s, Key_cfg)) return Tid_cfg;
else if (String_.Eq(s, Key_atr)) return Tid_atr;
else if (String_.Eq(s, Key_bin)) return Tid_bin;
else throw Err_.new_unhandled(s);
}
public static String Xto_key(byte v) {
switch (v) {
case Tid_cfg: return Key_cfg;
case Tid_atr: return Key_atr;
case Tid_bin: return Key_bin;
default: throw Err_.new_unhandled(v);
}
}
}

45
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_diff_regy_exec_cmd_tst.java
View File

@@ -1,45 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*;
import gplx.fsdb.*;
public class Xob_diff_regy_exec_cmd_tst {
private Xob_diff_regy_exec_cmd_fxt fxt = new Xob_diff_regy_exec_cmd_fxt();
@Test public void Xto_commons() {
fxt.Test_build_url("enwiki", 1, "atr", "enwiki-001-atr.sql");
fxt.Test_parse_url("/file/enwiki-001-atr.sql", "enwiki", 1, Fsdb_db_tid_.Tid_atr);
}
// @Test public void Smoke() {
// Xoae_app app = Xoa_app_fxt.Make__app__edit(Io_url_.new_dir_("C:\\xowa\\"), "wnt");
// Xowe_wiki wiki = Xoa_app_fxt.Make__wiki__edit(app);
// Xob_diff_regy_make_cmd cmd = new Xob_diff_regy_make_cmd(app.Bldr(), wiki);
// cmd.Cmd_run();
// }
}
class Xob_diff_regy_exec_cmd_fxt {
public void Test_build_url(String wiki_domain, int fsdb_db_id, String fsdb_db_type, String expd) {
Tfds.Eq(expd, Xob_diff_regy_sql_runner.Build_url(wiki_domain, fsdb_db_id, fsdb_db_type));
}
public void Test_parse_url(String raw, String expd_wiki_domain, int expd_fsdb_db_id, byte expd_fsdb_db_tid) {
Xob_diff_regy_sql_runner runner = new Xob_diff_regy_sql_runner();
runner.Parse_url(Io_url_.new_any_(raw));
Tfds.Eq(expd_wiki_domain, runner.Wiki_domain());
Tfds.Eq(expd_fsdb_db_id, runner.Fsdb_db_id());
Tfds.Eq(expd_fsdb_db_tid, runner.Fsdb_db_tid());
}
}

139
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_diff_regy_make_cmd.java
View File

@@ -1,139 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.stores.*;
import gplx.dbs.*; import gplx.fsdb.*; import gplx.dbs.engines.sqlite.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_diff_regy_make_cmd extends Xob_itm_basic_base implements Xob_cmd {
public Xob_diff_regy_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_diff_regy_make;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_run() {Exec_main();}
public void Cmd_end() {}
public void Cmd_term() {}
private void Exec_main() {
Db_conn make_db_provider = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
this.Make_join_indexes(make_db_provider);
this.Make_diff_regy(make_db_provider);
this.Make_delete_sql(make_db_provider);
}
private void Make_join_indexes(Db_conn make_db_provider) {
try {
make_db_provider.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_name("fsdb_regy", "fsdb_regy__join", "fsdb_name", "fsdb_is_orig", "fsdb_repo", "fsdb_w", "fsdb_time", "fsdb_page"));
make_db_provider.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_name("xfer_regy", "xfer_regy__join", "lnki_ttl", "file_is_orig", "orig_repo", "file_w", "lnki_time", "lnki_page"));
}
catch (Exception exc) {
app.Usr_dlg().Warn_many("", "", "error while making indexes: err=~{0}", Err_.Message_gplx_full(exc));
}
}
private void Make_diff_regy(Db_conn make_db_provider) {
Sqlite_engine_.Tbl_create_and_delete(make_db_provider, Xob_diff_regy_tbl.Tbl_name, Xob_diff_regy_tbl.Tbl_sql);
make_db_provider.Exec_sql(Xob_diff_regy_tbl.Make_diff_regy);
Sqlite_engine_.Idx_create(make_db_provider, Xob_diff_regy_tbl.Idx_diff_regy__load);
}
private void Make_delete_sql(Db_conn make_db_provider) {
DataRdr rdr = make_db_provider.Exec_sql_as_old_rdr(Xob_diff_regy_tbl.Make_deletes);
int cur_db_id = -1, cur_count = 0;
Bry_bfr atr_bfr = Bry_bfr.new_(), bin_bfr = Bry_bfr.new_();
Io_url sql_tmp_dir = wiki.App().Fsys_mgr().File_dir().GenSubDir_nest(wiki.Domain_str(), "tmp_sql");
while (rdr.MoveNextPeer()) {
byte diff_is_orig = rdr.ReadByte("diff_is_orig");
int diff_db_id = rdr.ReadInt("diff_db_id");
int diff_fil_id = rdr.ReadInt("diff_fil_id");
int diff_thm_id = rdr.ReadInt("diff_thm_id");
if (cur_db_id != diff_db_id) {
Make_delete_sql_file(atr_bfr, bin_bfr, sql_tmp_dir, cur_db_id, cur_count);
cur_db_id = diff_db_id;
cur_count = 0;
}
Make_delete_sql_item(atr_bfr, bin_bfr, diff_is_orig, diff_db_id, diff_fil_id, diff_thm_id);
++cur_count;
}
}
private void Make_delete_sql_file(Bry_bfr atr_bfr, Bry_bfr bin_bfr, Io_url sql_tmp_dir, int cur_db_id, int cur_count) {
Make_delete_sql_file(atr_bfr, sql_tmp_dir, cur_db_id, cur_count, Fsdb_db_tid_.Tid_atr);
Make_delete_sql_file(bin_bfr, sql_tmp_dir, cur_db_id, cur_count, Fsdb_db_tid_.Tid_bin);
app.Usr_dlg().Note_many("", "", "file.diff:sql generated: db_id=~{0} count=~{1}", Int_.To_str_pad_bgn_space(cur_db_id, 3), Int_.To_str_pad_bgn_space(cur_count, 7));
}
private void Make_delete_sql_file(Bry_bfr bfr, Io_url sql_dir, int db_id, int cur_count, byte db_tid) {
if (db_id != -1 && cur_count > 0) { // do not write 1st bfr
bfr.Add_str_a7("COMMIT;\n");
String sql_url_name = String_.Format("{0}-{1}-{2}.sql", wiki.Domain_str(), Int_.To_str_pad_bgn_zero(db_id, 3), Fsdb_db_tid_.Xto_key(db_tid));
Io_url sql_url = sql_dir.GenSubFil(sql_url_name);
Io_mgr.Instance.SaveFilBfr(sql_url, bfr);
}
bfr.Clear(); // clear bfr if cur_count == 0
bfr.Add_str_a7("BEGIN TRANSACTION;\n");
}
private void Make_delete_sql_item(Bry_bfr atr_bfr, Bry_bfr bin_bfr, byte diff_is_orig, int diff_db_id, int diff_fil_id, int diff_thm_id) {
if (diff_is_orig == Byte_.Zero) {
atr_bfr.Add_str_a7("DELETE FROM fsdb_xtn_thm WHERE thm_id = " + Int_.To_str(diff_thm_id) + ";\n");
bin_bfr.Add_str_a7("DELETE FROM fsdb_bin WHERE bin_owner_id = " + Int_.To_str(diff_thm_id) + ";\n");
}
else {
atr_bfr.Add_str_a7("UPDATE fsdb_fil SET fil_bin_db_id = -1 WHERE fil_id = " + Int_.To_str(diff_fil_id) + ";\n");
bin_bfr.Add_str_a7("DELETE FROM fsdb_bin WHERE bin_owner_id = " + Int_.To_str(diff_fil_id) + ";\n");
}
}
}
class Xob_diff_regy_tbl {
public static final String Tbl_name = "diff_regy";
public static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE diff_regy"
, "( diff_id integer NOT NULL PRIMARY KEY"
, ", diff_is_orig tinyint NOT NULL "
, ", diff_status integer NOT NULL"
, ", diff_db_id integer NOT NULL"
, ", diff_fil_id integer NOT NULL"
, ", diff_thm_id integer NOT NULL"
, ", diff_name varchar(255) NOT NULL"
, ", diff_repo tinyint NOT NULL"
, ", diff_w integer NOT NULL"
, ", diff_time double NOT NULL"
, ", diff_page integer NOT NULL"
, ", diff_size bigint NOT NULL"
, ");"
);
// public static final Db_idx_itm Idx_fsdb_regy__join = Db_idx_itm.sql_("CREATE INDEX fsdb_regy__join ON fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page);");
// public static final Db_idx_itm Idx_xfer_regy__join = Db_idx_itm.sql_("CREATE INDEX xfer_regy__join ON xfer_regy (lnki_ttl , file_is_orig, orig_repo, file_w, lnki_time, lnki_page);");
public static final Db_idx_itm Idx_diff_regy__load = Db_idx_itm.sql_("CREATE INDEX diff_regy__load ON diff_regy (diff_status, diff_db_id, diff_is_orig, diff_fil_id, diff_thm_id);");
public static final String
Make_diff_regy = String_.Concat_lines_nl
( "INSERT INTO diff_regy "
, " (diff_id, diff_is_orig, diff_db_id, diff_fil_id, diff_thm_id, diff_name, diff_repo, diff_w, diff_time, diff_page, diff_size, diff_status)"
, "SELECT "
, " f.fsdb_id, f.fsdb_is_orig, f.fsdb_db_id, f.fsdb_fil_id, f.fsdb_thm_id, f.fsdb_name, f.fsdb_repo, f.fsdb_w, f.fsdb_time, f.fsdb_page, f.fsdb_size, CASE WHEN x.lnki_ttl IS NULL THEN 0 ELSE 1 END"
, "FROM fsdb_regy f"
, " LEFT JOIN xfer_regy x "
, " ON f.fsdb_name = x.lnki_ttl"
, " AND f.fsdb_is_orig = x.file_is_orig"
, " AND f.fsdb_repo = x.orig_repo"
, " AND f.fsdb_w = x.file_w"
, " AND f.fsdb_time = x.lnki_time"
, " AND f.fsdb_page = x.lnki_page"
, ";"
)
, Make_deletes = String_.Concat_lines_nl
( "SELECT diff_db_id, diff_is_orig, diff_fil_id, diff_thm_id"
, "FROM diff_regy"
, "WHERE diff_status = 0"
, "ORDER BY diff_db_id, diff_is_orig, diff_fil_id, diff_thm_id"
)
;
}

380
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_fsdb_make_cmd.java
View File

@@ -1,380 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.stores.*; import gplx.core.envs.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.dbs.engines.sqlite.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.dbs.*; import gplx.fsdb.*; import gplx.core.ios.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.bins.*; import gplx.xowa.files.fsdb.*;
import gplx.fsdb.data.*; import gplx.fsdb.meta.*;
public class Xob_fsdb_make_cmd extends Xob_itm_basic_base implements Xob_cmd {
private Db_conn bldr_conn; private Db_cfg_tbl bldr_cfg_tbl;
private Xof_bin_mgr src_bin_mgr; private Xof_bin_wkr__fsdb_sql src_fsdb_wkr; private boolean src_bin_mgr__cache_enabled = Bool_.N; private String src_bin_mgr__fsdb_version; private String[] src_bin_mgr__fsdb_skip_wkrs; private boolean src_bin_mgr__wmf_enabled;
private Fsm_mnt_itm trg_mnt_itm; private Fsm_cfg_mgr trg_cfg_mgr; private Fsm_atr_fil trg_atr_fil; private Fsm_bin_fil trg_bin_fil; private long trg_bin_db_max; private String trg_bin_mgr__fsdb_version;
private final Xof_bin_updater trg_bin_updater = new Xof_bin_updater(); private Xob_bin_db_mgr bin_db_mgr; private int[] ns_ids; private int prv_lnki_tier_id = -1;
private long download_size_max = Io_mgr.Len_mb_long * 5; private int[] download_keep_tier_ids = Int_.Ary(0);
private Xobu_poll_mgr poll_mgr; private int poll_interval; private long time_bgn;
private int select_interval = 2500, progress_interval = 1, commit_interval = 1, delete_interval = 5000;
private boolean exec_done, resume_enabled; private int exec_count, exec_count_max = Int_.Max_value, exec_fail, exec_fail_max = 10000; // 115 errors over 900k images
private int tier_id_bmk = -1, tier_id_val = -1; private int page_id_bmk = -1, page_id_val = -1, page_id_end = Int_.Max_value; private int lnki_id_bmk = -1, lnki_id_val = -1;
private boolean exit_after_commit, exit_now;
public Xob_fsdb_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {
this.Cmd_ctor(bldr, wiki);
this.poll_mgr = new Xobu_poll_mgr(bldr.App());
wiki.File__fsdb_mode().Tid_v2_bld_y_();
this.src_bin_mgr = new Xof_bin_mgr(new Fsm_mnt_mgr(), wiki.File__repo_mgr(), app.File__img_mgr().Wkr_resize_img(), app.Wmf_mgr().Download_wkr().Download_xrg().Download_fmt());
}
public String Cmd_key() {return Xob_cmd_keys.Key_file_fsdb_make;}
public void Cmd_bgn(Xob_bldr bldr) {
wiki.Init_assert();
this.poll_interval = poll_mgr.Poll_interval();
this.bin_db_mgr = new Xob_bin_db_mgr(ns_ids);
// src_bin_mgr
if (src_bin_mgr__fsdb_version != null) {
this.src_fsdb_wkr = Xof_bin_wkr__fsdb_sql.new_(wiki.File__mnt_mgr());
src_bin_mgr.Wkrs__add(src_fsdb_wkr);
src_fsdb_wkr.Mnt_mgr().Ctor_by_load(new_src_bin_db_mgr(wiki, src_bin_mgr__fsdb_version));
src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Txn_bgn(); // NOTE: txn on atr speeds up from 50 -> 300; DATE:2015-03-21
if (src_bin_mgr__fsdb_skip_wkrs != null) {
src_fsdb_wkr.Skip_mgr_init(src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Cfg_mgr(), src_bin_mgr__fsdb_skip_wkrs);
}
if (src_bin_mgr__cache_enabled) {
usr_dlg.Prog_many("", "", "src_bin_mgr.cache.bgn");
src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Atr_mgr().Db__core().Fil_cache_enabled_y_();
usr_dlg.Prog_many("", "", "src_bin_mgr.cache.end");
}
}
if (src_bin_mgr__wmf_enabled) {
Xof_bin_wkr__http_wmf wmf_wkr = Xof_bin_wkr__http_wmf.new_(wiki);
src_bin_mgr.Wkrs__add(wmf_wkr);
wmf_wkr.Fail_timeout_(0); // 1000; NOTE: set Fail_timeout here; DATE:2014-06-21; NOTE: do not put in ctor, or else will be 1st wkr; DATE:2014-06-28
}
// trg_mnt_itm
this.trg_bin_db_max = app.Api_root().Bldr().Wiki().Import().File_db_max();
Io_url trg_file_dir_v1 = String_.Eq(trg_bin_mgr__fsdb_version, "v1") ? wiki.Fsys_mgr().File_dir().GenNewNameOnly(wiki.Domain_str() + "-prv") : wiki.Fsys_mgr().File_dir(); // NOTE: convoluted way of setting trg to -prv if trg_bin_mgr__fsdb_version_v1 is set; otherwise set to "en.wikipedia.org" which will noop; DATE:2015-12-02
Fsdb_db_mgr trg_db_mgr = Fsdb_db_mgr_.new_detect(wiki, wiki.Fsys_mgr().Root_dir(), trg_file_dir_v1);
if (trg_db_mgr == null) trg_db_mgr = Fsdb_db_mgr__v2_bldr.Instance.Get_or_make(wiki, Bool_.Y);
Fsm_mnt_mgr trg_mnt_mgr = new Fsm_mnt_mgr(); trg_mnt_mgr.Ctor_by_load(trg_db_mgr);
trg_mnt_mgr.Mnts__get_insert_idx_(Fsm_mnt_mgr.Mnt_idx_main); // NOTE: do not delete; mnt_mgr default to Mnt_idx_user; DATE:2014-04-25
this.trg_mnt_itm = trg_mnt_mgr.Mnts__get_insert();
Fsm_mnt_mgr.Patch(trg_mnt_itm.Cfg_mgr().Tbl()); // NOTE: always patch again; fsdb_make may be run separately without lnki_temp; DATE:2014-04-26
this.trg_atr_fil = trg_mnt_itm.Atr_mgr().Db__core();
this.trg_cfg_mgr = trg_mnt_itm.Cfg_mgr();
bin_db_mgr.Init_by_mnt_mgr(trg_mnt_mgr);
trg_atr_fil.Conn().Txn_bgn("bldr__fsdb_make__trg_atr_fil");
if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn())) // need to create txn for v1; DATE:2015-07-04
trg_cfg_mgr.Tbl().Conn().Txn_bgn("bldr__fsdb_make__trg_cfg_fil");
// bldr_db
Xob_db_file bldr_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
this.bldr_conn = bldr_db.Conn();
this.bldr_cfg_tbl = bldr_db.Tbl__cfg(); // NOTE: cfg and atr is in same db; use it
bldr_cfg_tbl.Conn().Txn_bgn("bldr__fsdb_make__bldr_cfg_tbl");
}
public void Cmd_run() {
Init_bldr_bmks();
this.time_bgn = Env_.TickCount();
int total_pending = Xob_xfer_regy_tbl.Select_total_pending(bldr_conn);
// if (total_pending > 250000 && src_bin_mgr__fsdb_version == null)
usr_dlg.Note_many("", "", "total pending: ~{0}", total_pending);
List_adp list = List_adp_.new_();
boolean loop = true;
while (loop) {
byte rslt = Select_fsdb_itms(list);
switch (rslt) {
case Select_rv_stop:
if (bin_db_mgr.Tier_id_is_last(tier_id_val))
loop = false;
else {
++tier_id_val;
page_id_val = 0;
continue;
}
break;
case Select_rv_next_page: ++page_id_val; lnki_id_val = 0; continue;
case Select_rv_process: break;
}
if (!loop) break; // no more ttls found
int len = list.Count();
usr_dlg.Prog_many("", "", "fetched pages: ~{0}", len);
for (int i = 0; i < len; ++i) {
Xodb_tbl_oimg_xfer_itm fsdb = (Xodb_tbl_oimg_xfer_itm)list.Get_at(i);
Download_itm(fsdb);
if ( exit_now
|| exec_count >= exec_count_max
|| exec_fail >= exec_fail_max
|| page_id_val >= page_id_end
) {
this.Txn_sav();
return;
}
}
}
exec_done = true;
}
private void Init_bldr_bmks() {
if (!resume_enabled) // clear cfg entries if resume disabled; note that disabled by default; DATE:2014-10-24
bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make);
Db_cfg_hash bmk_hash = bldr_cfg_tbl.Select_as_hash(Cfg_fsdb_make);
String tier_id_str = bmk_hash.Get_by(Cfg_tier_id_bmk).To_str_or(null);
if (tier_id_str == null) { // bmks not found; new db;
bldr_conn.Txn_bgn("bldr__fsdb_make__bldr_conn");
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_tier_id_bmk , tier_id_bmk);
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_page_id_bmk , page_id_bmk);
bldr_cfg_tbl.Insert_int(Cfg_fsdb_make, Cfg_lnki_id_bmk , lnki_id_bmk);
bldr_conn.Txn_end();
if (tier_id_bmk == -1) tier_id_bmk = 0;
if (page_id_bmk == -1) page_id_bmk = 0;
if (lnki_id_bmk == -1) lnki_id_bmk = 0;
}
else {
if (tier_id_bmk == -1) {
tier_id_bmk = Int_.parse(tier_id_str);
usr_dlg.Note_many("", "", "restoring from bmk: tier_id=~{0}", tier_id_bmk);
}
if (page_id_bmk == -1) {
page_id_bmk = bmk_hash.Get_by(Cfg_page_id_bmk).To_int();
usr_dlg.Note_many("", "", "restoring from bmk: page_id=~{0}", page_id_bmk);
}
if (lnki_id_bmk == -1) {
lnki_id_bmk = bmk_hash.Get_by(Cfg_lnki_id_bmk).To_int();
usr_dlg.Note_many("", "", "restoring from bmk: lnki_id=~{0}", lnki_id_bmk);
}
}
tier_id_val = tier_id_bmk;
page_id_val = page_id_bmk;
lnki_id_val = lnki_id_bmk;
}
private byte Select_fsdb_itms(List_adp list) {
list.Clear();
boolean pages_found = false, links_found = false;
DataRdr rdr = Xob_xfer_regy_tbl.Select_by_tier_page(bldr_conn, tier_id_val, page_id_val, select_interval);
try {
while (rdr.MoveNextPeer()) {
pages_found = true; // at least one page found; set true
Xodb_tbl_oimg_xfer_itm itm = Xodb_tbl_oimg_xfer_itm.new_rdr_( rdr);
if ( itm.Lnki_page_id() == page_id_val // same page_id
&& itm.Lnki_id() <= lnki_id_val // ... but lnki_id < last
)
continue; // ... ignore; note that select is by page_id, not page_id + link_id; needed else restarts would not resume exactly at same point;
links_found = true;
list.Add(itm);
}
} finally {rdr.Rls();}
if (pages_found && !links_found) return Select_rv_next_page; // pages found, but all links processed
else if (!pages_found) return Select_rv_stop; // no more pages found
else return Select_rv_process; // pages and links found
}
private void Download_itm(Xodb_tbl_oimg_xfer_itm fsdb) {
try {
tier_id_val = fsdb.Lnki_tier_id();
page_id_val = fsdb.Lnki_page_id();
lnki_id_val = fsdb.Lnki_id();
fsdb.Orig_repo_name_(fsdb.Orig_repo_id() == Xof_repo_itm_.Repo_local ? wiki.Domain_bry() : Xow_domain_itm_.Bry__commons);
Download_exec(fsdb);
++exec_count;
if (exec_count % progress_interval == 0) Print_progress(fsdb);
if (exec_count % poll_interval == 0) poll_mgr.Poll();
if (exec_count % commit_interval == 0) Txn_sav();
if (exec_count % delete_interval == 0) Delete_files();
}
catch (Exception exc) {
++exec_fail;
usr_dlg.Warn_many("", "", "download error; ttl=~{0} w=~{1} err=~{2}", fsdb.Orig_ttl(), fsdb.Lnki_w(), Err_.Message_gplx_full(exc));
}
}
private void Download_exec(Xodb_tbl_oimg_xfer_itm fsdb) {
Io_stream_rdr src_rdr = src_bin_mgr.Find_as_rdr(Xof_exec_tid.Tid_wiki_page, fsdb);
try {
if (src_rdr == Io_stream_rdr_.Noop) { // download failed
++exec_fail;
usr_dlg.Warn_many("", "", "failed: ttl=~{0}", String_.Format("[[File:{0}|{1}px]]", fsdb.Orig_ttl(), fsdb.Html_w()));
Print_progress(fsdb);
}
else { // download passed
long src_rdr_len = src_rdr.Len();
int lnki_tier_id = fsdb.Lnki_tier_id();
if ( src_rdr_len > download_size_max
&& !Int_.In(lnki_tier_id, download_keep_tier_ids)) {
usr_dlg.Warn_many("", "", "skipped; ttl=~{0} w=~{1} size=~{2} tier=~{3}", fsdb.Orig_ttl(), fsdb.Lnki_w(), src_rdr_len, lnki_tier_id);
return;
}
if (trg_bin_fil == null) // no trg_bin_fil
Make_trg_bin_file(Bool_.Y, fsdb, src_rdr_len);
else if (trg_bin_fil.Bin_len() + src_rdr_len > trg_bin_db_max) // or trg_bin_fil is out of space
Make_trg_bin_file(Bool_.N, fsdb, src_rdr_len);
else if (prv_lnki_tier_id != lnki_tier_id) { // or tier has changed
if ( prv_lnki_tier_id != -1
&& !bin_db_mgr.Schema_is_1()) // do not increment dbs for v1
Make_trg_bin_file(Bool_.Y, fsdb, src_rdr_len);
prv_lnki_tier_id = lnki_tier_id;
}
trg_bin_updater.Save_bin(trg_mnt_itm, trg_atr_fil, trg_bin_fil, fsdb, src_rdr, src_rdr_len);
}
}
finally {src_rdr.Rls();}
}
private void Make_trg_bin_file(boolean try_nth, Xodb_tbl_oimg_xfer_itm fsdb, long src_rdr_len) {
boolean is_solo = trg_mnt_itm.Db_mgr().File__solo_file();
boolean make = true, use_txn = !is_solo; // solo file; should never open txn
if (trg_bin_fil != null && use_txn) // pre-existing bin_file;
trg_bin_fil.Conn().Txn_end(); // close txn before making new db
int tier_id = fsdb.Lnki_tier_id();
Xob_bin_db_itm nth_bin_db = bin_db_mgr.Get_nth_by_tier(tier_id);
if (try_nth) { // try_nth is true; occurs for new runs or changed tier
if ( nth_bin_db.Id() != -1 // nth exists;
&& nth_bin_db.Db_len() + src_rdr_len < trg_bin_db_max) { // if src_rdr_len exceeds
make = false; // do not make; use existing
}
}
if (make) { // no nth; make it;
int ns_id = bin_db_mgr.Get_ns_id(tier_id);
int pt_id = bin_db_mgr.Increment_pt_id(nth_bin_db);
String new_bin_db_name = bin_db_mgr.Gen_name(wiki.Domain_str(), ns_id, pt_id);
this.trg_bin_fil = trg_mnt_itm.Bin_mgr().Dbs__make(new_bin_db_name);
if (!trg_mnt_itm.Db_mgr().File__solo_file()) {
Fsdb_db_file trg_bin_db = trg_mnt_itm.Db_mgr().File__bin_file__at(trg_mnt_itm.Id(), trg_bin_fil.Id(), new_bin_db_name);
if (!bin_db_mgr.Schema_is_1())
Fsdb_db_mgr__v2_bldr.Make_cfg_data(wiki, trg_atr_fil.Url_rel(), trg_bin_db, Xowd_db_file_.Tid_file_data, trg_bin_fil.Id() + List_adp_.Base1);
}
}
else { // nth available; use it
this.trg_bin_fil = trg_mnt_itm.Bin_mgr().Dbs__get_at(nth_bin_db.Id());
trg_bin_fil.Bin_len_(nth_bin_db.Db_len());
}
if (use_txn)
trg_bin_fil.Conn().Txn_bgn("bldr__fsdb_make__trg_bin_fil");
}
private void Txn_sav() {
usr_dlg.Prog_many("", "", "committing data: count=~{0} failed=~{1}", exec_count, exec_fail);
bldr_cfg_tbl.Update_int(Cfg_fsdb_make, Cfg_page_id_bmk, page_id_val);
bldr_cfg_tbl.Update_int(Cfg_fsdb_make, Cfg_lnki_id_bmk, lnki_id_val);
bldr_cfg_tbl.Conn().Txn_sav();
trg_cfg_mgr.Next_id_commit();
trg_atr_fil.Conn().Txn_sav();
if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn())) // need to create txn for v1
trg_cfg_mgr.Tbl().Conn().Txn_sav();
if (src_bin_mgr__fsdb_version != null && src_bin_mgr__fsdb_skip_wkrs != null) {
src_fsdb_wkr.Skip_mgr().Skip_term(src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Cfg_mgr());
}
if (!trg_mnt_itm.Db_mgr().File__solo_file())
trg_bin_fil.Conn().Txn_sav();
if (exit_after_commit) exit_now = true;
}
public void Cmd_end() {
usr_dlg.Note_many("", "", "fsdb_make.done: count=~{0} rate=~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, Env_.TickCount_elapsed_in_sec(time_bgn)).To_str("#,###.000"));
if (src_fsdb_wkr != null) {
src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Txn_end(); // NOTE: src_fsdb_wkr will be null if no src db defined
}
trg_atr_fil.Conn().Txn_end(); trg_atr_fil.Conn().Rls_conn();
if (!trg_atr_fil.Conn().Eq(trg_cfg_mgr.Tbl().Conn())) // need to create txn for v1
trg_cfg_mgr.Tbl().Conn().Txn_end();
trg_cfg_mgr.Tbl().Conn().Rls_conn();
if (!trg_mnt_itm.Db_mgr().File__solo_file()) {
trg_bin_fil.Conn().Txn_end(); trg_bin_fil.Conn().Rls_conn();
}
if (exec_done) {
bldr_cfg_tbl.Delete_grp(Cfg_fsdb_make); // delete bmks for future reruns; DATE:2014-08-20
Io_mgr.Instance.DeleteFil_args(wiki.Fsys_mgr().Root_dir().GenSubFil("xowa.file.make.cfg.gfs")).MissingFails_off().Exec();
}
bldr_conn.Rls_conn();
}
private void Print_progress(Xodb_tbl_oimg_xfer_itm itm) {
int time_elapsed = Env_.TickCount_elapsed_in_sec(time_bgn);
usr_dlg.Prog_many("", "", "prog: num=~{0} err=~{1} time=~{2} rate=~{3} page=~{4} lnki=~{5} ttl=~{6}", exec_count, exec_fail, time_elapsed, Math_.Div_safe_as_int(exec_count, time_elapsed), page_id_val, lnki_id_val, itm.Orig_ttl());
}
private void Delete_files() {}// TODO: purge /xowa/file/ dir to free up hard disk space
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_term() {}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_tier_id_bmk_)) tier_id_bmk = m.ReadInt("v");
else if (ctx.Match(k, Invk_page_id_bmk_)) page_id_bmk = m.ReadInt("v");
else if (ctx.Match(k, Invk_lnki_id_bmk_)) lnki_id_bmk = m.ReadInt("v");
else if (ctx.Match(k, Invk_select_interval_)) select_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_commit_interval_)) commit_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_progress_interval_)) progress_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_delete_interval_)) delete_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_exec_count_max_)) exec_count_max = m.ReadInt("v");
else if (ctx.Match(k, Invk_exec_fail_max_)) exec_fail_max = m.ReadInt("v");
else if (ctx.Match(k, Invk_exit_after_commit_)) exit_after_commit = m.ReadYn("v");
else if (ctx.Match(k, Invk_exit_now_)) exit_now = m.ReadYn("v");
else if (ctx.Match(k, Invk_resume_enabled_)) resume_enabled = m.ReadYn("v");
else if (ctx.Match(k, Invk_ns_ids_)) ns_ids = Int_.Ary_parse(m.ReadStr("v"), "|");
else if (ctx.Match(k, Invk_src_bin_mgr__fsdb_version_)) src_bin_mgr__fsdb_version = m.ReadStr("v");
else if (ctx.Match(k, Invk_src_bin_mgr__fsdb_skip_wkrs_)) src_bin_mgr__fsdb_skip_wkrs = m.ReadStrAry("v", "|");
else if (ctx.Match(k, Invk_src_bin_mgr__wmf_enabled_)) src_bin_mgr__wmf_enabled = m.ReadYn("v");
else if (ctx.Match(k, Invk_src_bin_mgr__cache_enabled_)) src_bin_mgr__cache_enabled = m.ReadYn("v");
else if (ctx.Match(k, Invk_trg_bin_mgr__fsdb_version_)) trg_bin_mgr__fsdb_version = m.ReadStr("v");
else if (ctx.Match(k, Invk_poll_mgr)) return poll_mgr;
else if (ctx.Match(k, Invk_download_keep_tier_ids)) download_keep_tier_ids = Int_.Ary_parse(m.ReadStr("v"), "|");
else if (ctx.Match(k, Invk_download_size_max)) download_size_max = Io_size_.To_long_by_msg_mb(m, download_size_max);
else return GfoInvkAble_.Rv_unhandled;
return this;
}
private static final String
Invk_tier_id_bmk_ = "tier_id_bmk_", Invk_page_id_bmk_ = "page_id_bmk_", Invk_lnki_id_bmk_ = "lnki_id_bmk_"
, Invk_select_interval_ = "select_interval_", Invk_commit_interval_ = "commit_interval_", Invk_progress_interval_ = "progress_interval_", Invk_delete_interval_ = "delete_interval_"
, Invk_exec_count_max_ = "exec_count_max_", Invk_exec_fail_max_ = "exec_fail_max_", Invk_exit_now_ = "exit_now_", Invk_exit_after_commit_ = "exit_after_commit_"
, Invk_resume_enabled_ = "resume_enabled_", Invk_poll_mgr = "poll_mgr"
, Invk_src_bin_mgr__fsdb_version_ = "src_bin_mgr__fsdb_version_", Invk_src_bin_mgr__fsdb_skip_wkrs_ = "src_bin_mgr__fsdb_skip_wkrs_"
, Invk_src_bin_mgr__wmf_enabled_ = "src_bin_mgr__wmf_enabled_"
, Invk_src_bin_mgr__cache_enabled_ = "src_bin_mgr__cache_enabled_", Invk_ns_ids_ = "ns_ids_"
, Invk_trg_bin_mgr__fsdb_version_ = "trg_bin_mgr__fsdb_version_"
, Invk_download_size_max = "download_size_max", Invk_download_keep_tier_ids = "download_keep_tier_ids"
;
public static Fsdb_db_mgr new_src_bin_db_mgr(Xow_wiki wiki, String version) {
String domain_str = wiki.Domain_str();
Fsdb_db_mgr rv = null; Io_url url = null;
if (String_.Eq(version, "v1")) {
url = wiki.Fsys_mgr().File_dir().OwnerDir().GenSubDir(domain_str + "-prv"); // v1: EX: /xowa/file/en.wikipedia.org-prv/
rv = new Fsdb_db_mgr__v1(url);
}
else if (String_.Eq(version, "v2")) {
url = wiki.Fsys_mgr().Root_dir().GenSubDir("prv"); // v2: EX: /xowa/wiki/en.wikipedia.org/prv/
rv = Fsdb_db_mgr_.new_detect(wiki, url, url); // note that v2 is prioritized over v1
}
else throw Err_.new_wo_type("fsdb.make:unknown fsdb_type", "version", version);
if (rv == null) throw Err_.new_wo_type("fsdb.make:source fsdb not found", "version", version, "url", url.Raw());
return rv;
}
private static final byte Select_rv_stop = 0, Select_rv_process = 1, Select_rv_next_page = 2;
private static final String Cfg_fsdb_make = "bldr.fsdb_make", Cfg_tier_id_bmk = "tier_id_bmk", Cfg_page_id_bmk = "page_id_bmk", Cfg_lnki_id_bmk = "lnki_id_bmk";
public static byte Status_null = 0, Status_pass = 1, Status_fail = 2;
}
class Xodb_tbl_oimg_xfer_itm extends Xof_fsdb_itm { public int Lnki_id() {return lnki_id;} private int lnki_id;
public int Lnki_tier_id() {return lnki_tier_id;} private int lnki_tier_id;
public int Lnki_page_id() {return lnki_page_id;} private int lnki_page_id;
public static Xodb_tbl_oimg_xfer_itm new_rdr_(DataRdr rdr) {
Xodb_tbl_oimg_xfer_itm rv = new Xodb_tbl_oimg_xfer_itm();
rv.lnki_id = rdr.ReadInt(Xob_xfer_regy_tbl.Fld_lnki_id);
rv.lnki_page_id = rdr.ReadInt(Xob_xfer_regy_tbl.Fld_lnki_page_id);
rv.lnki_tier_id = rdr.ReadInt(Xob_xfer_regy_tbl.Fld_lnki_tier_id);
rv.Init_at_fsdb_make
( rdr.ReadBryByStr(Xob_xfer_regy_tbl.Fld_lnki_ttl)
, rdr.ReadInt(Xob_xfer_regy_tbl.Fld_lnki_ext)
, rdr.ReadInt(Xob_xfer_regy_tbl.Fld_file_w), rdr.ReadInt(Xob_xfer_regy_tbl.Fld_file_h) // set lnki_size; Xof_bin_mgr uses lnki_size
, Xof_lnki_time.Db_load_double(rdr, Xob_xfer_regy_tbl.Fld_lnki_time)
, Xof_lnki_page.Db_load_int(rdr, Xob_xfer_regy_tbl.Fld_lnki_page)
, rdr.ReadByte(Xob_xfer_regy_tbl.Fld_orig_repo)
, rdr.ReadInt(Xob_xfer_regy_tbl.Fld_orig_w)
, rdr.ReadInt(Xob_xfer_regy_tbl.Fld_orig_h)
, Bry_.Empty
, rdr.ReadByte(Xob_xfer_regy_tbl.Fld_file_is_orig) == Bool_.Y_byte
);
return rv;
}
}

64
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_fsdb_reduce_cmd.java
View File

@@ -1,64 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.envs.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.fsdb.meta.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_fsdb_reduce_cmd extends Xob_itm_basic_base implements Xob_cmd {
public Xob_fsdb_reduce_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_file_fsdb_reduce;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_run() {Exec_main();}
public void Cmd_end() {}
public void Cmd_term() {}
private void Exec_main() {
/*
Open_bin_dir
Iterate_files
Convert_bin
Mark_meta_record
*/
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
return this;
}
}
class Fsdb_reduce_wkr {
private final Process_adp convert_process;
private final Io_url src_url, trg_url;
private final int dpi, quality;
public Fsdb_reduce_wkr(Process_adp convert_process, Io_url tmp_dir, int dpi, int quality) {
this.convert_process = convert_process;
this.src_url = tmp_dir.GenSubFil("fsdb_reduce.src.bin");
this.trg_url = tmp_dir.GenSubFil("fsdb_reduce.trg.bin");
this.dpi = dpi;
this.quality = quality;
}
public byte[] Reduce(byte[] orig_bry) {
Io_mgr.Instance.SaveFilBry(src_url, orig_bry);
convert_process.Run(src_url, trg_url, dpi, quality); // -strip -quality 50% -density 72 -resample 72
if (!convert_process.Exit_code_pass()) {
// throw err with convert_process.Rslt_out();
}
byte[] rv = Io_mgr.Instance.LoadFilBry(trg_url);
// fail if 0; fail if greater than;
// warn if not between 50% - 70% of size
return rv;
}
}

33
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_regy_cmd.java
View File

@@ -1,33 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.xowa.wikis.dbs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_lnki_regy_cmd extends Xob_itm_basic_base implements Xob_cmd {
public Xob_lnki_regy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_file_lnki_regy;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
Db_conn conn = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
Xob_lnki_regy_tbl.Create_table(conn);
Xob_lnki_regy_tbl.Create_data(usr_dlg, conn, Xob_lnki_temp_wkr.Ns_file_is_case_match_all(wiki));
}
public void Cmd_run() {}
public void Cmd_end() {}
public void Cmd_term() {}
}

115
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_regy_tbl.java
View File

@@ -1,115 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*;
class Xob_lnki_regy_tbl {
public static void Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
public static void Create_data(Gfo_usr_dlg usr_dlg, Db_conn p, boolean wiki_ns_for_file_is_case_match_all) {
p.Exec_sql(Sql_create_data);
Sqlite_engine_.Idx_create(usr_dlg, p, "lnki_regy", Idx_ttl);
if (wiki_ns_for_file_is_case_match_all)
Sqlite_engine_.Idx_create(usr_dlg, p, "lnki_regy_commons", Idx_ttl_commons);
}
public static final String Tbl_name = "lnki_regy"
, Fld_lnki_id = "lnki_id", Fld_lnki_tier_id = "lnki_tier_id", Fld_lnki_page_id = "lnki_page_id", Fld_lnki_page_ns = "lnki_page_ns"
, Fld_lnki_ttl = "lnki_ttl", Fld_lnki_commons_ttl = "lnki_commons_ttl"
, Fld_lnki_ext = "lnki_ext", Fld_lnki_type = "lnki_type", Fld_lnki_src_tid = "lnki_src_tid"
, Fld_lnki_w = "lnki_w", Fld_lnki_h = "lnki_h", Fld_lnki_upright = "lnki_upright", Fld_lnki_time = "lnki_time", Fld_lnki_page = "lnki_page"
, Fld_lnki_count = "lnki_count"
;
private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS lnki_regy"
, "( lnki_id integer NOT NULL PRIMARY KEY"
, ", lnki_tier_id integer NOT NULL"
, ", lnki_page_id integer NOT NULL"
, ", lnki_ttl varchar(255) NOT NULL"
, ", lnki_commons_ttl varchar(255) NULL"
, ", lnki_commons_flag integer NULL"
, ", lnki_ext integer NOT NULL"
, ", lnki_type integer NOT NULL"
, ", lnki_src_tid integer NOT NULL"
, ", lnki_w integer NOT NULL"
, ", lnki_h integer NOT NULL"
, ", lnki_upright double NOT NULL"
, ", lnki_time double NOT NULL"
, ", lnki_page integer NOT NULL"
, ", lnki_count integer NOT NULL"
, ");"
);
public static final String Sql_create_data = String_.Concat_lines_nl
( "INSERT INTO lnki_regy (lnki_id, lnki_tier_id, lnki_page_id, lnki_ttl, lnki_commons_ttl, lnki_ext, lnki_type, lnki_src_tid, lnki_w, lnki_h, lnki_upright, lnki_time, lnki_page, lnki_count)"
, "SELECT Min(lnki_id)"
, ", Min(lnki_tier_id)"
, ", Min(lnki_page_id)"
, ", lnki_ttl"
, ", lnki_commons_ttl"
, ", lnki_ext"
, ", lnki_type"
, ", lnki_src_tid"
, ", lnki_w"
, ", lnki_h"
, ", lnki_upright"
, ", lnki_time"
, ", lnki_page"
, ", Count(lnki_ttl)"
, "FROM lnki_temp"
, "GROUP BY"
, " lnki_ttl"
, ", lnki_commons_ttl"
, ", lnki_ext"
, ", lnki_type"
, ", lnki_src_tid"
, ", lnki_w"
, ", lnki_h"
, ", lnki_upright"
, ", lnki_time"
, ", lnki_page"
, ";"
)
, Sql_cs_mark_changed = String_.Concat_lines_nl
( "REPLACE INTO lnki_regy"
, "SELECT l.lnki_id"
, ", l.lnki_tier_id"
, ", l.lnki_page_id"
, ", l.lnki_ttl"
, ", l.lnki_commons_ttl"
, ", CASE WHEN o.lnki_ttl IS NULL THEN NULL ELSE 1 END"
, ", l.lnki_ext"
, ", l.lnki_type"
, ", l.lnki_src_tid"
, ", l.lnki_w"
, ", l.lnki_h"
, ", l.lnki_upright"
, ", l.lnki_time"
, ", l.lnki_page"
, ", l.lnki_count"
, "FROM lnki_regy l"
, " LEFT JOIN orig_regy o ON o.lnki_ttl = l.lnki_commons_ttl AND o.orig_commons_flag = 2"
, ";"
)
, Sql_cs_update_ttls = String_.Concat_lines_nl
( "UPDATE lnki_regy"
, "SET lnki_ttl = lnki_commons_ttl"
, "WHERE lnki_commons_flag = 1"
, ";"
);
private static final Db_idx_itm
Idx_ttl = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS lnki_regy__ttl ON lnki_regy (lnki_ttl, lnki_ext, lnki_id, lnki_page_id);")
, Idx_ttl_commons = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS lnki_regy__ttl ON lnki_regy (lnki_commons_ttl, lnki_ext, lnki_id, lnki_page_id);")
;
}

62
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_temp_tbl.java
View File

@@ -1,62 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.xowa.files.*;
class Xob_lnki_temp_tbl {
private static final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
private static final String Tbl_name = "lnki_temp";
public static final String
Fld_lnki_id = flds.Add_int_pkey_autonum("lnki_id"); // NOTE: insertion order index; public b/c not used and want to bypass warning
private static final String
Fld_lnki_tier_id = flds.Add_int("lnki_tier_id")
, Fld_lnki_page_id = flds.Add_int("lnki_page_id")
, Fld_lnki_ttl = flds.Add_str("lnki_ttl", 255)
, Fld_lnki_commons_ttl = flds.Add_str_null("lnki_commons_ttl", 255)
, Fld_lnki_ext = flds.Add_int("lnki_ext")
, Fld_lnki_type = flds.Add_int("lnki_type")
, Fld_lnki_src_tid = flds.Add_int("lnki_src_tid")
, Fld_lnki_w = flds.Add_int("lnki_w")
, Fld_lnki_h = flds.Add_int("lnki_h")
, Fld_lnki_upright = flds.Add_double("lnki_upright")
, Fld_lnki_time = flds.Add_double("lnki_time") // NOTE: thumbtime is float; using double b/c upright is double and would like to keep datatypes same; https://bugzilla.wikimedia.org/show_bug.cgi?id=39014
, Fld_lnki_page = flds.Add_int("lnki_page")
;
private Db_stmt stmt_insert;
public Xob_lnki_temp_tbl(Db_conn conn) {this.conn = conn;}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(Tbl_name, flds));}
public void Insert_bgn() {conn.Txn_bgn("bldr__lnki_temp"); stmt_insert = conn.Stmt_insert(Tbl_name, flds);}
public void Insert_commit() {conn.Txn_sav();}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public void Insert_cmd_by_batch(int tier_id, int page_id, byte[] ttl, byte[] ttl_commons, byte ext_id, byte img_type, byte lnki_src_tid, int w, int h, double upright, double time, int page) {
stmt_insert.Clear()
.Val_int (Fld_lnki_tier_id , tier_id)
.Val_int (Fld_lnki_page_id , page_id)
.Val_bry_as_str (Fld_lnki_ttl , ttl)
.Val_bry_as_str (Fld_lnki_commons_ttl , ttl_commons)
.Val_byte (Fld_lnki_ext , ext_id)
.Val_byte (Fld_lnki_type , img_type)
.Val_int (Fld_lnki_src_tid , lnki_src_tid)
.Val_int (Fld_lnki_w , w)
.Val_int (Fld_lnki_h , h)
.Val_double (Fld_lnki_upright , upright)
.Val_double (Fld_lnki_time , Xof_lnki_time.Db_save_double(time))
.Val_int (Fld_lnki_page , page)
.Exec_insert();
}
}

174
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_temp_wkr.java
View File

@@ -1,174 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.pages.*;
import gplx.xowa.files.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*; import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.xndes.*;
import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.wdatas.*;
import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*; import gplx.fsdb.*;
import gplx.xowa.langs.vnts.*; import gplx.xowa.parsers.vnts.*;
import gplx.xowa.parsers.lnkis.files.*;
public class Xob_lnki_temp_wkr extends Xob_dump_mgr_base implements gplx.xowa.parsers.lnkis.files.Xop_file_logger {
private Xob_lnki_temp_tbl tbl; private boolean wdata_enabled = true, xtn_ref_enabled = true, gen_html, gen_hdump;
private Xop_log_invoke_wkr invoke_wkr; private Xop_log_property_wkr property_wkr;
private boolean ns_file_is_case_match_all = true; private Xowe_wiki commons_wiki;
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr(); private Vnt_convert_lang converter_lang;
public Xob_lnki_temp_wkr(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_file_lnki_temp;}
@Override public byte Init_redirect() {return Bool_.N_byte;} // lnki_temp does not look at redirect pages
@Override public int[] Init_ns_ary() {return ns_ids;} private int[] ns_ids = Int_.Ary(Xow_ns_.Tid__main);
@Override protected void Init_reset(Db_conn conn) {
Db_cfg_tbl cfg_tbl = new Db_cfg_tbl(conn, "xowa_cfg");
cfg_tbl.Delete_all();
invoke_wkr.Init_reset();
property_wkr.Init_reset();
}
@Override protected Db_conn Init_db_file() {
ctx.Lnki().File_logger_(this);
Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
Db_conn make_conn = make_db.Conn();
this.tbl = new Xob_lnki_temp_tbl(make_conn); tbl.Create_tbl();
this.gen_hdump = hdump_bldr.Init(wiki, make_conn);
Xol_vnt_mgr vnt_mgr = wiki.Lang().Vnt_mgr();
if (vnt_mgr.Enabled()) {
this.converter_lang = vnt_mgr.Convert_lang();
converter_lang.Log__init(make_conn);
}
return make_conn;
}
@Override protected void Cmd_bgn_end() {
ns_file_is_case_match_all = Ns_file_is_case_match_all(wiki); // NOTE: must call after wiki.init
wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_page_.Tid_read).Ctgs_enabled_(false); // disable categories else progress messages written (also for PERF)
if (wiki.File__bin_mgr() != null)
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // remove wmf wkr, else will try to download images during parsing
commons_wiki = app.Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons);
Xop_log_mgr log_mgr = ctx.App().Log_mgr();
log_mgr.Log_dir_(wiki.Fsys_mgr().Root_dir()); // put log in wiki dir, instead of user.temp
invoke_wkr = this.Invoke_wkr(); // set member reference
invoke_wkr = log_mgr.Make_wkr_invoke();
property_wkr = this.Property_wkr(); // set member reference
property_wkr = log_mgr.Make_wkr_property();
wiki.Appe().Wiki_mgr().Wdata_mgr().Enabled_(wdata_enabled);
if (!xtn_ref_enabled) gplx.xowa.xtns.cites.References_nde.Enabled = false;
gplx.xowa.xtns.gallery.Gallery_xnde.Log_wkr = log_mgr.Make_wkr().Save_src_str_(Bool_.Y);
gplx.xowa.xtns.imaps.Imap_xnde.Log_wkr = log_mgr.Make_wkr();
gplx.xowa.parsers.xndes.Xop_xnde_wkr.Timeline_log_wkr = log_mgr.Make_wkr();
gplx.xowa.xtns.scores.Score_xnde.Log_wkr = log_mgr.Make_wkr();
gplx.xowa.xtns.hieros.Hiero_xnde.Log_wkr = log_mgr.Make_wkr();
gplx.xowa.xtns.math.Math_nde.Log_wkr = log_mgr.Make_wkr().Save_src_str_(Bool_.Y); // enabled; DATE:2015-10-10
Xof_fsdb_mgr__sql trg_fsdb_mgr = new Xof_fsdb_mgr__sql();
wiki.File__fsdb_mode().Tid_v2_bld_y_();
Fsdb_db_mgr__v2 fsdb_core = Fsdb_db_mgr__v2_bldr.Instance.Get_or_make(wiki, Bool_.Y);
trg_fsdb_mgr.Init_by_wiki(wiki);
Fsm_mnt_mgr trg_mnt_mgr = trg_fsdb_mgr.Mnt_mgr();
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__orig_mgr().Wkrs_del(gplx.xowa.files.origs.Xof_orig_wkr_.Tid_wmf_api);
trg_mnt_mgr = new Fsm_mnt_mgr(); trg_mnt_mgr.Ctor_by_load(fsdb_core);
trg_mnt_mgr.Mnts__get_insert_idx_(Fsm_mnt_mgr.Mnt_idx_main);
Fsm_mnt_mgr.Patch(trg_mnt_mgr.Mnts__get_main().Cfg_mgr().Tbl()); // NOTE: see fsdb_make; DATE:2014-04-26
tbl.Insert_bgn();
log_mgr.Txn_bgn();
}
@Override public void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm db_page, byte[] page_src) {
Xoa_ttl ttl = Xoa_ttl.parse(wiki, ns.Gen_ttl(db_page.Ttl_page_db()));
byte[] ttl_bry = ttl.Page_db();
byte page_tid = Xow_page_tid.Identify(wiki.Domain_tid(), ns.Id(), ttl_bry);
if (page_tid != Xow_page_tid.Tid_wikitext) return; // ignore js, css, lua, json
Xoae_page page = ctx.Page();
page.Clear_all();
page.Bldr__ns_ord_(ns_ord);
page.Ttl_(ttl).Revision_data().Id_(db_page.Id());
page.Redlink_lnki_list().Clear();
page.Url_(Xoa_url.new_(wiki.Domain_bry(), ttl.Full_db()));
if (ns.Id_is_tmpl())
parser.Parse_text_to_defn_obj(ctx, ctx.Tkn_mkr(), wiki.Ns_mgr().Ns_template(), ttl_bry, page_src);
else {
parser.Parse_page_all_clear(root, ctx, ctx.Tkn_mkr(), page_src);
if (gen_html && !page.Redirected())
wiki.Html_mgr().Page_wtr_mgr().Gen(ctx.Page().Root_(root), Xopg_page_.Tid_read);
if (gen_hdump)
hdump_bldr.Insert(page.Root_(root));
root.Clear();
}
}
@Override public void Exec_commit_hook() {
tbl.Conn().Txn_sav();
// if (converter_lang != null) converter_lang.Log__save();
if (gen_hdump) {
hdump_bldr.Commit();
}
}
@Override public void Exec_end_hook() {
// if (converter_lang != null) converter_lang.Log__rls();
if (gen_hdump) {
hdump_bldr.Bld_term();
}
String err_filter_mgr = invoke_wkr.Err_filter_mgr().Print();
if (String_.Len_gt_0(err_filter_mgr)) usr_dlg.Warn_many("", "", err_filter_mgr);
wiki.Appe().Log_mgr().Txn_end();
tbl.Insert_end();
}
public void Log_file(Xop_ctx ctx, Xop_lnki_tkn lnki, byte caller_tid) {
if (lnki.Ttl().ForceLiteralLink()) return; // ignore literal links which creat a link to file, but do not show the image; EX: [[:File:A.png|thumb|120px]] creates a link to File:A.png, regardless of other display-oriented args
byte[] ttl = lnki.Ttl().Page_db();
Xof_ext ext = Xof_ext_.new_by_ttl_(ttl);
double lnki_time = lnki.Time();
int lnki_page = lnki.Page();
byte[] ttl_commons = Xto_commons(ns_file_is_case_match_all, commons_wiki, ttl);
if ( Xof_lnki_page.Null_n(lnki_page) // page set
&& Xof_lnki_time.Null_n(lnki_time)) // thumbtime set
usr_dlg.Warn_many("", "", "page and thumbtime both set; this may be an issue with fsdb: page=~{0} ttl=~{1}", ctx.Page().Ttl().Page_db_as_str(), String_.new_u8(ttl));
if (lnki.Ns_id() == Xow_ns_.Tid__media)
caller_tid = Xop_file_logger_.Tid__media;
tbl.Insert_cmd_by_batch(ctx.Page().Bldr__ns_ord(), ctx.Page().Revision_data().Id(), ttl, ttl_commons, Byte_.By_int(ext.Id()), lnki.Lnki_type(), caller_tid, lnki.W(), lnki.H(), lnki.Upright(), lnki_time, lnki_page);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_wdata_enabled_)) wdata_enabled = m.ReadYn("v");
else if (ctx.Match(k, Invk_xtn_ref_enabled_)) xtn_ref_enabled = m.ReadYn("v");
else if (ctx.Match(k, Invk_ns_ids_)) ns_ids = Int_.Ary_parse(m.ReadStr("v"), "|");
else if (ctx.Match(k, Invk_ns_ids_by_aliases)) ns_ids = Xob_lnki_temp_wkr_.Ns_ids_by_aliases(wiki, m.ReadStrAry("v", "|"));
else if (ctx.Match(k, Invk_gen_html_)) gen_html = m.ReadYn("v");
else if (ctx.Match(k, Invk_hdump_bldr)) return hdump_bldr;
else if (ctx.Match(k, Invk_property_wkr)) return this.Property_wkr();
else if (ctx.Match(k, Invk_invoke_wkr)) return this.Invoke_wkr();
else return super.Invk(ctx, ikey, k, m);
return this;
}
private static final String Invk_wdata_enabled_ = "wdata_enabled_", Invk_xtn_ref_enabled_ = "xtn_ref_enabled_", Invk_gen_html_ = "gen_html_"
, Invk_ns_ids_ = "ns_ids_", Invk_ns_ids_by_aliases = "ns_ids_by_aliases"
, Invk_invoke_wkr = "invoke_wkr", Invk_property_wkr = "property_wkr", Invk_hdump_bldr = "hdump_bldr"
;
private Xop_log_invoke_wkr Invoke_wkr() {
if (invoke_wkr == null) invoke_wkr = ((Scrib_xtn_mgr)bldr.App().Xtn_mgr().Get_or_fail(Scrib_xtn_mgr.XTN_KEY)).Invoke_wkr_or_new();
return invoke_wkr;
}
private Xop_log_property_wkr Property_wkr() {
if (property_wkr == null) property_wkr = bldr.App().Wiki_mgr().Wdata_mgr().Property_wkr_or_new();
return property_wkr;
}
public static byte[] Xto_commons(boolean ns_file_is_case_match_all, Xowe_wiki commons_wiki, byte[] ttl_bry) {
if (!ns_file_is_case_match_all) return null; // return "" if wiki matches common
Xoa_ttl ttl = Xoa_ttl.parse(commons_wiki, Xow_ns_.Tid__file, ttl_bry);
byte[] rv = ttl.Page_db();
return Bry_.Eq(rv, ttl_bry) ? null : rv;
}
public static boolean Ns_file_is_case_match_all(Xow_wiki wiki) {return wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all;}
}

53
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_temp_wkr_.java
View File

@@ -1,53 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.wikis.nss.*;
class Xob_lnki_temp_wkr_ {
public static int[] Ns_ids_by_aliases(Xowe_wiki wiki, String[] aliases) {
int[] rv = Xob_lnki_temp_wkr_.Ids_by_aliases(wiki.Ns_mgr(), aliases);
int aliases_len = aliases.length;
int ids_len = rv.length;
for (int i = 0; i < aliases_len; i++) {
String alias = aliases[i];
int id = i < ids_len ? rv[i] : -1;
wiki.Appe().Usr_dlg().Note_many("", "", "ns: ~{0} <- ~{1}", Int_.To_str_fmt(id, "0000"), alias);
}
if (aliases_len != ids_len) throw Err_.new_wo_type("mismatch in aliases and ids", "aliases", aliases_len, "ids", ids_len);
return rv;
}
private static int[] Ids_by_aliases(Xow_ns_mgr ns_mgr, String[] aliases) {
List_adp list = List_adp_.new_();
int len = aliases.length;
for (int i = 0; i < len; i++) {
String alias = aliases[i];
if (String_.Eq(alias, Xow_ns_.Key__main))
list.Add(ns_mgr.Ns_main());
else {
Xow_ns ns = ns_mgr.Names_get_or_null(Bry_.new_u8(alias));
if (ns != null)
list.Add(ns);
}
}
len = list.Count();
int[] rv = new int[len];
for (int i = 0; i < len; i++) {
rv[i] = ((Xow_ns)list.Get_at(i)).Id();
}
return rv;
}
}

43
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_lnki_temp_wkr_tst.java
View File

@@ -1,43 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*;
public class Xob_lnki_temp_wkr_tst {
private Xob_lnki_temp_wkr_fxt fxt = new Xob_lnki_temp_wkr_fxt();
@Test public void Xto_commons() {
fxt.Init_Xto_commons(true);
fxt.Test_Xto_commons("a", "A");
fxt.Test_Xto_commons("A", null);
fxt.Init_Xto_commons(false);
fxt.Test_Xto_commons("a", null);
fxt.Test_Xto_commons("A", null);
}
}
class Xob_lnki_temp_wkr_fxt {
private boolean wiki_ns_file_is_case_match_all;
private Xowe_wiki commons_wiki;
public Xob_lnki_temp_wkr_fxt Init_Xto_commons(boolean wiki_ns_file_is_case_match_all) {
Xoae_app app = Xoa_app_fxt.Make__app__edit();
this.wiki_ns_file_is_case_match_all = wiki_ns_file_is_case_match_all;
this.commons_wiki = Xoa_app_fxt.Make__wiki__edit(app); // commons_wiki will default to Xow_ns.Id_file of case_match_1st
return this;
}
public void Test_Xto_commons(String ttl, String expd) {
Tfds.Eq(expd, String_.new_u8(Xob_lnki_temp_wkr.Xto_commons(wiki_ns_file_is_case_match_all, commons_wiki, Bry_.new_u8(ttl))));
}
}

48
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_orig_regy_cmd.java
View File

@@ -1,48 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_orig_regy_cmd extends Xob_itm_basic_base implements Xob_cmd {
private boolean repo_0_is_remote = false;
public Xob_orig_regy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_file_orig_regy;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
Db_conn conn = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
Xob_orig_regy_tbl.Create_table(conn);
Xowe_wiki commons_wiki = bldr.App().Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons).Init_assert();
Xowe_wiki repo_0 = wiki, repo_1 = commons_wiki;
if (repo_0_is_remote) { // NOTE: default is false; local_wiki will be preferred over commons_wiki
repo_0 = commons_wiki;
repo_1 = wiki;
}
repo_0.Init_assert(); repo_1.Init_assert();
Xob_db_file file_registry_db = Xob_db_file.New__page_regy(commons_wiki.Fsys_mgr().Root_dir());
Xob_orig_regy_tbl.Create_data(bldr.Usr_dlg(), conn, file_registry_db, repo_0_is_remote, repo_0, repo_1, Xob_lnki_temp_wkr.Ns_file_is_case_match_all(wiki));
}
public void Cmd_run() {}
public void Cmd_end() {}
public void Cmd_term() {}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_repo_0_is_remote_)) repo_0_is_remote = m.ReadYn("v");
else return super.Invk(ctx, ikey, k, m);
return this;
} private static final String Invk_repo_0_is_remote_ = "repo_0_is_remote_";
}

221
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_orig_regy_tbl.java
View File

@@ -1,221 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.xowa.files.repos.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.domains.*;
import gplx.dbs.engines.sqlite.*;
class Xob_orig_regy_tbl {
public static void Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
public static void Create_data(Gfo_usr_dlg usr_dlg, Db_conn p, Xob_db_file file_registry_db, boolean repo_0_is_remote, Xowe_wiki repo_0_wiki, Xowe_wiki repo_1_wiki, boolean wiki_ns_for_file_is_case_match_all) {
usr_dlg.Prog_many("", "", "inserting lnki_regy");
p.Exec_sql(Sql_create_data);
Sqlite_engine_.Idx_create(usr_dlg, p, "orig_regy", Idx_ttl_local);
Sqlite_engine_.Db_attach(p, "page_db", file_registry_db.Url().Raw());
Io_url repo_0_dir = repo_0_wiki.Fsys_mgr().Root_dir(), repo_1_dir = repo_1_wiki.Fsys_mgr().Root_dir();
byte repo_0_tid = Xof_repo_itm_.Repo_local, repo_1_tid = Xof_repo_itm_.Repo_remote;
boolean local_is_remote = Bry_.Eq(repo_0_wiki.Domain_bry(), repo_1_wiki.Domain_bry());
Xowe_wiki local_wiki = repo_0_wiki;
if ( repo_0_is_remote // .gfs manually marked specifes repo_0 as remote
|| ( Bry_.Eq(repo_0_wiki.Domain_bry(), Xow_domain_itm_.Bry__commons) // repo_0 = commons; force repo_0 to be remote; else all orig_repo will be 1; DATE:2014-02-01
&& local_is_remote // repo_0 = repo_1
)
) {
repo_0_tid = Xof_repo_itm_.Repo_remote;
repo_1_tid = Xof_repo_itm_.Repo_local;
local_wiki = repo_1_wiki;
}
Create_data_for_repo(usr_dlg, p, local_wiki, Byte_.By_int(repo_0_tid), repo_0_dir.GenSubFil(Xob_db_file.Name__wiki_image));
if (!local_is_remote) { // only run for repo_1 if local != remote; only affects commons
Create_data_for_repo(usr_dlg, p, local_wiki, Byte_.By_int(repo_1_tid), repo_1_dir.GenSubFil(Xob_db_file.Name__wiki_image));
if (wiki_ns_for_file_is_case_match_all) {
Io_url repo_remote_dir = repo_0_is_remote ? repo_0_dir : repo_1_dir;
Create_data_for_cs(usr_dlg, p, local_wiki, repo_remote_dir);
}
}
Sqlite_engine_.Db_detach(p, "page_db");
Sqlite_engine_.Idx_create(usr_dlg, p, "orig_regy", Idx_xfer_temp);
}
private static void Create_data_for_repo(Gfo_usr_dlg usr_dlg, Db_conn conn, Xowe_wiki local_wiki, byte repo_tid, Io_url join) {
usr_dlg.Note_many("", "", "inserting page for xowa.wiki.image: ~{0}", join.OwnerDir().NameOnly());
boolean wiki_has_cs_file = repo_tid == Xof_repo_itm_.Repo_remote && local_wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all;
String lnki_ttl_fld = wiki_has_cs_file ? "Coalesce(o.lnki_commons_ttl, o.lnki_ttl)" : "o.lnki_ttl"; // NOTE: use lnki_commons_ttl if [[File]] is cs PAGE:en.d:water EX:[[image:wikiquote-logo.png|50px|none|alt=]]; DATE:2014-09-05
if (wiki_has_cs_file)
Sqlite_engine_.Idx_create(usr_dlg, conn, "orig_regy", Idx_ttl_remote);
new Db_attach_mgr(conn, new Db_attach_itm("image_db", join))
.Exec_sql_w_msg("orig_regy:updating page" , Sql_update_repo_page, repo_tid, lnki_ttl_fld)
.Exec_sql_w_msg("orig_regy:updating redirect" , Sql_update_repo_redirect, repo_tid, lnki_ttl_fld);
}
private static void Create_data_for_cs(Gfo_usr_dlg usr_dlg, Db_conn p, Xowe_wiki local_wiki, Io_url repo_remote_dir) {
p.Exec_sql(Xob_orig_regy_tbl.Sql_cs_mark_dupes); // orig_regy: find dupes; see note in SQL
p.Exec_sql(Xob_orig_regy_tbl.Sql_cs_update_ttls); // orig_regy: update lnki_ttl with lnki_commons_ttl
Create_data_for_repo(usr_dlg, p, local_wiki, Xof_repo_itm_.Repo_remote, repo_remote_dir.GenSubFil(Xob_db_file.Name__wiki_image));
p.Exec_sql(Xob_lnki_regy_tbl.Sql_cs_mark_changed); // lnki_regy: update lnki_commons_flag
p.Exec_sql(Xob_lnki_regy_tbl.Sql_cs_update_ttls); // lnki_regy: update cs
}
public static final String Tbl_name = "orig_regy"
, Fld_lnki_id = "lnki_id", Fld_lnki_ttl = "lnki_ttl", Fld_lnki_ext = "lnki_ext", Fld_lnki_count = "lnki_count"
, Fld_orig_repo = "orig_repo", Fld_orig_page_id = "orig_page_id"
, Fld_orig_redirect_id = "orig_redirect_id", Fld_orig_redirect_ttl = "orig_redirect_ttl", Fld_orig_file_id = "orig_file_id", Fld_orig_file_ttl = "orig_file_ttl"
, Fld_orig_size = "orig_size", Fld_orig_w = "orig_w", Fld_orig_h = "orig_h", Fld_orig_bits = "orig_bits"
, Fld_orig_media_type = "orig_media_type", Fld_orig_minor_mime = "orig_minor_mime", Fld_orig_file_ext = "orig_file_ext", Fld_orig_timestamp = "orig_timestamp"
;
private static final Db_idx_itm
Idx_ttl_local = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS orig_regy__ttl_local ON orig_regy (lnki_ttl);")
, Idx_ttl_remote = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS orig_regy__ttl_remote ON orig_regy (lnki_commons_ttl, lnki_ttl);")
, Idx_xfer_temp = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS orig_regy__xfer_temp ON orig_regy (lnki_ttl, orig_file_ttl, orig_repo, orig_timestamp);")
;
private static final String
Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS orig_regy"
, "( lnki_id integer NOT NULL PRIMARY KEY" // NOTE: must be PRIMARY KEY, else later REPLACE INTO will create dupe rows
, ", lnki_ttl varchar(256) NOT NULL"
, ", lnki_commons_ttl varchar(256) NULL"
, ", orig_commons_flag integer NULL"
, ", lnki_ext integer NOT NULL"
, ", lnki_count integer NOT NULL"
, ", orig_repo integer NULL"
, ", orig_page_id integer NULL"
, ", orig_redirect_id integer NULL"
, ", orig_redirect_ttl varchar(256) NULL"
, ", orig_file_id integer NULL"
, ", orig_file_ttl varchar(256) NULL"
, ", orig_file_ext integer NULL"
, ", orig_size integer NULL"
, ", orig_w integer NULL"
, ", orig_h integer NULL"
, ", orig_bits smallint NULL"
, ", orig_media_type varchar(64) NULL"
, ", orig_minor_mime varchar(32) NULL"
, ", orig_timestamp varchar(14) NULL"
, ");"
)
, Sql_create_data = String_.Concat_lines_nl
( "INSERT INTO orig_regy (lnki_id, lnki_ttl, lnki_commons_ttl, orig_commons_flag, lnki_ext, lnki_count, orig_timestamp)"
, "SELECT Min(lnki_id)"
, ", lnki_ttl"
, ", lnki_commons_ttl"
, ", NULL"
, ", lnki_ext"
, ", Sum(lnki_count)"
, ", ''"
, "FROM lnki_regy"
, "GROUP BY lnki_ttl"
, ", lnki_ext"
, "ORDER BY 1" // must order by lnki_id since it is PRIMARY KEY, else sqlite will spend hours shuffling rows in table
, ";"
)
, Sql_update_repo_page = String_.Concat_lines_nl
( "REPLACE INTO orig_regy"
, "SELECT o.lnki_id"
, ", o.lnki_ttl"
, ", o.lnki_commons_ttl"
, ", o.orig_commons_flag"
, ", o.lnki_ext"
, ", o.lnki_count"
, ", {0}"
, ", m.src_id"
, ", NULL"
, ", NULL"
, ", m.src_id"
, ", m.src_ttl"
, ", i.img_ext_id"
, ", i.img_size"
, ", i.img_width"
, ", i.img_height"
, ", i.img_bits"
, ", i.img_media_type"
, ", i.img_minor_mime"
, ", i.img_timestamp"
, "FROM orig_regy o"
, " JOIN <image_db>image i ON {1} = i.img_name"
, " JOIN page_db.page_regy m ON m.repo_id = {0} AND m.itm_tid = 0 AND {1} = m.src_ttl"
, "WHERE o.orig_file_ttl IS NULL" // NOTE: only insert if file doesn't exist; changed from timestamp b/c old images may exist in both wikis; EX:ar.n:File:Facebook.png; DATE:2014-08-20
// , "WHERE i.img_timestamp > o.orig_timestamp" // NOTE: this handles an image in local and remote by taking later version; DATE:2014-07-22
, "ORDER BY 1" // must order by lnki_id since it is PRIMARY KEY, else sqlite will spend hours shuffling rows in table
, ";"
)
, Sql_update_repo_redirect = String_.Concat_lines_nl
( "REPLACE INTO orig_regy"
, "SELECT o.lnki_id"
, ", o.lnki_ttl"
, ", o.lnki_commons_ttl"
, ", o.orig_commons_flag"
, ", o.lnki_ext"
, ", o.lnki_count"
, ", {0}"
, ", m.src_id"
, ", m.trg_id"
, ", m.trg_ttl"
, ", m.trg_id"
, ", m.trg_ttl"
, ", i.img_ext_id"
, ", i.img_size"
, ", i.img_width"
, ", i.img_height"
, ", i.img_bits"
, ", i.img_media_type"
, ", i.img_minor_mime"
, ", i.img_timestamp"
, "FROM orig_regy o"
, " JOIN page_db.page_regy m ON m.repo_id = {0} AND m.itm_tid = 1 AND {1} = m.src_ttl"
, " JOIN <image_db>image i ON m.trg_ttl = i.img_name"
, "WHERE o.orig_file_ttl IS NULL" // NOTE: only insert if file doesn't exist; changed from timestamp b/c old images may exist in both wikis; EX:ar.n:File:Facebook.png; DATE:2014-08-20
// , "WHERE i.img_timestamp > o.orig_timestamp" // NOTE: this handles an image in local and remote by taking later version; DATE:2014-07-22
, "ORDER BY 1" // must order by lnki_id since it is PRIMARY KEY, else sqlite will spend hours shuffling rows in table
, ";"
)
, Sql_cs_mark_dupes = String_.Concat_lines_nl
( "REPLACE INTO orig_regy"
, "SELECT o.lnki_id"
, ", o.lnki_ttl"
, ", o.lnki_commons_ttl"
, ", 1"
, ", o.lnki_ext"
, ", o.lnki_count"
, ", o.orig_repo"
, ", o.orig_page_id"
, ", o.orig_redirect_id"
, ", o.orig_redirect_ttl"
, ", o.orig_file_id"
, ", o.orig_file_ttl"
, ", o.orig_file_ext"
, ", o.orig_size"
, ", o.orig_w"
, ", o.orig_h"
, ", o.orig_bits"
, ", o.orig_media_type"
, ", o.orig_minor_mime"
, ", o.orig_timestamp"
, "FROM orig_regy o"
, " JOIN orig_regy o2 ON o.lnki_commons_ttl = o2.lnki_ttl" // EX: 2 rows in table (1) A.jpg; and (2) "a.jpg,A.jpg"; do not insert row (2) b/c row (1) exists;
, "WHERE o.orig_file_ttl IS NULL" // NOTE: don't use timestamp logic here
, "ORDER BY 1" // must order by lnki_id since it is PRIMARY KEY, else sqlite will spend hours shuffling rows in table
, ";"
)
, Sql_cs_update_ttls = String_.Concat_lines_nl
( "UPDATE orig_regy"
, "SET lnki_ttl = lnki_commons_ttl"
, ", orig_commons_flag = 2"
, "WHERE orig_file_ttl IS NULL" // orig not found
, "AND lnki_commons_ttl IS NOT NULL" // orig commons ttl exists
, "AND orig_commons_flag IS NULL" // orig is not dupe
, ";"
)
;
}

100
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_orig_regy_update_bmk_mgr.java
View File

@@ -1,100 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_orig_regy_update_bmk_mgr implements GfoInvkAble {
private Db_cfg_tbl cfg_tbl;
private String cfg_grp;
public byte Repo_prv() {return repo_prv;} private byte repo_prv;
public int Ns_prv() {return ns_prv;} private int ns_prv;
public byte[] Ttl_prv() {return ttl_prv;} private byte[] ttl_prv;
private boolean repo_enable, ns_enable, ttl_enable;
private boolean repo_dirty, ns_dirty, ttl_dirty;
public Xob_orig_regy_update_bmk_mgr Init(Db_conn p, String grp, boolean repo_enable, boolean ns_enable, boolean ttl_enable) {
this.cfg_grp = grp;
cfg_tbl = new Db_cfg_tbl(p, "xowa_cfg");
this.repo_enable = repo_enable;
this.ns_enable = ns_enable;
this.ttl_enable = ttl_enable;
return this;
}
public void Term() {
this.Save(); // flush existing values
cfg_tbl.Rls();
}
public void Update_repo_ttl(byte repo, byte[] ttl) {Update(repo, Null_ns, ttl);}
public void Update(byte repo, int ns, byte[] ttl) {
if (repo_enable) {
if (repo_prv != repo) {
repo_dirty = true;
this.repo_prv = repo;
}
}
if (ns_enable) {
if (ns_prv != ns) {
ns_dirty = true;
this.ns_prv = ns;
}
}
if (ttl_enable) {
if (!Bry_.Eq(ttl_prv, ttl)) {
ttl_dirty = true;
this.ttl_prv = ttl;
}
}
}
public Xob_orig_regy_update_bmk_mgr Load() {
if (repo_enable)
repo_prv = cfg_tbl.Assert_byte(cfg_grp, Cfg_repo_prv, Byte_.Zero);
if (ns_enable)
ns_prv = cfg_tbl.Assert_int(cfg_grp, Cfg_ns_prv, 0);
if (ttl_enable)
ttl_prv = cfg_tbl.Assert_bry(cfg_grp, Cfg_ttl_prv, Bry_.Empty);
repo_dirty = ns_dirty = ttl_dirty = false;
return this;
}
public void Save() {
if (repo_enable && repo_dirty) {
Save(Cfg_repo_prv, Byte_.To_str(repo_prv));
repo_dirty = false;
}
if (ns_enable && ns_dirty) {
Save(Cfg_ns_prv, Int_.To_str(ns_prv));
ns_dirty = false;
}
if (ttl_enable && ttl_dirty) {
Save(Cfg_ttl_prv, String_.new_u8(ttl_prv));
ttl_dirty = false;
}
}
private void Save(String cfg_key, String cfg_val) {
cfg_tbl.Update_str(cfg_grp, cfg_key, cfg_val);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_repo_prv_)) repo_prv = m.ReadByte("v");
else if (ctx.Match(k, Invk_ns_prv_)) ns_prv = m.ReadInt("v");
else if (ctx.Match(k, Invk_ttl_prv_)) ttl_prv = m.ReadBry("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
}
private static final String Invk_repo_prv_ = "repo_prv_", Invk_ns_prv_ = "ns_prv_", Invk_ttl_prv_ = "ttl_prv_";
private static final String Cfg_repo_prv = "repo_prv", Cfg_ns_prv = "ns_prv", Cfg_ttl_prv = "ttl_prv";
private static final int Null_ns = -1;
// public static final byte Null_repo = Byte_.Max_value_127;
// public static final byte[ Null_ttl = null;
}

66
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_orig_regy_update_cmd.java
View File

@@ -1,66 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.stores.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.files.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.files.origs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_orig_regy_update_cmd extends Xob_itm_basic_base implements Xob_cmd { // downloads latest orig data
public Xob_orig_regy_update_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return KEY_oimg;} public static final String KEY_oimg = "oimg.orig_qry";
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
// Xof_orig_mgr qry_mgr = new Xof_orig_mgr();
Db_conn conn = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
Xob_orig_regy_update_bmk_mgr bmk = new Xob_orig_regy_update_bmk_mgr();
bmk.Init(conn, this.Cmd_key(), true, false, true);
bmk.Load();
// Xof_fsdb_itm itm = new Xof_fsdb_itm();
DataRdr rdr = DataRdr_.Null;
try {
// rdr = Select(conn, bmk.Repo_prv(), bmk.Ttl_prv());
while (rdr.MoveNextPeer()) {
// Load_itm(itm, rdr);
// QueryItm
}
}
finally {
rdr.Rls();
}
/*
DataRdr rdr = Get(repo_prv, ttl_prv);
while (rdr.MoveNextPeer()) {
Itm itm = Itm.load_(rdr);
qry.Call(itm);
}
*/
}
public DataRdr Select(Db_conn conn, byte prv_repo_id, byte[] prv_ttl) {
String sql = String_.Concat_lines_nl_skip_last
( "SELECT lnki_ttl"
, "FROM orig_regy"
, "WHERE lnki_repo >= '" + Byte_.To_str(prv_repo_id) + "'"
, "AND lnki_ttl > '" + prv_ttl + "'"
, "AND oimg_orig_page_id = -1;"
);
Db_qry select_qry = Db_qry_sql.rdr_(sql);
return conn.Exec_qry_as_old_rdr(select_qry);
}
public void Cmd_run() {}
public void Cmd_end() {}
public void Cmd_term() {}
}

52
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_page_regy_cmd.java
View File

@@ -1,52 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.files.repos.*;
import gplx.xowa.wikis.domains.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_page_regy_cmd extends Xob_itm_basic_base implements Xob_cmd {
public Xob_page_regy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
private boolean build_commons = false;
public String Cmd_key() {return Xob_cmd_keys.Key_file_page_regy;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
Xowe_wiki commons_wiki = bldr.App().Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons).Init_assert();
Db_conn page_regy_provider = Xob_db_file.New__page_regy(commons_wiki.Fsys_mgr().Root_dir()).Conn();
commons_wiki.Init_assert();
if (build_commons) {
Xob_page_regy_tbl.Reset_table(page_regy_provider);
Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_itm_.Repo_remote, commons_wiki);
Sqlite_engine_.Idx_create(usr_dlg, page_regy_provider, "repo_page", Xob_page_regy_tbl.Idx_main);
}
else {
if (!Bry_.Eq(commons_wiki.Domain_bry(), wiki.Domain_bry())) { // skip local wiki if cur wiki is commons
wiki.Init_assert();
Xob_page_regy_tbl.Delete_local(page_regy_provider);
Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_itm_.Repo_local, wiki);
}
}
}
public void Cmd_run() {}
public void Cmd_end() {}
public void Cmd_term() {}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_build_commons_)) build_commons = m.ReadYn("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_build_commons_ = "build_commons_";
}

86
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_page_regy_tbl.java
View File

@@ -1,86 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.xowa.files.repos.*; import gplx.dbs.engines.sqlite.*;
class Xob_page_regy_tbl {
public static void Reset_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
public static void Create_data(Gfo_usr_dlg usr_dlg, Db_conn p, byte repo_tid, Xowe_wiki wiki) {
Xowd_db_file db_core = wiki.Db_mgr_as_sql().Core_data_mgr().Db__core();
Create_data__insert_page(usr_dlg, p, repo_tid, db_core.Url());
Create_data__insert_redirect(usr_dlg, p, repo_tid, wiki.Fsys_mgr().Root_dir().GenSubFil(Xob_db_file.Name__wiki_redirect));
}
public static void Delete_local(Db_conn p) {
p.Exec_sql("DELETE FROM page_regy WHERE repo_id = " + Xof_repo_itm_.Repo_local);
}
private static void Create_data__insert_page(Gfo_usr_dlg usr_dlg, Db_conn cur, byte repo_tid, Io_url join) {
usr_dlg.Note_many("", "", "inserting page: ~{0}", join.NameOnly());
Sqlite_engine_.Db_attach(cur, "page_db", join.Raw());
cur.Exec_sql(String_.Format(Sql_create_page, repo_tid));
Sqlite_engine_.Db_detach(cur, "page_db");
}
private static void Create_data__insert_redirect(Gfo_usr_dlg usr_dlg, Db_conn cur, byte repo_tid, Io_url join) {
if (!Io_mgr.Instance.ExistsFil(join)) return; // redirect_db will not exist when commons.wikimedia.org is set up on new machine
usr_dlg.Note_many("", "", "inserting redirect: ~{0}", join.OwnerDir().NameOnly());
Sqlite_engine_.Db_attach(cur, "redirect_db", join.Raw());
cur.Exec_sql(String_.Format(Sql_create_redirect, repo_tid));
Sqlite_engine_.Db_detach(cur, "redirect_db");
}
public static final String Tbl_name = "page_regy"
, Fld_uid = "uid", Fld_repo_id = "repo_id", Fld_itm_tid = "itm_tid"
, Fld_src_id = "src_id", Fld_src_ttl = "src_ttl"
, Fld_trg_id = "trg_id", Fld_trg_ttl = "trg_ttl"
;
public static final Db_idx_itm
Idx_main = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS page_regy__main ON page_regy (repo_id, itm_tid, src_ttl, src_id, trg_id, trg_ttl);")
;
private static final String
Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS page_regy"
, "( uid integer NOT NULL PRIMARY KEY AUTOINCREMENT" // NOTE: must be PRIMARY KEY, else later REPLACE INTO will create dupe rows
, ", repo_id integer NOT NULL"
, ", itm_tid tinyint NOT NULL"
, ", src_id integer NOT NULL"
, ", src_ttl varchar(255) NOT NULL"
, ", trg_id integer NOT NULL"
, ", trg_ttl varchar(255) NOT NULL"
, ");"
)
, Sql_create_page = String_.Concat_lines_nl
( "INSERT INTO page_regy (repo_id, itm_tid, src_id, src_ttl, trg_id, trg_ttl)"
, "SELECT {0}"
, ", 0" // 0=page
, ", p.page_id"
, ", p.page_title"
, ", -1"
, ", ''"
, "FROM page_db.page p"
, ";"
)
, Sql_create_redirect = String_.Concat_lines_nl
( "INSERT INTO page_regy (repo_id, itm_tid, src_id, src_ttl, trg_id, trg_ttl)"
, "SELECT {0}"
, ", 1" // 1=redirect
, ", r.src_id"
, ", r.src_ttl"
, ", r.trg_id"
, ", r.trg_ttl"
, "FROM redirect_db.redirect r"
, ";"
)
;
}

37
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_regy_cmd.java
View File

@@ -1,37 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_xfer_regy_cmd extends Xob_itm_basic_base implements Xob_cmd {
public Xob_xfer_regy_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_regy;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_run() {
Db_conn conn = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
conn.Txn_bgn("bldr__xfer_regy");
Xob_xfer_regy_tbl.Create_table(conn);
Xob_xfer_regy_tbl.Create_data(usr_dlg, conn);
Xob_xfer_regy_tbl.Create_index(usr_dlg, conn);
Xob_xfer_regy_log_tbl.Create_table(conn);
conn.Txn_end();
}
public void Cmd_end() {}
public void Cmd_term() {}
}

158
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_regy_tbl.java
View File

@@ -1,158 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.stores.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.dbs.engines.sqlite.*;
public class Xob_xfer_regy_tbl {
public static final String Tbl_name = "xfer_regy"
, Fld_lnki_id = "lnki_id", Fld_lnki_tier_id = "lnki_tier_id", Fld_lnki_page_id = "lnki_page_id", Fld_lnki_ttl = "lnki_ttl", Fld_lnki_ext = "lnki_ext"
, Fld_lnki_time = "lnki_time", Fld_lnki_page = "lnki_page", Fld_lnki_count = "lnki_count"
, Fld_orig_repo = "orig_repo", Fld_orig_page_id = "orig_page_id", Fld_orig_redirect_src = "orig_redirect_src", Fld_orig_media_type = "orig_media_type"
, Fld_orig_w = "orig_w", Fld_orig_h = "orig_h"
, Fld_file_w = "file_w", Fld_file_h = "file_h", Fld_file_is_orig = "file_is_orig"
, Fld_xfer_status = "xfer_status"
;
public static void Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
public static void Create_data(Gfo_usr_dlg usr_dlg, Db_conn p) {
p.Exec_sql(Sql_create_data_orig);
p.Exec_sql(Sql_create_data_thumb);
}
public static void Create_index(Gfo_usr_dlg usr_dlg, Db_conn p) {Sqlite_engine_.Idx_create(usr_dlg, p, Xob_db_file.Name__file_make, Idx_lnki_page_id, Idx_lnki_ttl);}
public static DataRdr Select(Db_conn conn, byte repo_id, byte[] ttl, int limit) {
Db_qry qry = Db_qry_.select_().Cols_all_()
.From_(Tbl_name)
.Where_(gplx.core.criterias.Criteria_.And_many(Db_crt_.New_mte(Fld_orig_repo, repo_id), Db_crt_.New_mt(Fld_lnki_ttl, String_.new_u8(ttl)), Db_crt_.New_eq(Fld_xfer_status, 0)))
.Order_asc_many_(Fld_xfer_status, Fld_orig_repo, Fld_lnki_ttl, Fld_file_w)
.Limit_(limit)
;
return conn.Exec_qry_as_old_rdr(qry);
}
public static Db_stmt Select_by_page_id_stmt(Db_conn p) {return p.Stmt_sql(Sql_select_clause);}
public static DataRdr Select_by_page_id(Db_stmt stmt, int page_id, int limit) {return stmt.Val_int(page_id).Val_int(limit).Exec_select();}
private static final String
Sql_select_clause = String_.Concat_lines_nl
( "SELECT *"
, "FROM xfer_regy"
, "WHERE xfer_status = 0"
, "AND lnki_page_id >= ?"
, "ORDER BY lnki_tier_id, lnki_page_id, lnki_id"
, "LIMIT ?"
)
, Sql_select_total_pending = String_.Concat_lines_nl
( "SELECT Count(*) AS CountAll"
, "FROM xfer_regy"
, "WHERE xfer_status = 0"
)
;
public static DataRdr Select_by_tier_page(Db_conn conn, int tier_id, int page_id, int select_interval) {
Db_qry qry = Db_qry_.select_().Cols_all_()
.From_(Tbl_name)
.Where_(gplx.core.criterias.Criteria_.And_many(Db_crt_.New_eq(Fld_xfer_status, 0), Db_crt_.New_eq(Fld_lnki_tier_id, tier_id), Db_crt_.New_mte(Fld_lnki_page_id, page_id)))
.Order_asc_many_(Fld_lnki_tier_id, Fld_lnki_page_id, Fld_lnki_id)
.Limit_(select_interval)
;
return conn.Exec_qry_as_old_rdr(qry);
}
public static int Select_total_pending(Db_conn p) {
DataRdr rdr = p.Exec_sql_as_old_rdr(Sql_select_total_pending);
int rv = 0;
if (rdr.MoveNextPeer())
rv = rdr.ReadInt("CountAll");
rdr.Rls();
return rv;
}
private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS xfer_regy"
, "( lnki_id integer NOT NULL PRIMARY KEY"
, ", lnki_tier_id integer NOT NULL"
, ", lnki_page_id integer NOT NULL"
, ", lnki_ttl varchar(255) NOT NULL"
, ", lnki_ext integer NOT NULL"
, ", lnki_time double NOT NULL"
, ", lnki_page integer NOT NULL"
, ", lnki_count integer NOT NULL"
, ", orig_repo tinyint NOT NULL"
, ", orig_page_id integer NOT NULL"
, ", orig_redirect_src varchar(255) NOT NULL"
, ", orig_media_type varchar(64) NOT NULL"
, ", orig_w integer NOT NULL"
, ", orig_h integer NOT NULL"
, ", file_is_orig byte NOT NULL"
, ", file_w integer NOT NULL"
, ", file_h integer NOT NULL"
, ", xfer_status integer NOT NULL"
, ");"
);
private static final String Sql_create_data_orig = String_.Concat_lines_nl
( "INSERT INTO xfer_regy "
, "( lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
, ", xfer_status"
, ")"
, "SELECT "
, " Min(lnki_id), Min(lnki_tier_id), Min(lnki_page_id), Min(orig_page_id), orig_repo, lnki_ttl, Max(orig_redirect_src), lnki_ext, orig_media_type" // NOTE: Max(orig_redirect_src) not Min (else would get '')
, ", file_is_orig, orig_w, orig_h, -1, -1, lnki_time, lnki_page, Sum(lnki_count)"
, ", 0"
, "FROM xfer_temp x"
, "WHERE file_is_orig = 1"
, "GROUP BY orig_repo, lnki_ttl, lnki_ext, orig_media_type, file_is_orig, orig_w, orig_h, lnki_time, lnki_page"
);
private static final String Sql_create_data_thumb = String_.Concat_lines_nl
( "INSERT INTO xfer_regy "
, "( lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
, ", xfer_status"
, ")"
, "SELECT "
, " Min(lnki_id), Min(lnki_tier_id), Min(lnki_page_id), Min(orig_page_id), orig_repo, lnki_ttl, Max(orig_redirect_src), lnki_ext, orig_media_type"
, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, Sum(lnki_count)"
, ", 0"
, "FROM xfer_temp x"
, "WHERE file_is_orig = 0"
, "GROUP BY orig_repo, lnki_ttl, lnki_ext, orig_media_type, file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page"
);
private static final Db_idx_itm
// Idx_select = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS xfer_regy__select ON xfer_regy (xfer_status, orig_repo, lnki_ttl, file_w);")
Idx_lnki_page_id = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS xfer_regy__lnki_page_id ON xfer_regy (xfer_status, lnki_tier_id, lnki_page_id, lnki_id, orig_repo, file_is_orig, lnki_ttl, lnki_ext, lnki_time, lnki_page, file_w, file_h);")
, Idx_lnki_ttl = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS xfer_regy__lnki_ttl ON xfer_regy (lnki_ttl);") // needed for troubleshooting
;
public static byte Status_todo = 0, Status_pass = 1, Status_fail = 2, Status_ignore_processed = 3;
}
class Xob_xfer_regy_log_tbl {
public static void Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
public static Db_stmt Insert_stmt(Db_conn p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_lnki_id, Fld_xfer_status, Fld_xfer_bin_tid, Fld_xfer_bin_msg);}
public static void Insert(Db_stmt stmt, byte status, int id, byte wkr_tid, String wkr_msg) {
stmt.Clear()
.Val_int(id)
.Val_byte(status)
.Val_byte(wkr_tid)
.Val_str(wkr_msg)
.Exec_insert();
}
private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS xfer_regy_log"
, "( lnki_id integer NOT NULL"
, ", xfer_status tinyint NOT NULL" // 0=todo; 1=fail; 2=pass; 3=done
, ", xfer_bin_tid tinyint NOT NULL"
, ", xfer_bin_msg varchar(255) NOT NULL"
, ");"
);
public static final String Tbl_name = "xfer_regy_log"
, Fld_lnki_id = "lnki_id", Fld_xfer_status = "xfer_status", Fld_xfer_bin_tid = "xfer_bin_tid", Fld_xfer_bin_msg = "xfer_bin_msg"
;
}

174
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_regy_update_cmd.java
View File

@@ -1,174 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.fsdb.meta.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_xfer_regy_update_cmd extends Xob_itm_basic_base implements Xob_cmd {
public Xob_xfer_regy_update_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_regy_update;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_run() {Exec_main();}
public void Cmd_end() {}
public void Cmd_term() {}
private void Exec_main() {
wiki.Init_assert(); // NOTE: must init wiki to set up db_core; DATE:2015-08-17
Db_conn make_db_provider = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
this.Copy_atrs_to_make_db(make_db_provider);
this.Update_status(make_db_provider);
}
private void Copy_atrs_to_make_db(Db_conn make_db_provider) {
wiki.File_mgr().Init_file_mgr_by_load(wiki); // NOTE: this gets current fsdb to update from xfer_regy; this never uses -prv/ or /prv/ databases; DATE:2015-09-10
Fsm_mnt_itm fsdb_abc_mgr = wiki.File_mgr().Fsdb_mgr().Mnt_mgr().Mnts__get_main(); // 0 = fsdb.main
Db_conn conn = fsdb_abc_mgr.Atr_mgr().Db__core().Conn(); // 0 = fsdb.atr.00
Io_url fsdb_atr_url = ((gplx.dbs.engines.sqlite.Sqlite_conn_info)conn.Conn_info()).Url();
Sqlite_engine_.Tbl_create_and_delete(make_db_provider, Xob_fsdb_regy_tbl.Tbl_name, Xob_fsdb_regy_tbl.Tbl_sql);
Sqlite_engine_.Db_attach(make_db_provider, "fsdb_db", fsdb_atr_url.Raw());
make_db_provider.Txn_bgn("bldr__xfer_regy_update");
make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Update_regy_nil);
make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Insert_fsdb_fil);
String fsdb_thm_tbl = fsdb_abc_mgr.Db_mgr().File__schema_is_1() ? "fsdb_xtn_thm" : "fsdb_thm";
String insert_sql_fsdb_thm = wiki.File_mgr().Fsdb_mgr().Mnt_mgr().Mnts__get_main().Cfg_mgr().Schema_thm_page() // Cfg_get(Fsm_cfg_mgr.Grp_core).Get_yn_or_n(Fsm_cfg_mgr.Key_schema_thm_page)
? String_.Format(Xob_fsdb_regy_tbl.Insert_fsdb_thm, fsdb_thm_tbl)
: Xob_fsdb_regy_tbl.Insert_fsdb_thm_v0
;
make_db_provider.Exec_sql(insert_sql_fsdb_thm);
make_db_provider.Txn_end();
Sqlite_engine_.Idx_create(make_db_provider, Xob_fsdb_regy_tbl.Idx_main);
Sqlite_engine_.Db_detach(make_db_provider, "fsdb_db");
}
private void Update_status(Db_conn make_db_provider) {
make_db_provider.Txn_bgn("bldr__xfer_regy_update_status");
make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Update_regy_fil);
make_db_provider.Exec_sql(Xob_fsdb_regy_tbl.Update_regy_thm);
make_db_provider.Txn_end();
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
return this;
}
}
class Xob_fsdb_regy_tbl {
public static final String Tbl_name = "fsdb_regy";
public static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE fsdb_regy "
, "( fsdb_id integer NOT NULL PRIMARY KEY AUTOINCREMENT"
, ", fsdb_name varchar(255) NOT NULL"
, ", fsdb_is_orig tinyint NOT NULL"
, ", fsdb_repo tinyint NOT NULL"
, ", fsdb_w integer NOT NULL"
, ", fsdb_time double NOT NULL"
, ", fsdb_page integer NOT NULL"
, ", fsdb_db_id integer NOT NULL"
, ", fsdb_size bigint NOT NULL"
, ", fsdb_status tinyint NOT NULL"
, ", fsdb_fil_id integer NOT NULL"
, ", fsdb_thm_id integer NOT NULL"
, ");"
);
public static final Db_idx_itm Idx_main = Db_idx_itm.sql_("CREATE INDEX fsdb_regy__main ON fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page);");
public static final String
Insert_fsdb_fil = String_.Concat_lines_nl
( "INSERT INTO fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page, fsdb_db_id, fsdb_size, fsdb_status, fsdb_fil_id, fsdb_thm_id)"
, "SELECT f.fil_name"
, ", 1"
, ", CASE WHEN d.dir_name = 'commons.wikimedia.org' THEN 0 ELSE 1 END"
, ", -1"
, ", -1"
, ", -1"
, ", f.fil_bin_db_id"
, ", f.fil_size"
, ", 0"
, ", f.fil_id"
, ", -1"
, "FROM fsdb_db.fsdb_fil f"
, " JOIN fsdb_db.fsdb_dir d ON f.fil_owner_id = d.dir_id"
, "WHERE f.fil_bin_db_id != -1"
, ";"
)
, Insert_fsdb_thm = String_.Concat_lines_nl
( "INSERT INTO fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page, fsdb_db_id, fsdb_size, fsdb_status, fsdb_fil_id, fsdb_thm_id)"
, "SELECT f.fil_name"
, ", 0"
, ", CASE WHEN d.dir_name = 'commons.wikimedia.org' THEN 0 ELSE 1 END"
, ", t.thm_w"
, ", t.thm_time"
, ", t.thm_page"
, ", t.thm_bin_db_id"
, ", t.thm_size"
, ", 0"
, ", f.fil_id"
, ", t.thm_id"
, "FROM fsdb_db.fsdb_fil f"
, " JOIN fsdb_db.{0} t ON f.fil_id = t.thm_owner_id"
, " JOIN fsdb_db.fsdb_dir d ON f.fil_owner_id = d.dir_id"
, ";"
)
, Insert_fsdb_thm_v0 = String_.Concat_lines_nl
( "INSERT INTO fsdb_regy (fsdb_name, fsdb_is_orig, fsdb_repo, fsdb_w, fsdb_time, fsdb_page, fsdb_db_id, fsdb_size, fsdb_status, fsdb_fil_id, fsdb_thm_id)"
, "SELECT f.fil_name"
, ", 0"
, ", CASE WHEN d.dir_name = 'commons.wikimedia.org' THEN 0 ELSE 1 END"
, ", t.thm_w"
, ", t.thm_thumbtime"
, ", -1"
, ", t.thm_bin_db_id"
, ", t.thm_size"
, ", 0"
, ", f.fil_id"
, ", t.thm_id"
, "FROM fsdb_db.fsdb_fil f"
, " JOIN fsdb_db.{0} t ON f.fil_id = t.thm_owner_id"
, " JOIN fsdb_db.fsdb_dir d ON f.fil_owner_id = d.dir_id"
, ";"
)
, Update_regy_nil = "UPDATE xfer_regy SET xfer_status = 0;"
, Update_regy_fil = String_.Concat_lines_nl
( "REPLACE INTO xfer_regy "
, "( lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
, ", xfer_status"
, ")"
, "SELECT "
, " lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
, ", CASE WHEN f.fsdb_name IS NOT NULL THEN 1 ELSE 0 END"
, "FROM xfer_regy x"
, " LEFT JOIN fsdb_regy f ON x.lnki_ttl = f.fsdb_name"
, "WHERE x.file_is_orig = 1 AND f.fsdb_is_orig = 1"
, "AND x.orig_repo = f.fsdb_repo"
, ";"
)
, Update_regy_thm = String_.Concat_lines_nl
( "REPLACE INTO xfer_regy "
, "( lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
, ", xfer_status"
, ")"
, "SELECT "
, " lnki_id, lnki_tier_id, lnki_page_id, orig_page_id, orig_repo, lnki_ttl, orig_redirect_src, lnki_ext, orig_media_type"
, ", file_is_orig, orig_w, orig_h, file_w, file_h, lnki_time, lnki_page, lnki_count"
, ", CASE WHEN f.fsdb_name IS NOT NULL THEN 1 ELSE 0 END"
, "FROM xfer_regy x"
, " LEFT JOIN fsdb_regy f ON x.lnki_ttl = f.fsdb_name AND x.file_w = f.fsdb_w"
, " AND x.lnki_time = f.fsdb_time AND x.lnki_page = f.fsdb_page"
, "WHERE x.file_is_orig = 0 AND f.fsdb_is_orig = 0"
, "AND x.orig_repo = f.fsdb_repo"
, ";"
)
;
}

106
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_cmd_orig.java
View File

@@ -1,106 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.stores.*;
import gplx.dbs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.files.*; import gplx.xowa.files.exts.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.parsers.lnkis.*;
public class Xob_xfer_temp_cmd_orig extends Xob_itm_basic_base implements Xob_cmd {
private byte[] ext_rules_key = Bry_.Empty;
public Xob_xfer_temp_cmd_orig(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_temp_orig;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
Db_conn conn = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
Xob_xfer_temp_tbl.Create_table(conn);
Db_stmt trg_stmt = Xob_xfer_temp_tbl.Insert_stmt(conn);
conn.Txn_bgn("bldr__xfer_temp");
DataRdr rdr = conn.Exec_sql_as_old_rdr(Sql_select_clause);
long[] ext_maxs = Calc_ext_max();
while (rdr.MoveNextPeer()) {
int lnki_ext = rdr.ReadByte(Xob_lnki_regy_tbl.Fld_lnki_ext);
String orig_media_type = rdr.ReadStrOr(Xob_orig_regy_tbl.Fld_orig_media_type, ""); // convert nulls to ""
lnki_ext = rdr.ReadInt(Xob_orig_regy_tbl.Fld_orig_file_ext);
int lnki_id = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_id);
int lnki_tier_id = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_tier_id);
byte orig_repo = rdr.ReadByte(Xob_orig_regy_tbl.Fld_orig_repo);
int orig_page_id = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_page_id, -1);
if (orig_page_id == -1) continue; // no orig found; ignore
String join_ttl = rdr.ReadStr(Xob_orig_regy_tbl.Fld_orig_file_ttl);
String redirect_src = rdr.ReadStr(Xob_orig_regy_tbl.Fld_lnki_ttl);
if (String_.Eq(join_ttl, redirect_src)) // lnki_ttl is same as redirect_src; not a redirect
redirect_src = "";
int orig_w = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_w, -1);
int orig_h = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_h, -1);
int orig_size = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_size, -1);
if (orig_size > ext_maxs[lnki_ext]) continue;
int lnki_page_id = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_page_id);
Xob_xfer_temp_tbl.Insert(trg_stmt, lnki_id, lnki_tier_id, lnki_page_id, orig_repo, orig_page_id, join_ttl, redirect_src, lnki_ext, Xop_lnki_type.Id_none, orig_media_type
, Bool_.Y // orig is y
, orig_w, orig_h
, orig_w, orig_h // file_w, file_h is same as orig_w,orig_h; i.e.: make same file_w as orig_w
, Xof_img_size.Null, Xof_img_size.Null // html_w, html_h is -1; i.e.: will not be displayed in page at specific size (this matches logic in Xob_xfer_temp_cmd_thumb)
, Xof_lnki_time.Null
, Xof_lnki_page.Null
, 0);
}
conn.Txn_end();
}
private long[] Calc_ext_max() {
Xof_rule_grp ext_rules = wiki.Appe().File_mgr().Ext_rules().Get_or_new(ext_rules_key);
long[] rv = new long[Xof_ext_.Id__max];
for (int i = 0; i < Xof_ext_.Id__max; i++) {
byte[] ext = Xof_ext_.Get_ext_by_id_(i);
Xof_rule_itm ext_rule = ext_rules.Get_or_null(ext);
long max = ext_rule == null ? 0 : ext_rule.Make_max();
rv[i] = max;
}
return rv;
}
public void Cmd_run() {}
public void Cmd_end() {}
public void Cmd_term() {}
private static final String
Sql_select_clause = String_.Concat_lines_nl
( "SELECT DISTINCT"
, " l.lnki_id"
// , ", lnki_ttl"
, ", l.lnki_ext"
, ", l.lnki_page_id"
, ", o.orig_repo"
, ", o.orig_page_id"
// , ", orig_file_id"
, ", o.orig_file_ttl"
, ", o.orig_file_ext"
, ", o.lnki_ttl"
, ", o.orig_size"
, ", o.orig_w"
, ", o.orig_h"
, ", o.orig_media_type"
// , ", orig_bits"
, "FROM lnki_regy l"
, " JOIN orig_regy o ON o.lnki_ttl = l.lnki_ttl"
, "WHERE o.orig_file_ttl IS NOT NULL"
, "ORDER BY o.orig_file_ttl DESC"
);
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_ext_rules_)) ext_rules_key = m.ReadBry("v");
else return super.Invk (ctx, ikey, k, m);
return this;
} private static final String Invk_ext_rules_ = "ext_rules_";
}

83
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_cmd_thumb.java
View File

@@ -1,83 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.stores.*;
import gplx.dbs.*; import gplx.xowa.files.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_xfer_temp_cmd_thumb extends Xob_itm_basic_base implements Xob_cmd {
public Xob_xfer_temp_cmd_thumb(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_temp_thumb;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
Db_conn conn = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
Xob_xfer_temp_tbl.Create_table(conn);
Db_stmt trg_stmt = Xob_xfer_temp_tbl.Insert_stmt(conn);
conn.Txn_bgn("bldr__xfer_temp_thumb");
DataRdr rdr = conn.Exec_sql_as_old_rdr(Sql_select_clause);
Xob_xfer_temp_itm temp_itm = new Xob_xfer_temp_itm();
Xof_img_size img_size = new Xof_img_size();
byte[] cur_ttl = Bry_.Empty; byte cur_repo = Byte_.Max_value_127;
while (rdr.MoveNextPeer()) {
temp_itm.Clear();
temp_itm.Load(rdr);
if (Bry_.Eq(cur_ttl, temp_itm.Orig_file_ttl())) { // same ttl; DATE:2015-03-22
if (temp_itm.Orig_repo() != cur_repo) // if repo is different, ignore 2nd; handles images in both repos; take 1st only (which should be local)
continue;
}
else { // new ttl; update ttl, repo
cur_ttl = temp_itm.Orig_file_ttl();
cur_repo = temp_itm.Orig_repo();
}
if (temp_itm.Chk(img_size))
temp_itm.Insert(trg_stmt, img_size);
}
conn.Txn_end();
}
public void Cmd_run() {}
public void Cmd_end() {}
public void Cmd_term() {}
private static final String
Sql_select_clause = String_.Concat_lines_nl
( "SELECT l.lnki_id"
, ", l.lnki_tier_id"
, ", l.lnki_page_id"
, ", l.lnki_ext"
, ", l.lnki_type"
, ", l.lnki_src_tid"
, ", l.lnki_w"
, ", l.lnki_h"
, ", l.lnki_upright"
, ", l.lnki_time"
, ", l.lnki_page"
, ", l.lnki_count"
, ", o.orig_repo"
, ", o.orig_page_id"
, ", o.orig_file_ttl"
, ", o.orig_file_ext"
, ", o.orig_file_id"
, ", o.lnki_ttl"
, ", o.orig_w"
, ", o.orig_h"
, ", o.orig_media_type"
, ", o.orig_minor_mime"
, "FROM lnki_regy l"
, " JOIN orig_regy o ON o.lnki_ttl = l.lnki_ttl"
, "WHERE o.orig_file_ttl IS NOT NULL"
, "ORDER BY o.orig_file_ttl, o.orig_repo DESC, l.lnki_w DESC" // NOTE: local=1,common=0; DATE:2015-03-22
);
}

133
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_itm.java
View File

@@ -1,133 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.stores.*;
import gplx.dbs.*; import gplx.xowa.files.*;
import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.lnkis.files.*;
class Xob_xfer_temp_itm {
public int Lnki_id() {return lnki_id;} private int lnki_id;
public int Lnki_tier_id() {return lnki_tier_id;} private int lnki_tier_id;
public byte Orig_repo() {return orig_repo;} private byte orig_repo;
public int Lnki_ext() {return lnki_ext;} private int lnki_ext;
public byte[] Orig_file_ttl() {return orig_file_ttl;} private byte[] orig_file_ttl;
public String Orig_media_type() {return orig_media_type;} private String orig_media_type;
public String Orig_minor_mime() {return orig_minor_mime;} private String orig_minor_mime;
public byte Orig_media_type_tid() {return orig_media_type_tid;} private byte orig_media_type_tid;
public int Orig_page_id() {return orig_page_id;} private int orig_page_id;
public String Join_ttl() {return join_ttl;} private String join_ttl;
public String Redirect_src() {return redirect_src;} private String redirect_src;
public byte Lnki_type() {return lnki_type;} private byte lnki_type;
public byte Lnki_src_tid() {return lnki_src_tid;} private byte lnki_src_tid;
public int Lnki_w() {return lnki_w;} private int lnki_w;
public int Lnki_h() {return lnki_h;} private int lnki_h;
public int Lnki_count() {return lnki_count;} private int lnki_count;
public int Lnki_page_id() {return lnki_page_id;} private int lnki_page_id;
public int Orig_w() {return orig_w;} private int orig_w;
public int Orig_h() {return orig_h;} private int orig_h;
public int Orig_ext_id() {return orig_ext_id;} private int orig_ext_id;
public double Lnki_upright() {return lnki_upright;} private double lnki_upright;
public double Lnki_thumbtime() {return lnki_thumbtime;} private double lnki_thumbtime;
public int Lnki_page() {return lnki_page;} private int lnki_page;
public void Clear() {
orig_file_ttl = null;
lnki_ext = lnki_type = lnki_src_tid
= orig_repo = orig_media_type_tid = Byte_.Max_value_127;
chk_tid = Chk_tid_none;
lnki_id = lnki_tier_id = lnki_w = lnki_h = lnki_count = lnki_page_id
= orig_w = orig_h = orig_page_id = Int_.Neg1;
join_ttl = redirect_src = orig_media_type = null;
lnki_upright = Xop_lnki_tkn.Upright_null;
lnki_thumbtime = Xof_lnki_time.Null;
lnki_page = Xof_lnki_page.Null;
}
public void Load(DataRdr rdr) {
lnki_id = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_id);
lnki_tier_id = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_tier_id);
lnki_page_id = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_page_id);
lnki_ext = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_ext);
lnki_type = rdr.ReadByte(Xob_lnki_regy_tbl.Fld_lnki_type);
lnki_src_tid = rdr.ReadByte(Xob_lnki_regy_tbl.Fld_lnki_src_tid);
lnki_w = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_w);
lnki_h = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_h);
lnki_upright = rdr.ReadDouble(Xob_lnki_regy_tbl.Fld_lnki_upright);
lnki_thumbtime = Xof_lnki_time.Db_load_double(rdr, Xob_lnki_regy_tbl.Fld_lnki_time);
lnki_page = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_page);
lnki_count = rdr.ReadInt(Xob_lnki_regy_tbl.Fld_lnki_count);
orig_file_ttl = rdr.ReadBryByStr(Xob_lnki_regy_tbl.Fld_lnki_ttl);
orig_repo = rdr.ReadByte(Xob_orig_regy_tbl.Fld_orig_repo);
orig_page_id = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_page_id, -1);
join_ttl = rdr.ReadStr(Xob_orig_regy_tbl.Fld_orig_file_ttl);
redirect_src = rdr.ReadStr(Xob_orig_regy_tbl.Fld_lnki_ttl);
orig_w = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_w, -1);
orig_h = rdr.ReadIntOr(Xob_orig_regy_tbl.Fld_orig_h, -1);
orig_media_type = rdr.ReadStrOr(Xob_orig_regy_tbl.Fld_orig_media_type, ""); // convert nulls to ""
orig_minor_mime = rdr.ReadStrOr(Xob_orig_regy_tbl.Fld_orig_minor_mime, ""); // convert nulls to ""
orig_ext_id = rdr.ReadInt(Xob_orig_regy_tbl.Fld_orig_file_ext);
}
public static final byte
Chk_tid_none = 0
, Chk_tid_orig_page_id_is_null = 1
, Chk_tid_orig_media_type_is_audio = 2
, Chk_tid_ns_is_media = 3
, Chk_tid_orig_w_is_0 = 4
;
public byte Chk_tid() {return chk_tid;} private byte chk_tid;
public boolean Chk(Xof_img_size img_size) {
if (String_.Eq(join_ttl, redirect_src)) // join_ttl is same as redirect_src; not a redirect; EX:(direct) join="A.png";redirect_src="A.png"; (redirect) join="A.png";redirect_src="B.png" (i.e.: B redirects to A)
redirect_src = "";
// else { // redirect; make sure extension matches; EX: A.png redirects to B.png; lnki_ext will be .png (the lnki's ext); should be .png (the actual file's ext)
// Xof_ext join_ext = Xof_ext_.new_by_ttl_(Bry_.new_u8(join_ttl));
// lnki_ext = join_ext.Id();
// }
lnki_ext = orig_ext_id;
orig_media_type_tid = Xof_media_type.Xto_byte(orig_media_type);
if ( Xof_lnki_time.Null_n(lnki_thumbtime) // thumbtime defined
&& orig_media_type_tid != Xof_media_type.Tid_video // video can have thumbtime
)
lnki_thumbtime = Xof_lnki_time.Null; // set thumbtime to NULL; actually occurs for one file: [[File:Crash.arp.600pix.jpg|thumb|thumbtime=2]]
if ( Xof_lnki_page.Null_n(lnki_page)
&& !Xof_ext_.Id_supports_page(orig_ext_id)) // djvu / pdf can have page parameters, which are currently being stored in thumbtime; DATE:2014-01-18
lnki_page = Xof_lnki_page.Null;
if (orig_page_id == -1) { // no orig found (i.e.: not in local's / remote's image.sql);
chk_tid = Chk_tid_orig_page_id_is_null;
return false;
}
if (orig_media_type_tid == Xof_media_type.Tid_audio) { // ignore: audio will never have thumbs
chk_tid = Chk_tid_orig_media_type_is_audio;
return false;
}
if (orig_w <= 0) { // ignore files that have an orig_w of 0; note that ogg files that are sometimes flagged as VIDEO; EX:2009_10_08_Marc_Randazza_interview.ogg; DATE:2014-08-20
chk_tid = Chk_tid_orig_w_is_0;
return false;
}
if (lnki_ext == Xof_ext_.Id_mid) { // NOTE: .mid does not have orig_media_type of "AUDIO"
chk_tid = Chk_tid_orig_media_type_is_audio;
return false;
}
if (lnki_src_tid == Xop_file_logger_.Tid__media) {
chk_tid = Chk_tid_ns_is_media;
return false;
}
int upright_patch = Xof_patch_upright_tid_.Tid_all; // all future blds will have upright_patch
img_size.Html_size_calc(Xof_exec_tid.Tid_wiki_page, lnki_w, lnki_h, lnki_type, upright_patch, lnki_upright, lnki_ext, orig_w, orig_h, Xof_img_size.Thumb_width_img);
return true;
}
public void Insert(Db_stmt stmt, Xof_img_size img_size) {
Xob_xfer_temp_tbl.Insert(stmt, lnki_id, lnki_tier_id, lnki_page_id, orig_repo, orig_page_id, join_ttl, redirect_src, lnki_ext, lnki_type, orig_media_type, img_size.File_is_orig(), orig_w, orig_h, img_size.File_w(), img_size.File_h(), img_size.Html_w(), img_size.Html_h(), lnki_thumbtime, lnki_page, lnki_count);
}
}

191
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_itm_tst.java
View File

@@ -1,191 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*;
import gplx.core.gfo_ndes.*;
import gplx.core.stores.*; import gplx.xowa.files.*; import gplx.xowa.files.repos.*;
import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.lnkis.files.*;
public class Xob_xfer_temp_itm_tst {
private Xob_xfer_temp_itm_fxt fxt = new Xob_xfer_temp_itm_fxt();
@Before public void init() {fxt.Reset();}
@Test public void Pass() {fxt.Test_pass().Test_itm_chk_fail_id_none();}
@Test public void Missing_orig() {fxt.Test_fail(Xob_xfer_temp_itm.Chk_tid_orig_page_id_is_null , Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_page_id, null));}
@Test public void File_is_audio() {fxt.Test_fail(Xob_xfer_temp_itm.Chk_tid_orig_media_type_is_audio , Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_media_type, Xof_media_type.Name_audio));}
@Test public void File_is_mid() {
fxt.Test_fail(Xob_xfer_temp_itm.Chk_tid_orig_media_type_is_audio , Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext, Xof_ext_.Id_mid));
}
@Test public void Redirect_src_is_empty() { // orig_cmd sets all direct files to have "orig_join" == "lnki_ttl"
fxt.Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_file_ttl , "A.png")
, Keyval_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl , "A.png")
);
fxt.Test_lnki_redirect_src(""); // confirm redirect_src set to ""
}
@Test public void Redirect_src_has_val() { // orig_cmd sets all redirect files to have "orig_join" = redirect and "lnki_ttl" as orig; EX: A.png redirects to B.png; orig_join will be B.png (the actual image) and redirect_src will be A.png (the original lnki)
fxt.Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_file_ttl , "B.png")
, Keyval_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl , "A.png")
);
fxt.Test_lnki_redirect_src("A.png"); // confirm redirect_src set to ""
}
@Test public void Redirect_should_take_trg_ext() {// if "A.png" redirects to "B.jpg", ext_id should be ".jpg" (the actual file) not ".png (lnki_ext_id)
fxt.Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_file_ttl , "B.jpg")
, Keyval_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl , "A.png")
, Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext , Xof_ext_.Id_jpg) // .png b/c B.jpg
);
fxt.Test_lnki_ext_id(Xof_ext_.Id_jpg); // confirm ext changed to .jpg
}
@Test public void Thumbtime_check() {// PURPOSE: one image actually had a thumbtime defined; EX: General_Dynamics_F-16_Fighting_Falcon; [[File:Crash.arp.600pix.jpg|thumb|thumbtime=2]]
fxt.Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext , Xof_ext_.Id_jpg)
, Keyval_.new_(Xob_lnki_regy_tbl.Fld_lnki_time , (double)3)
);
fxt.Test_lnki_thumbtime(Xof_lnki_time.Null);
fxt.Reset().Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_media_type , Xof_media_type.Name_video)
, Keyval_.new_(Xob_lnki_regy_tbl.Fld_lnki_time , (double)3)
);
fxt.Test_lnki_thumbtime(3);
}
@Test public void Page_check() {
fxt.Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext , Xof_ext_.Id_jpg)
, Keyval_.new_(Xob_lnki_regy_tbl.Fld_lnki_page , 3)
);
fxt.Test_lnki_page(Xof_lnki_page.Null);
fxt.Reset().Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext , Xof_ext_.Id_pdf)
, Keyval_.new_(Xob_lnki_regy_tbl.Fld_lnki_page , 3)
);
fxt.Test_lnki_page(3);
fxt.Reset().Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_file_ext , Xof_ext_.Id_djvu)
, Keyval_.new_(Xob_lnki_regy_tbl.Fld_lnki_page , 3)
);
fxt.Test_lnki_page(3);
}
@Test public void Media_should_be_ignored() {// ignore [[Media:]] for xfer_thumb (needed for xfer_orig)
fxt.Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl , "A.png")
, Keyval_.new_(Xob_lnki_regy_tbl.Fld_lnki_src_tid , Xop_file_logger_.Tid__media)
);
fxt.Test_itm_chk_fail_id(Xob_xfer_temp_itm.Chk_tid_ns_is_media);
}
@Test public void Orig_width_is_0() {// PURPOSE: ignore files with an orig width of 0; note that ogg files that are sometimes flagged as VIDEO; EX:2009_10_08_Marc_Randazza_interview.ogg; DATE:2014-08-20
fxt.Test_bgn
( Keyval_.new_(Xob_orig_regy_tbl.Fld_lnki_ttl , "A.ogg")
, Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_media_type , Xof_media_type.Name_video) // VIDEO
, Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_w , 0) // no width defined in image table
, Keyval_.new_(Xob_orig_regy_tbl.Fld_orig_h , 0)
);
fxt.Test_itm_chk_fail_id(Xob_xfer_temp_itm.Chk_tid_orig_w_is_0);
}
}
class Xob_xfer_temp_itm_fxt {
private Xob_xfer_temp_itm itm = new Xob_xfer_temp_itm();
private Xof_img_size img_size = new Xof_img_size();
private DataRdr_mem rdr;
private GfoNde nde;
public static String[] Flds = new String[]
{ Xob_lnki_regy_tbl.Fld_lnki_ext
, Xob_lnki_regy_tbl.Fld_lnki_id
, Xob_lnki_regy_tbl.Fld_lnki_tier_id
, Xob_orig_regy_tbl.Fld_orig_repo
, Xob_orig_regy_tbl.Fld_orig_file_ttl
, Xob_orig_regy_tbl.Fld_orig_file_ext
, Xob_orig_regy_tbl.Fld_lnki_ttl
, Xob_lnki_regy_tbl.Fld_lnki_type
, Xob_lnki_regy_tbl.Fld_lnki_src_tid
, Xob_lnki_regy_tbl.Fld_lnki_w
, Xob_lnki_regy_tbl.Fld_lnki_h
, Xob_lnki_regy_tbl.Fld_lnki_count
, Xob_lnki_regy_tbl.Fld_lnki_page_id
, Xob_orig_regy_tbl.Fld_orig_w
, Xob_orig_regy_tbl.Fld_orig_h
, Xob_orig_regy_tbl.Fld_orig_page_id
, Xob_lnki_regy_tbl.Fld_lnki_upright
, Xob_lnki_regy_tbl.Fld_lnki_time
, Xob_lnki_regy_tbl.Fld_lnki_page
, Xob_orig_regy_tbl.Fld_orig_media_type
, Xob_orig_regy_tbl.Fld_orig_minor_mime
}
;
public Xob_xfer_temp_itm_fxt Reset() {
itm.Clear();
return this;
}
public Xob_xfer_temp_itm_fxt Init_rdr_image() {
GfoFldList flds = GfoFldList_.str_(Flds);
nde = GfoNde_.vals_(flds, Object_.Ary
( Xof_ext_.Id_png, 1, 1, Xof_repo_itm_.Repo_remote
, "A.png", Xof_ext_.Id_png, "A.png", Xop_lnki_type.Id_thumb, Xop_file_logger_.Tid__file
, 220, 200, 1, 2, 440, 400, 3
, Xop_lnki_tkn.Upright_null, Xof_lnki_time.Null, Xof_lnki_page.Null
, Xof_media_type.Name_bitmap, "png"
));
GfoNdeList subs = GfoNdeList_.new_();
subs.Add(nde);
GfoNde root = GfoNde_.root_(nde);
rdr = DataRdr_mem.new_(root, flds, subs);
rdr.MoveNextPeer();
return this;
}
public Xob_xfer_temp_itm_fxt Init_rdr(String key, Object val) {
nde.Write(key, val);
return this;
}
public Xob_xfer_temp_itm_fxt Test_pass() {return Test_bgn();}
public Xob_xfer_temp_itm_fxt Test_fail(byte fail_tid, Keyval... kvs) {
Test_bgn(kvs);
this.Test_itm_chk_fail_id(fail_tid);
return this;
}
public Xob_xfer_temp_itm_fxt Test_bgn(Keyval... kvs) {
Init_rdr_image();
int len = kvs.length;
for (int i = 0; i < len; i++) {
Keyval kv = kvs[i];
Init_rdr(kv.Key(), kv.Val());
}
this.Exec_load();
this.Exec_chk();
return this;
}
public Xob_xfer_temp_itm_fxt Test_atr(String key, Object val) {
Tfds.Eq(rdr.Read(key), val);
return this;
}
public Xob_xfer_temp_itm_fxt Exec_load() {
itm.Load(rdr);
return this;
}
public Xob_xfer_temp_itm_fxt Exec_chk() {
itm.Chk(img_size);
return this;
}
public Xob_xfer_temp_itm_fxt Test_lnki_ext_id(int expd) {Tfds.Eq(expd, itm.Lnki_ext()); return this;}
public Xob_xfer_temp_itm_fxt Test_lnki_thumbtime(double expd) {Tfds.Eq(expd, itm.Lnki_thumbtime()); return this;}
public Xob_xfer_temp_itm_fxt Test_lnki_page(int expd) {Tfds.Eq(expd, itm.Lnki_page()); return this;}
public Xob_xfer_temp_itm_fxt Test_lnki_redirect_src(String expd) {Tfds.Eq(expd, itm.Redirect_src()); return this;}
public Xob_xfer_temp_itm_fxt Test_itm_chk_fail_id_none() {return Test_itm_chk_fail_id(Xob_xfer_temp_itm.Chk_tid_none);}
public Xob_xfer_temp_itm_fxt Test_itm_chk_fail_id(byte expd) {
Tfds.Eq(expd, itm.Chk_tid());
return this;
}
}

79
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_temp_tbl.java
View File

@@ -1,79 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.files.*;
class Xob_xfer_temp_tbl {
public static void Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql);}
public static Db_stmt Insert_stmt(Db_conn p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_lnki_id, Fld_lnki_tier_id, Fld_lnki_page_id, Fld_orig_repo, Fld_orig_page_id, Fld_lnki_ttl, Fld_orig_redirect_src, Fld_lnki_ext, Fld_lnki_type, Fld_orig_media_type, Fld_file_is_orig, Fld_orig_w, Fld_orig_h, Fld_file_w, Fld_file_h, Fld_html_w, Fld_html_h, Fld_lnki_time, Fld_lnki_page, Fld_lnki_count);}
public static void Insert(Db_stmt stmt, int lnki_id, int lnki_tier_id, int lnki_page_id, byte repo_id, int page_id, String ttl, String redirect_src, int ext_id, byte lnki_type, String orig_media_type, boolean file_is_orig, int orig_w, int orig_h, int html_w, int html_h, int file_w, int file_h, double thumbtime, int page, int count) {
stmt.Clear()
.Val_int(lnki_id)
.Val_int(lnki_tier_id)
.Val_int(lnki_page_id)
.Val_byte(repo_id)
.Val_int(page_id)
.Val_str(ttl)
.Val_str(redirect_src)
.Val_int(ext_id)
.Val_byte(lnki_type)
.Val_str(orig_media_type)
.Val_bool_as_byte(file_is_orig)
.Val_int(orig_w)
.Val_int(orig_h)
.Val_int(file_w)
.Val_int(file_h)
.Val_int(html_w)
.Val_int(html_h)
.Val_double(Xof_lnki_time.Db_save_double(thumbtime))
.Val_int(page)
.Val_int(count)
.Exec_insert();
}
public static final String Tbl_name = "xfer_temp"
, Fld_lnki_id = "lnki_id", Fld_lnki_tier_id = "lnki_tier_id", Fld_lnki_page_id = "lnki_page_id", Fld_lnki_ttl = "lnki_ttl", Fld_lnki_ext = "lnki_ext", Fld_lnki_type = "lnki_type"
, Fld_lnki_time = "lnki_time", Fld_lnki_page = "lnki_page", Fld_lnki_count = "lnki_count"
, Fld_orig_repo = "orig_repo", Fld_orig_page_id = "orig_page_id", Fld_orig_redirect_src = "orig_redirect_src", Fld_orig_media_type = "orig_media_type"
, Fld_orig_w = "orig_w", Fld_orig_h = "orig_h"
, Fld_file_w = "file_w", Fld_file_h = "file_h", Fld_file_is_orig = "file_is_orig"
, Fld_html_w = "html_w", Fld_html_h = "html_h"
;
private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS xfer_temp"
, "( lnki_id integer NOT NULL PRIMARY KEY"
, ", lnki_tier_id integer NOT NULL"
, ", lnki_page_id integer NOT NULL"
, ", lnki_ttl varchar(255) NOT NULL"
, ", lnki_ext integer NOT NULL"
, ", lnki_type integer NOT NULL"
, ", lnki_time double NOT NULL"
, ", lnki_page integer NOT NULL"
, ", lnki_count integer NOT NULL"
, ", orig_repo integer NOT NULL"
, ", orig_page_id integer NOT NULL"
, ", orig_redirect_src varchar(255) NOT NULL"
, ", orig_media_type varchar(64) NOT NULL"
, ", orig_w integer NOT NULL"
, ", orig_h integer NOT NULL"
, ", file_is_orig tinyint NOT NULL"
, ", file_w integer NOT NULL"
, ", file_h integer NOT NULL"
, ", html_w integer NOT NULL"
, ", html_h integer NOT NULL"
, ");"
);
}

82
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xob_xfer_update_cmd.java
View File

@@ -1,82 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.files.*; import gplx.xowa.bldrs.cmds.files.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_xfer_update_cmd extends Xob_itm_basic_base implements Xob_cmd {
private Io_url prv_url;
public Xob_xfer_update_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_file_xfer_update;}
public void Cmd_run() {
// init vars
Xob_db_file cur_file = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
Db_conn conn = cur_file.Conn();
if (prv_url == null) {
prv_url = wiki.Appe().Fsys_mgr().File_dir().GenSubFil_nest(wiki.Domain_str(), "bldr", Xob_db_file.Name__file_make);
}
// run sql
Sqlite_engine_.Tbl_rename(conn, "xfer_regy", "xfer_regy_old");
Xob_xfer_regy_tbl.Create_table(conn);
Sqlite_engine_.Db_attach(conn, "old_db", prv_url.Raw());
conn.Exec_sql(Sql_update);
Sqlite_engine_.Db_detach(conn, "old_db");
Sqlite_engine_.Tbl_delete(conn, "xfer_regy_old");
Xob_xfer_regy_tbl.Create_index(usr_dlg, conn);
// // rotate db
// DateAdp wiki_date = wiki.Db_mgr().Dump_date_query();
// Io_url archive_url = prv_url.GenNewNameOnly("oimg_lnki_" + wiki_date.XtoStr_fmt("yyyyMMdd"));
// Io_mgr.Instance.CopyFil(cur_file.Url(), archive_url, true);
// Io_mgr.Instance.CopyFil(cur_file.Url(), prv_url, true);
}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_end() {}
public void Cmd_term() {}
public static final String Sql_update = String_.Concat_lines_nl
( "INSERT INTO xfer_regy"
, "SELECT cur.lnki_id"
, ", cur.orig_page_id"
, ", cur.orig_repo"
, ", cur.lnki_ttl"
, ", cur.orig_redirect_src"
, ", cur.lnki_ext"
, ", cur.orig_media_type"
, ", cur.file_is_orig"
, ", cur.orig_w"
, ", cur.orig_h"
, ", cur.file_w"
, ", cur.file_h"
, ", cur.lnki_time"
, ", cur.lnki_count"
, ", CASE"
, " WHEN old.lnki_ttl IS NULL THEN" // not in old table; mark todo
, " " + Byte_.To_str(Xob_xfer_regy_tbl.Status_todo)
, " ELSE" // in old table; mark processed
, " " + Byte_.To_str(Xob_xfer_regy_tbl.Status_ignore_processed)
, " END"
, ", cur.xfer_bin_tid"
, ", cur.xfer_bin_msg"
, "FROM xfer_regy_old cur"
, " LEFT JOIN old_db.xfer_regy old ON cur.lnki_ttl = old.lnki_ttl AND cur.file_w = old.file_w"
);
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_previous_url_)) prv_url = m.ReadIoUrl("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk_previous_url_ = "previous_url_";
}

38
400_xowa/src/gplx/xowa/bldrs/cmds/files/Xobu_poll_mgr.java
View File

@@ -1,38 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
public class Xobu_poll_mgr implements GfoInvkAble {
public Xobu_poll_mgr(Xoae_app app) {this.app = app;} private Xoae_app app;
public int Poll_interval() {return poll_interval;} private int poll_interval = 1000;
private Io_url poll_file;
public void Poll() {
if (poll_file == null) poll_file = app.Fsys_mgr().Root_dir().GenSubFil("bldr_poll.gfs");
if (!Io_mgr.Instance.ExistsFil(poll_file)) return; // file doesn't exist
String poll_text = Io_mgr.Instance.LoadFilStr(poll_file);
Io_mgr.Instance.DeleteFil(poll_file);
app.Usr_dlg().Note_many("", "", "poll file found: ~{0}", poll_file.Raw());
app.Gfs_mgr().Run_str(poll_text);
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_poll_interval_)) poll_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk_poll_file_)) poll_file = m.ReadIoUrl("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
}
private static final String Invk_poll_interval_ = "poll_interval_", Invk_poll_file_ = "poll_file_";
}

8
400_xowa/src/gplx/xowa/bldrs/cmds/texts/Xob_init_base.java
View File

@@ -16,16 +16,18 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.xtns.wdatas.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.xmls.*; import gplx.xowa.bldrs.cmds.texts.xmls.*;
import gplx.xowa.xtns.wdatas.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.xmls.*; import gplx.xowa.bldrs.cmds.texts.xmls.*;
import gplx.xowa.bldrs.css.*; import gplx.xowa.wikis.domains.*;
public abstract class Xob_init_base implements Xob_cmd, GfoInvkAble {
private Xob_bldr bldr; private Xowe_wiki wiki; private Gfo_usr_dlg usr_dlg;
private byte wbase_enabled = Bool_.__byte;
public Xob_init_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki; this.usr_dlg = wiki.Appe().Usr_dlg(); return this;}
public abstract String Cmd_key();
public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public abstract void Cmd_ini_wdata(Xob_bldr bldr, Xowe_wiki wiki);
public abstract void Cmd_run_end(Xowe_wiki wiki);
public void Cmd_init(Xob_bldr bldr) { // add other cmds; EX: wikidata
@gplx.Virtual public void Cmd_init(Xob_bldr bldr) { // add other cmds; EX: wikidata
bldr.Import_marker().Bgn(wiki);
if (wbase_enabled == Bool_.__byte) wbase_enabled = wiki.Domain_tid() == Xow_domain_tid_.Int__wikidata ? Bool_.Y_byte : Bool_.N_byte; // if wbase_enabled not explicitly set, set it to y if wiki is "www.wikidata.org"
if (wbase_enabled == Bool_.Y_byte) // if wbase_enabled, auto-add wdata_wkrs bldr
@@ -45,7 +47,7 @@ public abstract class Xob_init_base implements Xob_cmd, GfoInvkAble {
Io_mgr.Instance.DeleteFil_args(url).MissingFails_off().Exec();
}
}
public void Cmd_term() {}
@gplx.Virtual public void Cmd_term() {}
@gplx.Virtual public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_src_xml_fil_)) wiki.Import_cfg().Src_fil_xml_(m.ReadIoUrl("v"));
else if (ctx.Match(k, Invk_src_bz2_fil_)) wiki.Import_cfg().Src_fil_bz2_(m.ReadIoUrl("v"));

3
400_xowa/src/gplx/xowa/bldrs/cmds/texts/Xob_term_base.java
View File

@@ -16,10 +16,11 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.bldrs.xmls.*; import gplx.xowa.xtns.wdatas.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.dbs.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.xmls.*; import gplx.xowa.xtns.wdatas.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.dbs.*;
public abstract class Xob_term_base implements Xob_cmd, GfoInvkAble {
public Xob_term_base Ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.wiki = wiki; return this;} private Xowe_wiki wiki;
public abstract String Cmd_key();
public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_run() {}

5
400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_css_cmd.java
View File

@@ -16,12 +16,13 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.xowa.bldrs.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.htmls.css.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.htmls.css.*;
public class Xob_css_cmd implements Xob_cmd {
private final Xob_bldr bldr; private final Xowe_wiki wiki; private final Gfo_usr_dlg usr_dlg;
private final Xob_bldr bldr; private final Xowe_wiki wiki; private final Gfo_usr_dlg usr_dlg;
private Io_url css_dir; private String css_key;
public Xob_css_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki; this.usr_dlg = wiki.Appe().Usr_dlg();}
public String Cmd_key() {return Xob_cmd_keys.Key_text_css;}
public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public void Cmd_init(Xob_bldr bldr) {
if (css_dir == null) css_dir = wiki.App().Fsys_mgr().Wiki_css_dir(wiki.Domain_str()); // EX: /xowa/user/anonymous/wiki/en.wikipedia.org
if (css_key == null) css_key = Xowd_css_core_mgr.Key_default;

10
400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_init_cmd.java
View File

@@ -25,9 +25,19 @@ public class Xob_init_cmd extends Xob_init_base {
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_wbase_qid);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_wbase_pid);
}
@Override public void Cmd_init(Xob_bldr bldr) {
super.Cmd_init(bldr);
// gplx.dbs.qrys.bats.Db_batch__journal_wal.Batch__init(gplx.dbs.Db_conn_pool.Instance.Batch_mgr());
}
@Override public void Cmd_run_end(Xowe_wiki wiki) {
if (gplx.xowa.wikis.Xow_fsys_mgr.Find_core_fil(wiki) != null)
throw wiki.Appe().Bldr().Usr_dlg().Fail_many("", "", "directory must not contain any .xowa or .sqlite3 files: dir=~{0}", wiki.Fsys_mgr().Root_dir().Raw());
Xowe_wiki_.Create(wiki, wiki.Import_cfg().Src_rdr_len(), wiki.Import_cfg().Src_fil().NameOnly());
}
@Override public void Cmd_term() {
super.Cmd_term();
// gplx.dbs.qrys.bats.Db_batch__journal_wal.Batch__term(gplx.dbs.Db_conn_pool.Instance.Batch_mgr());
// gplx.dbs.Db_conn_pool.Instance.Rls_all();
}
}

4
400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_page_cmd.java
View File

@@ -16,12 +16,12 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.dbs.*; import gplx.core.ios.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.wikis.*;
import gplx.dbs.*; import gplx.core.ios.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.dbs.*;
import gplx.xowa.wikis.*; import gplx.xowa.bldrs.filters.dansguardians.*; import gplx.xowa.apps.apis.xowa.bldrs.imports.*;
import gplx.xowa.parsers.utils.*;
import gplx.xowa.parsers.utils.*; import gplx.xowa.addons.builds.files.cmds.*; import gplx.xowa.addons.builds.files.dbs.*;
public class Xob_page_cmd extends Xob_itm_basic_base implements Xob_page_wkr, GfoInvkAble {
private Xowd_db_mgr db_mgr; private Db_idx_mode idx_mode = Db_idx_mode.Itm_end; private Xowd_page_tbl page_core_tbl; private Io_stream_zip_mgr text_zip_mgr; private byte text_zip_tid;
private Xop_redirect_mgr redirect_mgr; private Xob_redirect_tbl redirect_tbl; private boolean redirect_id_enabled;

5
400_xowa/src/gplx/xowa/bldrs/cmds/texts/sqls/Xob_page_delete_cmd.java
View File

@@ -17,8 +17,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.dbs.*; import gplx.xowa.wikis.data.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_page_delete_cmd extends Xob_cmd_base {
private final Xow_wiki wiki;
private final Xow_wiki wiki;
public Xob_page_delete_cmd(Xob_bldr bldr, Xow_wiki wiki) {this.wiki = wiki;}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_text_delete_page;}
@Override public void Cmd_run() {
@@ -74,7 +75,7 @@ public class Xob_page_delete_cmd extends Xob_cmd_base {
}
} catch (Exception e) {Gfo_usr_dlg_.Instance.Warn_many("", "", "fatal error during page deletion: err=~{0}", Err_.Message_gplx_log(e));}
core_db_conn.Exec_sql_plog_ntx("deleting from table: page", "DELETE FROM page WHERE page_id IN (SELECT page_id FROM page_filter);");
// core_db_conn.Ddl_delete_tbl("page_filter");
// core_db_conn.Meta_tbl_delete("page_filter");
core_db_conn.Env_vacuum();
usr_dlg.Plog_many("", "", "");
}

2
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_search_base_tst.java
View File

@@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import org.junit.*; import gplx.xowa.wikis.tdbs.*; import gplx.xowa.bldrs.*; import gplx.xowa.wikis.nss.*;
import gplx.xowa.addons.searchs.dbs.bldrs.*;
import gplx.xowa.addons.apps.searchs.bldrs.*;
public class Xob_search_base_tst {
@Test public void Split() {
tst_Split("a", "a");

2
400_xowa/src/gplx/xowa/bldrs/cmds/texts/tdbs/Xob_search_tdb.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.texts.tdbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.cmds.texts.*;
import gplx.xowa.addons.searchs.dbs.bldrs.*;
import gplx.xowa.addons.apps.searchs.bldrs.*;
public class Xob_search_tdb extends Srch_bldr_wkr_base {
public Xob_search_tdb(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
@Override public String Wkr_key() {return Xob_cmd_keys.Key_tdb_make_search_title;}

15
400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_alert_cmd.java
View File

@@ -20,12 +20,21 @@ import gplx.xowa.bldrs.wkrs.*;
import gplx.gfui.*;
public class Xob_alert_cmd extends Xob_cmd__base implements Xob_cmd {
public Xob_alert_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
public Xob_alert_cmd Msg_(String v) {this.msg = v; return this;} private String msg = "developer forgot to include message";
@Override public String Cmd_key() {return Xob_cmd_keys.Key_util_alert;}
public Xob_alert_cmd Msg_(String v) {this.msg = v; return this;} private String msg = "no message specified";
@Override public void Cmd_run() {
Gfui_kit kit = app.Gui_mgr().Kit();
if (kit.Tid() != Gfui_kit_.Swt_tid) return;
kit.Ask_ok("", "", msg);
Xoa_app_.Usr_dlg().Prog_many("", "", msg);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return GfoInvkAble_.Rv_unhandled;}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__text_)) this.msg = m.ReadStr("v");
else return GfoInvkAble_.Rv_unhandled;
return this;
} private static final String Invk__text_ = "text_";
public static final String BLDR_CMD_KEY = "ui.alert";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
public static final Xob_cmd Prototype = new Xob_alert_cmd(null, null);
@Override public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return new Xob_alert_cmd(bldr, wiki);}
}

2
400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_exec_sql_cmd.java
View File

@@ -16,10 +16,12 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.dbs.*;
public class Xob_exec_sql_cmd implements Xob_cmd {
private Xowe_wiki wiki; private int file_idx = -1; private String sql;
public Xob_exec_sql_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.wiki = wiki;}
public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public String Cmd_key() {return Xob_cmd_keys.Key_exec_sql;}
public void Cmd_run() {
Xoae_app app = wiki.Appe();

5
400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_site_meta_cmd.java
View File

@@ -17,13 +17,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.core.net.*;
import gplx.dbs.*; import gplx.xowa.bldrs.*;
import gplx.xowa.bldrs.wms.*; import gplx.xowa.bldrs.wms.sites.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.wms.*; import gplx.xowa.bldrs.wms.sites.*;
import gplx.xowa.wikis.domains.*; import gplx.xowa.apps.site_cfgs.*;
public class Xob_site_meta_cmd implements Xob_cmd {
private final Xob_bldr bldr;
private String[] wikis; private Io_url db_url; private DateAdp cutoff_time;
public Xob_site_meta_cmd(Xob_bldr bldr, Xow_wiki wiki) {this.bldr = bldr;}
public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public String Cmd_key() {return Xob_cmd_keys.Key_site_meta;}
public void Cmd_run() {
Xoa_app app = bldr.App();

7
400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_xml_dumper_cmd.java
View File

@@ -16,13 +16,14 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.bldrs.xmls.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.xmls.*;
import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_xml_dumper_cmd implements Xob_cmd {
private final Xowe_wiki wiki; private final Gfo_usr_dlg usr_dlg;
private final Xob_xml_dumper xml_dumper = new Xob_xml_dumper(); private int commit_interval = 1000;
private final Xowe_wiki wiki; private final Gfo_usr_dlg usr_dlg;
private final Xob_xml_dumper xml_dumper = new Xob_xml_dumper(); private int commit_interval = 1000;
private Io_url dump_url;
public Xob_xml_dumper_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.wiki = wiki; this.usr_dlg = wiki.Appe().Usr_dlg();}
public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public String Cmd_key() {return Xob_cmd_keys.Key_util_xml_dump;}
public void Cmd_init(Xob_bldr bldr) {
dump_url = wiki.Fsys_mgr().Root_dir().GenSubFil(wiki.Domain_str() + "-dump.xml");

118
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_image_cmd.java
View File

@@ -1,118 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.core.ios.*; import gplx.xowa.files.*;
import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
public class Xob_image_cmd extends Xob_itm_dump_base implements Xob_cmd, GfoInvkAble, Sql_file_parser_cmd {
private Db_conn conn = null; private Db_stmt stmt = null;
private Xob_image_tbl tbl_image = new Xob_image_tbl();
private byte[] cur_ttl, cur_media_type, cur_minor_mime, cur_timestamp; private int cur_size, cur_width, cur_height, cur_bits, cur_ext_id;
private int commit_count = 10000;
public Xob_image_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_wiki_image;}
public Io_url Src_fil() {return src_fil;} public Xob_image_cmd Src_fil_(Io_url v) {src_fil = v; return this;} private Io_url src_fil;
public Sql_file_parser Parser() {return parser;} private Sql_file_parser parser = new Sql_file_parser();
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql
Init_dump(Xob_cmd_keys.Key_wiki_image);
if (src_fil == null) {
src_fil = Xob_page_wkr_cmd.Find_fil_by(wiki.Fsys_mgr().Root_dir(), "*-image.sql");
if (src_fil == null) throw Err_.new_wo_type(".sql file not found in dir", "dir", wiki.Fsys_mgr().Root_dir());
}
parser.Src_fil_(src_fil).Trg_fil_gen_(dump_url_gen).Fld_cmd_(this).Flds_req_idx_(20, Fld_img_name, Fld_img_size, Fld_img_width, Fld_img_height, Fld_img_bits, Fld_img_media_type, Fld_img_minor_mime, Fld_img_timestamp);
this.conn = Xob_db_file.New__wiki_image(wiki.Fsys_mgr().Root_dir()).Conn();
conn.Txn_bgn("bldr__image");
this.tbl_image = new Xob_image_tbl();
tbl_image.Create_table(conn);
this.stmt = tbl_image.Insert_stmt(conn);
}
public void Cmd_run() {
parser.Parse(bldr.Usr_dlg());
tbl_image.Create_index(conn);
conn.Txn_end();
}
public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
switch (fld_idx) {
case Fld_img_name: cur_ttl = Bry_.Mid(src, fld_bgn, fld_end); break;
case Fld_img_size: cur_size = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld_img_width: cur_width = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld_img_height: cur_height = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld_img_bits: cur_bits = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld_img_media_type: cur_media_type = Bry_.Mid(src, fld_bgn, fld_end); break;
case Fld_img_minor_mime: cur_minor_mime = Bry_.Mid(src, fld_bgn, fld_end); break;
case Fld_img_timestamp: cur_timestamp = Bry_.Mid(src, fld_bgn, fld_end);
cur_ext_id = Calc_ext_id(show_issues ? app.Usr_dlg() : Gfo_usr_dlg_.Noop, cur_ttl, cur_media_type, cur_minor_mime, cur_width, cur_height);
tbl_image.Insert(stmt, cur_ttl, cur_media_type, cur_minor_mime, cur_size, cur_width, cur_height, cur_bits, cur_ext_id, cur_timestamp);
++commit_count;
if ((commit_count % 10000) == 0) {
usr_dlg.Prog_many("", "", "committing: count=~{0} last=~{1}", commit_count, String_.new_u8(cur_ttl));
conn.Txn_sav();
}
break;
}
}
public void Cmd_end() {}
public void Cmd_term() {}
private boolean show_issues = true;
private static final int Fld_img_name = 0, Fld_img_size = 1, Fld_img_width = 2, Fld_img_height = 3, Fld_img_bits = 5, Fld_img_media_type = 6, Fld_img_minor_mime = 8, Fld_img_timestamp = 12;
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_src_fil_)) src_fil = m.ReadIoUrl("v");
else if (ctx.Match(k, Invk_show_issues_)) show_issues = m.ReadYn("v");
else return super.Invk(ctx, ikey, k, m);
return this;
} private static final String Invk_src_fil_ = "src_fil_", Invk_show_issues_ = "show_issues_";
public static int Calc_ext_id(Gfo_usr_dlg usr_dlg, byte[] file, byte[] media_type, byte[] minor_mime, int w, int h) {
Xof_ext file_ext = Xof_ext_.new_by_ttl_(file); int file_ext_id = file_ext.Id();
Xof_ext mime_ext = Xof_mime_minor_.ext_(minor_mime); int mime_ext_id = mime_ext.Id();
int media_type_id = Xof_media_type.Xto_byte(String_.new_u8(media_type));
if (file_ext_id != mime_ext_id) { // file_ext_id != mime_ext_id; EX: "A.png" actually has a minor_mime of "jpg"
boolean update = false, notify = true;
switch (file_ext_id) {
case Xof_ext_.Id_jpg: case Xof_ext_.Id_jpeg:
if (Int_.In(mime_ext_id, Xof_ext_.Id_jpg, Xof_ext_.Id_jpeg)) notify = false; // skip: both jpg
break;
case Xof_ext_.Id_tif: case Xof_ext_.Id_tiff:
if (Int_.In(mime_ext_id, Xof_ext_.Id_tif, Xof_ext_.Id_tiff)) notify = false; // skip: both tif
break;
case Xof_ext_.Id_ogg: case Xof_ext_.Id_oga: case Xof_ext_.Id_ogv:
if (Int_.In(mime_ext_id, Xof_ext_.Id_ogg, Xof_ext_.Id_oga, Xof_ext_.Id_ogv)) notify = false; // skip: both tif
break;
case Xof_ext_.Id_png:
if (Int_.In(mime_ext_id, Xof_ext_.Id_jpg, Xof_ext_.Id_jpeg))
update = true;
break;
}
if (update)
file_ext_id = mime_ext_id;
else {
if (notify)
usr_dlg.Log_many("", "", "image.ext_calc.mismatch_exts: file=~{0} mime=~{1}", String_.new_u8(file), String_.new_u8(minor_mime));
}
}
if ( file_ext_id == Xof_ext_.Id_ogg // file_ext is ".ogg"
&& media_type_id == Xof_media_type.Tid_video // media_type is "VIDEO"
) {
if (w > 0 && h > 0) // some .ogg files are "VIDEO" but have 0 width, 0 height
file_ext_id = Xof_ext_.Id_ogv; // manually specify ogv
else
usr_dlg.Log_many("", "", "image.ext_calc.ogg_video_with_null_size: media_type=~{0} minor_mime=~{1} w=~{2} h=~{3} file=~{4}", String_.new_u8(media_type), String_.new_u8(minor_mime), w, h, String_.new_u8(file));
}
return file_ext_id;
}
}

51
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_image_cmd_tst.java
View File

@@ -1,51 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import org.junit.*; import gplx.core.ios.*; import gplx.xowa.files.*;
public class Xob_image_cmd_tst {
private Xob_image_cmd_fxt fxt = new Xob_image_cmd_fxt();
@Test public void Basic() {fxt.Init_("A.png" , Xof_media_type.Name_bitmap , Xof_ext_.Bry_png, 220, 110) .Test(Xof_ext_.Id_png);} // A.png -> png
@Test public void Ogg_VIDEO() {fxt.Init_("A.ogg" , Xof_media_type.Name_video , Xof_ext_.Bry_ogg, 220, 110) .Test(Xof_ext_.Id_ogv);} // A.ogg and VIDEO -> ogv
@Test public void Ogg_VIDEO_null_size() {fxt.Init_("A.ogg" , Xof_media_type.Name_video , Xof_ext_.Bry_ogg, 0, 0) .Test(Xof_ext_.Id_ogg);} // A.ogg but 0,0 -> ogg (not ogv)
@Test public void Png_is_jpg() {fxt.Init_("A.png" , Xof_media_type.Name_bitmap , Xof_ext_.Bry_jpg, 220, 110) .Test(Xof_ext_.Id_jpg);} // A.png and jpg -> jpg
@Test public void Jpeg_is_jpeg() {fxt.Init_("A.jpeg" , Xof_media_type.Name_bitmap , Xof_ext_.Bry_jpg, 220, 110) .Test(Xof_ext_.Id_jpeg);} // A.jpeg and jpg -> jpeg (unchanged)
}
class Xob_image_cmd_fxt {
private byte[] name, media_type, minor_mime; int w, h;
public Xob_image_cmd_fxt Init_png() {Name_("A.png").Media_type_(Xof_media_type.Name_bitmap).Minor_mime_(Xof_ext_.Bry_png).W_(220).H_(110);
return this;
}
public Xob_image_cmd_fxt Init_(String name, String media_type, byte[] minor_mime, int w, int h) {
Name_(name);
Media_type_(media_type);
Minor_mime_(minor_mime);
W_(w);
H_(h);
return this;
}
public Xob_image_cmd_fxt Name_(String v) {name = Bry_.new_a7(v); return this;}
public Xob_image_cmd_fxt Media_type_(String v) {media_type = Bry_.new_a7(v); return this;}
public Xob_image_cmd_fxt Minor_mime_(byte[] v) {minor_mime = v; return this;}
public Xob_image_cmd_fxt Minor_mime_(String v) {return Minor_mime_(Bry_.new_a7(v));}
public Xob_image_cmd_fxt W_(int v) {w = v; return this;}
public Xob_image_cmd_fxt H_(int v) {h = v; return this;}
public Xob_image_cmd_fxt Test(int expd) {
Tfds.Eq(expd, Xob_image_cmd.Calc_ext_id(Gfo_usr_dlg_.Noop, name, media_type, minor_mime, w, h));
return this;
}
}

58
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_image_tbl.java
View File

@@ -1,58 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*;
public class Xob_image_tbl {
public Xob_image_tbl Create_table(Db_conn p) {Sqlite_engine_.Tbl_create_and_delete(p, Tbl_name, Tbl_sql); return this;}
public Xob_image_tbl Create_index(Db_conn p) {Sqlite_engine_.Idx_create(p, Idx_img_name); return this;}
public Db_stmt Insert_stmt(Db_conn p) {return Db_stmt_.new_insert_(p, Tbl_name, Fld_img_name, Fld_img_media_type, Fld_img_minor_mime, Fld_img_size, Fld_img_width, Fld_img_height, Fld_img_bits, Fld_img_ext_id, Fld_img_timestamp);}
public void Insert(Db_stmt stmt, byte[] ttl, byte[] media_type, byte[] minor_mime, int size, int w, int h, int bits, int ext_id, byte[] img_timestamp) {
stmt.Clear()
.Val_bry_as_str(ttl)
.Val_bry_as_str(media_type)
.Val_bry_as_str(minor_mime)
.Val_int(size)
.Val_int(w)
.Val_int(h)
.Val_int(bits)
.Val_int(ext_id)
.Val_bry_as_str(img_timestamp)
.Exec_insert();
}
public static final String Tbl_name = "image"
, Fld_img_name = "img_name", Fld_img_media_type = "img_media_type", Fld_img_minor_mime = "img_minor_mime"
, Fld_img_size = "img_size", Fld_img_width = "img_width", Fld_img_height = "img_height", Fld_img_bits = "img_bits", Fld_img_ext_id = "img_ext_id"
, Fld_img_timestamp = "img_timestamp"
;
private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS image"
, "( img_name varchar(255) NOT NULL -- varbinary(255)"
, ", img_media_type varchar(64) NOT NULL -- enum('UNKNOWN','BITMAP','DRAWING','AUDIO','VIDEO','MULTIMEDIA','OFFICE','TEXT','EXECUTABLE','ARCHIVE')"
, ", img_minor_mime varchar(32) NOT NULL -- DEFAULT 'unknown'"
, ", img_size integer NOT NULL -- int(8) unsigned"
, ", img_width integer NOT NULL -- int(5)"
, ", img_height integer NOT NULL -- int(5)"
, ", img_bits smallint NOT NULL -- int(3)"
, ", img_ext_id int NOT NULL -- xowa"
, ", img_timestamp varchar(14) NOT NULL -- 20140101155749"
, ");"
);
private static final Db_idx_itm
Idx_img_name = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS image__img_name ON image (img_name, img_timestamp);")
;
}

43
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_page_dump_cmd_drop.java
View File

@@ -1,43 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.wikis.dbs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_page_dump_cmd_drop extends Xob_itm_basic_base implements Xob_cmd {
public Xob_page_dump_cmd_drop(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_wiki_page_dump_drop;}
public void Cmd_run() {
wiki.Init_assert();
Xowd_db_mgr db_mgr = wiki.Data__core_mgr();
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; i++) {
Xowd_db_file db_file = db_mgr.Dbs__get_at(i);
switch (db_file.Tid()) {
case Xowd_db_file_.Tid_wiki_solo:
case Xowd_db_file_.Tid_text_solo:
case Xowd_db_file_.Tid_text:
db_file.Conn().Meta_tbl_drop(Xobd_page_dump_tbl.Tbl_name);
break;
}
}
}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_end() {}
public void Cmd_term() {}
}

45
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_page_dump_cmd_make.java
View File

@@ -1,45 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.xowa.wikis.data.*; import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.wikis.dbs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_page_dump_cmd_make extends Xob_itm_basic_base implements Xob_cmd {
public Xob_page_dump_cmd_make(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return Xob_cmd_keys.Key_wiki_page_dump_make;}
public void Cmd_run() {
wiki.Init_assert();
Xowd_db_mgr db_mgr = wiki.Data__core_mgr();
Io_url page_db_url = db_mgr.Db__core().Url();
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; i++) {
Xowd_db_file db_file = db_mgr.Dbs__get_at(i);
switch (db_file.Tid()) {
case Xowd_db_file_.Tid_wiki_solo:
case Xowd_db_file_.Tid_text_solo:
case Xowd_db_file_.Tid_text:
Xobd_page_dump_tbl tbl = new Xobd_page_dump_tbl(db_file.Conn());
tbl.Create_data(page_db_url, db_file.Id());
break;
}
}
}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {}
public void Cmd_end() {}
public void Cmd_term() {}
}

47
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_page_dump_tbl.java
View File

@@ -1,47 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*;
class Xobd_page_dump_tbl {
public final static String Tbl_name = "page_dump";
private final String fld_id, fld_title, fld_namespace, fld_is_redirect;
private final Db_conn conn; private final Dbmeta_fld_list flds = Dbmeta_fld_list.new_();
public Xobd_page_dump_tbl(Db_conn conn) {
this.conn = conn;
this.fld_id = flds.Add_int_pkey("page_id");
this.fld_title = flds.Add_str("page_title", 255);
this.fld_namespace = flds.Add_int("page_namespace");
this.fld_is_redirect = flds.Add_int("page_is_redirect");
}
public void Create_data(Io_url page_db_url, int text_db_id) {
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(Tbl_name, flds));
conn.Stmt_delete(Tbl_name).Exec_delete(); // always clear tables again; allows commands to be rerun; DATE:2015-08-04
new Db_attach_mgr(conn, new Db_attach_itm("page_db", page_db_url))
.Exec_sql_w_msg("text_db_prep.clone_page", Sql_insert_data, text_db_id);
conn.Meta_idx_create(Dbmeta_idx_itm.new_unique_by_tbl(Tbl_name, "main", fld_id, fld_namespace, fld_is_redirect, fld_title));
}
private static final String Sql_insert_data = String_.Concat_lines_nl
( "INSERT INTO page_dump (page_id, page_title, page_namespace, page_is_redirect)"
, "SELECT p.page_id"
, ", p.page_title"
, ", p.page_namespace"
, ", p.page_is_redirect"
, "FROM <page_db>page p"
, "WHERE p.page_text_db_id = {0};"
);
}

59
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_redirect_cmd.java
View File

@@ -1,59 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.langs.htmls.encoders.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.parsers.utils.*;
public class Xob_redirect_cmd extends Xob_dump_mgr_base {
private Db_conn conn; private Xob_redirect_tbl redirect_tbl;
private Xodb_mgr_sql db_mgr; private Xop_redirect_mgr redirect_mgr; private Gfo_url_encoder encoder;
public Xob_redirect_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.Reset_db_y_();}
@Override public String Cmd_key() {return Xob_cmd_keys.Key_wiki_redirect;}
@Override public int[] Init_ns_ary() {return Int_.Ary(Xow_ns_.Tid__file);} // restrict to file ns
@Override public byte Init_redirect() {return Bool_.Y_byte;} // restrict to redirects
@Override protected void Init_reset(Db_conn conn) {
Db_cfg_tbl cfg_tbl = new Db_cfg_tbl(conn, "xowa_cfg");
cfg_tbl.Delete_all();
conn.Exec_sql("DELETE FROM " + Xob_redirect_tbl.Tbl_name);
}
@Override protected Db_conn Init_db_file() {
this.db_mgr = wiki.Db_mgr_as_sql();
redirect_mgr = wiki.Redirect_mgr();
encoder = gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url_ttl;
redirect_tbl = new Xob_redirect_tbl(wiki.Fsys_mgr().Root_dir(), gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url_ttl).Create_table();
conn = redirect_tbl.Conn();
conn.Txn_bgn("bldr__redirect");
return conn;
}
@Override protected void Cmd_bgn_end() {}
@Override public void Exec_pg_itm_hook(int ns_ord, Xow_ns ns, Xowd_page_itm page, byte[] page_src) {
Xoa_ttl redirect_ttl = redirect_mgr.Extract_redirect(page_src);
byte[] redirect_ttl_bry = Xoa_ttl.Replace_spaces(redirect_ttl.Page_db()); // NOTE: spaces can still exist b/c redirect is scraped from #REDIRECT which sometimes has a mix; EX: "A_b c"
redirect_ttl_bry = encoder.Decode(redirect_ttl_bry);
redirect_tbl.Insert(page.Id(), Xoa_ttl.Replace_spaces(page.Ttl_page_db()), -1, redirect_ttl.Ns().Id(), redirect_ttl_bry, redirect_ttl.Anch_txt(), 1);
}
@Override public void Exec_commit_hook() {
conn.Txn_sav();
}
@Override public void Exec_end_hook() {
conn.Txn_end();
redirect_tbl.Create_indexes(usr_dlg);
redirect_tbl.Update_trg_redirect_id(db_mgr.Core_data_mgr().Db__core().Url(), 4);
}
}

166
400_xowa/src/gplx/xowa/bldrs/cmds/wikis/Xob_redirect_tbl.java
View File

@@ -1,166 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.cmds.wikis; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*;
import gplx.langs.htmls.encoders.*;
public class Xob_redirect_tbl {
private Gfo_url_encoder encoder; private Db_stmt insert_stmt;
public Xob_redirect_tbl(Io_url root_dir, Gfo_url_encoder encoder) {
this.db_file = Xob_db_file.New__wiki_redirect(root_dir);
this.conn = db_file.Conn();
this.encoder = encoder;
}
public Xob_db_file Db_file() {return db_file;} private Xob_db_file db_file;
public Db_conn Conn() {return conn;} private Db_conn conn;
public Xob_redirect_tbl Create_table() {Sqlite_engine_.Tbl_create(conn, Tbl_name, Tbl_sql); return this;}
public void Create_indexes(Gfo_usr_dlg usr_dlg) {
Sqlite_engine_.Idx_create(usr_dlg, conn, Xob_db_file.Name__wiki_redirect, Idx_trg_id, Idx_trg_ttl);
}
public void Update_trg_redirect_id(Io_url core_url, int max_redirected_depth) {
Sqlite_engine_.Db_attach(conn, "page_db", core_url.Raw()); // link database with page table
conn.Exec_sql(Sql_get_page_data); // fill in page_id, page_ns, page_is_redirect for trg_ttl; EX: Page_A has "#REDIRECT Page_B"; Page_B is in redirect tbl; find its id, ttl, redirect status
for (int i = 0; i < max_redirected_depth; i++) { // loop to find redirected redirects; note that it is bounded by depth (to guard against circular redirects)
int affected = conn.Exec_sql(Sql_get_redirect_redirects); // find redirects that are also redirects
if (affected == 0) break; // no more redirected redirects; stop
conn.Exec_sql(Sql_get_redirect_page_data); // get page data for redirects
}
Sqlite_engine_.Db_detach(conn, "page_db");
}
public void Update_src_redirect_id(Io_url core_url, Db_conn core_provider) {
// core_provider.Exec_sql(Sql_ddl__page_redirect_id); // create page.page_redirect_id
Sqlite_engine_.Idx_create(conn, Idx_trg_src);
Sqlite_engine_.Db_attach(conn, "page_db", core_url.Raw()); // link database with page table
conn.Exec_sql(Sql_update_redirect_id); // update page_redirect_id
Sqlite_engine_.Db_detach(conn, "page_db");
}
public void Insert(int src_id, byte[] src_bry, Xoa_ttl trg_ttl) {
byte[] redirect_ttl_bry = Xoa_ttl.Replace_spaces(trg_ttl.Page_db()); // NOTE: spaces can still exist b/c redirect is scraped from #REDIRECT which sometimes has a mix; EX: "A_b c"
redirect_ttl_bry = encoder.Decode(redirect_ttl_bry);
this.Insert(src_id, Xoa_ttl.Replace_spaces(src_bry), -1, trg_ttl.Ns().Id(), redirect_ttl_bry, trg_ttl.Anch_txt(), 1);
}
public void Insert(int src_id, byte[] src_ttl, int trg_id, int trg_ns, byte[] trg_ttl, byte[] trg_anchor, int count) {
if (insert_stmt == null) insert_stmt = Db_stmt_.new_insert_(conn, Tbl_name, Fld_src_id, Fld_src_ttl, Fld_trg_id, Fld_trg_ns, Fld_trg_ttl, Fld_trg_anchor, Fld_trg_is_redirect, Fld_redirect_count);
insert_stmt.Clear()
.Val_int(src_id)
.Val_bry_as_str(src_ttl)
.Val_int(trg_id)
.Val_int(trg_ns)
.Val_bry_as_str(trg_ttl)
.Val_bry_as_str(trg_anchor)
.Val_byte((byte)1)
.Val_int(count)
.Exec_insert();
}
public void Rls_all() {
insert_stmt.Rls();
conn.Rls_conn();
}
public static final String Tbl_name = "redirect";
private static final String
Fld_src_id = "src_id", Fld_src_ttl = "src_ttl", Fld_trg_id = "trg_id", Fld_trg_ns = "trg_ns", Fld_trg_ttl = "trg_ttl", Fld_trg_anchor = "trg_anchor"
, Fld_trg_is_redirect = "trg_is_redirect", Fld_redirect_count = "redirect_count";
private static final String Tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE IF NOT EXISTS redirect"
, "( src_id integer NOT NULL PRIMARY KEY"
, ", src_ttl varchar(255) NOT NULL"
, ", trg_id integer NOT NULL"
, ", trg_ns integer NOT NULL"
, ", trg_ttl varchar(255) NOT NULL"
, ", trg_anchor varchar(255) NOT NULL"
, ", trg_is_redirect tinyint NOT NULL"
, ", redirect_count integer NOT NULL"
, ");"
);
private static final Db_idx_itm
Idx_trg_ttl = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS redirect__trg_ttl ON redirect (trg_ttl);")
, Idx_trg_id = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS redirect__trg_id ON redirect (trg_id);")
, Idx_trg_src = Db_idx_itm.sql_("CREATE INDEX IF NOT EXISTS redirect__trg_src ON redirect (src_id, trg_id);")
;
// public static final String
// Sql_ddl__page_redirect_id = "ALTER TABLE page ADD COLUMN page_redirect_id integer NOT NULL DEFAULT '-1'"
// ;
private static final String
Sql_get_page_data = String_.Concat_lines_nl // get data from page table for initial redirect dump
( "REPLACE INTO redirect "
, "SELECT t.src_id"
, ", t.src_ttl"
, ", j.page_id"
, ", t.trg_ns"
, ", t.trg_ttl"
, ", t.trg_anchor"
, ", j.page_is_redirect"
, ", t.redirect_count"
, "FROM redirect t"
, " JOIN page_db.page j "
, " ON t.trg_ns = j.page_namespace"
, " AND t.trg_ttl = j.page_title"
, " AND t.trg_is_redirect = 1 -- limit to redirects"
, ";"
)
, Sql_get_redirect_redirects = String_.Concat_lines_nl // find redirects that are redirected
( "REPLACE INTO redirect"
, "SELECT t.src_id"
, ", t.src_ttl"
, ", j.trg_id"
, ", -1"
, ", ''"
, ", ''"
, ", 1"
, ", t.redirect_count + 1"
, "FROM redirect t"
, " JOIN redirect j "
, " ON t.trg_id = j.src_id"
, " AND t.trg_is_redirect = 1"
, ";"
, ""
)
, Sql_get_redirect_page_data = String_.Concat_lines_nl // get data from page table for redirected redirects
( "REPLACE INTO redirect"
, "SELECT t.src_id"
, ", t.src_ttl"
, ", t.trg_id"
, ", j.page_namespace"
, ", j.page_title"
, ", t.trg_anchor"
, ", j.page_is_redirect"
, ", t.redirect_count"
, "FROM redirect t"
, " JOIN page_db.page j "
, " ON t.trg_id = j.page_id "
, " AND t.trg_is_redirect = 1 -- limit to redirects"
, ";"
)
, Sql_update_redirect_id = String_.Concat_lines_nl_skip_last
( "REPLACE INTO"
, " page_db.page (page_id, page_namespace, page_title, page_is_redirect, page_touched, page_len, page_random_int, page_text_db_id, page_html_db_id, page_redirect_id, page_score)"
, "SELECT p.page_id"
, ", p.page_namespace"
, ", p.page_title"
, ", p.page_is_redirect"
, ", p.page_touched"
, ", p.page_len"
, ", p.page_random_int"
, ", p.page_text_db_id"
, ", p.page_html_db_id"
, ", r.trg_id"
, ", p.page_score"
, "FROM redirect r"
, " JOIN page_db.page p ON r.src_id = p.page_id"
)
;
}

16
400_xowa/src/gplx/xowa/bldrs/infos/Xob_info_file.java
View File

@@ -22,14 +22,14 @@ public class Xob_info_file {
this.id = id; this.type = type; this.ns_ids = ns_ids; this.part_id = part_id; this.guid = guid;
this.schema_version = schema_version; this.core_file_name = core_file_name; this.orig_file_name = orig_file_name;
}
public int Id() {return id;} private final int id;
public String Type() {return type;} private final String type;
public String Ns_ids() {return ns_ids;} private final String ns_ids;
public int Part_id() {return part_id;} private final int part_id;
public Guid_adp Guid() {return guid;} private final Guid_adp guid;
public int Schema_version() {return schema_version;} private final int schema_version;
public String Core_file_name() {return core_file_name;} private final String core_file_name;
public String Orig_file_name() {return orig_file_name;} private final String orig_file_name;
public int Id() {return id;} private final int id;
public String Type() {return type;} private final String type;
public String Ns_ids() {return ns_ids;} private final String ns_ids;
public int Part_id() {return part_id;} private final int part_id;
public Guid_adp Guid() {return guid;} private final Guid_adp guid;
public int Schema_version() {return schema_version;} private final int schema_version;
public String Core_file_name() {return core_file_name;} private final String core_file_name;
public String Orig_file_name() {return orig_file_name;} private final String orig_file_name;
public void Save(Db_cfg_tbl tbl) {
tbl.Conn().Txn_bgn("make__info__file");
tbl.Insert_int (Cfg_grp, Cfg_key__id , id);

14
400_xowa/src/gplx/xowa/bldrs/infos/Xob_info_session.java
View File

@@ -21,12 +21,12 @@ public class Xob_info_session {
Xob_info_session(String user, String version, String wiki_domain, String dump_name, DateAdp time, Guid_adp guid) {
this.user = user; this.version = version; this.wiki_domain = wiki_domain; this.dump_name = dump_name; this.time = time; this.guid = guid;
}
public String User() {return user;} private final String user;
public String Version() {return version;} private final String version;
public String Wiki_domain() {return wiki_domain;} private final String wiki_domain;
public String Dump_name() {return dump_name;} private final String dump_name;
public DateAdp Time() {return time;} private final DateAdp time;
public Guid_adp Uuid() {return guid;} private final Guid_adp guid;
public String User() {return user;} private final String user;
public String Version() {return version;} private final String version;
public String Wiki_domain() {return wiki_domain;} private final String wiki_domain;
public String Dump_name() {return dump_name;} private final String dump_name;
public DateAdp Time() {return time;} private final DateAdp time;
public Guid_adp Uuid() {return guid;} private final Guid_adp guid;
public void Save(Db_cfg_tbl tbl) {
tbl.Conn().Txn_bgn("make__info__session");
tbl.Insert_str (Cfg_grp, Cfg_key__user , user);
@@ -57,5 +57,5 @@ public class Xob_info_session {
, Cfg_key__guid = "guid" // EX: 00000000-0000-0000-0000-000000000000
;
public static Xob_info_session new_(String user, String wiki_domain, String dump_name) {return new Xob_info_session(user, Xoa_app_.Version, wiki_domain, dump_name, DateAdp_.Now(), Guid_adp_.new_());}
public static final Xob_info_session Test = new_("anonymous", "en.wikipedia.org", "enwiki-latest-pages-articles");
public static final Xob_info_session Test = new_("anonymous", "en.wikipedia.org", "enwiki-latest-pages-articles");
}

2
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_base.java
View File

@@ -103,7 +103,7 @@ class Xoi_cmd_search2_build extends Xoi_cmd_base {
@Override public void Process_async_init(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {
if (app.Setup_mgr().Dump_mgr().Wiki_storage_type_is_sql()) {
wiki.Db_mgr_as_sql().Category_version_update(false);
gplx.xowa.addons.searchs.dbs.bldrs.Srch_bldr_mgr_.Setup(wiki);
gplx.xowa.addons.apps.searchs.bldrs.Srch_bldr_mgr_.Setup(wiki);
}
}
@Override public void Process_async_done(Xoae_app app, Xowe_wiki wiki, Xob_bldr bldr) {

4
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_cmd_wiki_import.java
View File

@@ -64,8 +64,8 @@ class Xoi_cmd_wiki_import implements Gfo_thread_cmd {
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_core);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_cat_link);
}
if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.addons.searchs.specials.Srch_special_page.Version_2)
gplx.xowa.addons.searchs.dbs.bldrs.Srch_bldr_mgr_.Setup(wiki);
if (wiki.Appe().Setup_mgr().Dump_mgr().Search_version() == gplx.xowa.addons.apps.searchs.specials.Srch_special_page.Version_2)
gplx.xowa.addons.apps.searchs.bldrs.Srch_bldr_mgr_.Setup(wiki);
bldr.Cmd_mgr().Add_cmd(wiki, Xob_cmd_keys.Key_text_term);
}
private void Process_async() {

6
400_xowa/src/gplx/xowa/bldrs/installs/Xoi_dump_mgr.java
View File

@@ -31,7 +31,7 @@ public class Xoi_dump_mgr implements GfoInvkAble {
public boolean Css_wiki_update() {return css_wiki_update;} private boolean css_wiki_update = true;
public boolean Css_commons_download() {return css_commons_download;} private boolean css_commons_download = true; // changed from false to true; DATE:2014-10-19
public boolean Delete_xml_file() {return delete_xml_file;} private boolean delete_xml_file = true;
public byte Search_version() {return search_version;} private byte search_version = gplx.xowa.addons.searchs.specials.Srch_special_page.Version_2;
public byte Search_version() {return search_version;} private byte search_version = gplx.xowa.addons.apps.searchs.specials.Srch_special_page.Version_2;
public boolean Import_bz2_by_stdout() {return import_bz2_by_stdout;} private boolean import_bz2_by_stdout = true;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_server_urls)) return String_.Concat_with_str(",\n", server_urls);
@@ -79,7 +79,7 @@ public class Xoi_dump_mgr implements GfoInvkAble {
;
private static Keyval[] Options_data_storage_format_list = Keyval_.Ary(Keyval_.new_(".xdat", "text"), Keyval_.new_(".gz", "gzip"), Keyval_.new_(".bz2", "bzip2")); // removed .zip; DATE:2014-05-13; updated aliases; DATE:2014-06-20
static final byte Wiki_storage_type_xdat = 1, Wiki_storage_type_sqlite = 2;
private static final Keyval[] Options_storage_type_list = Keyval_.Ary(Keyval_.new_("sqlite"), Keyval_.new_("xdat")); // DEPRECATED: Keyval_.new_("xdat"); DATE:2015-03-30
private static final Keyval[] Options_storage_type_list = Keyval_.Ary(Keyval_.new_("sqlite"), Keyval_.new_("xdat")); // DEPRECATED: Keyval_.new_("xdat"); DATE:2015-03-30
public static String Wiki_storage_type_str(byte v) {
switch (v) {
case Xoi_dump_mgr.Wiki_storage_type_xdat : return "xdat";
@@ -92,7 +92,7 @@ public class Xoi_dump_mgr implements GfoInvkAble {
else if (String_.Eq(v, "sqlite")) return Xoi_dump_mgr.Wiki_storage_type_sqlite;
else throw Err_.new_unhandled(v);
}
private static final Keyval[] Options_search_version_list = Keyval_.Ary(Keyval_.new_("1"), Keyval_.new_("2"));
private static final Keyval[] Options_search_version_list = Keyval_.Ary(Keyval_.new_("1"), Keyval_.new_("2"));
public static String Options_search_version_str(byte v) {return Byte_.To_str(v);}
public static byte Options_search_version_parse(String v) {return Byte_.parse(v);}
}

2
400_xowa/src/gplx/xowa/bldrs/setups/maints/Xoa_maint_wikis_mgr.java
View File

@@ -17,8 +17,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.setups.maints; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.setups.*;
public class Xoa_maint_wikis_mgr implements GfoInvkAble {
private final Ordered_hash hash = Ordered_hash_.New_bry();
public Xoa_maint_wikis_mgr(Xoae_app app) {this.app = app;} private Xoae_app app;
private Ordered_hash hash = Ordered_hash_.New_bry();
public int Len() {return hash.Count();}
public Xowe_wiki Get_at(int i) {
if (init) Init();

15
400_xowa/src/gplx/xowa/bldrs/Xob_cmd.java → 400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_cmd.java
View File

@@ -15,12 +15,13 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public interface Xob_cmd extends GfoInvkAble {
String Cmd_key();
void Cmd_init(Xob_bldr bldr);
void Cmd_bgn(Xob_bldr bldr);
void Cmd_run();
void Cmd_end();
void Cmd_term();
String Cmd_key();
Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki);
void Cmd_init(Xob_bldr bldr);
void Cmd_bgn(Xob_bldr bldr);
void Cmd_run();
void Cmd_end();
void Cmd_term();
}

26
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_cmd__base.java
View File

@@ -16,14 +16,20 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public abstract class Xob_cmd__base implements GfoInvkAble {
protected final Xoae_app app; protected final Xob_bldr bldr; protected final Xowe_wiki wiki; protected final Gfo_usr_dlg usr_dlg;
public Xob_cmd__base(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki; this.app = bldr.App(); usr_dlg = bldr.Usr_dlg();}
public abstract String Cmd_key();
public abstract void Cmd_run();
@gplx.Virtual public void Cmd_init(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_bgn(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_end() {}
@gplx.Virtual public void Cmd_term() {}
public abstract Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m);
public abstract class Xob_cmd__base implements Xob_cmd, GfoInvkAble {
protected final Xoae_app app; protected final Xob_bldr bldr; protected Xowe_wiki wiki; protected final Gfo_usr_dlg usr_dlg;
public Xob_cmd__base(Xob_bldr bldr, Xowe_wiki wiki) {
this.bldr = bldr;
this.wiki = wiki;
this.app = bldr == null ? null : bldr.App();;
this.usr_dlg = bldr == null ? null : bldr.Usr_dlg();
}
public abstract String Cmd_key();
@gplx.Virtual public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
public abstract void Cmd_run();
@gplx.Virtual public void Cmd_init(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_bgn(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_end() {}
@gplx.Virtual public void Cmd_term() {}
@gplx.Virtual public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return GfoInvkAble_.Rv_unhandled;}
}

3
400_xowa/src/gplx/xowa/bldrs/Xob_cmd_base.java → 400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_cmd_base.java
View File

@@ -15,9 +15,10 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*;
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public abstract class Xob_cmd_base implements Xob_cmd {
public abstract String Cmd_key();
@gplx.Virtual public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
@gplx.Virtual public void Cmd_init(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_bgn(Xob_bldr bldr) {}
@gplx.Virtual public void Cmd_run() {}

10
400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_itm_basic_base.java
View File

@@ -18,7 +18,15 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
public abstract class Xob_itm_basic_base implements GfoInvkAble {
protected Xoae_app app; protected Xob_bldr bldr; protected Xowe_wiki wiki; protected Gfo_usr_dlg usr_dlg;
public void Cmd_ctor(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki; this.app = bldr.App(); usr_dlg = bldr.Usr_dlg(); this.Cmd_ctor_end(bldr, wiki);}
public void Cmd_ctor(Xob_bldr bldr, Xowe_wiki wiki) {
this.bldr = bldr;
this.wiki = wiki;
this.app = bldr == null ? null : bldr.App();
this.usr_dlg = bldr == null ? null : bldr.Usr_dlg();
if (bldr != null && wiki != null)
this.Cmd_ctor_end(bldr, wiki);
}
@gplx.Virtual public Xob_cmd Cmd_new(Xob_bldr bldr, Xowe_wiki wiki) {return null;}
@gplx.Virtual protected void Cmd_ctor_end(Xob_bldr bldr, Xowe_wiki wiki) {}
@gplx.Virtual public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_owner)) return bldr.Cmd_mgr();