Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.9.4.1'

Browse Source
This commit is contained in:
gnosygnu
2016-09-25 22:05:47 -04:00
parent 35d78f6106
commit e3b393650d
211 changed files with 3148 additions and 2197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__download.java
View File

@@ -34,25 +34,35 @@ public class Xobc_cmd__download extends Xobc_cmd__base {
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
int error_wait = 10000, error_tries_max = 6, error_tries_cur = 0; // retry every 10 seconds for a total of 6 tries (1 min)
// loop while "server ... " error
while (true) {
long trg_size_bgn = Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size();
long trg_size_bgn = Tmp_url_size();
byte status = wkr.Exec(this, src_url, trg_url, expd_size);
if (status == Gfo_prog_ui_.Status__fail) {
// check if anything more downloaded; if so, then reset to 0; DATE:2016-09-03
long trg_size_cur = Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size();
if (trg_size_cur > trg_size_bgn) {
error_tries_cur = 0;
trg_size_bgn = trg_size_cur;
// check if server error; note: must not loop if bad size; DATE:2016-09-24
String fail_msg = wkr.Fail_msg();
if (String_.Has_at_bgn(fail_msg, Http_download_wkr__jre.Err__server_download_failed)) {
// check if anything more downloaded; if so, then reset to 0; DATE:2016-09-03
long trg_size_cur = Tmp_url_size();
if (trg_size_cur > trg_size_bgn) {
error_tries_cur = 0;
trg_size_bgn = trg_size_cur;
}
// retry
if (error_tries_cur++ < error_tries_max) {
task_mgr.Work_mgr().On_stat(this.Task_id(), String_.Format("connection interrupted: retrying in {0} seconds; attempt {1} of {2}", error_wait / 1000, error_tries_cur, error_tries_max));
Gfo_usr_dlg_.Instance.Log_many("", "", "xobc_cmd task download interrupted; ~{0} ~{1} ~{2} ~{3}", this.Task_id(), this.Step_id(), trg_url, Io_mgr.Instance.QueryFil(trg_url).Size());
gplx.core.threads.Thread_adp_.Sleep(error_wait);
continue;
}
}
// retry
if (error_tries_cur++ < error_tries_max) {
task_mgr.Work_mgr().On_stat(this.Task_id(), String_.Format("connection interrupted: retrying in {0} seconds; attempt {1} of {2}", error_wait / 1000, error_tries_cur, error_tries_max));
Gfo_usr_dlg_.Instance.Log_many("", "", "xobc_cmd task download interrupted; ~{0} ~{1} ~{2} ~{3}", this.Task_id(), this.Step_id(), trg_url, Io_mgr.Instance.QueryFil(trg_url).Size());
gplx.core.threads.Thread_adp_.Sleep(error_wait);
continue;
}
this.Cmd_exec_err_(wkr.Fail_msg());
// otherewise exit loop
this.Cmd_exec_err_(fail_msg);
break;
}
else
@@ -60,6 +70,9 @@ public class Xobc_cmd__download extends Xobc_cmd__base {
}
Gfo_log_.Instance.Info("xobc_cmd task download", "task_id", this.Task_id(), "step_id", this.Step_id(), "trg_url", trg_url, "trg_len", Io_mgr.Instance.QueryFil(trg_url).Size());
}
private long Tmp_url_size() {
return wkr.Tmp_url() == null ? 0 : Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size(); // NOTE: wkr.Tmp_url is null in some extreme exceptions; DATE:2016-09-24
}
@Override public void Cmd_cleanup() {
wkr.Exec_cleanup();
}

46
400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__fsdb_delete.java Normal file
View File

@@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
import gplx.dbs.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.addons.bldrs.updates.files.*;
public class Xobc_cmd__fsdb_delete extends Xobc_cmd__base {
private final Io_url deletion_db_url;
public Xobc_cmd__fsdb_delete(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url deletion_db_url) {super(task_mgr, task_id, step_id, cmd_idx);
this.deletion_db_url = deletion_db_url;
}
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.fsdb.delete";
@Override public String Cmd_name() {return "deleting old files";}
@Override public boolean Cmd_suspendable() {return true;}
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
if (!Io_mgr.Instance.ExistsFil(deletion_db_url)) throw Err_.New("deletion db does not exist; file={0}", deletion_db_url.Raw());
boolean pass = false;
try {
new Xodel_exec_mgr().Exec_delete(this, ctx.App().Bldr(), deletion_db_url);
pass = true;
}
catch (Exception e) {
this.Cmd_exec_err_(Err_.Message_gplx_log(e));
}
Gfo_log_.Instance.Info("xobc_cmd task delete", "task_id", this.Task_id(), "step_id", this.Step_id(), "delete_url", deletion_db_url.Raw(), "pass", pass);
}
@Override public void Cmd_cleanup() {
if (Io_mgr.Instance.ExistsFil(deletion_db_url))
new Xodel_exec_mgr().Exec_cleanup(deletion_db_url);
}
}

4
400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__verify_dir.java
View File

@@ -20,9 +20,9 @@ import gplx.core.progs.*;
import gplx.core.security.*; import gplx.core.security.files.*;
public class Xobc_cmd__verify_dir extends Xobc_cmd__base {
private final Io_url delete_fil, checksum_fil;
public Xobc_cmd__verify_dir(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url dir_url, String checksum_fil_name, Io_url delete_fil) {super(task_mgr, task_id, step_id, cmd_idx);
public Xobc_cmd__verify_dir(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url checksum_fil, Io_url delete_fil) {super(task_mgr, task_id, step_id, cmd_idx);
this.checksum_fil = checksum_fil;
this.delete_fil = delete_fil;
this.checksum_fil = dir_url.GenSubFil(checksum_fil_name);
}
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.hash_dir";
@Override public String Cmd_name() {return "verify";}

18
400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java
View File

@@ -18,13 +18,15 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.centrals.dbs.datas.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
public class Xobc_import_type {
public static final int // SERIALIZED: bc_db; import_step
Tid__ignore = 0
, Tid__pack = 1
, Tid__wiki__core = 2
, Tid__wiki__srch = 3
, Tid__wiki__html = 4
, Tid__wiki__text = 5
, Tid__file__core = 6
, Tid__file__data = 7
Tid__ignore = 0
, Tid__pack = 1
, Tid__wiki__core = 2
, Tid__wiki__srch = 3
, Tid__wiki__html = 4
, Tid__wiki__text = 5
, Tid__file__core = 6
, Tid__file__data = 7
, Tid__fsdb__delete = 8
, Tid__wiki__ctg = 9
;
}

27
400_xowa/src/gplx/xowa/addons/bldrs/centrals/steps/Xobc_step_factory.java
View File

@@ -20,6 +20,9 @@ import gplx.core.brys.evals.*; import gplx.core.primitives.*;
import gplx.xowa.addons.bldrs.centrals.tasks.*; import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.utils.*;
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
import gplx.xowa.addons.bldrs.exports.merges.*;
import gplx.xowa.addons.bldrs.updates.files.*;
import gplx.xowa.addons.bldrs.exports.packs.files.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.wikis.domains.*;
public class Xobc_step_factory {
private final Xobc_task_mgr task_mgr;
@@ -60,16 +63,24 @@ public class Xobc_step_factory {
Io_url zip_file_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__download_fil, wiki_domain, file_name));
Io_url unzip_dir_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__unzip_dir, wiki_domain, file_name));
Io_url wiki_dir_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__wiki_dir, wiki_domain, file_name));
list.Add(new Xobc_cmd__download (task_mgr, task_id, step_id, 0, src_http_url, zip_file_url, import_itm.Import_size_zip));
list.Add(new Xobc_cmd__verify_fil (task_mgr, task_id, step_id, 1, zip_file_url, import_itm.Import_md5, import_itm.Import_size_zip));
list.Add(new Xobc_cmd__unzip (task_mgr, task_id, step_id, 2, zip_file_url, unzip_dir_url, import_itm.Import_size_raw));
list.Add(new Xobc_cmd__verify_dir (task_mgr, task_id, step_id, 3, unzip_dir_url, String_.Replace(file_name, ".zip", ".md5"), zip_file_url));
// list.Add(new Xobc_cmd__wiki_merge (task_mgr, task_id, step_id, 4, merge_mgr, wiki_domain, unzip_dir_url, import_itm.Import_prog_data_max, import_itm.Import_prog_row_max, step_seqn));
list.Add(new Xobc_cmd__move_fils (task_mgr, task_id, step_id, 4, unzip_dir_url, wiki_dir_url));
if (import_itm.Import_type == Xobc_import_type.Tid__wiki__core) {
list.Add(new Xobc_cmd__wiki_reg (task_mgr, task_id, step_id, 5, wiki_dir_url, wiki_domain));
Io_url checksum_url = unzip_dir_url.GenSubFil(file_name + ".md5");
int cmd_idx = 0;
list.Add(new Xobc_cmd__download (task_mgr, task_id, step_id, cmd_idx++, src_http_url, zip_file_url, import_itm.Import_size_zip));
list.Add(new Xobc_cmd__verify_fil (task_mgr, task_id, step_id, cmd_idx++, zip_file_url, import_itm.Import_md5, import_itm.Import_size_zip));
list.Add(new Xobc_cmd__unzip (task_mgr, task_id, step_id, cmd_idx++, zip_file_url, unzip_dir_url, import_itm.Import_size_raw));
list.Add(new Xobc_cmd__verify_dir (task_mgr, task_id, step_id, cmd_idx++, checksum_url, zip_file_url));
// list.Add(new Xobc_cmd__wiki_merge (task_mgr, task_id, step_id, cmd_idx++, merge_mgr, wiki_domain, unzip_dir_url, import_itm.Import_prog_data_max, import_itm.Import_prog_row_max, step_seqn));
list.Add(new Xobc_cmd__move_fils (task_mgr, task_id, step_id, cmd_idx++, unzip_dir_url, wiki_dir_url));
switch (import_itm.Import_type) {
case Xobc_import_type.Tid__wiki__core: list.Add(new Xobc_cmd__wiki_reg (task_mgr, task_id, step_id, cmd_idx++, wiki_dir_url, wiki_domain)); break;
case Xobc_import_type.Tid__fsdb__delete: list.Add(new Xobc_cmd__fsdb_delete (task_mgr, task_id, step_id, cmd_idx++, Pack_zip_name_bldr.To_wiki_url(wiki_dir_url, zip_file_url.OwnerDir()))); break;
}
return (Xobc_cmd_itm[])list.To_ary_and_clear(Xobc_cmd_itm.class);
}
private Io_url Eval_url(String src) {return Io_url_.new_any_(String_.new_u8(eval_mgr.Eval(Bry_.new_u8(src))));}
public static Xow_wiki Get_wiki_by_abrv(Xoa_app app, byte[] wiki_abrv) {
Xow_domain_itm domain_itm = Xow_abrv_xo_.To_itm(wiki_abrv);
return app.Wiki_mgri().Get_by_or_make_init_y(domain_itm.Domain_bry());
}
}

2
400_xowa/src/gplx/xowa/addons/bldrs/centrals/tasks/Xobc_task_regy__work.java
View File

@@ -79,8 +79,8 @@ public class Xobc_task_regy__work extends Xobc_task_regy__base {
thread_mgr.Halt(cmd_uid, Thread_halt_cbk_.Noop);
}
private void Redo_task(Xobc_task_itm task, Xobc_step_itm step, Xobc_cmd_itm cmd) {
cmd.Cmd_clear();
cmd = step.Step_fallback_to(cmd.Cmd_fallback());
cmd.Cmd_clear();
task.Task_status_(gplx.core.progs.Gfo_prog_ui_.Status__working);
task_mgr.Send_json("xo.bldr.work.prog__start__recv", task.Save_to(Gfobj_nde.New()));
thread_mgr.Add(cmd.Cmd_uid(), Thread_adp_.Start_by_val("xobc: " + cmd.Cmd_name(), cmd, cmd, Xobc_cmd__base.Invk__exec, ctx));

4
400_xowa/src/gplx/xowa/addons/bldrs/centrals/utils/Bry_eval_wkr__builder_central.java
View File

@@ -22,7 +22,7 @@ public class Bry_eval_wkr__builder_central implements Bry_eval_wkr {
public Bry_eval_wkr__builder_central(Io_url wiki_dir) {this.wiki_dir = wiki_dir.RawBry();}
public String Key() {return "builder_central";}
public void Resolve(Bry_bfr rv, byte[] src, int args_bgn, int args_end) {
// EX: "~{builder_central|download_fil|en.wikipedia.org|en.wikipedia.org-2016.05-html-ns.000-db.001.zip}" -> "/xowa/wiki/en.wikipedia.org/tmp/bldr/en.wikipedia.org-2016.05-html-ns.000-db.001.zip/download.zip"
// EX: "~{builder_central|download_fil|en.wikipedia.org|Xowa_enwiki_2016-05_html_ns.000_db.001.zip}" -> "/xowa/wiki/en.wikipedia.org/tmp/bldr/Xowa_enwiki_2016-05_html_ns.000_db.001.zip/download.zip"
byte[][] args = Bry_split_.Split(src, args_bgn, args_end, Byte_ascii.Pipe, Bool_.N);
int type = hash.Get_as_byte_or(args[0], Byte_.Max_value_127);
if (type == Byte_.Max_value_127) throw Err_.new_wo_type("unknown eval type", "src", src);
@@ -32,7 +32,7 @@ public class Bry_eval_wkr__builder_central implements Bry_eval_wkr {
if (type == Type__wiki_dir) return;
rv.Add_str_a7("tmp").Add_byte(dir_spr); // "tmp/"
rv.Add_str_a7("bldr").Add_byte(dir_spr); // "bldr/"
rv.Add(args[2]).Add_byte(dir_spr); // "en.wikipedia.org-2016.05-html-ns.000-db.001.zip/"
rv.Add(args[2]).Add_byte(dir_spr); // "Xowa_enwiki_2016-05_html_ns.000_db.001.zip/"
switch (type) {
case Type__download_fil: rv.Add_str_a7("download.zip"); break;
case Type__unzip_dir: rv.Add_str_a7("unzip").Add_byte(dir_spr); break;

3
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java
View File

@@ -21,6 +21,7 @@ public class Pack_file_bldr_cfg implements Gfo_invk {
public Io_url Deploy_dir() {return deploy_dir;} private Io_url deploy_dir;
public boolean Pack_html() {return pack_html;} private boolean pack_html = true;
public boolean Pack_file() {return pack_file;} private boolean pack_file = true;
public boolean Pack_fsdb_delete() {return pack_fsdb_delete;} private boolean pack_fsdb_delete;
public DateAdp Pack_file_cutoff() {return pack_file_cutoff;} private DateAdp pack_file_cutoff = null;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
@@ -28,10 +29,12 @@ public class Pack_file_bldr_cfg implements Gfo_invk {
else if (ctx.Match(k, Invk__pack_html_)) pack_html = m.ReadYn("v");
else if (ctx.Match(k, Invk__pack_file_)) pack_file = m.ReadYn("v");
else if (ctx.Match(k, Invk__pack_file_cutoff_)) pack_file_cutoff = m.ReadDate("v");
else if (ctx.Match(k, Invk__pack_fsdb_delete_)) pack_fsdb_delete = m.ReadYn("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk__deploy_dir_ = "deploy_dir_"
, Invk__pack_html_ = "pack_html_", Invk__pack_file_ = "pack_file_", Invk__pack_file_cutoff_ = "pack_file_cutoff_"
, Invk__pack_fsdb_delete_ = "pack_fsdb_delete_"
;
}

35
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java
View File

@@ -27,7 +27,7 @@ class Pack_file_mgr {
Io_url pack_dir = wiki_dir.GenSubDir_nest("tmp", "pack");
Io_mgr.Instance.DeleteDirDeep(pack_dir); Io_mgr.Instance.CreateDirIfAbsent(pack_dir);
String wiki_date = wiki.Props().Modified_latest().XtoStr_fmt("yyyy.MM");
Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg.Pack_html(), cfg.Pack_file(), cfg.Pack_file_cutoff());
Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg.Pack_html(), cfg.Pack_file(), cfg.Pack_file_cutoff(), cfg.Pack_fsdb_delete());
// get import_tbl
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
@@ -51,9 +51,9 @@ class Pack_file_mgr {
// build tasks
if (cfg.Pack_html())
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html);
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html, Xobc_import_type.Tid__wiki__ctg);
if (cfg.Pack_file())
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data);
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data); // , Xobc_import_type.Tid__fsdb__delete
bc_conn.Txn_end();
// deploy
@@ -163,7 +163,7 @@ class Pack_file_mgr {
}
}
class Pack_hash_bldr {
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff) {
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff, boolean pack_fsdb_delete) {
Pack_hash rv = new Pack_hash();
Pack_zip_name_bldr zip_name_bldr = new Pack_zip_name_bldr(pack_dir, wiki.Domain_str(), String_.new_a7(wiki.Domain_itm().Abrv_wm()), wiki_date);
Xow_db_mgr db_mgr = wiki.Data__core_mgr();
@@ -200,6 +200,13 @@ class Pack_hash_bldr {
}
}
}
// bld pack_fsdb_delete
if (pack_fsdb_delete) {
gplx.xowa.bldrs.Xob_db_file fsdb_deletion_db = gplx.xowa.bldrs.Xob_db_file.New__deletion_db(wiki);
if (!Io_mgr.Instance.ExistsFil(fsdb_deletion_db.Url())) throw Err_.new_wo_type("deletion db does not exists: url=" + fsdb_deletion_db.Url().Raw());
rv.Add(zip_name_bldr, Xobc_import_type.Tid__fsdb__delete, fsdb_deletion_db.Url());
}
return rv;
}
private static int Get_pack_tid(byte db_file_tid) {
@@ -209,6 +216,9 @@ class Pack_hash_bldr {
case Xow_db_file_.Tid__search_link: return Xobc_import_type.Tid__wiki__srch;
case Xow_db_file_.Tid__html_solo:
case Xow_db_file_.Tid__html_data: return Xobc_import_type.Tid__wiki__html;
case Xow_db_file_.Tid__cat:
case Xow_db_file_.Tid__cat_core:
case Xow_db_file_.Tid__cat_link: return Xobc_import_type.Tid__wiki__ctg;
case Xow_db_file_.Tid__file_core: return Xobc_import_type.Tid__file__core;
case Xow_db_file_.Tid__file_solo:
case Xow_db_file_.Tid__file_data: return Xobc_import_type.Tid__file__data;
@@ -216,20 +226,3 @@ class Pack_hash_bldr {
}
}
}
class Pack_zip_name_bldr { // en.wikipedia.org-file-ns.000-db.001.xowa -> Xowa_enwiki_2016-05_file_ns.000_db.001.zip
private final Io_url pack_dir;
private final byte[] wiki_domain, zip_name_prefix;
public Pack_zip_name_bldr(Io_url pack_dir, String wiki_domain_str, String wiki_abrv, String wiki_date) {
this.pack_dir = pack_dir;
this.wiki_domain = Bry_.new_u8(wiki_domain_str);
this.zip_name_prefix = Bry_.new_u8("Xowa_" + wiki_abrv + "_" + String_.Replace(wiki_date, ".", "-"));
}
public Io_url Bld(Io_url orig_url) {
String orig_str = String_.Replace(orig_url.NameAndExt(), ".xowa", ".zip");
byte[] orig_bry = Bry_.new_u8(orig_str);
orig_bry = Bry_.Replace(orig_bry, Byte_ascii.Dash, Byte_ascii.Underline);
orig_bry = Bry_.Replace(orig_bry, wiki_domain, zip_name_prefix);
return pack_dir.GenSubFil(String_.new_u8(orig_bry));
}
}

56
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr.java Normal file
View File

@@ -0,0 +1,56 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
public class Pack_zip_name_bldr { // en.wikipedia.org-file-ns.000-db.001.xowa -> Xowa_enwiki_2016-05_file_ns.000_db.001.zip
private final Io_url pack_dir;
private final byte[] wiki_domain, zip_name_prefix;
public Pack_zip_name_bldr(Io_url pack_dir, String wiki_domain_str, String wiki_abrv, String wiki_date) {
this.pack_dir = pack_dir;
this.wiki_domain = Bry_.new_u8(wiki_domain_str);
this.zip_name_prefix = Bry_.new_u8("Xowa_" + wiki_abrv + "_" + String_.Replace(wiki_date, ".", "-"));
}
public Io_url Bld(Io_url orig_url) {
String orig_str = orig_url.NameOnly() + ".zip";
byte[] orig_bry = Bry_.new_u8(orig_str);
orig_bry = Bry_.Replace(orig_bry, Byte_ascii.Dash, Byte_ascii.Underline);
orig_bry = Bry_.Replace(orig_bry, wiki_domain, zip_name_prefix);
return pack_dir.GenSubFil(String_.new_u8(orig_bry));
}
public static Io_url To_wiki_url(Io_url wiki_dir, Io_url zip_dir) {
// get wiki_url based on wiki_dir and xobc_zip_fil; EX: "/wiki/en.wikipedia.org/", "/wiki/tmp/Xowa_enwiki_2016-09_file_core_deletion_2016-09/" -> "/wiki/en.wikipedia.org-file-core-deletion-2016.09.zip"
String name_str = zip_dir.NameOnly() + ".xowa";
byte[] name_bry = Bry_.new_u8(name_str);
int pos = Bry_find__Find_fwd_idx(name_bry, Byte_ascii.Underline, 2);
name_bry = Bry_.Mid(name_bry, pos, name_bry.length);
name_bry = Bry_.Add(Bry_.new_u8(wiki_dir.NameOnly()), name_bry);
name_bry = Bry_.Replace(name_bry, Byte_ascii.Underline, Byte_ascii.Dash);
return wiki_dir.GenSubFil(String_.new_u8(name_bry));
}
private static int Bry_find__Find_fwd_idx(byte[] src, byte val, int find_max) {
int src_len = src.length;
int find_cur = 0;
int cur_pos = 0;
while (true) {
int new_pos = Bry_find_.Find_fwd(src, val, cur_pos, src_len);
if (new_pos == -1) throw Err_.New("failed to find value; src={0} val={1}", src, val);
if (find_cur++ == find_max) return new_pos;
cur_pos = new_pos + 1;
}
}
}

30
400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr__tst.java Normal file
View File

@@ -0,0 +1,30 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
import org.junit.*; import gplx.core.tests.*;
public class Pack_zip_name_bldr__tst {
private Pack_zip_name_bldr__fxt fxt = new Pack_zip_name_bldr__fxt();
@Test public void Basic() {
fxt.Test__to_wiki_url("mem/wiki/en.wikipedia.org/", "mem/wiki/en.wikipedia.org/tmp/Xowa_enwiki_2016-09_file_deletion_2016.09/", "mem/wiki/en.wikipedia.org/en.wikipedia.org-file-deletion-2016.09.xowa");
}
}
class Pack_zip_name_bldr__fxt {
public void Test__to_wiki_url(String wiki_dir, String zip_fil, String expd) {
Gftest.Eq__str(expd, Pack_zip_name_bldr.To_wiki_url(Io_url_.mem_fil_(wiki_dir), Io_url_.mem_dir_(zip_fil)).Raw(), "wiki_url");
}
}

2
400_xowa/src/gplx/xowa/addons/bldrs/exports/splits/files/Split_meta_wkr__fil.java
View File

@@ -47,7 +47,7 @@ class Split_meta_wkr__fil extends Split_meta_wkr_base {
);
}
@Override protected Object Load_itm(Db_rdr rdr) {
return tbl.Load_by_rdr__full(Fsm_mnt_mgr.Mnt_idx_main, rdr);
return tbl.New_by_rdr(Fsm_mnt_mgr.Mnt_idx_main, rdr);
}
@Override protected void Save_itm(Split_ctx ctx, Split_rslt_mgr rslt_mgr, Object itm_obj) {
Fsd_fil_itm itm = (Fsd_fil_itm)itm_obj;

2
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__fsdb_db__create_data.java
View File

@@ -185,7 +185,7 @@ public class Xobldr__fsdb_db__create_data extends Xob_cmd__base implements Xob_c
tier_id_val = fsdb.Lnki_tier_id();
page_id_val = fsdb.Lnki_page_id();
lnki_id_val = fsdb.Lnki_id();
fsdb.Orig_repo_name_(fsdb.Orig_repo_id() == Xof_repo_itm_.Repo_local ? wiki.Domain_bry() : Xow_domain_itm_.Bry__commons);
fsdb.Orig_repo_name_(fsdb.Orig_repo_id() == Xof_repo_tid_.Tid__local ? wiki.Domain_bry() : Xow_domain_itm_.Bry__commons);
Download_exec(fsdb);
++exec_count;
if (exec_count % progress_interval == 0) Print_progress(fsdb);

35
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__image__create.java
View File

@@ -17,16 +17,20 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
import gplx.dbs.*; import gplx.core.ios.*; import gplx.xowa.files.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sql_dumps.*;
import gplx.xowa.addons.bldrs.files.dbs.*;
public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd, Gfo_invk, Sql_file_parser_cmd {
public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd, Gfo_invk, Xosql_dump_cbk {
private Xosql_dump_parser parser;
private Db_conn conn = null; private Db_stmt stmt = null;
private Xob_image_tbl tbl_image = new Xob_image_tbl();
private byte[] cur_ttl, cur_media_type, cur_minor_mime, cur_timestamp; private int cur_size, cur_width, cur_height, cur_bits, cur_ext_id;
private int commit_count = 10000;
public Xobldr__image__create(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public Xobldr__image__create(Xob_bldr bldr, Xowe_wiki wiki) {
this.parser = new Xosql_dump_parser(this, "img_name", "img_size", "img_width", "img_height", "img_bits", "img_media_type", "img_minor_mime", "img_timestamp");
this.Cmd_ctor(bldr, wiki);
}
public Io_url Src_fil() {return src_fil;} public Xobldr__image__create Src_fil_(Io_url v) {src_fil = v; return this;} private Io_url src_fil;
public Sql_file_parser Parser() {return parser;} private Sql_file_parser parser = new Sql_file_parser();
public Xosql_dump_parser Parser() {return parser;}
public void Cmd_init(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql
@@ -35,7 +39,7 @@ public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd,
src_fil = Xob_page_wkr_cmd.Find_fil_by(wiki.Fsys_mgr().Root_dir(), "*-image.sql");
if (src_fil == null) throw Err_.new_wo_type(".sql file not found in dir", "dir", wiki.Fsys_mgr().Root_dir());
}
parser.Src_fil_(src_fil).Trg_fil_gen_(dump_url_gen).Fld_cmd_(this).Flds_req_idx_(20, Fld_img_name, Fld_img_size, Fld_img_width, Fld_img_height, Fld_img_bits, Fld_img_media_type, Fld_img_minor_mime, Fld_img_timestamp);
parser.Src_fil_(src_fil);
this.conn = Xob_db_file.New__wiki_image(wiki.Fsys_mgr().Root_dir()).Conn();
conn.Txn_bgn("bldr__image");
this.tbl_image = new Xob_image_tbl();
@@ -47,16 +51,16 @@ public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd,
tbl_image.Create_index(conn);
conn.Txn_end();
}
public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
public void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end) {
switch (fld_idx) {
case Fld_img_name: cur_ttl = Bry_.Mid(src, fld_bgn, fld_end); break;
case Fld_img_size: cur_size = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld_img_width: cur_width = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld_img_height: cur_height = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld_img_bits: cur_bits = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld_img_media_type: cur_media_type = Bry_.Mid(src, fld_bgn, fld_end); break;
case Fld_img_minor_mime: cur_minor_mime = Bry_.Mid(src, fld_bgn, fld_end); break;
case Fld_img_timestamp: cur_timestamp = Bry_.Mid(src, fld_bgn, fld_end);
case Fld_img_name: cur_ttl = Bry_.Mid(src, val_bgn, val_end); break;
case Fld_img_size: cur_size = Bry_.To_int_or(src, val_bgn, val_end, -1); break;
case Fld_img_width: cur_width = Bry_.To_int_or(src, val_bgn, val_end, -1); break;
case Fld_img_height: cur_height = Bry_.To_int_or(src, val_bgn, val_end, -1); break;
case Fld_img_bits: cur_bits = Bry_.To_int_or(src, val_bgn, val_end, -1); break;
case Fld_img_media_type: cur_media_type = Bry_.Mid(src, val_bgn, val_end); break;
case Fld_img_minor_mime: cur_minor_mime = Bry_.Mid(src, val_bgn, val_end); break;
case Fld_img_timestamp: cur_timestamp = Bry_.Mid(src, val_bgn, val_end);
cur_ext_id = Calc_ext_id(show_issues ? app.Usr_dlg() : Gfo_usr_dlg_.Noop, cur_ttl, cur_media_type, cur_minor_mime, cur_width, cur_height);
tbl_image.Insert(stmt, cur_ttl, cur_media_type, cur_minor_mime, cur_size, cur_width, cur_height, cur_bits, cur_ext_id, cur_timestamp);
++commit_count;
@@ -70,7 +74,8 @@ public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd,
public void Cmd_end() {}
public void Cmd_term() {}
private boolean show_issues = true;
private static final int Fld_img_name = 0, Fld_img_size = 1, Fld_img_width = 2, Fld_img_height = 3, Fld_img_bits = 5, Fld_img_media_type = 6, Fld_img_minor_mime = 8, Fld_img_timestamp = 12;
private static final int Fld_img_name = 0, Fld_img_size = 1, Fld_img_width = 2, Fld_img_height = 3, Fld_img_bits = 4, Fld_img_media_type = 5, Fld_img_minor_mime = 6, Fld_img_timestamp = 7;
// Fld_img_name = 0, Fld_img_size = 1, Fld_img_width = 2, Fld_img_height = 3, Fld_img_bits = 5, Fld_img_media_type = 6, Fld_img_minor_mime = 8, Fld_img_timestamp = 12;
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_src_fil_)) src_fil = m.ReadIoUrl("v");
else if (ctx.Match(k, Invk_show_issues_)) show_issues = m.ReadYn("v");

4
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__page_regy__create.java
View File

@@ -28,14 +28,14 @@ public class Xobldr__page_regy__create extends Xob_cmd__base {
commons_wiki.Init_assert();
if (build_commons) {
Xob_page_regy_tbl.Reset_table(page_regy_provider);
Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_itm_.Repo_remote, commons_wiki);
Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_tid_.Tid__remote, commons_wiki);
Sqlite_engine_.Idx_create(usr_dlg, page_regy_provider, "repo_page", Xob_page_regy_tbl.Idx_main);
}
else {
if (!Bry_.Eq(commons_wiki.Domain_bry(), wiki.Domain_bry())) { // skip local wiki if cur wiki is commons
wiki.Init_assert();
Xob_page_regy_tbl.Delete_local(page_regy_provider);
Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_itm_.Repo_local, wiki);
Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_tid_.Tid__local, wiki);
}
}
}

14
400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_orig_regy_tbl.java
View File

@@ -29,7 +29,7 @@ public class Xob_orig_regy_tbl {
Sqlite_engine_.Idx_create(usr_dlg, p, "orig_regy", Idx_ttl_local);
Sqlite_engine_.Db_attach(p, "page_db", file_registry_db.Url().Raw());
Io_url repo_0_dir = repo_0_wiki.Fsys_mgr().Root_dir(), repo_1_dir = repo_1_wiki.Fsys_mgr().Root_dir();
byte repo_0_tid = Xof_repo_itm_.Repo_local, repo_1_tid = Xof_repo_itm_.Repo_remote;
byte repo_0_tid = Xof_repo_tid_.Tid__local, repo_1_tid = Xof_repo_tid_.Tid__remote;
boolean local_is_remote = Bry_.Eq(repo_0_wiki.Domain_bry(), repo_1_wiki.Domain_bry());
Xowe_wiki local_wiki = repo_0_wiki;
if ( repo_0_is_remote // .gfs manually marked specifes repo_0 as remote
@@ -37,8 +37,8 @@ public class Xob_orig_regy_tbl {
&& local_is_remote // repo_0 = repo_1
)
) {
repo_0_tid = Xof_repo_itm_.Repo_remote;
repo_1_tid = Xof_repo_itm_.Repo_local;
repo_0_tid = Xof_repo_tid_.Tid__remote;
repo_1_tid = Xof_repo_tid_.Tid__local;
local_wiki = repo_1_wiki;
}
Create_data_for_repo(usr_dlg, p, local_wiki, Byte_.By_int(repo_0_tid), repo_0_dir.GenSubFil(Xob_db_file.Name__wiki_image));
@@ -54,18 +54,18 @@ public class Xob_orig_regy_tbl {
}
private static void Create_data_for_repo(Gfo_usr_dlg usr_dlg, Db_conn conn, Xowe_wiki local_wiki, byte repo_tid, Io_url join) {
usr_dlg.Note_many("", "", "inserting page for xowa.wiki.image: ~{0}", join.OwnerDir().NameOnly());
boolean wiki_has_cs_file = repo_tid == Xof_repo_itm_.Repo_remote && local_wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all;
boolean wiki_has_cs_file = repo_tid == Xof_repo_tid_.Tid__remote && local_wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all;
String lnki_ttl_fld = wiki_has_cs_file ? "Coalesce(o.lnki_commons_ttl, o.lnki_ttl)" : "o.lnki_ttl"; // NOTE: use lnki_commons_ttl if [[File]] is cs PAGE:en.d:water EX:[[image:wikiquote-logo.png|50px|none|alt=]]; DATE:2014-09-05
if (wiki_has_cs_file)
Sqlite_engine_.Idx_create(usr_dlg, conn, "orig_regy", Idx_ttl_remote);
new Db_attach_mgr(conn, new Db_attach_itm("image_db", join))
.Exec_sql_w_msg("orig_regy:updating page" , Sql_update_repo_page, repo_tid, lnki_ttl_fld)
.Exec_sql_w_msg("orig_regy:updating redirect" , Sql_update_repo_redirect, repo_tid, lnki_ttl_fld);
.Exec_sql_w_msg("orig_regy:updating page" , Sql_update_repo_page , repo_tid, lnki_ttl_fld)
.Exec_sql_w_msg("orig_regy:updating redirect" , Sql_update_repo_redirect , repo_tid, lnki_ttl_fld);
}
private static void Create_data_for_cs(Gfo_usr_dlg usr_dlg, Db_conn p, Xowe_wiki local_wiki, Io_url repo_remote_dir) {
p.Exec_sql(Xob_orig_regy_tbl.Sql_cs_mark_dupes); // orig_regy: find dupes; see note in SQL
p.Exec_sql(Xob_orig_regy_tbl.Sql_cs_update_ttls); // orig_regy: update lnki_ttl with lnki_commons_ttl
Create_data_for_repo(usr_dlg, p, local_wiki, Xof_repo_itm_.Repo_remote, repo_remote_dir.GenSubFil(Xob_db_file.Name__wiki_image));
Create_data_for_repo(usr_dlg, p, local_wiki, Xof_repo_tid_.Tid__remote, repo_remote_dir.GenSubFil(Xob_db_file.Name__wiki_image));
p.Exec_sql(Xob_lnki_regy_tbl.Sql_cs_mark_changed); // lnki_regy: update lnki_commons_flag
p.Exec_sql(Xob_lnki_regy_tbl.Sql_cs_update_ttls); // lnki_regy: update cs
}

2
400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_page_regy_tbl.java
View File

@@ -26,7 +26,7 @@ public class Xob_page_regy_tbl {
Create_data__insert_redirect(usr_dlg, p, repo_tid, wiki.Fsys_mgr().Root_dir().GenSubFil(Xob_db_file.Name__wiki_redirect));
}
public static void Delete_local(Db_conn p) {
p.Exec_sql("DELETE FROM page_regy WHERE repo_id = " + Xof_repo_itm_.Repo_local);
p.Exec_sql("DELETE FROM page_regy WHERE repo_id = " + Xof_repo_tid_.Tid__local);
}
private static void Create_data__insert_page(Gfo_usr_dlg usr_dlg, Db_conn cur, byte repo_tid, Io_url join) {
usr_dlg.Note_many("", "", "inserting page: ~{0}", join.NameOnly());

3
400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_bin_db_mgr.java
View File

@@ -41,7 +41,8 @@ public class Xob_bin_db_mgr {
Xob_bin_db_itm itm = schema_is_1 ? Xob_bin_db_itm.new_v1(fil) : Xob_bin_db_itm.new_v2(fil);
int ns_id = itm.Ns_id();
Xob_bin_db_itm nth = (Xob_bin_db_itm)nth_hash.Get_by(tier_key.Val_(ns_id));
if (itm.Pt_id() > nth.Pt_id()) // update max pt_id
if ( nth != null // occurs when existing fsdb_dbb has "file-ns.014-db.001", but 14 no longer specified in fsdb_make; DATE:2016-09-23
&& itm.Pt_id() > nth.Pt_id()) // update max pt_id
nth.Set(itm.Id(), itm.Pt_id(), itm.Db_url()); // note that ns_id is same
}
len = nth_hash.Count();

2
400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_xfer_temp_itm_tst.java
View File

@@ -134,7 +134,7 @@ class Xob_xfer_temp_itm_fxt {
public Xob_xfer_temp_itm_fxt Init_rdr_image() {
GfoFldList flds = GfoFldList_.str_(Flds);
nde = GfoNde_.vals_(flds, Object_.Ary
( Xof_ext_.Id_png, 1, 1, Xof_repo_itm_.Repo_remote
( Xof_ext_.Id_png, 1, 1, Xof_repo_tid_.Tid__remote
, "A.png", Xof_ext_.Id_png, "A.png", Xop_lnki_type.Id_thumb, Xop_file_logger_.Tid__file
, 220, 200, 1, 2, 440, 400, 3
, Xop_lnki_tkn.Upright_null, Xof_lnki_time.Null, Xof_lnki_page.Null

18
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_mgr_db.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*; import gplx.xowa.addons.bldrs.mass_parses.parses.locks.*;
public class Xomp_mgr_db {
public Xomp_mgr_db(Io_url url) {
@@ -24,18 +24,20 @@ public class Xomp_mgr_db {
this.conn = Db_conn_bldr.Instance.Get_or_autocreate(true, url);
this.page_tbl = new Xomp_page_tbl(conn);
this.wkr_tbl = new Xomp_wkr_tbl(conn);
this.cfg_tbl = new Db_cfg_tbl(conn, "xowa_cfg");
// this.lock_mgr = new Xomp_lock_mgr__db(conn, 5000);
this.lock_mgr = new Xomp_lock_mgr__fsys(5000, this.Dir());
}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public Io_url Url() {return url;} private final Io_url url;
public Io_url Dir() {return url.OwnerDir();}
public Xomp_page_tbl Page_tbl() {return page_tbl;} private final Xomp_page_tbl page_tbl;
public Xomp_wkr_tbl Wkr_tbl() {return wkr_tbl;} private final Xomp_wkr_tbl wkr_tbl;
public Xomp_lock_mgr Lock_mgr() {return lock_mgr;} private final Xomp_lock_mgr lock_mgr;
public Db_conn Conn() {return conn;} private final Db_conn conn;
public Io_url Url() {return url;} private final Io_url url;
public Io_url Dir() {return url.OwnerDir();}
public Xomp_page_tbl Tbl__page() {return page_tbl;} private final Xomp_page_tbl page_tbl;
public Xomp_wkr_tbl Tbl__wkr() {return wkr_tbl;} private final Xomp_wkr_tbl wkr_tbl;
public Db_cfg_tbl Tbl__cfg() {return cfg_tbl;} private final Db_cfg_tbl cfg_tbl;
public Xomp_lock_mgr Lock_mgr() {return lock_mgr;} private final Xomp_lock_mgr lock_mgr;
public void Remake() {
conn.Meta_tbl_remake_many(page_tbl, wkr_tbl);
conn.Meta_tbl_remake_many(page_tbl, wkr_tbl, cfg_tbl);
lock_mgr.Remake();
}

7
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_mgr.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.inits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
class Xomp_init_mgr {
@@ -27,9 +27,12 @@ class Xomp_init_mgr {
Xomp_mgr_db mgr_db = Xomp_mgr_db.New__make(wiki);
Db_conn mgr_conn = mgr_db.Conn();
// remake all
// remake all tbls
mgr_db.Remake();
// insert ns into cfg; need for lnki_temp.tier in xomp.make
mgr_db.Tbl__cfg().Insert_str("", gplx.xowa.addons.bldrs.mass_parses.parses.wkrs.Xomp_parse_wkr.Cfg__ns_ids, Int_.Ary_concat("|", cfg.Ns_ids()));
// fill page tbl
Db_attach_mgr attach_mgr = new Db_attach_mgr(mgr_conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
int[] ns_ary = cfg.Ns_ids();

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_rdr.java
View File

@@ -23,7 +23,7 @@ class Xomp_html_db_rdr {
private final Xomp_mgr_db mgr_db;
public Xomp_html_db_rdr(Xowe_wiki wiki) {
this.mgr_db = Xomp_mgr_db.New__load(wiki);
this.src_tbls = new Xowd_html_tbl[mgr_db.Wkr_tbl().Select_count()];
this.src_tbls = new Xowd_html_tbl[mgr_db.Tbl__wkr().Select_count()];
}
public void Rows__get(Xowd_html_row rv, int wkr_uid, int page_id) {
Xowd_html_tbl src_tbl = src_tbls[wkr_uid];

28
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_wtr.java
View File

@@ -17,11 +17,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.wikis.data.*;
import gplx.xowa.bldrs.cmds.*;
class Xomp_html_db_wtr {
private final long len_max;
private final Xowe_wiki wiki; private final Xow_db_mgr db_mgr;
private long len_cur;
private int prv_ns_id = -1;
private Xow_db_file html_db; private Xowd_html_tbl html_tbl;
private Xob_ns_file_itm ns_itm;
public Xomp_html_db_wtr(Xowe_wiki wiki) {
this.wiki = wiki; this.db_mgr = wiki.Data__core_mgr();
this.len_max = wiki.Appe().Api_root().Bldr().Wiki().Import().Html_db_max();
@@ -34,18 +37,19 @@ class Xomp_html_db_wtr {
public Xowd_html_tbl Tbls__get_or_new(int ns_id, long html_len) {
long len_new = len_cur + html_len;
boolean not_inited = html_tbl == null, out_of_space = len_new > len_max;
if (not_inited || out_of_space) {
if (out_of_space)
Commit();
if ( db_mgr.Props().Layout_html().Tid_is_all_or_few() // is not "lot"
&& not_inited // not_inited; set html_db
boolean is_all_or_few = db_mgr.Props().Layout_html().Tid_is_all_or_few();
boolean ns_changed = ns_id != prv_ns_id;
if (not_inited || out_of_space || ns_changed) {
Commit();
if ( is_all_or_few // is not "lot"
&& not_inited // not_inited; set html_db
) {
this.html_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__html_data);
if (html_db == null)
this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
Make_html_db(is_all_or_few, ns_id);
}
else
this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data);
Make_html_db(is_all_or_few, ns_id);
this.html_tbl = new Xowd_html_tbl(html_db.Conn());
html_tbl.Create_tbl();
@@ -56,10 +60,20 @@ class Xomp_html_db_wtr {
len_cur = len_new;
return html_tbl;
}
private void Make_html_db(boolean is_all_or_few, int ns_id) {
if (prv_ns_id != ns_id
|| ns_itm == null) {
prv_ns_id = ns_id;
ns_itm = new Xob_ns_file_itm(Xow_db_file_.Tid__html_data, "ns." + Int_.To_str_pad_bgn_zero(ns_id, 3), Int_.Ary(ns_id));
}
String file_name = is_all_or_few ? "-html.xowa" : ns_itm.Make_file_name();
this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data, Int_.To_str(ns_id), ns_itm.Nth_db_idx(), file_name);
}
public void Rls() {
this.Commit();
}
private void Commit() {
if (html_tbl == null) return;
html_tbl.Conn().Txn_end();
html_tbl.Conn().Rls_conn();

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_html.java
View File

@@ -26,10 +26,10 @@ class Xomp_make_html {
Db_conn mgr_conn = mgr_db.Conn();
// update wkr_uid; note that this cannot be done in parse_wkr, b/c multiple-writer-errors for xomp.db|page
int wkrs_len = mgr_db.Wkr_tbl().Select_count();
int wkrs_len = mgr_db.Tbl__wkr().Select_count();
for (int i = 0; i < wkrs_len; ++i) {
Xomp_wkr_db wkr_db = Xomp_wkr_db.New(mgr_db.Dir(), i);
mgr_db.Page_tbl().Update_wkr_uid(i, wkr_db.Conn());
mgr_db.Tbl__page().Update_wkr_uid(i, wkr_db.Conn());
}
// init more

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java
View File

@@ -30,7 +30,7 @@ class Xomp_make_lnki {
lnki_temp_tbl.Insert_bgn();
// create ary; add index
int wkr_count = xomp_db.Wkr_tbl().Select_count();
int wkr_count = xomp_db.Tbl__wkr().Select_count();
Xomp_wkr_db[] db_ary = new Xomp_wkr_db[wkr_count];
for (int i = 0; i < wkr_count; ++i) {
Xomp_wkr_db wkr_db = Xomp_wkr_db.New(xomp_db.Dir(), i);

7
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
View File

@@ -51,10 +51,13 @@ public class Xomp_parse_mgr {
// assert wkr_tbl
int wkr_len = cfg.Num_wkrs();
int wkr_uid_bgn = mgr_db.Wkr_tbl().Init_wkrs(cfg.Wkr_machine_name(), wkr_len);
int wkr_uid_bgn = mgr_db.Tbl__wkr().Init_wkrs(cfg.Wkr_machine_name(), wkr_len);
latch = new Gfo_countdown_latch(wkr_len);
Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len];
// init ns_ord_mgr
Xomp_ns_ord_mgr ns_ord_mgr = new Xomp_ns_ord_mgr(Int_.Ary_parse(mgr_db.Tbl__cfg().Select_str("", Xomp_parse_wkr.Cfg__ns_ids), "|"));
// init parse_wkrs
for (int i = 0; i < wkr_len; ++i) {
// make wiki
@@ -62,7 +65,7 @@ public class Xomp_parse_mgr {
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
// make wkr
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, cfg, mgr_db, page_pool, prog_mgr, file_orig_wkr, wkr_wiki, i + wkr_uid_bgn);
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, cfg, mgr_db, page_pool, prog_mgr, file_orig_wkr, ns_ord_mgr, wkr_wiki, i + wkr_uid_bgn);
wkrs[i] = wkr;
}

29
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_ns_ord_mgr.java Normal file
View File

@@ -0,0 +1,29 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.core.primitives.*; import gplx.core.lists.hashs.*;
public class Xomp_ns_ord_mgr {
private final Hash_adp__int hash = new Hash_adp__int();
public Xomp_ns_ord_mgr(int[] ns_id_ary) {
int len = ns_id_ary.length;
for (int i = 0; i < len; ++i) {
hash.Add(ns_id_ary[i], new Int_obj_val(i));
}
}
public int Get_ord_by_ns_id(int ns_id) {return ((Int_obj_val)hash.Get_by_or_fail(ns_id)).Val();}
}

24
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
View File

@@ -28,6 +28,7 @@ public class Xomp_parse_wkr implements Gfo_invk {
private final Xomp_prog_mgr prog_mgr;
private final Xomp_page_pool page_pool;
private final Xof_orig_wkr file_orig_wkr;
private final Xomp_ns_ord_mgr ns_ord_mgr;
// cfg vars
private final Xomp_parse_mgr_cfg cfg;
@@ -42,10 +43,11 @@ public class Xomp_parse_wkr implements Gfo_invk {
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private int done_count; private long done_time;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg, Xomp_mgr_db mgr_db, Xomp_page_pool page_pool, Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xowe_wiki wiki, int uid) {
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg, Xomp_mgr_db mgr_db, Xomp_page_pool page_pool, Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xomp_ns_ord_mgr ns_ord_mgr, Xowe_wiki wiki, int uid) {
// mgr vars
this.mgr = mgr; this.mgr_db = mgr_db;
this.page_pool = page_pool; this.prog_mgr = prog_mgr; this.file_orig_wkr = file_orig_wkr;
this.ns_ord_mgr = ns_ord_mgr;
// cfg vars
this.cfg = cfg;
@@ -67,7 +69,7 @@ public class Xomp_parse_wkr implements Gfo_invk {
wiki.File__orig_mgr().Wkrs__set(file_orig_wkr);
wiki.File_mgr().Fsdb_mode().Tid__v2__mp__y_();
// disable categories b/c categories will be retrieved at run-time
// enable disable categories according to flag
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(cfg.Hdump_catboxs());
// enable lnki_temp
@@ -84,14 +86,15 @@ public class Xomp_parse_wkr implements Gfo_invk {
wkr_db.Conn().Txn_bgn("xomp");
// set status to running
mgr_db.Wkr_tbl().Update_status(uid, Xomp_wkr_tbl.Status__running);
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running);
// main loop
int prv_ns = -1;
while (true) {
// get page from page pool
Xomp_page_itm ppg = Get_next();
if (ppg == Xomp_page_itm.Null) {
mgr_db.Wkr_tbl().Update_status(uid, Xomp_wkr_tbl.Status__sleeping);
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__sleeping);
break; // no more pages
}
if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
@@ -99,8 +102,16 @@ public class Xomp_parse_wkr implements Gfo_invk {
try {
// init page
long done_bgn = gplx.core.envs.System_.Ticks();
Xoa_ttl ttl = wiki.Ttl_parse(ppg.Ns_id(), ppg.Ttl_bry());
int cur_ns = ppg.Ns_id();
Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
// if ns changed and prv_ns is main
if (cur_ns != prv_ns) {
if (prv_ns == gplx.xowa.wikis.nss.Xow_ns_.Tid__main)
wiki.Cache_mgr().Free_mem_all(Bool_.Y); // NOTE: this clears all caches, include imglinks
prv_ns = cur_ns;
}
Xoae_page wpg = Xoae_page.New(wiki, ttl);
wpg.Bldr__ns_ord_(ns_ord_mgr.Get_ord_by_ns_id(cur_ns)); // NOTE: must set ns_id for tier_id in lnki_temp; DATE:2016-09-19
wpg.Db().Text().Text_bry_(ppg.Text());
wpg.Db().Page().Id_(ppg.Id());
@@ -153,7 +164,7 @@ public class Xomp_parse_wkr implements Gfo_invk {
}
private Xomp_page_itm Get_next() {
if (list_idx == list_len) {
mgr_db.Wkr_tbl().Update_exec(uid, done_count, done_time);
mgr_db.Tbl__wkr().Update_exec(uid, done_count, done_time);
list.Clear();
page_pool.Get_next(mgr_db, cfg.Wkr_machine_name(), list);
list_len = list.Len();
@@ -168,4 +179,5 @@ public class Xomp_parse_wkr implements Gfo_invk {
return this;
}
public static final String Invk__exec = "exec";
public static final String Cfg__ns_ids = "xomp.ns_ids";
}

139
400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__exec.java
View File

@@ -1,139 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*;
import gplx.fsdb.*; import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.files.cmds.*;
public class Xobldr__deletion_db__exec extends Xob_cmd__base {
private Io_url deletion_db_url;
public Xobldr__deletion_db__exec(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, null);}
@Override public void Cmd_run() {
if (deletion_db_url == null) throw Err_.new_("bldr", "no file specified");
Db_conn conn = Db_conn_bldr.Instance.Get_or_noop(deletion_db_url);
if (conn == Db_conn_.Noop) throw Err_.new_("bldr", "file does not exist", "file", deletion_db_url.Raw());
Db_cfg_tbl cfg_tbl = gplx.xowa.wikis.data.Xowd_cfg_tbl_.Get_or_null(conn);
if (cfg_tbl == null) throw Err_.new_("bldr", "xowa_cfg tbl does not exist", "file", deletion_db_url.Raw());
byte[] domain_bry = cfg_tbl.Select_bry_or("", Xobldr__deletion_db__make.Cfg__deletion_db__domain, null);
if (domain_bry == null) throw Err_.new_("bldr", "no domain found in deletion db", "file", deletion_db_url.Raw());
this.wiki = bldr.App().Wiki_mgr().Get_by_or_make(domain_bry);
wiki.Init_assert();
Delete_by_url(wiki, deletion_db_url);
conn.Rls_conn();
Io_mgr.Instance.DeleteFil(deletion_db_url);
}
private void Delete_by_url(Xowe_wiki wiki, Io_url delete_url) {
Xof_fsdb_mgr fsdb_mgr = wiki.File_mgr().Fsdb_mgr();
Fsm_mnt_itm mnt_itm = fsdb_mgr.Mnt_mgr().Mnts__get_main();
Db_conn core_conn = mnt_itm.Atr_mgr().Db__core().Conn();
Db_conn delete_conn = Db_conn_bldr.Instance.Get_or_new(delete_url).Conn();
core_conn.Env_db_attach("delete_db", delete_conn);
int dbs_len = mnt_itm.Bin_mgr().Dbs__len();
String dbs_len_str = Int_.To_str(dbs_len - Int_.Base1);
for (int i = 0; i < dbs_len; ++i) {
Fsm_bin_fil bin_db = mnt_itm.Bin_mgr().Dbs__get_at(i);
Delete_by_db(core_conn, bin_db, dbs_len_str);
}
core_conn.Env_db_detach("delete_db");
}
private void Delete_by_db(Db_conn conn, Fsm_bin_fil bin_db, String dbs_len_str) {
Gfo_usr_dlg usr_dlg = Xoa_app_.Usr_dlg();
// get rows to delete in db
List_adp list = List_adp_.New();
int bin_db_id = bin_db.Id();
String bin_db_id_str = Int_.To_str(bin_db_id);
usr_dlg.Prog_many("", "", "processing files for deletion in database " + bin_db_id_str + " of " + dbs_len_str);
Db_rdr rdr = conn.Exec_rdr(String_.Concat_lines_nl
( "SELECT ff.fil_id AS item_id"
, ", 1 AS item_is_orig"
, "FROM fsdb_fil ff"
, "JOIN delete_db.delete_regy dr ON ff.fil_id = dr.fil_id AND dr.thm_id = -1"
, "WHERE ff.fil_bin_db_id = " + bin_db_id_str
, "UNION"
, "SELECT ft.thm_id AS item_id"
, ", 0 AS item_is_orig"
, "FROM fsdb_thm ft"
, " JOIN delete_db.delete_regy dr ON ft.thm_owner_id = dr.fil_id AND ft.thm_id = dr.thm_id"
, "WHERE ft.thm_bin_db_id = " + bin_db_id_str
));
try {
while (rdr.Move_next()) {
int item_id = rdr.Read_int("item_id");
int item_is_orig = rdr.Read_int("item_is_orig");
list.Add(new Xob_img_prune_itm(item_id, item_is_orig == 1));
}
} finally {rdr.Rls();}
int len = list.Count();
if (len == 0) return; // no files; exit, else will vacuum below
conn.Env_db_attach("bin_db", bin_db.Conn());
conn.Txn_bgn("img_prune__" + bin_db_id_str);
// delete bin
Db_stmt delete_bin_stmt = conn.Stmt_sql("DELETE FROM bin_db.fsdb_bin WHERE bin_owner_id = ?");
for (int i = 0; i < len; ++i) {
Xob_img_prune_itm itm = (Xob_img_prune_itm)list.Get_at(i);
delete_bin_stmt.Clear().Crt_int("bin_owner_id", itm.Item_id);
delete_bin_stmt.Exec_delete();
if (i % 10000 == 0) usr_dlg.Prog_many("", "", "deleting data in database " + bin_db_id_str + " of " + dbs_len_str);
}
delete_bin_stmt.Rls();
// delete meta
Db_stmt delete_fil_stmt = conn.Stmt_sql("DELETE FROM fsdb_fil WHERE fil_id = ?");
Db_stmt delete_thm_stmt = conn.Stmt_sql("DELETE FROM fsdb_thm WHERE thm_id = ?");
for (int i = 0; i < len; ++i) {
Xob_img_prune_itm itm = (Xob_img_prune_itm)list.Get_at(i);
if (itm.Item_is_orig) {
delete_fil_stmt.Clear().Crt_int("fil_id", itm.Item_id);
delete_fil_stmt.Exec_delete();
}
else {
delete_thm_stmt.Clear().Crt_int("thm_id", itm.Item_id);
delete_thm_stmt.Exec_delete();
}
if (i % 10000 == 0) usr_dlg.Prog_many("", "", "deleting meta in database " + bin_db_id_str + " of " + dbs_len_str);
}
delete_fil_stmt.Rls();
delete_thm_stmt.Rls();
conn.Txn_end();
conn.Env_db_detach("bin_db");
bin_db.Conn().Env_vacuum();
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__file_)) this.deletion_db_url = Io_url_.new_any_(m.ReadStr("v"));
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk__file_ = "file_";
public static final String BLDR_CMD_KEY = "fsdb.deletion_db.exec";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
public static final Xob_cmd Prototype = new Xobldr__deletion_db__exec(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xobldr__deletion_db__exec(bldr, wiki);}
}
class Xob_img_prune_itm {
public Xob_img_prune_itm(int item_id, boolean item_is_orig) {
this.Item_id = item_id;
this.Item_is_orig = item_is_orig;
}
public final int Item_id;
public final boolean Item_is_orig;
}

8
400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xoax_updates_files_addon.java → 400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_addon.java
View File

@@ -18,12 +18,12 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.utils_rankings.bldrs.*;
public class Xoax_updates_files_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
public class Xodel_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
public Xob_cmd[] Bldr_cmds() {
return new Xob_cmd[]
{ Xobldr__deletion_db__make.Prototype
, Xobldr__deletion_db__exec.Prototype
, Xobldr__deletion_db__small_files.Prototype
{ Xodel_make_cmd.Prototype
, Xodel_exec_cmd.Prototype
, Xodel_small_cmd.Prototype
};
}

36
400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_exec_cmd.java Normal file
View File

@@ -0,0 +1,36 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xodel_exec_cmd extends Xob_cmd__base {
private Io_url deletion_db_url;
public Xodel_exec_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
new Xodel_exec_mgr().Exec_delete(gplx.core.progs.Gfo_prog_ui_.Noop, bldr, deletion_db_url);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__file_)) this.deletion_db_url = Io_url_.new_any_(m.ReadStr("v"));
else return Gfo_invk_.Rv_unhandled;
return this;
} private static final String Invk__file_ = "file_";
public static final String BLDR_CMD_KEY = "fsdb.deletion_db.exec";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
public static final Xob_cmd Prototype = new Xodel_exec_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xodel_exec_cmd(bldr, wiki);}
}

134
400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_exec_mgr.java Normal file
View File

@@ -0,0 +1,134 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*;
import gplx.core.progs.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.wikis.data.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.files.fsdb.*; import gplx.fsdb.meta.*;
public class Xodel_exec_mgr {
public void Exec_delete(Gfo_prog_ui prog_ui, Xob_bldr bldr, Io_url deletion_db_url) {
// get domain bry from deletion_conn's cfg_tbl
Db_conn deletion_conn = Db_conn_bldr.Instance.Get_or_fail(deletion_db_url);
Db_cfg_tbl cfg_tbl = Xowd_cfg_tbl_.Get_or_fail(deletion_conn);
byte[] domain_bry = cfg_tbl.Select_bry("", Xodel_exec_mgr.Cfg__deletion_db__domain);
// get wiki; init it; do delete
Xowe_wiki wiki = bldr.App().Wiki_mgr().Get_by_or_make(domain_bry);
wiki.Init_assert();
Delete_by_url(prog_ui, wiki, deletion_conn, cfg_tbl);
// cleanup
deletion_conn.Rls_conn();
Io_mgr.Instance.DeleteFil(deletion_db_url);
}
public void Exec_cleanup(Io_url deletion_db_url) {
Db_conn deletion_conn = Db_conn_bldr.Instance.Get_or_fail(deletion_db_url);
Db_cfg_tbl cfg_tbl = Xowd_cfg_tbl_.Get_or_fail(deletion_conn);
cfg_tbl.Delete_val("", Xodel_exec_mgr.Cfg__deletion_db__db_bgn);
}
private void Delete_by_url(Gfo_prog_ui prog_ui, Xowe_wiki wiki, Db_conn deletion_conn, Db_cfg_tbl cfg_tbl) {
// get fsdb_mgr
Xof_fsdb_mgr fsdb_mgr = wiki.File_mgr().Fsdb_mgr();
Fsm_mnt_itm mnt_itm = fsdb_mgr.Mnt_mgr().Mnts__get_main();
Db_conn core_conn = mnt_itm.Atr_mgr().Db__core().Conn();
try {
core_conn.Env_db_attach("delete_db", deletion_conn);
// loop dbs
int dbs_len = mnt_itm.Bin_mgr().Dbs__len();
String dbs_len_str = Int_.To_str(dbs_len - Int_.Base1);
int db_bgn = cfg_tbl.Select_int_or("", Cfg__deletion_db__db_bgn, 0);
for (int i = db_bgn; i < dbs_len; ++i) {
if (prog_ui.Prog_notify_and_chk_if_suspended(i, dbs_len)) return;
Fsm_bin_fil bin_db = mnt_itm.Bin_mgr().Dbs__get_at(i);
Delete_by_db(core_conn, bin_db, dbs_len_str);
cfg_tbl.Upsert_int("", Cfg__deletion_db__db_bgn, i + 1);
}
} finally {core_conn.Env_db_detach("delete_db");}
}
private void Delete_by_db(Db_conn deletion_conn, Fsm_bin_fil bin_db, String dbs_len_str) {
Gfo_usr_dlg usr_dlg = Xoa_app_.Usr_dlg();
// get rows to delete in db
List_adp list = List_adp_.New();
int bin_db_id = bin_db.Id();
String bin_db_id_str = Int_.To_str(bin_db_id);
usr_dlg.Prog_many("", "", "processing files for deletion in database " + bin_db_id_str + " of " + dbs_len_str);
Db_rdr rdr = deletion_conn.Exec_rdr(String_.Concat_lines_nl
( "SELECT ff.fil_id AS item_id"
, ", 1 AS item_is_orig"
, "FROM fsdb_fil ff"
, "JOIN delete_db.delete_regy dr ON ff.fil_id = dr.fil_id AND dr.thm_id = -1"
, "WHERE ff.fil_bin_db_id = " + bin_db_id_str
, "UNION"
, "SELECT ft.thm_id AS item_id"
, ", 0 AS item_is_orig"
, "FROM fsdb_thm ft"
, " JOIN delete_db.delete_regy dr ON ft.thm_owner_id = dr.fil_id AND ft.thm_id = dr.thm_id"
, "WHERE ft.thm_bin_db_id = " + bin_db_id_str
));
try {
while (rdr.Move_next()) {
int item_id = rdr.Read_int("item_id");
int item_is_orig = rdr.Read_int("item_is_orig");
list.Add(new Xodel_prune_itm(item_id, item_is_orig == 1));
}
} finally {rdr.Rls();}
int len = list.Count();
if (len == 0) return; // no files; exit, else will vacuum below
deletion_conn.Env_db_attach("bin_db", bin_db.Conn());
deletion_conn.Txn_bgn("img_prune__" + bin_db_id_str);
// delete bin
Db_stmt delete_bin_stmt = deletion_conn.Stmt_sql("DELETE FROM bin_db.fsdb_bin WHERE bin_owner_id = ?");
for (int i = 0; i < len; ++i) {
Xodel_prune_itm itm = (Xodel_prune_itm)list.Get_at(i);
delete_bin_stmt.Clear().Crt_int("bin_owner_id", itm.Item_id);
delete_bin_stmt.Exec_delete();
if (i % 10000 == 0) usr_dlg.Prog_many("", "", "deleting data in database " + bin_db_id_str + " of " + dbs_len_str);
}
delete_bin_stmt.Rls();
// delete meta
Db_stmt delete_fil_stmt = deletion_conn.Stmt_sql("DELETE FROM fsdb_fil WHERE fil_id = ?");
Db_stmt delete_thm_stmt = deletion_conn.Stmt_sql("DELETE FROM fsdb_thm WHERE thm_id = ?");
for (int i = 0; i < len; ++i) {
Xodel_prune_itm itm = (Xodel_prune_itm)list.Get_at(i);
if (itm.Item_is_orig) {
delete_fil_stmt.Clear().Crt_int("fil_id", itm.Item_id);
delete_fil_stmt.Exec_delete();
}
else {
delete_thm_stmt.Clear().Crt_int("thm_id", itm.Item_id);
delete_thm_stmt.Exec_delete();
}
if (i % 10000 == 0) usr_dlg.Prog_many("", "", "deleting meta in database " + bin_db_id_str + " of " + dbs_len_str);
}
delete_fil_stmt.Rls();
delete_thm_stmt.Rls();
// cleanup
deletion_conn.Txn_end();
deletion_conn.Env_db_detach("bin_db");
bin_db.Conn().Env_vacuum();
}
public static final String Cfg__deletion_db__domain = "file.deletion_db.domain", Cfg__deletion_db__db_bgn = "file.deletion_db.db_bgn";
}

31
400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_make_cmd.java Normal file
View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xodel_make_cmd extends Xob_cmd__base implements Xob_cmd {
public Xodel_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
new Xodel_make_mgr().Exec(wiki);
}
public static final String BLDR_CMD_KEY = "file.deletion_db.make";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
public static final Xob_cmd Prototype = new Xodel_make_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xodel_make_cmd(bldr, wiki);}
}

28
400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__make.java → 400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_make_mgr.java
View File

@@ -16,13 +16,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*;
import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.addons.bldrs.files.dbs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.fsdb.meta.*;
public class Xobldr__deletion_db__make extends Xob_cmd__base implements Xob_cmd {
public Xobldr__deletion_db__make(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
import gplx.dbs.*;
import gplx.xowa.bldrs.*;
class Xodel_make_mgr {
public void Exec(Xow_wiki wiki) {
// mark the records deleted
Db_conn make_conn = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn();
make_conn.Exec_sql_concat_w_msg
@@ -39,16 +36,15 @@ public class Xobldr__deletion_db__make extends Xob_cmd__base implements Xob_cmd
, "AND xr.file_w = fsdb_regy.fsdb_w"
, "AND xr.lnki_time = fsdb_regy.fsdb_time"
, "AND xr.lnki_page = fsdb_regy.fsdb_page"
, ")"
);
, ")");
// create deletion db
Xob_db_file deletion_db = Xob_db_file.New__deletion_db(wiki);
Db_conn deletion_conn = deletion_db.Conn();
deletion_db.Tbl__cfg().Upsert_str("", Xobldr__deletion_db__make.Cfg__deletion_db__domain, wiki.Domain_str());
deletion_db.Tbl__cfg().Upsert_str("", Xodel_exec_mgr.Cfg__deletion_db__domain, wiki.Domain_str());
// copy records over to it
Xob_delete_regy delete_regy_tbl = new Xob_delete_regy(deletion_conn);
Xob_delete_regy_tbl delete_regy_tbl = new Xob_delete_regy_tbl(deletion_conn);
deletion_conn.Meta_tbl_remake(delete_regy_tbl.Meta());
deletion_conn.Env_db_attach("make_db", make_conn);
deletion_conn.Exec_sql_concat_w_msg
@@ -64,19 +60,13 @@ public class Xobldr__deletion_db__make extends Xob_cmd__base implements Xob_cmd
deletion_conn.Env_db_detach("make_db");
deletion_conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(delete_regy_tbl.tbl_name, "main", delete_regy_tbl.fld_fil_id, delete_regy_tbl.fld_thm_id));
}
public static final String Cfg__deletion_db__domain = "file.deletion_db.domain";
public static final String BLDR_CMD_KEY = "file.deletion_db.make";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
public static final Xob_cmd Prototype = new Xobldr__deletion_db__make(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xobldr__deletion_db__make(bldr, wiki);}
}
class Xob_delete_regy {
class Xob_delete_regy_tbl {
public final String tbl_name = "delete_regy";
public final Dbmeta_fld_list flds = new Dbmeta_fld_list();
public final String fld_fil_id, fld_thm_id, fld_reason;
public final Db_conn conn;
public Xob_delete_regy(Db_conn conn) {
public Xob_delete_regy_tbl(Db_conn conn) {
this.conn = conn;
this.fld_fil_id = flds.Add_int("fil_id");
this.fld_thm_id = flds.Add_int("thm_id");

26
400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_prune_itm.java Normal file
View File

@@ -0,0 +1,26 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*;
class Xodel_prune_itm {
public Xodel_prune_itm(int item_id, boolean item_is_orig) {
this.Item_id = item_id;
this.Item_is_orig = item_is_orig;
}
public final int Item_id;
public final boolean Item_is_orig;
}

33
400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_small_cmd.java Normal file
View File

@@ -0,0 +1,33 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xodel_small_cmd extends Xob_cmd__base {
public Xodel_small_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
private final int[] ext_max_ary = Xobldr__fsdb_db__delete_small_files_.New_ext_max_ary();
@Override public void Cmd_run() {
wiki.Init_assert();
new Xodel_small_mgr().Exec(wiki, ext_max_ary);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return this;}
public static final String BLDR_CMD_KEY = "file.deletion_db.small_files";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
public static final Xob_cmd Prototype = new Xodel_small_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xodel_small_cmd(bldr, wiki);}
}

16
400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__small_files.java → 400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_small_mgr.java
View File

@@ -17,12 +17,10 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.bldrs.*;
import gplx.fsdb.*; import gplx.fsdb.meta.*; import gplx.xowa.files.*;
public class Xobldr__deletion_db__small_files extends Xob_cmd__base {
public Xobldr__deletion_db__small_files(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
private final int[] ext_max_ary = Xobldr__fsdb_db__delete_small_files_.New_ext_max_ary();
@Override public void Cmd_run() {
class Xodel_small_mgr {
public void Exec(Xowe_wiki wiki, int[] ext_max_ary) {
wiki.Init_assert();
// get atr_conn
Fsdb_db_mgr db_core_mgr = Fsdb_db_mgr_.new_detect(wiki, wiki.Fsys_mgr().Root_dir(), wiki.Fsys_mgr().File_dir());
@@ -54,13 +52,7 @@ public class Xobldr__deletion_db__small_files extends Xob_cmd__base {
, "AND t.thm_size <= " + Int_.To_str(max)
);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return this;}
public static final String BLDR_CMD_KEY = "file.deletion_db.small_files";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
public static final Xob_cmd Prototype = new Xobldr__deletion_db__small_files(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xobldr__deletion_db__small_files(bldr, wiki);}
}
}
class Xobldr__fsdb_db__delete_small_files_ {
public static int[] New_ext_max_ary() {
int[] rv = new int[Xof_ext_.Id__max];

14
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_cmd.java
View File

@@ -17,24 +17,24 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
public class Imglnk_bldr_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd {
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sql_dumps.*;
public class Imglnk_bldr_cmd extends Xob_sql_dump_base implements Xosql_dump_cbk {
private Imglnk_bldr_mgr mgr;
private int tmp_page_id;
private int rows = 0;
public Imglnk_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
@Override public String Sql_file_name() {return "imagelinks";}
@Override protected Xosql_dump_parser New_parser() {return new Xosql_dump_parser(this, "il_from", "il_to");}
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
parser.Save_csv_n_().Fld_cmd_(this).Flds_req_idx_(3, 0, 1);
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Xosql_dump_parser parser) {
mgr = new Imglnk_bldr_mgr(wiki);
}
public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
public void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end) {
switch (fld_idx) {
case Fld__il_from: this.tmp_page_id = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld__il_from: this.tmp_page_id = Bry_.To_int_or(src, val_bgn, val_end, -1); break;
case Fld__il_to:
mgr.Tmp_tbl().Insert_by_batch(tmp_page_id, Bry_.Mid(src, fld_bgn, fld_end));
mgr.Tmp_tbl().Insert_by_batch(tmp_page_id, Bry_.Mid(src, val_bgn, val_end));
if (++rows % 100000 == 0) usr_dlg.Prog_many("", "", "reading row ~{0}", Int_.To_str_fmt(rows, "#,##0"));
break;
}

4
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_mgr.java
View File

@@ -41,8 +41,8 @@ class Imglnk_bldr_mgr {
Imglnk_reg_tbl reg_tbl = new Imglnk_reg_tbl(conn);
conn.Meta_tbl_remake(reg_tbl);
reg_tbl.Create_idx__src_ttl();
reg_tbl.Insert(conn, Xof_repo_itm_.Repo_local , wiki);
reg_tbl.Insert(conn, Xof_repo_itm_.Repo_remote, wiki.Appe().Wiki_mgr().Wiki_commons());
reg_tbl.Insert(conn, Xof_repo_tid_.Tid__local , wiki);
reg_tbl.Insert(conn, Xof_repo_tid_.Tid__remote, wiki.Appe().Wiki_mgr().Wiki_commons());
reg_tbl.Create_idx__trg_ttl();
// Imglnk_bulk_cmd__img_id.Bulk_exec(conn, Bool_.Y, wiki);

6
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Xof_orig_wkr__img_links_.java
View File

@@ -23,8 +23,8 @@ public class Xof_orig_wkr__img_links_ {
public static void Load_all(Xof_orig_wkr__img_links wkr) {
Xowe_wiki wiki = wkr.Wiki();
Db_conn conn = Xob_db_file.New__img_link(wiki).Conn();
Load_all_by_wiki(wkr, conn, Xof_repo_itm_.Repo_local , wiki);
Load_all_by_wiki(wkr, conn, Xof_repo_itm_.Repo_remote, wiki.Appe().Wiki_mgr().Wiki_commons());
Load_all_by_wiki(wkr, conn, Xof_repo_tid_.Tid__local , wiki);
Load_all_by_wiki(wkr, conn, Xof_repo_tid_.Tid__remote, wiki.Appe().Wiki_mgr().Wiki_commons());
}
public static Xof_orig_itm Load_itm(Xof_orig_wkr__img_links wkr, Db_conn conn, Xowe_wiki wiki, byte[] ttl) {
Imglnk_reg_tbl imglnk_reg_tbl = wkr.Tbl__imglnk_reg();
@@ -39,7 +39,7 @@ public class Xof_orig_wkr__img_links_ {
else // ttl missing; EX:</*_File:Chehov_v_serpuhove11.JPG; DATE:2016-08-10
return Xof_orig_itm.Null;
} finally {rdr.Rls();}
Xowe_wiki image_wiki = img_repo == Xof_repo_itm_.Repo_local ? wiki : wiki.Appe().Wiki_mgr().Wiki_commons();
Xowe_wiki image_wiki = img_repo == Xof_repo_tid_.Tid__local ? wiki : wiki.Appe().Wiki_mgr().Wiki_commons();
return Load_itm_by_wiki(wkr, conn, image_wiki, img_repo, ttl, img_trg);
}
private static Xof_orig_itm Load_itm_by_wiki(Xof_orig_wkr__img_links wkr, Db_conn conn, Xowe_wiki wiki, byte repo_id, byte[] img_src, byte[] img_trg) {

16
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/pagelinks/bldrs/Pglnk_bldr_cmd.java
View File

@@ -16,18 +16,18 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.pagelinks.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*; import gplx.xowa.addons.bldrs.wmdumps.pagelinks.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sql_dumps.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.*; import gplx.xowa.addons.bldrs.wmdumps.pagelinks.dbs.*;
public class Pglnk_bldr_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd {
public class Pglnk_bldr_cmd extends Xob_sql_dump_base implements Xosql_dump_cbk {
private Db_conn conn;
private Pglnk_page_link_temp_tbl temp_tbl;
private int tmp_src_id, tmp_trg_ns;
private int rows = 0;
public Pglnk_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
@Override public String Sql_file_name() {return Dump_type_key;} public static final String Dump_type_key = "pagelinks";
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
@Override protected Xosql_dump_parser New_parser() {return new Xosql_dump_parser(this, "pl_from", "pl_namespace", "pl_title");}
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Xosql_dump_parser parser) {
wiki.Init_assert();
parser.Fld_cmd_(this).Flds_req_idx_(4, 0, 1, 2);
Xob_db_file page_link_db = Xob_db_file.New__page_link(wiki);
this.conn = page_link_db.Conn();
this.temp_tbl = new Pglnk_page_link_temp_tbl(conn);
@@ -49,12 +49,12 @@ public class Pglnk_bldr_cmd extends Xob_sql_dump_base implements Sql_file_parser
actl_tbl.Create_idx__trg_src();
conn.Env_vacuum();
}
public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
public void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end) {
switch (fld_idx) {
case Fld__pl_from: this.tmp_src_id = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld__pl_namespace: this.tmp_trg_ns = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld__pl_from: this.tmp_src_id = Bry_.To_int_or(src, val_bgn, val_end, -1); break;
case Fld__pl_namespace: this.tmp_trg_ns = Bry_.To_int_or(src, val_bgn, val_end, -1); break;
case Fld__pl_title:
byte[] tmp_trg_ttl = Bry_.Mid(src, fld_bgn, fld_end);
byte[] tmp_trg_ttl = Bry_.Mid(src, val_bgn, val_end);
temp_tbl.Insert(tmp_src_id, tmp_trg_ns, tmp_trg_ttl);
if (++rows % 100000 == 0) usr_dlg.Prog_many("", "", "reading row ~{0}", Int_.To_str_fmt(rows, "#,##0"));
break;