Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.9.4.1'

Browse Source
This commit is contained in:
gnosygnu
2016-09-25 22:05:47 -04:00
parent 35d78f6106
commit e3b393650d
211 changed files with 3148 additions and 2197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__download.java
View File

@@ -34,25 +34,35 @@ public class Xobc_cmd__download extends Xobc_cmd__base {
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
int error_wait = 10000, error_tries_max = 6, error_tries_cur = 0; // retry every 10 seconds for a total of 6 tries (1 min)
// loop while "server ... " error
while (true) {
long trg_size_bgn = Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size();
long trg_size_bgn = Tmp_url_size();
byte status = wkr.Exec(this, src_url, trg_url, expd_size);
if (status == Gfo_prog_ui_.Status__fail) {
// check if anything more downloaded; if so, then reset to 0; DATE:2016-09-03
long trg_size_cur = Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size();
if (trg_size_cur > trg_size_bgn) {
error_tries_cur = 0;
trg_size_bgn = trg_size_cur;
// check if server error; note: must not loop if bad size; DATE:2016-09-24
String fail_msg = wkr.Fail_msg();
if (String_.Has_at_bgn(fail_msg, Http_download_wkr__jre.Err__server_download_failed)) {
// check if anything more downloaded; if so, then reset to 0; DATE:2016-09-03
long trg_size_cur = Tmp_url_size();
if (trg_size_cur > trg_size_bgn) {
error_tries_cur = 0;
trg_size_bgn = trg_size_cur;
}
// retry
if (error_tries_cur++ < error_tries_max) {
task_mgr.Work_mgr().On_stat(this.Task_id(), String_.Format("connection interrupted: retrying in {0} seconds; attempt {1} of {2}", error_wait / 1000, error_tries_cur, error_tries_max));
Gfo_usr_dlg_.Instance.Log_many("", "", "xobc_cmd task download interrupted; ~{0} ~{1} ~{2} ~{3}", this.Task_id(), this.Step_id(), trg_url, Io_mgr.Instance.QueryFil(trg_url).Size());
gplx.core.threads.Thread_adp_.Sleep(error_wait);
continue;
}
}
// retry
if (error_tries_cur++ < error_tries_max) {
task_mgr.Work_mgr().On_stat(this.Task_id(), String_.Format("connection interrupted: retrying in {0} seconds; attempt {1} of {2}", error_wait / 1000, error_tries_cur, error_tries_max));
Gfo_usr_dlg_.Instance.Log_many("", "", "xobc_cmd task download interrupted; ~{0} ~{1} ~{2} ~{3}", this.Task_id(), this.Step_id(), trg_url, Io_mgr.Instance.QueryFil(trg_url).Size());
gplx.core.threads.Thread_adp_.Sleep(error_wait);
continue;
}
this.Cmd_exec_err_(wkr.Fail_msg());
// otherewise exit loop
this.Cmd_exec_err_(fail_msg);
break;
}
else
@@ -60,6 +70,9 @@ public class Xobc_cmd__download extends Xobc_cmd__base {
}
Gfo_log_.Instance.Info("xobc_cmd task download", "task_id", this.Task_id(), "step_id", this.Step_id(), "trg_url", trg_url, "trg_len", Io_mgr.Instance.QueryFil(trg_url).Size());
}
private long Tmp_url_size() {
return wkr.Tmp_url() == null ? 0 : Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size(); // NOTE: wkr.Tmp_url is null in some extreme exceptions; DATE:2016-09-24
}
@Override public void Cmd_cleanup() {
wkr.Exec_cleanup();
}

46
400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__fsdb_delete.java Normal file
View File

@@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
import gplx.dbs.*;
import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*;
import gplx.xowa.addons.bldrs.updates.files.*;
public class Xobc_cmd__fsdb_delete extends Xobc_cmd__base {
private final Io_url deletion_db_url;
public Xobc_cmd__fsdb_delete(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url deletion_db_url) {super(task_mgr, task_id, step_id, cmd_idx);
this.deletion_db_url = deletion_db_url;
}
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.fsdb.delete";
@Override public String Cmd_name() {return "deleting old files";}
@Override public boolean Cmd_suspendable() {return true;}
@Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) {
if (!Io_mgr.Instance.ExistsFil(deletion_db_url)) throw Err_.New("deletion db does not exist; file={0}", deletion_db_url.Raw());
boolean pass = false;
try {
new Xodel_exec_mgr().Exec_delete(this, ctx.App().Bldr(), deletion_db_url);
pass = true;
}
catch (Exception e) {
this.Cmd_exec_err_(Err_.Message_gplx_log(e));
}
Gfo_log_.Instance.Info("xobc_cmd task delete", "task_id", this.Task_id(), "step_id", this.Step_id(), "delete_url", deletion_db_url.Raw(), "pass", pass);
}
@Override public void Cmd_cleanup() {
if (Io_mgr.Instance.ExistsFil(deletion_db_url))
new Xodel_exec_mgr().Exec_cleanup(deletion_db_url);
}
}

4
400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__verify_dir.java
View File

@@ -20,9 +20,9 @@ import gplx.core.progs.*;
import gplx.core.security.*; import gplx.core.security.files.*;
public class Xobc_cmd__verify_dir extends Xobc_cmd__base {
private final Io_url delete_fil, checksum_fil;
public Xobc_cmd__verify_dir(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url dir_url, String checksum_fil_name, Io_url delete_fil) {super(task_mgr, task_id, step_id, cmd_idx);
public Xobc_cmd__verify_dir(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url checksum_fil, Io_url delete_fil) {super(task_mgr, task_id, step_id, cmd_idx);
this.checksum_fil = checksum_fil;
this.delete_fil = delete_fil;
this.checksum_fil = dir_url.GenSubFil(checksum_fil_name);
}
@Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.hash_dir";
@Override public String Cmd_name() {return "verify";}

18
400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java
View File

@@ -18,13 +18,15 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.centrals.dbs.datas.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*;
public class Xobc_import_type {
public static final int // SERIALIZED: bc_db; import_step
Tid__ignore = 0
, Tid__pack = 1
, Tid__wiki__core = 2
, Tid__wiki__srch = 3
, Tid__wiki__html = 4
, Tid__wiki__text = 5
, Tid__file__core = 6
, Tid__file__data = 7
Tid__ignore = 0
, Tid__pack = 1
, Tid__wiki__core = 2
, Tid__wiki__srch = 3
, Tid__wiki__html = 4
, Tid__wiki__text = 5
, Tid__file__core = 6
, Tid__file__data = 7
, Tid__fsdb__delete = 8
, Tid__wiki__ctg = 9
;
}

27
400_xowa/src/gplx/xowa/addons/bldrs/centrals/steps/Xobc_step_factory.java
View File

@@ -20,6 +20,9 @@ import gplx.core.brys.evals.*; import gplx.core.primitives.*;
import gplx.xowa.addons.bldrs.centrals.tasks.*; import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.utils.*;
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
import gplx.xowa.addons.bldrs.exports.merges.*;
import gplx.xowa.addons.bldrs.updates.files.*;
import gplx.xowa.addons.bldrs.exports.packs.files.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.wikis.domains.*;
public class Xobc_step_factory {
private final Xobc_task_mgr task_mgr;
@@ -60,16 +63,24 @@ public class Xobc_step_factory {
Io_url zip_file_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__download_fil, wiki_domain, file_name));
Io_url unzip_dir_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__unzip_dir, wiki_domain, file_name));
Io_url wiki_dir_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__wiki_dir, wiki_domain, file_name));
list.Add(new Xobc_cmd__download (task_mgr, task_id, step_id, 0, src_http_url, zip_file_url, import_itm.Import_size_zip));
list.Add(new Xobc_cmd__verify_fil (task_mgr, task_id, step_id, 1, zip_file_url, import_itm.Import_md5, import_itm.Import_size_zip));
list.Add(new Xobc_cmd__unzip (task_mgr, task_id, step_id, 2, zip_file_url, unzip_dir_url, import_itm.Import_size_raw));
list.Add(new Xobc_cmd__verify_dir (task_mgr, task_id, step_id, 3, unzip_dir_url, String_.Replace(file_name, ".zip", ".md5"), zip_file_url));
// list.Add(new Xobc_cmd__wiki_merge (task_mgr, task_id, step_id, 4, merge_mgr, wiki_domain, unzip_dir_url, import_itm.Import_prog_data_max, import_itm.Import_prog_row_max, step_seqn));
list.Add(new Xobc_cmd__move_fils (task_mgr, task_id, step_id, 4, unzip_dir_url, wiki_dir_url));
if (import_itm.Import_type == Xobc_import_type.Tid__wiki__core) {
list.Add(new Xobc_cmd__wiki_reg (task_mgr, task_id, step_id, 5, wiki_dir_url, wiki_domain));
Io_url checksum_url = unzip_dir_url.GenSubFil(file_name + ".md5");
int cmd_idx = 0;
list.Add(new Xobc_cmd__download (task_mgr, task_id, step_id, cmd_idx++, src_http_url, zip_file_url, import_itm.Import_size_zip));
list.Add(new Xobc_cmd__verify_fil (task_mgr, task_id, step_id, cmd_idx++, zip_file_url, import_itm.Import_md5, import_itm.Import_size_zip));
list.Add(new Xobc_cmd__unzip (task_mgr, task_id, step_id, cmd_idx++, zip_file_url, unzip_dir_url, import_itm.Import_size_raw));
list.Add(new Xobc_cmd__verify_dir (task_mgr, task_id, step_id, cmd_idx++, checksum_url, zip_file_url));
// list.Add(new Xobc_cmd__wiki_merge (task_mgr, task_id, step_id, cmd_idx++, merge_mgr, wiki_domain, unzip_dir_url, import_itm.Import_prog_data_max, import_itm.Import_prog_row_max, step_seqn));
list.Add(new Xobc_cmd__move_fils (task_mgr, task_id, step_id, cmd_idx++, unzip_dir_url, wiki_dir_url));
switch (import_itm.Import_type) {
case Xobc_import_type.Tid__wiki__core: list.Add(new Xobc_cmd__wiki_reg (task_mgr, task_id, step_id, cmd_idx++, wiki_dir_url, wiki_domain)); break;
case Xobc_import_type.Tid__fsdb__delete: list.Add(new Xobc_cmd__fsdb_delete (task_mgr, task_id, step_id, cmd_idx++, Pack_zip_name_bldr.To_wiki_url(wiki_dir_url, zip_file_url.OwnerDir()))); break;
}
return (Xobc_cmd_itm[])list.To_ary_and_clear(Xobc_cmd_itm.class);
}
private Io_url Eval_url(String src) {return Io_url_.new_any_(String_.new_u8(eval_mgr.Eval(Bry_.new_u8(src))));}
public static Xow_wiki Get_wiki_by_abrv(Xoa_app app, byte[] wiki_abrv) {
Xow_domain_itm domain_itm = Xow_abrv_xo_.To_itm(wiki_abrv);
return app.Wiki_mgri().Get_by_or_make_init_y(domain_itm.Domain_bry());
}
}

2
400_xowa/src/gplx/xowa/addons/bldrs/centrals/tasks/Xobc_task_regy__work.java
View File

@@ -79,8 +79,8 @@ public class Xobc_task_regy__work extends Xobc_task_regy__base {
thread_mgr.Halt(cmd_uid, Thread_halt_cbk_.Noop);
}
private void Redo_task(Xobc_task_itm task, Xobc_step_itm step, Xobc_cmd_itm cmd) {
cmd.Cmd_clear();
cmd = step.Step_fallback_to(cmd.Cmd_fallback());
cmd.Cmd_clear();
task.Task_status_(gplx.core.progs.Gfo_prog_ui_.Status__working);
task_mgr.Send_json("xo.bldr.work.prog__start__recv", task.Save_to(Gfobj_nde.New()));
thread_mgr.Add(cmd.Cmd_uid(), Thread_adp_.Start_by_val("xobc: " + cmd.Cmd_name(), cmd, cmd, Xobc_cmd__base.Invk__exec, ctx));

4
400_xowa/src/gplx/xowa/addons/bldrs/centrals/utils/Bry_eval_wkr__builder_central.java
View File

@@ -22,7 +22,7 @@ public class Bry_eval_wkr__builder_central implements Bry_eval_wkr {
public Bry_eval_wkr__builder_central(Io_url wiki_dir) {this.wiki_dir = wiki_dir.RawBry();}
public String Key() {return "builder_central";}
public void Resolve(Bry_bfr rv, byte[] src, int args_bgn, int args_end) {
// EX: "~{builder_central|download_fil|en.wikipedia.org|en.wikipedia.org-2016.05-html-ns.000-db.001.zip}" -> "/xowa/wiki/en.wikipedia.org/tmp/bldr/en.wikipedia.org-2016.05-html-ns.000-db.001.zip/download.zip"
// EX: "~{builder_central|download_fil|en.wikipedia.org|Xowa_enwiki_2016-05_html_ns.000_db.001.zip}" -> "/xowa/wiki/en.wikipedia.org/tmp/bldr/Xowa_enwiki_2016-05_html_ns.000_db.001.zip/download.zip"
byte[][] args = Bry_split_.Split(src, args_bgn, args_end, Byte_ascii.Pipe, Bool_.N);
int type = hash.Get_as_byte_or(args[0], Byte_.Max_value_127);
if (type == Byte_.Max_value_127) throw Err_.new_wo_type("unknown eval type", "src", src);
@@ -32,7 +32,7 @@ public class Bry_eval_wkr__builder_central implements Bry_eval_wkr {
if (type == Type__wiki_dir) return;
rv.Add_str_a7("tmp").Add_byte(dir_spr); // "tmp/"
rv.Add_str_a7("bldr").Add_byte(dir_spr); // "bldr/"
rv.Add(args[2]).Add_byte(dir_spr); // "en.wikipedia.org-2016.05-html-ns.000-db.001.zip/"
rv.Add(args[2]).Add_byte(dir_spr); // "Xowa_enwiki_2016-05_html_ns.000_db.001.zip/"
switch (type) {
case Type__download_fil: rv.Add_str_a7("download.zip"); break;
case Type__unzip_dir: rv.Add_str_a7("unzip").Add_byte(dir_spr); break;