Archives
/
gnosygnu_xowa
mirror of https://github.com/gnosygnu/xowa
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

'v3.8.5.1'

Browse Source
v3.3.4 v3.8.5.1
gnosygnu 8 years ago
parent e4a2af026b
commit 232838c732
292 changed files with 4500 additions and 1836 deletions
Show Stats Download Patch File Download Diff File

3
100_core/src/gplx/Bry_.java
Unescape View File

@ -1054,4 +1054,7 @@ public class Bry_ {
for (int i = 0; i < len; ++i)
bry[i] = Byte_.Zero;
}
public static byte[] Replace_nl_w_tab(byte[] src, int bgn, int end) {
return Bry_.Replace(Bry_.Mid(src, bgn, end), Byte_ascii.Nl, Byte_ascii.Tab);
}
}

2
100_core/src/gplx/Bry_find_.java
Unescape View File

@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx;
public class Bry_find_ {
public static final int Not_found = -1;
public static final int Not_found = -1;
public static int Find_fwd(byte[] src, byte lkp) {return Find_fwd(src, lkp, 0, src.length);}
public static int Find_fwd(byte[] src, byte lkp, int cur) {return Find_fwd(src, lkp, cur, src.length);}
public static int Find_fwd(byte[] src, byte lkp, int cur, int end) {

5
100_core/src/gplx/Byte_.java
Unescape View File

@ -42,6 +42,11 @@ public class Byte_ {
if (v == itm) return true;
return false;
}
public static boolean Eq_many(byte v, byte... ary) {
for (byte itm : ary)
if (v != itm) return false;
return true;
}
public static int Compare(byte lhs, byte rhs) {
if (lhs == rhs) return CompareAble_.Same;
else if (lhs < rhs) return CompareAble_.Less;

3
100_core/src/gplx/Datetime_now.java
Unescape View File

@ -48,6 +48,9 @@ public class Datetime_now {
if (autoincrement) manual = rv.Add_minute(1); // simulate passage of manual by increasing manual by 1 minute with each call
return rv;
}
public static DateAdp Get_force() { // ignore manual and force get of real time
return new DateAdp(new GregorianCalendar());
}
// private static final DateAdp manual_time_dflt = DateAdp_.parse_gplx("2001-01-01 00:00:00.000");
// private static DateAdp manual_time;
// static boolean Now_enabled() {return now_enabled;} private static boolean now_enabled;

4
100_core/src/gplx/Gfo_usr_dlg_.java
Unescape View File

@ -19,7 +19,7 @@ package gplx;
public class Gfo_usr_dlg_ {
private static Gfo_usr_dlg_base test__list, test__show;
public static Gfo_usr_dlg Instance = Gfo_usr_dlg_noop.Instance; // NOTE: global instance which can be reassigned
public static final Gfo_usr_dlg Noop = Gfo_usr_dlg_noop.Instance;
public static final Gfo_usr_dlg Noop = Gfo_usr_dlg_noop.Instance;
public static Gfo_usr_dlg__gui Test__list__init() {
if (test__list == null)
test__list = new Gfo_usr_dlg_base(Gfo_usr_dlg__log_.Noop, Gfo_usr_dlg__gui_.Test);
@ -70,5 +70,5 @@ class Gfo_usr_dlg_noop implements Gfo_usr_dlg {
public String Log_direct(String msg) {return "";}
public String Plog_many(String grp_key, String msg_key, String fmt, Object... args) {return "";}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return this;}
public static final Gfo_usr_dlg_noop Instance = new Gfo_usr_dlg_noop(); Gfo_usr_dlg_noop() {}
public static final Gfo_usr_dlg_noop Instance = new Gfo_usr_dlg_noop(); Gfo_usr_dlg_noop() {}
}

2
100_core/src/gplx/Gfo_usr_dlg__log_base.java
Unescape View File

@ -83,7 +83,7 @@ public class Gfo_usr_dlg__log_base implements Gfo_usr_dlg__log {
}
catch (Exception e) {Err_.Noop(e);} // java.lang.StringBuilder can throw exceptions in some situations when called on a different thread; ignore errors
} private String_bldr sb = String_bldr_.new_thread(); // NOTE: use java.lang.StringBuffer to try to avoid random exceptions when called on a different thread
private String Bld_msg(String s) {return sb.Add(Datetime_now.Get().XtoUtc().XtoStr_fmt_yyyyMMdd_HHmmss_fff()).Add(" ").Add(s).Add_char_nl().To_str_and_clear();}
private String Bld_msg(String s) {return sb.Add(Datetime_now.Get_force().XtoUtc().XtoStr_fmt_yyyyMMdd_HHmmss_fff()).Add(" ").Add(s).Add_char_nl().To_str_and_clear();}
private void Log_msg(Io_url url, String txt) {
if (queue_enabled) {
String url_raw = url == null ? "mem" : url.Raw();

39
100_core/src/gplx/Io_url_.java
Unescape View File

@ -27,9 +27,9 @@ public class Io_url_ {
if (usr_dir == null) {
switch (Op_sys.Cur().Tid()) {
case Op_sys.Tid_wnt: usr_dir = Io_url_.new_inf_("C:\\", IoUrlInfo_.Wnt); break;
case Op_sys.Tid_lnx: usr_dir = Io_url_.new_inf_(String_.Format("/home/{0}/", Env_.UserName()), IoUrlInfo_.Lnx); break;
case Op_sys.Tid_osx: usr_dir = Io_url_.new_inf_(String_.Format("/Users/{0}/", Env_.UserName()), IoUrlInfo_.Lnx); break;
case Op_sys.Tid_drd: usr_dir = Io_url_.new_inf_(String_.Format("/mnt/{0}/", Env_.UserName()), IoUrlInfo_.Lnx); break;
case Op_sys.Tid_lnx: usr_dir = Io_url_.new_inf_(String_.Format("/home/{0}/", System_.Prop__user_name()), IoUrlInfo_.Lnx); break;
case Op_sys.Tid_osx: usr_dir = Io_url_.new_inf_(String_.Format("/Users/{0}/", System_.Prop__user_name()), IoUrlInfo_.Lnx); break;
case Op_sys.Tid_drd: usr_dir = Io_url_.new_inf_(String_.Format("/mnt/{0}/", System_.Prop__user_name()), IoUrlInfo_.Lnx); break;
default: throw Err_.new_unhandled(Op_sys.Cur().Tid());
}
}
@ -49,22 +49,31 @@ public class Io_url_ {
public static Io_url new_dir_(String raw) {return new_any_(raw);} // NOTE: for now, same as new_fil; stack overflow when doing new_dir
public static Io_url new_any_(String raw) {return new_inf_(raw, IoUrlInfoRegy.Instance.Match(raw));}
public static Io_url new_inf_(String raw, IoUrlInfo info) {return String_.Eq(raw, "") ? Io_url_.Empty : new Io_url(raw, info);}
public static Io_url http_any_(String src, boolean wnt) {
return new_any_(parse_http_file(src, wnt));
public static Io_url New__http_or_fail(String raw) {return New__http_or_fail(Bry_.new_u8(raw));}
public static Io_url New__http_or_fail(byte[] raw) {
Io_url rv = New__http_or_null(raw);
if (rv == null) throw Err_.new_wo_type("url:invalid http_file raw", "raw", raw);
return rv;
}
private static String parse_http_file(String v, boolean wnt) {
byte[] v_bry = Bry_.new_u8(v);
int v_len = v_bry.length;
if (Bry_.Has_at_bgn(v_bry, Io_url.Http_file_bry, 0, v_len)) {
byte[] rv = new byte[v_len - Io_url.Http_file_len];
for (int i = 0; i < rv.length; i++) {
byte b = v_bry[i + Io_url.Http_file_len];
if (wnt && b == Byte_ascii.Slash) b = Byte_ascii.Backslash;
public static Io_url New__http_or_null(String raw) {return New__http_or_null(Bry_.new_u8(raw));}
public static Io_url New__http_or_null(byte[] raw) {
int len = raw.length;
if (!Bry_.Has_at_bgn(raw, Io_url.Http_file_bry, 0, len)) return null; // doesn't start with "file:///"; return null;
// bld rv; note that wnt has to convert / to \
byte[] rv = null;
if (Op_sys.Cur().Tid_is_wnt()) {
int rv_len = len - Io_url.Http_file_len;
rv = new byte[rv_len];
for (int i = 0; i < rv_len; ++i) {
byte b = raw[i + Io_url.Http_file_len];
if (b == Op_sys.Dir_spr__lnx) b = Op_sys.Dir_spr__wnt;
rv[i] = b;
}
return String_.new_u8(rv);
}
return v;
else
rv = Bry_.Mid(raw, Io_url.Http_file_len);
return rv == null ? null : new_any_(String_.new_u8(rv));
}
public static Io_url store_orFail_(SrlMgr mgr, String key, Io_url v) {

32
100_core/src/gplx/Io_url__tst.java
Unescape View File

@ -0,0 +1,32 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx;
import org.junit.*; import gplx.core.tests.*; import gplx.core.envs.*;
public class Io_url__tst {
@Before public void init() {fxt.Clear();} private final Io_url__fxt fxt = new Io_url__fxt();
@Test public void Basic__lnx() {fxt.Test__New__http_or_null(Bool_.N, "file:///C:/a.txt", "C:/a.txt");}
@Test public void Basic__wnt() {fxt.Test__New__http_or_null(Bool_.Y, "file:///C:/a.txt", "C:\\a.txt");}
@Test public void Null() {fxt.Test__New__http_or_null(Bool_.N, "C:/a.txt", null);}
}
class Io_url__fxt {
public void Clear() {Io_mgr.Instance.InitEngine_mem();}
public void Test__New__http_or_null(boolean os_is_wnt, String raw, String expd) {
Op_sys.Cur_(os_is_wnt ? Op_sys.Tid_wnt : Op_sys.Tid_lnx);
Gftest.Eq__obj_or_null(expd, Io_url_.New__http_or_null(raw));
}
}

1
100_core/src/gplx/Tfds.java
Unescape View File

@ -24,6 +24,7 @@ public class Tfds { // URL:doc/gplx.tfds/Tfds.txt
public static void Eq_byte (byte expd , byte actl, String fmt, Object... args) {Eq_exec_y(expd, actl, fmt, args);}
public static void Eq_int (int expd , int actl) {Eq_exec_y(expd, actl, "", Object_.Ary_empty);}
public static void Eq_int (int expd , int actl, String fmt, Object... args) {Eq_exec_y(expd, actl, fmt, args);}
public static void Eq_double(double expd, double actl) {Eq_exec_y(expd, actl, "", Object_.Ary_empty);}
public static void Eq_str (byte[] expd, byte[] actl, String fmt, Object... args) {Eq_exec_y(String_.new_u8(expd), String_.new_u8(actl), fmt, args);}
public static void Eq_str (byte[] expd, String actl, String fmt, Object... args) {Eq_exec_y(String_.new_u8(expd), actl, fmt, args);}
public static void Eq_str (String expd, byte[] actl, String fmt, Object... args) {Eq_exec_y(expd, String_.new_u8(actl), fmt, args);}

2
100_core/src/gplx/Time_span_.java
Unescape View File

@ -33,7 +33,7 @@ public class Time_span_ {
long fracs = Merge_long(units, Time_span_.Divisors);
return Time_span_.fracs_(fracs);
}
public static Time_span from_(long bgn) {return Time_span_.fracs_(Env_.TickCount() - bgn);}
public static Time_span from_(long bgn) {return Time_span_.fracs_(System_.Ticks() - bgn);}
public static final long parse_null = Long_.Min_value;
public static Time_span parse(String raw) {
byte[] bry = Bry_.new_u8(raw);

39
100_core/src/gplx/core/brys/Bry_rdr.java
Unescape View File

@ -39,19 +39,37 @@ public class Bry_rdr {
public int Move_to(int v) {this.pos = v; return pos;}
public int Move_by_one() {return Move_by(1);}
public int Move_by(int v) {this.pos += v; return pos;}
public int Find_fwd_lr() {return Find_fwd(dflt_dlm , Bool_.Y, Bool_.N);}
public int Find_fwd_lr(byte find) {return Find_fwd(find , Bool_.Y, Bool_.N);}
public int Find_fwd_lr(byte[] find) {return Find_fwd(find , Bool_.Y, Bool_.N);}
public int Find_fwd_rr() {return Find_fwd(dflt_dlm , Bool_.N, Bool_.N);}
public int Find_fwd_rr(byte find) {return Find_fwd(find , Bool_.N, Bool_.N);}
public int Find_fwd_rr(byte[] find) {return Find_fwd(find , Bool_.N, Bool_.N);}
private int Find_fwd(byte find, boolean ret_lhs, boolean pos_lhs) {
int find_pos = Bry_find_.Find_fwd(src, find, pos, src_end); if (find_pos == Bry_find_.Not_found) {err_wkr.Fail("find failed", "find", Byte_ascii.To_str(find)); return Bry_find_.Not_found;}
public int Find_fwd_lr() {return Find_fwd(dflt_dlm , Bool_.Y, Bool_.N, Fail_if_missing);}
public int Find_fwd_lr(byte find) {return Find_fwd(find , Bool_.Y, Bool_.N, Fail_if_missing);}
public int Find_fwd_lr_or(byte find, int or)
{return Find_fwd(find , Bool_.Y, Bool_.N, or);}
public int Find_fwd_lr(byte[] find) {return Find_fwd(find , Bool_.Y, Bool_.N, Fail_if_missing);}
public int Find_fwd_rr() {return Find_fwd(dflt_dlm , Bool_.N, Bool_.N, Fail_if_missing);}
public int Find_fwd_rr(byte find) {return Find_fwd(find , Bool_.N, Bool_.N, Fail_if_missing);}
public int Find_fwd_rr(byte[] find) {return Find_fwd(find , Bool_.N, Bool_.N, Fail_if_missing);}
private int Find_fwd(byte find, boolean ret_lhs, boolean pos_lhs, int or) {
int find_pos = Bry_find_.Find_fwd(src, find, pos, src_end);
if (find_pos == Bry_find_.Not_found) {
if (or == Fail_if_missing) {
err_wkr.Fail("find failed", "find", Byte_ascii.To_str(find));
return Bry_find_.Not_found;
}
else
return or;
}
pos = find_pos + (pos_lhs ? 0 : 1);
return ret_lhs ? find_pos : pos;
}
private int Find_fwd(byte[] find, boolean ret_lhs, boolean pos_lhs) {
int find_pos = Bry_find_.Find_fwd(src, find, pos, src_end); if (find_pos == Bry_find_.Not_found) {err_wkr.Fail("find failed", "find", String_.new_u8(find)); return Bry_find_.Not_found;}
private int Find_fwd(byte[] find, boolean ret_lhs, boolean pos_lhs, int or) {
int find_pos = Bry_find_.Find_fwd(src, find, pos, src_end);
if (find_pos == Bry_find_.Not_found) {
if (or == Fail_if_missing) {
err_wkr.Fail("find failed", "find", String_.new_u8(find));
return Bry_find_.Not_found;
}
else
return or;
}
pos = find_pos + (pos_lhs ? 0 : find.length);
return ret_lhs ? find_pos : pos;
}
@ -203,4 +221,5 @@ public class Bry_rdr {
}
return this;
}
private static final int Fail_if_missing = Int_.Min_value;
}

36
100_core/src/gplx/core/encoders/Hex_utl_.java
Unescape View File

@ -17,8 +17,8 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.encoders; import gplx.*; import gplx.core.*;
public class Hex_utl_ {
static public int Parse_or(byte[] src, int or) {return Parse_or(src, 0, src.length, or);}
static public int Parse_or(byte[] src, int bgn, int end, int or) {
public static int Parse_or(byte[] src, int or) {return Parse_or(src, 0, src.length, or);}
public static int Parse_or(byte[] src, int bgn, int end, int or) {
int rv = 0; int factor = 1;
byte b = Byte_.Max_value_127;
for (int i = end - 1; i >= bgn; i--) {
@ -35,11 +35,11 @@ public class Hex_utl_ {
}
return rv;
}
static public int Parse(String src) {
public static int Parse(String src) {
int rv = Parse_or(src, -1); if (rv == -1) throw Err_.new_parse("HexDec", "src");
return rv;
}
static public int Parse_or(String src, int or) {
public static int Parse_or(String src, int or) {
int rv = 0; int digit = 0, factor = 1, len = String_.Len(src);
for (int i = len - 1; i > -1; --i) {
digit = To_int(String_.CharAt(src, i));
@ -49,7 +49,7 @@ public class Hex_utl_ {
}
return rv;
}
static public void Encode_bry(byte[] src, byte[] trg) {
public static void Encode_bry(byte[] src, byte[] trg) {
int src_len = src.length, trg_len = trg.length;
if (trg_len != src_len * 2) throw Err_.new_("hex", "trg.len must be src.len * 2", "src_len", src_len, "trg_len", trg_len);
int trg_idx = -1;
@ -59,7 +59,7 @@ public class Hex_utl_ {
trg[++trg_idx] = To_byte_lcase(0xf & src_byte);
}
}
static public String To_str(int val, int pad) {
public static String To_str(int val, int pad) {
char[] ary = new char[8]; int idx = 8; // 8 is max len of hexString; (2^4 * 8); EX: int.MaxValue = 7FFFFFFF
do {
int byt = val % 16;
@ -70,7 +70,7 @@ public class Hex_utl_ {
ary[--idx] = '0';
return String_.new_charAry_(ary, idx, 8-idx);
}
static public void Write(byte[] bry, int bgn, int end, int val) {
public static void Write(byte[] bry, int bgn, int end, int val) {
for (int i = end - 1; i > bgn - 1; i--) {
int b = val % 16;
bry[i] = To_byte(b);
@ -78,7 +78,21 @@ public class Hex_utl_ {
if (val == 0) break;
}
}
static private int To_int(char c) {
public static boolean Is_hex_many(byte... ary) {
for (byte itm : ary) {
switch (itm) {
case Byte_ascii.Num_0: case Byte_ascii.Num_1: case Byte_ascii.Num_2: case Byte_ascii.Num_3: case Byte_ascii.Num_4:
case Byte_ascii.Num_5: case Byte_ascii.Num_6: case Byte_ascii.Num_7: case Byte_ascii.Num_8: case Byte_ascii.Num_9:
case Byte_ascii.Ltr_A: case Byte_ascii.Ltr_B: case Byte_ascii.Ltr_C: case Byte_ascii.Ltr_D: case Byte_ascii.Ltr_E: case Byte_ascii.Ltr_F:
case Byte_ascii.Ltr_a: case Byte_ascii.Ltr_b: case Byte_ascii.Ltr_c: case Byte_ascii.Ltr_d: case Byte_ascii.Ltr_e: case Byte_ascii.Ltr_f:
break;
default:
return false;
}
}
return true;
}
private static int To_int(char c) {
switch (c) {
case '0': return 0; case '1': return 1; case '2': return 2; case '3': return 3; case '4': return 4;
case '5': return 5; case '6': return 6; case '7': return 7; case '8': return 8; case '9': return 9;
@ -87,7 +101,7 @@ public class Hex_utl_ {
default: return -1;
}
}
static private char To_char(int val) {
private static char To_char(int val) {
switch (val) {
case 0: return '0'; case 1: return '1'; case 2: return '2'; case 3: return '3'; case 4: return '4';
case 5: return '5'; case 6: return '6'; case 7: return '7'; case 8: return '8'; case 9: return '9';
@ -95,7 +109,7 @@ public class Hex_utl_ {
default: throw Err_.new_parse("hexstring", Int_.To_str(val));
}
}
static private byte To_byte(int v) {
private static byte To_byte(int v) {
switch (v) {
case 0: return Byte_ascii.Num_0; case 1: return Byte_ascii.Num_1; case 2: return Byte_ascii.Num_2; case 3: return Byte_ascii.Num_3; case 4: return Byte_ascii.Num_4;
case 5: return Byte_ascii.Num_5; case 6: return Byte_ascii.Num_6; case 7: return Byte_ascii.Num_7; case 8: return Byte_ascii.Num_8; case 9: return Byte_ascii.Num_9;
@ -103,7 +117,7 @@ public class Hex_utl_ {
default: throw Err_.new_parse("hexstring", Int_.To_str(v));
}
}
static private byte To_byte_lcase(int v) {
private static byte To_byte_lcase(int v) {
switch (v) {
case 0: return Byte_ascii.Num_0; case 1: return Byte_ascii.Num_1; case 2: return Byte_ascii.Num_2; case 3: return Byte_ascii.Num_3;
case 4: return Byte_ascii.Num_4; case 5: return Byte_ascii.Num_5; case 6: return Byte_ascii.Num_6; case 7: return Byte_ascii.Num_7;

63
100_core/src/gplx/core/envs/Env_.java
Unescape View File

@ -16,8 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.envs; import gplx.*; import gplx.core.*;
import gplx.core.threads.*;
public class Env_ {
public class Env_ {
public static void Init(String[] args, String appNameAndExt, Class<?> type) {
mode_testing = false;
mode_debug = String_.In("GPLX_DEBUG_MODE_ENABLED", args);
@ -33,67 +32,17 @@ public class Env_ {
public static void Init_drd() {
mode_testing = mode_debug = false;
}
public static void Init_testing() {mode_testing = true;}
public static void Init_testing_n_() {mode_testing = false;}
public static boolean Mode_testing() {return mode_testing;} static boolean mode_testing = true;
public static boolean Mode_debug() {return mode_debug;} static boolean mode_debug = false;
public static void Init_testing() {mode_testing = true;}
public static void Init_testing_n_() {mode_testing = false;}
public static boolean Mode_testing() {return mode_testing;} private static boolean mode_testing = true;
public static boolean Mode_debug() {return mode_debug;} private static boolean mode_debug = false;
public static String[] AppArgs() {return appArgs;} static String[] appArgs;
public static Io_url AppUrl() {
if (mode_testing) return Io_url_.mem_fil_("mem/testing.jar");
if (appUrl == Io_url_.Empty) throw Err_.new_wo_type("Env_.Init was not called");
return appUrl;
} static Io_url appUrl = Io_url_.Empty;
public static void Exit() {Exit_code(0);}
public static void Exit_code(int code) {System.exit(code);}
public static String UserName() {return System.getProperty("user.name");}
public static void GarbageCollect() {if (mode_testing) return; System.gc();}
public static long TickCount() {return TickCount_Test >= 0 ? TickCount_Test : System.currentTimeMillis();}
public static int TickCount_elapsed_in_sec(long time_bgn) {return (int)(Env_.TickCount() - time_bgn) / 1000;}
public static int TickCount_elapsed_in_frac(long time_bgn) {return (int)(Env_.TickCount() - time_bgn);}
public static long TickCount_Test = -1; // in milliseconds
public static void TickCount_normal() {TickCount_Test = -1;}
public static int System_cpu_count() {return Runtime.getRuntime().availableProcessors();}
public static long System_memory_max() {return Runtime.getRuntime().maxMemory();}
public static long System_memory_total() {return Runtime.getRuntime().totalMemory();}
public static long System_memory_free() {return Runtime.getRuntime().freeMemory();}
public static final String LocalHost = "127.0.0.1";
public static String NewLine_lang() {return mode_testing ? "\n" : "\n";}
public static String GenHdr(boolean forSourceCode, String programName, String hdr_bgn, String hdr_end) {
String newLine = Op_sys.Lnx.Nl_str();
String lineEnd = Op_sys.Lnx.Nl_str();
String codeBgn = forSourceCode ? "/*" + newLine : "";
String codeEnd = forSourceCode ? "*/" + newLine : "";
String codeHdr = forSourceCode ? "This file is part of {0}." + newLine + newLine : "";
String fmt = String_.Concat
( codeBgn
, codeHdr
, hdr_bgn
, "Copyright (c) 2012-2016 gnosygnu@gmail.com", newLine
, newLine
, "This program is free software: you can redistribute it and/or modify", lineEnd
, "it under the terms of the GNU Affero General Public License as", lineEnd
, "published by the Free Software Foundation, either version 3 of the", lineEnd
, "License, or (at your option) any later version.", newLine
, newLine
, "This program is distributed in the hope that it will be useful,", lineEnd
, "but WITHOUT ANY WARRANTY; without even the implied warranty of", lineEnd
, "MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the", lineEnd
, "GNU Affero General Public License for more details.", newLine
, newLine
, "You should have received a copy of the GNU Affero General Public License", lineEnd
, "along with this program. If not, see <http://www.gnu.org/licenses/>.", newLine
, codeEnd
, hdr_end
);
return String_.Format(fmt, programName);
}
public static String Env_prop__user_language() {return Env_prop(Env_prop_key__user_language);}
public static String Env_prop__java_version() {return Env_prop(Env_prop_key__java_version);}
public static String Env_prop(String key) {
return System.getProperty(key);
}
private static final String
Env_prop_key__user_language = "user.language"
, Env_prop_key__java_version = "java.version"
;
}

29
100_core/src/gplx/core/envs/Op_sys.java
Unescape View File

@ -20,16 +20,16 @@ public class Op_sys {
Op_sys(byte tid, byte sub_tid, String os_name, byte bitness, String nl_str, byte fsys_dir_spr_byte, boolean fsys_case_match) {
this.tid = tid; this.sub_tid = sub_tid; this.os_name = os_name; this.bitness = bitness; this.nl_str = nl_str; this.fsys_dir_spr_byte = fsys_dir_spr_byte; this.fsys_dir_spr_str = Char_.To_str((char)fsys_dir_spr_byte); this.fsys_case_match = fsys_case_match;
}
public byte Tid() {return tid;} private final byte tid;
public byte Sub_tid() {return sub_tid;} private final byte sub_tid;
public byte Tid() {return tid;} private final byte tid;
public byte Sub_tid() {return sub_tid;} private final byte sub_tid;
public String Os_name() {return os_name;} private String os_name;
public byte Bitness() {return bitness;} private final byte bitness;
public byte Bitness() {return bitness;} private final byte bitness;
public String Bitness_str() {return (bitness == Bitness_32 ? "32" : "64");}
public String Nl_str() {return nl_str;} private final String nl_str;
public String Fsys_dir_spr_str() {return fsys_dir_spr_str;} private final String fsys_dir_spr_str;
public byte Fsys_dir_spr_byte() {return fsys_dir_spr_byte;} private final byte fsys_dir_spr_byte;
public String Nl_str() {return nl_str;} private final String nl_str;
public String Fsys_dir_spr_str() {return fsys_dir_spr_str;} private final String fsys_dir_spr_str;
public byte Fsys_dir_spr_byte() {return fsys_dir_spr_byte;} private final byte fsys_dir_spr_byte;
public String Fsys_http_frag_to_url_str(String raw) {return fsys_dir_spr_byte == Byte_ascii.Slash ? raw : String_.Replace(raw, Lnx.Fsys_dir_spr_str(), fsys_dir_spr_str);}
public boolean Fsys_case_match() {return fsys_case_match;} private final boolean fsys_case_match;
public boolean Fsys_case_match() {return fsys_case_match;} private final boolean fsys_case_match;
public String Fsys_case_match_str(String s) {return String_.CaseNormalize(fsys_case_match, s);}
public boolean Tid_is_wnt() {return tid == Tid_wnt;}
public boolean Tid_is_lnx() {return tid == Tid_lnx;}
@ -40,11 +40,12 @@ public class Op_sys {
public static final byte Tid_nil = 0, Tid_wnt = 1, Tid_lnx = 2, Tid_osx = 3, Tid_drd = 4;
public static final byte Sub_tid_unknown = 0, Sub_tid_win_xp = 1, Sub_tid_win_7 = 2, Sub_tid_win_8 = 3;
public static final byte Bitness_32 = 1, Bitness_64 = 2;
public static final char Dir_spr_char_lnx = '\n';
public static final Op_sys Lnx = new_unx_flavor_(Tid_lnx, "linux", Bitness_32);
public static final Op_sys Osx = new_unx_flavor_(Tid_osx, "macosx", Bitness_32);
public static final Op_sys Drd = new_unx_flavor_(Tid_drd, "android", Bitness_32);
public static final Op_sys Wnt = new_wnt_(Sub_tid_unknown, Bitness_32);
public static final char Nl_char_lnx = '\n';
public static final byte Dir_spr__lnx = Byte_ascii.Slash, Dir_spr__wnt = Byte_ascii.Backslash;
public static final Op_sys Lnx = new_unx_flavor_(Tid_lnx, "linux", Bitness_32);
public static final Op_sys Osx = new_unx_flavor_(Tid_osx, "macosx", Bitness_32);
public static final Op_sys Drd = new_unx_flavor_(Tid_drd, "android", Bitness_32);
public static final Op_sys Wnt = new_wnt_(Sub_tid_unknown, Bitness_32);
public static Op_sys Cur() {return cur_op_sys;} static Op_sys cur_op_sys = new_auto_identify_();
public static String Fsys_path_to_lnx(String v) {
return cur_op_sys.Tid_is_wnt() ? String_.Replace(v, Wnt.fsys_dir_spr_str, Lnx.fsys_dir_spr_str) : v;
@ -52,8 +53,8 @@ public class Op_sys {
public static String Fsys_path_to_wnt(String v) {
return cur_op_sys.Tid_is_wnt() ? String_.Replace(v, Lnx.fsys_dir_spr_str, Wnt.fsys_dir_spr_str) : v;
}
private static Op_sys new_wnt_(byte bitness, byte sub_tid) {return new Op_sys(Tid_wnt , sub_tid , "windows", bitness, "\r\n", Byte_ascii.Backslash , Bool_.N);}
private static Op_sys new_unx_flavor_(byte tid, String os_name, byte bitness) {return new Op_sys(tid , Sub_tid_unknown , os_name , bitness, "\n" , Byte_ascii.Slash , Bool_.Y);}
private static Op_sys new_wnt_(byte bitness, byte sub_tid) {return new Op_sys(Tid_wnt , sub_tid , "windows", bitness, "\r\n", Dir_spr__wnt , Bool_.N);}
private static Op_sys new_unx_flavor_(byte tid, String os_name, byte bitness) {return new Op_sys(tid , Sub_tid_unknown , os_name , bitness, "\n" , Dir_spr__lnx , Bool_.Y);}
public static void Cur_(int tid) {
switch (tid) {
case Tid_wnt: cur_op_sys = Wnt; break;

4
100_core/src/gplx/core/envs/Process_adp.java
Unescape View File

@ -167,11 +167,11 @@ public class Process_adp implements Gfo_invk, Rls_able {
notify_fmtr.Fmt_(prog_fmt);
while (thread.isAlive()) {
thread_run = true;
long prv = Env_.TickCount();
long prv = System_.Ticks();
Thread_adp_.Sleep(thread_interval);
// try {thread.join(thread_interval);}
// catch (InterruptedException e) {throw Err_.err_key_(e, "gplx.ProcessAdp", "thread interrupted at join");}
long cur = Env_.TickCount();
long cur = System_.Ticks();
int dif = (int)(cur - prv);
elapsed += dif;
if (prog_dlg != null) {

25
100_core/src/gplx/core/envs/Runtime_.java
Unescape View File

@ -0,0 +1,25 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.envs; import gplx.*; import gplx.core.*;
public class Runtime_ {
// *** Hardware-related
public static int Cpu_count() {return Runtime.getRuntime().availableProcessors();}
public static long Memory_max() {return Runtime.getRuntime().maxMemory();}
public static long Memory_total() {return Runtime.getRuntime().totalMemory();}
public static long Memory_free() {return Runtime.getRuntime().freeMemory();}
}

63
100_core/src/gplx/core/envs/System_.java
Unescape View File

@ -0,0 +1,63 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.envs; import gplx.*; import gplx.core.*;
public class System_ {
// *** ticks
public static long Ticks() {return Ticks__test_val >= 0 ? Ticks__test_val : System.currentTimeMillis();}
public static int Ticks__elapsed_in_sec (long time_bgn) {return (int)(Ticks() - time_bgn) / 1000;}
public static int Ticks__elapsed_in_frac (long time_bgn) {return (int)(Ticks() - time_bgn);}
public static void Ticks__test_set(long v) {Ticks__test_val = v;}
public static void Ticks__test_add(long v) {Ticks__test_val += v;}
public static void Ticks__test_off() {Ticks__test_val = -1;}
private static long Ticks__test_val = -1; // in milliseconds
// *** misc methods: Exit / GC
public static void Exit() {Exit_by_code(0);}
public static void Exit_by_code(int code) {System.exit(code);}
public static void Garbage_collect() {if (Env_.Mode_testing()) return; System.gc();}
// *** java properties: getProperty; "-D" properties
public static String Prop__user_language() {return Prop__get(Prop_key__user_language);}
public static String Prop__user_name() {return Prop__get(Prop_key__user_name);}
public static String Prop__java_version() {return Prop__get(Prop_key__java_version);}
public static String Prop__java_home() {return Prop__get(Prop_key__java_home);}
private static String Prop__get(String key) {
return System.getProperty(key);
}
private static final String
Prop_key__user_language = "user.language"
, Prop_key__user_name = "user.name"
, Prop_key__java_version = "java.version"
, Prop_key__java_home = "java.version"
;
// *** environment variables: getenv
public static String Env__machine_name() {
String rv = "UNKNOWN_MACHINE_NAME";
rv = Env__get(Env_key__computername); if (String_.Len_gt_0(rv)) return rv;
rv = Env__get(Env_key__hostname); if (String_.Len_gt_0(rv)) return rv;
return rv;
}
private static String Env__get(String key) {
return System.getenv(key);
}
private static final String
Env_key__computername = "COMPUTERNAME"
, Env_key__hostname = "HOSTNAME"
;
}

2
100_core/src/gplx/core/ios/IoEngine_xrg_loadFilStr.java
Unescape View File

@ -33,7 +33,7 @@ public class IoEngine_xrg_loadFilStr {
public String[] ExecAsStrAryLnx() {
String raw = Exec();
if (String_.Len(raw) == 0) return String_.Ary_empty;
return String_.Split(raw, Op_sys.Dir_spr_char_lnx, false);
return String_.Split(raw, Op_sys.Nl_char_lnx, false);
}
int Bom_Utf8 = 65279; // U+FEFF; see http://en.wikipedia.org/wiki/Byte_order_mark
public static IoEngine_xrg_loadFilStr new_(Io_url url) {

6
100_core/src/gplx/core/ios/Io_download_fmt.java
Unescape View File

@ -59,11 +59,11 @@ public class Io_download_fmt {
prog_pct = 0;
prog_rate = 0;
prog_left = 0;
time_bgn = time_prv = Env_.TickCount();
time_bgn = time_prv = System_.Ticks();
time_checkpoint = 0;
}
public void Prog(int prog_read) {
time_now = Env_.TickCount();
time_now = System_.Ticks();
time_dif = time_now - time_bgn; if (time_dif == 0) time_dif = 1; // avoid div by zero error below
prog_done += prog_read;
time_checkpoint += time_now - time_prv;
@ -84,7 +84,7 @@ public class Io_download_fmt {
usr_dlg.Prog_none("", "prog", prog_msg);
}
public void Term() {
time_end = Env_.TickCount();
time_end = System_.Ticks();
// prog_rate = (prog_done * 1000) / (time_dif);
// prog_pct = (prog_done * 10000) / src_len; // 100 00 to get 2 decimal places; EX: .1234 -> 1234 -> 12.34%
// prog_left = (1000 * (src_len - prog_done)) / prog_rate;

4
100_core/src/gplx/core/ios/Io_download_fmt_tst.java
Unescape View File

@ -43,12 +43,12 @@ class Io_download_fmt_fxt {
if (fmt == null) {
fmt = new Io_download_fmt();
}
Env_.TickCount_Test = 0;
System_.Ticks__test_set(0);
prog_done = prog_rate = prog_pct = prog_left = -1;
prog_msg = null;
return this;
} Io_download_fmt fmt;
public Io_download_fmt_fxt Now_add_f(int v) {Env_.TickCount_Test += v; return this;}
public Io_download_fmt_fxt Now_add_f(int v) {System_.Ticks__test_add(v); return this;}
public Io_download_fmt_fxt Prog_done_(int v) {prog_done = v; return this;} long prog_done = -1;
public Io_download_fmt_fxt Prog_pct_ (int v) {prog_pct = v; return this;} long prog_pct = -1;
public Io_download_fmt_fxt Prog_rate_(int v) {prog_rate = v; return this;} long prog_rate = -1;

8
100_core/src/gplx/core/logs/Gfo_log__base.java
Unescape View File

@ -19,22 +19,22 @@ package gplx.core.logs; import gplx.*; import gplx.core.*;
public abstract class Gfo_log__base implements Gfo_log {
private long time_prv = 0;
public void Warn(String msg, Object... args) {
long time = gplx.core.envs.Env_.TickCount();
long time = gplx.core.envs.System_.Ticks();
long elapsed = time_prv == 0 ? 0 : time - time_prv;
Exec(Gfo_log_itm.Type__warn, time, elapsed, msg, args);
}
public void Note(String msg, Object... args) {
long time = gplx.core.envs.Env_.TickCount();
long time = gplx.core.envs.System_.Ticks();
long elapsed = time_prv == 0 ? 0 : time - time_prv;
Exec(Gfo_log_itm.Type__note, time, elapsed, msg, args);
}
public void Info(String msg, Object... args) {
long time = gplx.core.envs.Env_.TickCount();
long time = gplx.core.envs.System_.Ticks();
long elapsed = time_prv == 0 ? 0 : time - time_prv;
Exec(Gfo_log_itm.Type__info, time, elapsed, msg, args);
}
public void Prog(String msg, Object... args) {
long time = gplx.core.envs.Env_.TickCount();
long time = gplx.core.envs.System_.Ticks();
long elapsed = time_prv == 0 ? 0 : time - time_prv;
Exec(Gfo_log_itm.Type__prog, time, elapsed, msg, args);
}

1
100_core/src/gplx/core/primitives/Int_obj_ref.java
Unescape View File

@ -21,6 +21,7 @@ public class Int_obj_ref {
public int Val() {return val;} public Int_obj_ref Val_(int v) {val = v; return this;} int val;
public int Val_add() {val++; return val;}
public int Val_add_post() {return val++;}
public int Val_add_pre() {return ++val;}
public int Val_add(int v) {val += v; return val;}
public Int_obj_ref Val_zero_() {val = 0; return this;}
public Int_obj_ref Val_neg1_() {val = -1; return this;}

9
100_core/src/gplx/core/tests/Gftest.java
Unescape View File

@ -21,6 +21,7 @@ public class Gftest {
private static final Bry_bfr bfr = Bry_bfr_.New();
public static void Eq__ary(int[] expd, int[] actl, String msg_fmt, Object... msg_args) {Eq__array(Type_adp_.Tid__int, expd, actl, msg_fmt, msg_args);}
public static void Eq__ary(long[] expd, long[] actl, String msg_fmt, Object... msg_args) {Eq__array(Type_adp_.Tid__long, expd, actl, msg_fmt, msg_args);}
public static void Eq__ary__lines(String expd, byte[] actl, String msg_fmt, Object... msg_args) {Eq__ary__lines(expd, String_.new_u8(actl), msg_fmt, msg_args);}
public static void Eq__ary__lines(String expd, String actl, String msg_fmt, Object... msg_args) {Eq__array(Type_adp_.Tid__bry, Bry_split_.Split_lines(Bry_.new_u8(expd)), Bry_split_.Split_lines(Bry_.new_u8(actl)), msg_fmt, msg_args);}
public static void Eq__ary(String[] expd, String[] actl) {Eq__array(Type_adp_.Tid__bry, Bry_.Ary(expd), Bry_.Ary(actl), "no_msg");}
public static void Eq__ary(String[] expd, String[] actl, String msg_fmt, Object... msg_args) {Eq__array(Type_adp_.Tid__bry, Bry_.Ary(expd), Bry_.Ary(actl), msg_fmt, msg_args);}
@ -51,6 +52,11 @@ public class Gftest {
bfr.Add(Bry__line_end);
throw Err_.new_wo_type(bfr.To_str_and_clear());
}
public static void Eq__obj_or_null(Object expd, Object actl) {
if (expd == null) expd = Str__null;
if (actl == null) actl = Str__null;
Eq__str(Object_.Xto_str_or(expd, Str__null), Object_.Xto_str_or(actl, null), Str__null);
}
public static void Eq__str(String expd, byte[] actl, String msg_fmt, Object... msg_args) {Eq__str(expd, String_.new_u8(actl), msg_fmt, msg_args);}
public static void Eq__str(String expd, byte[] actl) {Eq__str(expd, String_.new_u8(actl), null);}
public static void Eq__str(String expd, String actl) {Eq__str(expd, actl, null);}
@ -181,8 +187,9 @@ public class Gftest {
}
return rv;
}
private static final String Str__null = "<<NULL>>";
private static final byte[] Bry__item__eq_n = Bry_.new_a7("!= ") // Bry__item__eq_y = Bry_.new_a7("== "),
, Bry__null = Bry_.new_a7("<<NULL>>")
, Bry__null = Bry_.new_a7(Str__null)
, Bry__line_bgn = Bry_.new_a7("\n************************************************************************************************\n")
, Bry__line_mid = Bry_.new_a7("\n------------------------------------------------------------------------------------------------\n")
, Bry__line_end = Bry_.new_a7( "________________________________________________________________________________________________")

4
100_core/src/gplx/core/tests/PerfLogMgr_fxt.java
Unescape View File

@ -60,7 +60,7 @@ public class PerfLogMgr_fxt {
}
class PerfLogTmr {
public void Bgn() {bgn = Env_.TickCount();} long bgn;
public long ElapsedMilliseconds() {return Env_.TickCount() - bgn; }
public void Bgn() {bgn = System_.Ticks();} long bgn;
public long ElapsedMilliseconds() {return System_.Ticks() - bgn; }
public static PerfLogTmr new_() {return new PerfLogTmr();} PerfLogTmr() {}
}

4
100_core/src/gplx/core/threads/Thread_halt_wkr.java
Unescape View File

@ -24,7 +24,7 @@ class Thread_halt_wkr implements Gfo_invk {
public Thread_halt_wkr(Thread_adp_mgr mgr, Thread_halt_itm itm, Thread_halt_cbk cbk, int sleep_time, int quit_time) {
this.mgr = mgr; this.thread = itm.Thread; this.thread_key = itm.Key; this.cbk = cbk;
this.sleep_time = sleep_time; this.quit_time = quit_time;
this.bgn_time = gplx.core.envs.Env_.TickCount();
this.bgn_time = gplx.core.envs.System_.Ticks();
}
private void Halt() {
// first, cancel the thread
@ -32,7 +32,7 @@ class Thread_halt_wkr implements Gfo_invk {
// now check if canceled; interrupt if not;
while (true) {
long time_now = gplx.core.envs.Env_.TickCount();
long time_now = gplx.core.envs.System_.Ticks();
boolean halted = false, interrupted = false;
if (thread.Thread__is_alive()) { // thread is still alive
if ( !thread.Thread__cancelable() // itm is not cancelable

6
110_gfml/tst/gplx/gfml/z901_perf_tst.java
Unescape View File

@ -91,7 +91,7 @@ public class z901_perf_tst {
// tmr.End_and_print("type");
// }
interface ClassType {int Type();}
class ClassType_ {public static final int Type_1 = 1, Type_2 = 2, Type_3 = 3;}
class ClassType_ {public static final int Type_1 = 1, Type_2 = 2, Type_3 = 3;}
class ClassType1 implements ClassType {public int Type() {return ClassType_.Type_1;}}
class ClassType2 implements ClassType {public int Type() {return ClassType_.Type_2;}}
class ClassType3 implements ClassType {public int Type() {return ClassType_.Type_3;}}
@ -102,8 +102,8 @@ class PerfFieldVsProc {
public int Val_proc() {return 1;}
}
class TimerWatch {
public void Bgn() {bgnTime = Env_.TickCount();} long bgnTime;
public void End() {duration = Env_.TickCount() - bgnTime;} long duration;
public void Bgn() {bgnTime = System_.Ticks();} long bgnTime;
public void End() {duration = System_.Ticks() - bgnTime;} long duration;
public void End_and_print(String text) {
this.End();
Tfds.Write(XtoStr_ms() + " " + text);

9
140_dbs/src/gplx/dbs/Db_conn.java
Unescape View File

@ -39,6 +39,7 @@ public class Db_conn {
public void Env_vacuum() {Exec_sql_plog_ntx("vacuuming: url=" + this.Conn_info().Db_api(), "VACUUM;");}
public void Meta_tbl_create(Dbmeta_tbl_itm meta) {engine.Meta_tbl_create(meta); engine.Meta_idx_create(Gfo_usr_dlg_.Noop, meta.Idxs().To_ary());}
public void Meta_tbl_delete(String tbl) {engine.Meta_tbl_delete(tbl);}
public void Meta_tbl_remake_many(Db_tbl... tbls) {for (Db_tbl tbl : tbls) Meta_tbl_remake(tbl);}
public void Meta_tbl_remake(Db_tbl tbl) {engine.Meta_tbl_delete(tbl.Tbl_name()); tbl.Create_tbl();}
public void Meta_tbl_remake(Dbmeta_tbl_itm meta) {engine.Meta_tbl_delete(meta.Name()); engine.Meta_tbl_create(meta);}
public void Meta_idx_assert(String tbl, String suffix, String... flds) {if (engine.Meta_idx_exists(tbl + "__" + suffix)) return; this.Meta_idx_create(tbl, suffix, flds);}
@ -110,6 +111,14 @@ public class Db_conn {
Gfo_usr_dlg_.Instance.Plog_many("", "", "done:" + msg);
return rv;
}
public int Exec_select_max_as_int (String tbl_name, String fld_name, int or) {
Object rv = Exec_select_as_obj(String_.Format("SELECT Max({0}) FROM {1}", fld_name, tbl_name));
return rv == null ? or : Int_.cast(rv);
}
public int Exec_select_count_as_int(String tbl_name, int or) {
Object rv = Exec_select_as_obj(String_.Format("SELECT Count(*) FROM {0}", tbl_name));
return rv == null ? or : Int_.cast(rv);
}
public int Exec_select_as_int (String sql, int or) {Object rv = Exec_select_as_obj(sql); return rv == null ? or : Int_.cast(rv);}
public double Exec_select_as_double (String sql, double or) {Object rv = Exec_select_as_obj(sql); return rv == null ? or : Double_.cast(rv);}
private Object Exec_select_as_obj(String sql) {

5
140_dbs/xtn/gplx/dbs/SqliteDbMain.java
Unescape View File

@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.dbs;
import gplx.*; import gplx.core.consoles.*; import gplx.core.envs.Env_;
import gplx.core.envs.System_;
import gplx.dbs.engines.sqlite.*;
import java.nio.ByteBuffer;
import java.nio.CharBuffer;
@ -49,7 +50,7 @@ public class SqliteDbMain {
// }
// }
private void CreateMany(int number, int base_val) {
long time_bgn = Env_.TickCount();
long time_bgn = System_.Ticks();
Db_conn provider = Db_conn_pool.Instance.Get_or_new(Db_conn_info_.sqlite_(Io_url_.new_fil_("E:\\test.sqlite3")));
String tbl_sql = String_.Concat_lines_nl
( "CREATE TABLE fsdb_xtn_thm"
@ -80,7 +81,7 @@ public class SqliteDbMain {
.Val_str("")
.Exec_insert();
}
long time_elapsed = (Env_.TickCount() - time_bgn);
long time_elapsed = (System_.Ticks() - time_bgn);
// provider.Txn_mgr().Txn_end();
provider.Rls_conn();
Tfds.Write(time_elapsed, number / time_elapsed);

2
150_gfui/src/gplx/gfui/controls/windows/GfoConsoleWin.java
Unescape View File

@ -216,7 +216,7 @@ class GfuiTextBoxLogger implements Gfo_invk {
if (!owner.Enabled()) return;
if (!win.Visible()) win.Visible_set(true);
textToShow += tbox.Text() + String_.as_(s)+ String_.CrLf;
long curTick = Env_.TickCount();
long curTick = System_.Ticks();
// if (curTick - lastTick > 500) {
WriteText(textToShow);
textToShow = "";

4
150_gfui/src/gplx/gfui/kits/swts/Swt_app_browser.java
Unescape View File

@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.gfui.kits.swts; import gplx.*; import gplx.gfui.*; import gplx.gfui.kits.*;
import gplx.*;
import gplx.core.envs.Env_;
import gplx.core.envs.System_;
import org.eclipse.swt.*;
import org.eclipse.swt.browser.*; import org.eclipse.swt.custom.*; import org.eclipse.swt.events.*; import org.eclipse.swt.graphics.*; import org.eclipse.swt.layout.*; import org.eclipse.swt.widgets.*;
@ -81,7 +81,7 @@ class Swt_app_browser_mgr {
// browser.setUrl("about:blank");
browser.dispose();
}
Env_.GarbageCollect();
System_.Garbage_collect();
browser = null;
}
}

4
150_gfui/src/gplx/gfui/kits/swts/Swt_html.java
Unescape View File

@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.gfui.kits.swts; import gplx.*; import gplx.gfui.*; import gplx.gfui.kits.*;
import gplx.core.envs.Env_;
import gplx.core.envs.System_;
import gplx.core.primitives.*;
import gplx.core.threads.Thread_adp_;
import gplx.gfui.controls.elems.GfuiElem;
@ -132,7 +132,7 @@ public class Swt_html implements Gxw_html, Swt_control, FocusListener, Gfo_evt_m
public void Html_dispose() {
browser.dispose();
delete_owner.SubElems().DelOrFail(delete_cur); // NOTE: must delete cur from owner, else new tab will fail after closing one; DATE:2014-07-09
Env_.GarbageCollect();
System_.Garbage_collect();
}
private GfuiElem delete_owner, delete_cur;
public void Delete_elems_(GfuiElem delete_owner, GfuiElem delete_cur) {this.delete_owner = delete_owner; this.delete_cur = delete_cur;} // HACK: set owner / cur so delete can work;

2
400_xowa/src/gplx/core/caches/Gfo_cache_mgr_bry.java
Unescape View File

@ -27,7 +27,7 @@ class Gfo_cache_itm {
public Object Key() {return key;} private Object key;
public Object Val() {return val;} private Object val;
public long Touched() {return touched;} private long touched;
public Gfo_cache_itm Touched_update() {touched = Env_.TickCount(); return this;}
public Gfo_cache_itm Touched_update() {touched = System_.Ticks(); return this;}
}
class Gfo_cache_itm_comparer implements gplx.core.lists.ComparerAble {
public int compare(Object lhsObj, Object rhsObj) {

43
400_xowa/src/gplx/core/intls/Gfo_i18n_itm.java
Unescape View File

@ -1,43 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.intls; import gplx.*; import gplx.core.*;
import gplx.core.brys.fmtrs.*;
class Gfo_i18n_itm {
public Gfo_i18n_itm(int src, byte[] key, byte[] val, boolean val_fmt_exists, Gfo_i18n_val_cmd val_cmd) {
this.src = src; this.key = key; this.val = val; this.val_fmt_exists = val_fmt_exists; this.val_cmd = val_cmd;
}
public int Src() {return src;} private final int src;
public byte[] Key() {return key;} private final byte[] key;
public byte[] Val() {return val;} private final byte[] val;
public boolean Val_fmt_exists() {return val_fmt_exists;} private final boolean val_fmt_exists;
public Gfo_i18n_val_cmd Val_cmd() {return val_cmd;} private final Gfo_i18n_val_cmd val_cmd;
public byte[] Bld_none() {
return val_cmd == null ? val : val_cmd.Process(src, key, val);
}
public byte[] Bld_many(Object... args) {
byte[] rv = null;
synchronized (tmp_fmtr) {
tmp_fmtr.Fmt_(val);
tmp_fmtr.Bld_bfr_many(tmp_bfr, args);
rv = tmp_bfr.To_bry_and_clear();
}
return val_cmd == null ? rv : val_cmd.Process(src, key, rv);
}
private static final Bry_fmtr tmp_fmtr = Bry_fmtr.new_();
private static final Bry_bfr tmp_bfr = Bry_bfr_.Reset(255);
}

41
400_xowa/src/gplx/core/intls/Gfo_i18n_mgr.java
Unescape View File

@ -1,41 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.intls; import gplx.*; import gplx.core.*;
public class Gfo_i18n_mgr {
public String Dflt() {return dflt;} private String dflt = "en";
public Gfo_i18n_mgr Add_txt_dflt(String key, String val) {return this;}
public Gfo_i18n_mgr Add_txt_one(String key, String lng, String val) {return this;}
public Gfo_i18n_mgr Add_txt_many(String key, String... ary) {return this;}
}
class Gfo_i18n_lng {
private Hash_adp_bry hash = Hash_adp_bry.cs();
public Gfo_i18n_lng(String lng) {this.lng = lng;}
public String Lng() {return lng;} private final String lng;
public void Add(int src, byte[] key, byte[] val, boolean val_fmt_exists, Gfo_i18n_val_cmd val_cmd) {
Gfo_i18n_itm itm = new Gfo_i18n_itm(src, key, val, val_fmt_exists, val_cmd);
hash.Add_bry_obj(key, itm);
}
public byte[] Bld_to_bry_none(byte[] key) {
Gfo_i18n_itm itm = (Gfo_i18n_itm)hash.Get_by_bry(key);
return itm.Bld_none();
}
public byte[] Bld_to_bry_many(byte[] key, Object... args) {
Gfo_i18n_itm itm = (Gfo_i18n_itm)hash.Get_by_bry(key);
return itm.Bld_many(args);
}
}

4
400_xowa/src/gplx/core/ios/Io_sort.java
Unescape View File

@ -37,13 +37,13 @@ public class Io_sort {
usr_dlg.Prog_one(GRP_KEY, "write", "writing chunk: ~{0}", trg_url.Raw());
Split_flush(trg_url, row_list, memory_max, bfr, rv);
row_list.Resize_bounds(16); // MEM: resize bounds manually; note that each Flush-set may have widely disparately #of rows (EX: 1 row with a million pages vs. 1 million rows with 1 page)
size_new = size_row; Env_.GarbageCollect();
size_new = size_row; System_.Garbage_collect();
if (!reading) break;
}
row_list.Add(new Io_sort_split_itm(rdr));
size_cur = size_new;
}
rdr.Rls(); bfr.Rls(); Env_.GarbageCollect();
rdr.Rls(); bfr.Rls(); System_.Garbage_collect();
return (Io_url[])rv.To_ary(Io_url.class);
}
public void Merge(Gfo_usr_dlg usr_dlg, Io_url[] src_ary, ComparerAble comparer, Io_line_rdr_key_gen key_gen, Io_sort_cmd cmd) {

4
400_xowa/src/gplx/dbs/percentiles/Percentile_rng.java
Unescape View File

@ -35,7 +35,7 @@ public class Percentile_rng {
public void Select_init(int total_needed, int prv_score_bgn, int prv_score_len, int score_len_adj) {
this.total_needed = total_needed;
this.found_all = 0;
this.prv_time = gplx.core.envs.Env_.TickCount();
this.prv_time = gplx.core.envs.System_.Ticks();
int score_unit = Calc_score_unit(total_needed, total_max, score_max);
if (prv_score_bgn == Score_null) {
score_len = score_unit + (score_unit * score_len_adj);
@ -69,7 +69,7 @@ public class Percentile_rng {
Rng_len_(Bool_.N);
// update times
long new_time = gplx.core.envs.Env_.TickCount();
long new_time = gplx.core.envs.System_.Ticks();
this.elapsed = Int_.Subtract_long(new_time, prv_time);
prv_time = new_time;
}

3
400_xowa/src/gplx/langs/gfs/Gfs_parser_ctx.java
Unescape View File

@ -122,5 +122,6 @@ class Gfs_err_mgr {
default: bfr.Add_byte(b); break;
}
}
} static final byte[] Esc_nl = Bry_.new_a7("\\n"), Esc_cr = Bry_.new_a7("\\r"), Esc_tab = Bry_.new_a7("\\t");
}
private static final byte[] Esc_nl = Bry_.new_a7("\\n"), Esc_cr = Bry_.new_a7("\\r"), Esc_tab = Bry_.new_a7("\\t");
}

131
400_xowa/src/gplx/langs/htmls/Gfh_tag_.java
Unescape View File

@ -18,62 +18,66 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.langs.htmls; import gplx.*; import gplx.langs.*;
public class Gfh_tag_ { // NOTE: not serialized; used by tag_rdr
public static final int
Id__comment = -3
, Id__eos = -2
, Id__any = -1
, Id__unknown = 0
, Id__h1 = 1
, Id__h2 = 2
, Id__h3 = 3
, Id__h4 = 4
, Id__h5 = 5
, Id__h6 = 6
, Id__a = 7
, Id__span = 8
, Id__div = 9
, Id__img = 10
, Id__ul = 11
, Id__ol = 12
, Id__li = 13
, Id__dd = 14
, Id__dt = 15
, Id__p = 16
, Id__br = 17
, Id__hr = 18
, Id__table = 19
, Id__tr = 20
, Id__td = 21
, Id__th = 22
, Id__thead = 23
, Id__tbody = 24
, Id__caption = 25
, Id__pre = 26
, Id__small = 27
, Id__i = 28
, Id__b = 29
, Id__sup = 30
, Id__sub = 31
, Id__bdi = 32
, Id__font = 33
, Id__strong = 34
, Id__s = 35
, Id__abbr = 36
, Id__cite = 37
, Id__var = 38
, Id__u = 39
, Id__big = 40
, Id__del = 41
, Id__strike = 42
, Id__tt = 43
, Id__code = 44
, Id__wbr = 45
, Id__center = 46 // en.v:Vandalism_in_progress
, Id__dfn = 47
, Id__kbd = 48
, Id__samp = 49
, Id__ins = 50
, Id__em = 51
Id__comment = -3
, Id__eos = -2
, Id__any = -1
, Id__unknown = 0
, Id__h1 = 1
, Id__h2 = 2
, Id__h3 = 3
, Id__h4 = 4
, Id__h5 = 5
, Id__h6 = 6
, Id__a = 7
, Id__span = 8
, Id__div = 9
, Id__img = 10
, Id__ul = 11
, Id__ol = 12
, Id__li = 13
, Id__dd = 14
, Id__dt = 15
, Id__p = 16
, Id__br = 17
, Id__hr = 18
, Id__table = 19
, Id__tr = 20
, Id__td = 21
, Id__th = 22
, Id__thead = 23
, Id__tbody = 24
, Id__caption = 25
, Id__pre = 26
, Id__small = 27
, Id__i = 28
, Id__b = 29
, Id__sup = 30
, Id__sub = 31
, Id__bdi = 32
, Id__font = 33
, Id__strong = 34
, Id__s = 35
, Id__abbr = 36
, Id__cite = 37
, Id__var = 38
, Id__u = 39
, Id__big = 40
, Id__del = 41
, Id__strike = 42
, Id__tt = 43
, Id__code = 44
, Id__wbr = 45
, Id__center = 46 // not HTML5, but used by en.v:Vandalism_in_progress
, Id__dfn = 47
, Id__kbd = 48
, Id__samp = 49
, Id__ins = 50
, Id__em = 51
, Id__blockquote = 52
, Id__map = 53
, Id__bdo = 54
;
// private static final int Id__ary_max = 54;
public static final byte[]
Bry__a = Bry_.new_a7("a")
, Bry__ul = Bry_.new_a7("ul")
@ -85,7 +89,18 @@ public class Gfh_tag_ { // NOTE: not serialized; used by tag_rdr
, Bry__script = Bry_.new_a7("script")
, Bry__xowa_any = Bry_.new_a7("xowa_any")
, Bry__xowa_comment = Bry_.new_a7("xowa_comment")
, Bry__img = Bry_.new_a7("img")
;
// private static final Gfh_tag_meta[] Ary = new Gfh_tag_meta[Id__ary_max];
// private static final Hash_adp_bry tags_by_bry = Hash_adp_bry.ci_a7();
// public static Gfh_tag_meta New_tag(int id, String key_str) {
// Gfh_tag_meta rv = new Gfh_tag_meta(id, key_str);
// Ary[id] = rv;
// tags_by_bry.Add_bry_int(rv.Key_bry(), id);
// return rv;
// }
public static final Hash_adp_bry Hash = Hash_adp_bry.ci_a7()
.Add_bry_int(Bry__a , Id__a)
.Add_str_int("h1" , Id__h1)
@ -138,6 +153,9 @@ public class Gfh_tag_ { // NOTE: not serialized; used by tag_rdr
.Add_str_int("samp" , Id__samp)
.Add_str_int("ins" , Id__ins)
.Add_str_int("em" , Id__em)
.Add_str_int("blockquote" , Id__blockquote)
.Add_str_int("map" , Id__map)
.Add_str_int("bdo" , Id__bdo)
;
public static String To_str(int tid) {
switch (tid) {
@ -196,6 +214,9 @@ public class Gfh_tag_ { // NOTE: not serialized; used by tag_rdr
case Id__samp: return "samp";
case Id__ins: return "ins";
case Id__em: return "em";
case Id__blockquote: return "blockquote";
case Id__map: return "map";
case Id__bdo: return "bdo";
default: throw Err_.new_unhandled(tid);
}
}

28
400_xowa/src/gplx/langs/htmls/Gfh_tag_meta.java
Unescape View File

@ -0,0 +1,28 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.langs.htmls; import gplx.*; import gplx.langs.*;
public class Gfh_tag_meta {
public Gfh_tag_meta(int id, String key_str) {
this.id = id;
this.key_str = key_str;
this.key_bry = Bry_.new_u8(key_str);
}
public int Id() {return id;} private final int id;
public String Key_str() {return key_str;} private final String key_str;
public byte[] Key_bry() {return key_bry;} private final byte[] key_bry;
}

2
400_xowa/src/gplx/langs/htmls/Gfh_utl.java
Unescape View File

@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.langs.htmls; import gplx.*; import gplx.langs.*;
import gplx.core.primitives.*; import gplx.core.btries.*; import gplx.langs.htmls.encoders.*;
import gplx.langs.htmls.entitys.*;
public class Gfh_utl {
public class Gfh_utl {// TS:Gfo_url_encoder is TS
private static final Gfo_url_encoder encoder_id = Gfo_url_encoder_.Id;
public static String Encode_id_as_str(byte[] key) {return String_.new_u8(Encode_id_as_bry(key));}
public static byte[] Encode_id_as_bry(byte[] key) {

2
400_xowa/src/gplx/langs/htmls/docs/Gfh_tag.java
Unescape View File

@ -20,6 +20,7 @@ import gplx.xowa.parsers.htmls.*; import gplx.langs.htmls.styles.*; import gplx.
public class Gfh_tag implements Mwh_atr_wkr {
private Gfh_tag_rdr tag_rdr;
private Ordered_hash atrs_hash; private boolean atrs_null; private int atrs_bgn, atrs_end;
private final Gfh_style_wkr__val_as_int style_wkr = new Gfh_style_wkr__val_as_int();
public Gfh_tag Init(Gfh_tag_rdr tag_rdr, byte[] src, boolean tag_is_tail, boolean tag_is_inline, int src_bgn, int src_end, int atrs_bgn, int atrs_end, int name_id, byte[] name_bry) {
this.tag_rdr = tag_rdr; this.src = src; this.atrs_null = true;
this.tag_is_tail = tag_is_tail; this.tag_is_inline = tag_is_inline;
@ -79,7 +80,6 @@ public class Gfh_tag implements Mwh_atr_wkr {
byte rv = Gfh_class_.Find_1st(src, cls_atr.Val_bgn(), cls_atr.Val_end(), hash); if (rv == Byte_.Max_value_127) return or;
return rv;
}
private static final Gfh_style_wkr__val_as_int style_wkr = new Gfh_style_wkr__val_as_int();
public int Atrs__style_get_as_int(byte[] key) {
if (atrs_null) Atrs__make();
Gfh_atr rv = (Gfh_atr)atrs_hash.Get_by(Gfh_atr_.Bry__style); if (rv == null) return -1;

1
400_xowa/src/gplx/langs/htmls/docs/Gfh_tag_rdr.java
Unescape View File

@ -113,6 +113,7 @@ public class Gfh_tag_rdr {
}
}
public Gfh_tag Tag__move_fwd_tail(byte[] find_tag_bry) {return Tag__find(Bool_.Y, Bool_.N, Bool_.Y, pos, src_end, find_tag_bry);}
public Gfh_tag Tag__peek_fwd_tail(byte[] find_tag_bry) {return Tag__find(Bool_.N, Bool_.N, Bool_.Y, pos, src_end, find_tag_bry);}
private Gfh_tag Tag__find(boolean move, boolean bwd, boolean tail, int rng_bgn, int rng_end, byte[] find_tag_bry) {
int tmp = rng_bgn;
int stop_pos = rng_end; int adj = 1;

2
400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder.java
Unescape View File

@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.langs.htmls.encoders; import gplx.*; import gplx.langs.*; import gplx.langs.htmls.*;
import gplx.core.btries.*;
import gplx.langs.htmls.*;
public class Gfo_url_encoder implements Url_encoder_interface {
public class Gfo_url_encoder implements Url_encoder_interface { // TS; Gfo_url_encoder_itm[] are read-only; anchor_encoder is effectively read-only
private final Gfo_url_encoder_itm[] encode_ary, decode_ary; private final Gfo_url_encoder anchor_encoder;
public Gfo_url_encoder(Gfo_url_encoder_itm[] encode_ary, Gfo_url_encoder_itm[] decode_ary, Gfo_url_encoder anchor_encoder) {
this.encode_ary = encode_ary; this.decode_ary = decode_ary; this.anchor_encoder = anchor_encoder;

4
400_xowa/src/gplx/langs/htmls/styles/Gfh_style_parser__tst.java
Unescape View File

@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.langs.htmls.styles; import gplx.*; import gplx.langs.*; import gplx.langs.htmls.*;
import org.junit.*;
public class Gfh_style_parser__tst {
private final Gfh_style_parser__fxt fxt = new Gfh_style_parser__fxt();
private final Gfh_style_parser__fxt fxt = new Gfh_style_parser__fxt();
@Test public void Basic() {
fxt.Test__parse("k_0:v_0" , fxt.Make("k_0", "v_0"));
fxt.Test__parse("k_0:v_0;" , fxt.Make("k_0", "v_0"));
@ -41,7 +41,7 @@ public class Gfh_style_parser__tst {
}
}
class Gfh_style_parser__fxt {
private final Gfh_style_wkr__ary wkr = Gfh_style_wkr__ary.Instance;
private final Gfh_style_wkr__ary wkr = new Gfh_style_wkr__ary();
public Gfh_style_itm Make(String k, String v) {return new Gfh_style_itm(-1, Bry_.new_u8(k), Bry_.new_u8(v));}
public void Test__parse(String src_str, Gfh_style_itm... expd) {
byte[] src_bry = Bry_.new_u8(src_str);

1
400_xowa/src/gplx/langs/htmls/styles/Gfh_style_wkr__ary.java
Unescape View File

@ -28,5 +28,4 @@ public class Gfh_style_wkr__ary implements Gfh_style_wkr {
Gfh_style_parser_.Parse(src, src_bgn, src_end, this);
return (Gfh_style_itm[])list.To_ary_and_clear(Gfh_style_itm.class);
}
public static final Gfh_style_wkr__ary Instance = new Gfh_style_wkr__ary(); Gfh_style_wkr__ary() {}
}

1
400_xowa/src/gplx/langs/jsons/Json_doc.java
Unescape View File

@ -50,6 +50,7 @@ public class Json_doc {
Json_itm rv = Find_nde(root_nde, tmp_qry_bry, 0, 0); if (rv == null) return null;
return (Json_grp)rv;
}
public Json_grp Get_grp_many(String... qry_ary) {return Get_grp_many(Bry_.Ary(qry_ary));}
public Json_grp Get_grp_many(byte[]... qry_bry) {
Json_itm rv = Find_nde(root_nde, qry_bry, qry_bry.length - 1, 0); if (rv == null) return null;
return (Json_grp)rv;

11
400_xowa/src/gplx/langs/phps/Php_srl_itm.java
Unescape View File

@ -25,10 +25,10 @@ interface Php_srl_itm {
void Clear();
}
class Php_srl_itm_ {
public static final Php_srl_itm[] Ary_empty = new Php_srl_itm[0];
public static final Php_srl_itm[] Ary_empty = new Php_srl_itm[0];
public static final byte Tid_unknown = 0, Tid_nil = 1, Tid_bool = 2, Tid_int = 3, Tid_double = 4, Tid_string = 5, Tid_array = 6, Tid_function = 7;
public static final byte[][] Names = Bry_.Ary("unknown", "nil", "boolean", "int", "double", "string", "array", "function");
public static final Object Val_nil = null, Val_table = null;
public static final byte[][] Names = Bry_.Ary("unknown", "nil", "boolean", "int", "double", "string", "array", "function");
public static final Object Val_nil = null, Val_table = null;
}
abstract class Php_srl_itm_base implements Php_srl_itm {
public abstract byte Tid();
@ -114,7 +114,8 @@ class Php_srl_itm_ary extends Php_srl_itm_base {
subs[i].Xto_bfr(bfr, depth + 1);
Php_srl_wtr.Indent(bfr, depth);
bfr.Add_byte(Byte_ascii.Curly_end).Add_byte_nl();
} static final byte[] CONST_ary_bgn = Bry_.new_a7("]{\n");
}
private static final byte[] CONST_ary_bgn = Bry_.new_a7("]{\n");
Php_srl_itm_kv[] subs = Php_srl_itm_kv.Ary_empty;
}
class Php_srl_itm_kv {
@ -129,7 +130,7 @@ class Php_srl_itm_kv {
key.Xto_bfr(bfr, depth);
val.Xto_bfr(bfr, depth);
}
public static final Php_srl_itm_kv[] Ary_empty = new Php_srl_itm_kv[0];
public static final Php_srl_itm_kv[] Ary_empty = new Php_srl_itm_kv[0];
}
class Php_srl_wtr {
public static void Indent(Bry_bfr bfr, int depth) {

2
400_xowa/src/gplx/langs/phps/Php_srl_parser.java
Unescape View File

@ -142,7 +142,7 @@ public class Php_srl_parser {
default: throw err_(raw, pos, "unexpected type: {0}", Char_.To_str(b));
}
return rv;
} static final byte[] CONST_funct_bgn = Bry_.new_a7("O:42:\"Scribunto_LuaStandaloneInterpreterFunction\":1:{s:2:\"id\";i:"), CONST_funct_end = Bry_.new_a7(";}");
}
int Parse_int_val(int bgn) {
pos = bgn;
pos = Chk(raw, pos + 1, Byte_ascii.Colon);

1
400_xowa/src/gplx/xowa/Xoa_app.java
Unescape View File

@ -43,7 +43,6 @@ public interface Xoa_app extends Gfo_invk {
Xof_img_mgr File__img_mgr();
Io_download_fmt File__download_fmt();
Xoh_href_parser Html__href_parser();
Xoh_lnki_bldr Html__lnki_bldr();
Xoa_css_extractor Html__css_installer();
Xoh_bridge_mgr Html__bridge_mgr();
Xog_cbk_mgr Gui__cbk_mgr();

2
400_xowa/src/gplx/xowa/Xoa_app_.java
Unescape View File

@ -32,7 +32,7 @@ public class Xoa_app_ {
}
}
public static final String Name = "xowa";
public static final String Version = "3.8.2.1";
public static final String Version = "3.8.5.1";
public static String Build_date = "2012-12-30 00:00:00";
public static String Op_sys_str;
public static String User_agent = "";

1
400_xowa/src/gplx/xowa/Xoa_page.java
Unescape View File

@ -27,7 +27,6 @@ public interface Xoa_page {
Xopg_hdump_data Hdump_mgr();
Xoa_page__commons_mgr Commons_mgr();
void Xtn_gallery_packed_exists_y_();
boolean Xtn__timeline_exists();
boolean Xtn__gallery_exists();
}

5
400_xowa/src/gplx/xowa/Xoae_app.java
Unescape View File

@ -31,7 +31,6 @@ import gplx.xowa.bldrs.wms.*;
import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.tdbs.hives.*; import gplx.xowa.wikis.xwikis.*;
import gplx.xowa.addons.*; import gplx.xowa.specials.mgrs.*;
public class Xoae_app implements Xoa_app, Gfo_invk {
private final Xoh_href_wtr html__href_wtr = new Xoh_href_wtr();
public Xoae_app(Gfo_usr_dlg usr_dlg, Xoa_app_mode mode, Io_url root_dir, Io_url wiki_dir, Io_url file_dir, Io_url user_dir, Io_url css_dir, String bin_dir_name) {
Xoa_app_.Usr_dlg_(usr_dlg);
this.mode = mode;
@ -65,7 +64,6 @@ public class Xoae_app implements Xoa_app, Gfo_invk {
http_server = new Http_server_mgr(this);
cfg_regy = new Xocfg_regy(this);
html_mgr = new Xoh_html_mgr(this);
this.html__lnki_bldr = new Xoh_lnki_bldr(this, html__href_wtr);
this.html__bridge_mgr = new Xoh_bridge_mgr(utl__json_parser);
this.site_cfg_mgr = new Xoa_site_cfg_mgr(this);
}
@ -76,7 +74,6 @@ public class Xoae_app implements Xoa_app, Gfo_invk {
public Xof_img_mgr File__img_mgr() {return file_mgr.Img_mgr();}
public Io_download_fmt File__download_fmt() {return wmf_mgr.Download_wkr().Download_xrg().Download_fmt();}
public Xoh_href_parser Html__href_parser() {return html__href_parser;} private final Xoh_href_parser html__href_parser = new Xoh_href_parser();
public Xoh_lnki_bldr Html__lnki_bldr() {return html__lnki_bldr;} private final Xoh_lnki_bldr html__lnki_bldr;
public Xoa_css_extractor Html__css_installer() {return html__css_installer;} private final Xoa_css_extractor html__css_installer = new Xoa_css_extractor();
public Xoh_bridge_mgr Html__bridge_mgr() {return html__bridge_mgr;} private final Xoh_bridge_mgr html__bridge_mgr;
public Xowmf_mgr Wmf_mgr() {return wmf_mgr;} private final Xowmf_mgr wmf_mgr = new Xowmf_mgr();
@ -191,7 +188,7 @@ public class Xoae_app implements Xoa_app, Gfo_invk {
public void Reset_all() {
this.Free_mem(true);
gplx.xowa.xtns.scribunto.Scrib_core_mgr.Term_all(this);
Env_.GarbageCollect();
System_.Garbage_collect();
}
public void Free_mem(boolean clear_ctx) {
this.Utl__bfr_mkr().Clear();

20
400_xowa/src/gplx/xowa/Xoae_page.java
Unescape View File

@ -27,7 +27,7 @@ public class Xoae_page implements Xoa_page {
this.wiki = wiki; this.ttl = ttl;
this.lang = wiki.Lang(); // default to wiki.lang; can be override later by wikitext
html.Init_by_page(ttl);
html.Toc_mgr().Init(wiki.Lang().Msg_mgr().Itm_by_id_or_null(gplx.xowa.langs.msgs.Xol_msg_itm_.Id_toc).Val(), ttl.Page_db());
html.Toc_mgr().Init(wiki.Html_mgr().Tidy_mgr(), wiki.Lang().Msg_mgr().Itm_by_id_or_null(gplx.xowa.langs.msgs.Xol_msg_itm_.Id_toc).Val(), ttl.Page_db());
Ttl_(ttl);
} Xoae_page() {} // called by Empty
public Xow_wiki Wiki() {return wiki;}
@ -61,23 +61,8 @@ public class Xoae_page implements Xoa_page {
public byte[][] Category_list() {return category_list;} public Xoae_page Category_list_(byte[][] v) {category_list = v; return this;} private byte[][] category_list = new byte[0][];
public List_adp Slink_list() {return slink_list;} private List_adp slink_list = List_adp_.New();
public Wdata_external_lang_links_data Wdata_external_lang_links() {return wdata_external_lang_links;} private Wdata_external_lang_links_data wdata_external_lang_links = new Wdata_external_lang_links_data();
public boolean Pages_recursed() {return pages_recursed;} public void Pages_recursed_(boolean v) {pages_recursed = v; } private boolean pages_recursed;
public int Bldr__ns_ord() {return bldr__ns_ord;} public void Bldr__ns_ord_(int v) {bldr__ns_ord = v;} private int bldr__ns_ord;
public Xopg_tmpl_prepend_mgr Tmpl_prepend_mgr() {return tmpl_prepend_mgr;} private Xopg_tmpl_prepend_mgr tmpl_prepend_mgr = new Xopg_tmpl_prepend_mgr();
public void Tmpl_stack_del() {--tmpl_stack_ary_len;}
public boolean Tmpl_stack_add(byte[] key) {
for (int i = 0; i < tmpl_stack_ary_len; i++) {
if (Bry_.Match(key, tmpl_stack_ary[i])) return false;
}
int new_len = tmpl_stack_ary_len + 1;
if (new_len > tmpl_stack_ary_max) {
tmpl_stack_ary_max = new_len * 2;
tmpl_stack_ary = (byte[][])Array_.Resize(tmpl_stack_ary, tmpl_stack_ary_max);
}
tmpl_stack_ary[tmpl_stack_ary_len] = key;
tmpl_stack_ary_len = new_len;
return true;
} private byte[][] tmpl_stack_ary = Bry_.Ary_empty; private int tmpl_stack_ary_len = 0, tmpl_stack_ary_max = 0;
public void Clear_all() {Clear(true);}
public void Clear(boolean clear_scrib) { // NOTE: this is called post-fetch but pre-wtxt; do not clear items set by post-fetch, such as id, ttl, redirected_ttls, data_raw
db.Clear();
@ -95,9 +80,6 @@ public class Xoae_page implements Xoa_page {
if (clear_scrib) wiki.Parser_mgr().Scrib().When_page_changed(this);
slink_list.Clear();
tab_data.Clear();
pages_recursed = false;
tmpl_stack_ary = Bry_.Ary_empty;
tmpl_stack_ary_len = tmpl_stack_ary_max = 0;
popup_mgr.Clear();
tmpl_prepend_mgr.Clear();
commons_mgr.Clear();

1
400_xowa/src/gplx/xowa/Xow_wiki.java
Unescape View File

@ -45,6 +45,7 @@ public interface Xow_wiki extends Xow_ttl_parser, Gfo_invk {
Xow_hdump_mgr Html__hdump_mgr();
Xoh_page_wtr_mgr Html__wtr_mgr();
Xoh_href_wtr Html__href_wtr();
Xoh_lnki_bldr Html__lnki_bldr();
boolean Html__css_installing(); void Html__css_installing_(boolean v);
Xow_mw_parser_mgr Mw_parser_mgr();
Xow_xwiki_mgr Xwiki_mgr();

4
400_xowa/src/gplx/xowa/Xowe_wiki.java
Unescape View File

@ -77,6 +77,7 @@ public class Xowe_wiki implements Xow_wiki, Gfo_invk, Gfo_evt_itm {
maint_mgr = new Xow_maint_mgr(this);
cache_mgr = new Xow_cache_mgr(this);
/*if (Bry_.Eq(domain_bry, Xow_domain_itm_.Bry__home))*/ xwiki_mgr.Add_by_atrs(domain_bry, domain_bry); // add full name to xwiki_mgr; needed for lookup in home ns; EX: [[home:Help/Contents]]
this.lnki_bldr = new Xoh_lnki_bldr(app, href_wtr);
}
public Gfo_evt_mgr Evt_mgr() {return ev_mgr;} private final Gfo_evt_mgr ev_mgr;
public Xow_ns_mgr Ns_mgr() {return ns_mgr;} private final Xow_ns_mgr ns_mgr;
@ -102,7 +103,8 @@ public class Xowe_wiki implements Xow_wiki, Gfo_invk, Gfo_evt_itm {
public Fsm_mnt_mgr File__mnt_mgr() {return file_mgr.Fsdb_mgr().Mnt_mgr();}
public boolean Html__hdump_enabled() {return html_mgr__hdump_enabled;} private boolean html_mgr__hdump_enabled = Bool_.N;
public Xoh_page_wtr_mgr Html__wtr_mgr() {return html_mgr.Page_wtr_mgr();}
public Xoh_href_wtr Html__href_wtr() {return html__href_wtr;} private final Xoh_href_wtr html__href_wtr = new Xoh_href_wtr();
public Xoh_lnki_bldr Html__lnki_bldr() {return lnki_bldr;} private final Xoh_lnki_bldr lnki_bldr;
public Xoh_href_wtr Html__href_wtr() {return href_wtr;} private final Xoh_href_wtr href_wtr = new Xoh_href_wtr();
public boolean Html__css_installing() {return html__css_installing;} public void Html__css_installing_(boolean v) {html__css_installing = v;} private boolean html__css_installing;
public Xow_url_parser Utl__url_parser() {return url__parser;} private final Xow_url_parser url__parser;
public Xow_mw_parser_mgr Mw_parser_mgr() {return mw_parser_mgr;} private final Xow_mw_parser_mgr mw_parser_mgr = new Xow_mw_parser_mgr();

2
400_xowa/src/gplx/xowa/Xowe_wiki_.java
Unescape View File

@ -27,7 +27,7 @@ public class Xowe_wiki_ {
}
public static void Rls_mem_if_needed(Xowe_wiki wiki) {
Xoae_app app = wiki.Appe();
if (gplx.core.envs.Env_.System_memory_free() < app.Sys_cfg().Free_mem_when()) // check if low in memory
if (gplx.core.envs.Runtime_.Memory_free() < app.Sys_cfg().Free_mem_when()) // check if low in memory
Xowe_wiki_.Rls_mem(wiki, false); // clear caches (which will clear bry_bfr_mkr)
else // not low in memory
wiki.Utl__bfr_mkr().Clear(); // clear bry_bfr_mkr only; NOTE: call before page parse, not when page is first added, else threading errors; DATE:2014-05-30

1
400_xowa/src/gplx/xowa/addons/Xoax_addon_mgr.java
Unescape View File

@ -53,6 +53,7 @@ public class Xoax_addon_mgr {
, new gplx.xowa.addons.bldrs.xodirs .Xobc_xodir_addon()
, new gplx.xowa.addons.bldrs.centrals .Xobc_task_addon()
, new gplx.xowa.addons.apps.helps.logs .Xolog_addon()
, new gplx.xowa.addons.wikis.pages.syncs .Sync_addon()
// jsons
);

11
400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__base.java
Unescape View File

@ -17,12 +17,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*;
import gplx.core.gfobjs.*; import gplx.core.progs.*; import gplx.core.progs.rates.*;
import gplx.xowa.apps.apis.*;
public abstract class Xobc_cmd__base implements Xobc_cmd_itm {
private final Xobc_task_mgr task_mgr;
private final Gfo_rate_list rate_list; private final long notify_delay = 1000;
private final double delta_threshold = .25d; // allow variance of up to 25% before updating rate
private long time_prv;
private double rate_cur;
private boolean log_verbose;
public Xobc_cmd__base(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_id) {
this.task_mgr = task_mgr; this.task_id = task_id; this.step_id = step_id; this.cmd_id = cmd_id;
this.cmd_uid = String_.Concat_with_str(":", Int_.To_str(task_id), Int_.To_str(step_id), Int_.To_str(cmd_id));
@ -51,9 +53,12 @@ public abstract class Xobc_cmd__base implements Xobc_cmd_itm {
public void Cmd_exec(Xobc_cmd_ctx ctx) {
// rate_list.Clear(); this.rate_cur = 0; // TOMBSTONE: do not reset rate else pause and resume will show different numbers
Xoapi_root api_root = task_mgr.App().Api_root();
if (api_root != null)
this.log_verbose = api_root.Addon().Bldr().Central().Log_verbose();
try {
Gfo_log_.Instance.Info("xobc_cmd task bgn", "task_id", task_id, "step_id", step_id, "cmd_id", cmd_id);
this.time_prv = gplx.core.envs.Env_.TickCount();
this.time_prv = gplx.core.envs.System_.Ticks();
this.status = Gfo_prog_ui_.Status__working;
this.Cmd_exec_hook(ctx);
Gfo_log_.Instance.Info("xobc_cmd task end", "task_id", task_id, "step_id", step_id, "cmd_id", cmd_id);
@ -106,7 +111,7 @@ public abstract class Xobc_cmd__base implements Xobc_cmd_itm {
public boolean Prog_notify_and_chk_if_suspended(long new_data_cur, long new_data_end) {
if (status == Gfo_prog_ui_.Status__suspended) return true; // task paused by ui; exit now;
long time_cur = gplx.core.envs.Env_.TickCount();
long time_cur = gplx.core.envs.System_.Ticks();
if (time_cur < time_prv + notify_delay) return false; // message came too soon. ignore it
// update rate
@ -130,6 +135,8 @@ public abstract class Xobc_cmd__base implements Xobc_cmd_itm {
task_mgr.Send_json("xo.bldr.work.prog__update__recv", Gfobj_nde.New()
.Add_int ("task_id", task_id).Add_long("prog_data_cur", data_cur).Add_long("prog_data_end", data_end).Add_int("prog_rate", (int)rate_cur));
if (log_verbose)
Gfo_usr_dlg_.Instance.Note_many("", "", "xobc:notify: task_id=~{0} cmd_id=~{1} prog_data_cur=~{2} prog_data_end=~{3} rate_cur=~{4}", task_id, cmd_id, data_cur, data_end, rate_cur);
return false;
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {

8
400_xowa/src/gplx/xowa/addons/bldrs/exports/merges/Merge2_mgr.java
Unescape View File

@ -43,7 +43,7 @@ public class Merge2_mgr {
// fails b/c no Main_Page; Gfo_invk_.Invk_by_msg(wiki.App().Gui__tab_mgr(), gplx.xowa.guis.tabs.Xog_tab_mgr_.Invk__new_tab, GfoMsg_.new_cast_("").Add("focus", true).Add("site", wiki.Domain_str()).Add("page", String_.new_u8(wiki.Props().Main_page())));
}
public void Merge_data(Xow_wiki wiki, Io_url src_url, int idx_cur) {
long all_time_bgn = gplx.core.envs.Env_.TickCount();
long all_time_bgn = gplx.core.envs.System_.Ticks();
wiki.Init_by_wiki();
Db_conn src_conn = Db_conn_bldr.Instance.Get_or_autocreate(false, src_url);
ctx.Init(wiki, src_conn);
@ -54,12 +54,12 @@ public class Merge2_mgr {
if (prog_wkr.Canceled()) break;
Merge2_wkr wkr = (Merge2_wkr)wkr_hash.Get_at(i);
// if (prog_wkr.Checkpoint__skip_wkr(src_url, wkr.Tbl_name())) continue;
long wkr_time_bgn = gplx.core.envs.Env_.TickCount();
long wkr_time_bgn = gplx.core.envs.System_.Ticks();
wkr.Merge_data(ctx, prog_wkr);
Gfo_log_.Instance.Info("merge.wkr.done", "data", src_url.NameAndExt() + "|" + wkr.Tbl().Tbl_name() + "|" + gplx.core.envs.Env_.TickCount_elapsed_in_frac(wkr_time_bgn));
Gfo_log_.Instance.Info("merge.wkr.done", "data", src_url.NameAndExt() + "|" + wkr.Tbl().Tbl_name() + "|" + gplx.core.envs.System_.Ticks__elapsed_in_frac(wkr_time_bgn));
}
if (ctx.Heap__copy_to_wiki()) ctx.Heap__increment_nxt();
Gfo_log_.Instance.Info("merge.wkr.done", "data", src_url.NameAndExt() + "|-1|" + gplx.core.envs.Env_.TickCount_elapsed_in_frac(all_time_bgn));
Gfo_log_.Instance.Info("merge.wkr.done", "data", src_url.NameAndExt() + "|-1|" + gplx.core.envs.System_.Ticks__elapsed_in_frac(all_time_bgn));
src_conn.Rls_conn(); // NOTE: must close conn else pack_conn will stay open
}
private static Ordered_hash Make_wkrs(Merge2_wkr... wkrs) {

4
400_xowa/src/gplx/xowa/addons/bldrs/exports/merges/Merge_prog_wkr.java
Unescape View File

@ -45,7 +45,7 @@ public class Merge_prog_wkr implements Db_bulk_prog {
this.cur_fil = fil;
this.cur_wkr_tid = wkr_tid;
this.Checkpoint__save();
time_nxt = gplx.core.envs.Env_.TickCount() + time_gap;
time_nxt = gplx.core.envs.System_.Ticks() + time_gap;
return false;
}
public void Checkpoint__save() {
@ -54,7 +54,7 @@ public class Merge_prog_wkr implements Db_bulk_prog {
}
public boolean Prog__insert_and_stop_if_suspended(int row_size) {
++prog_count_cur;
long time_cur = gplx.core.envs.Env_.TickCount();
long time_cur = gplx.core.envs.System_.Ticks();
if (time_cur < time_nxt) return false;
// gplx.core.threads.Thread_adp_.Sleep(10);
time_nxt = time_cur + time_gap;

6
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__fsdb_db__create_data.java
Unescape View File

@ -88,7 +88,7 @@ public class Xobldr__fsdb_db__create_data extends Xob_cmd__base implements Xob_c
}
@Override public void Cmd_run() {
Init_bldr_bmks();
this.time_bgn = Env_.TickCount();
this.time_bgn = System_.Ticks();
int total_pending = Xob_xfer_regy_tbl.Select_total_pending(bldr_conn);
// if (total_pending > 250000 && src_bin_mgr__fsdb_version == null)
usr_dlg.Note_many("", "", "total pending: ~{0}", total_pending);
@ -277,7 +277,7 @@ public class Xobldr__fsdb_db__create_data extends Xob_cmd__base implements Xob_c
if (exit_after_commit) exit_now = true;
}
@Override public void Cmd_end() {
usr_dlg.Note_many("", "", "fsdb_make.done: count=~{0} rate=~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, Env_.TickCount_elapsed_in_sec(time_bgn)).To_str("#,###.000"));
usr_dlg.Note_many("", "", "fsdb_make.done: count=~{0} rate=~{1}", exec_count, Decimal_adp_.divide_safe_(exec_count, System_.Ticks__elapsed_in_sec(time_bgn)).To_str("#,###.000"));
if (src_fsdb_wkr != null) {
src_fsdb_wkr.Mnt_mgr().Mnts__get_main().Txn_end(); // NOTE: src_fsdb_wkr will be null if no src db defined
}
@ -295,7 +295,7 @@ public class Xobldr__fsdb_db__create_data extends Xob_cmd__base implements Xob_c
bldr_conn.Rls_conn();
}
private void Print_progress(Xodb_tbl_oimg_xfer_itm itm) {
int time_elapsed = Env_.TickCount_elapsed_in_sec(time_bgn);
int time_elapsed = System_.Ticks__elapsed_in_sec(time_bgn);
usr_dlg.Prog_many("", "", "prog: num=~{0} err=~{1} time=~{2} rate=~{3} page=~{4} lnki=~{5} ttl=~{6}", exec_count, exec_fail, time_elapsed, Math_.Div_safe_as_int(exec_count, time_elapsed), page_id_val, lnki_id_val, itm.Orig_ttl());
}
private void Delete_files() {}// TODO_OLD: purge /xowa/file/ dir to free up hard disk space

12
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__lnki_temp__create.java
Unescape View File

@ -26,7 +26,8 @@ import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*; import gplx.fsdb.*;
import gplx.xowa.langs.vnts.*; import gplx.xowa.parsers.vnts.*;
import gplx.xowa.parsers.lnkis.files.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.files.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.xowa.addons.bldrs.files.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*;
import gplx.xowa.addons.bldrs.wmdumps.imglinks.*;
public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx.xowa.parsers.lnkis.files.Xop_file_logger {
private Xob_lnki_temp_tbl tbl; private boolean wdata_enabled = true, xtn_ref_enabled = true, gen_html, gen_hdump;
private Xop_log_invoke_wkr invoke_wkr; private Xop_log_property_wkr property_wkr;
@ -61,6 +62,13 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // remove wmf wkr, else will try to download images during parsing
commons_wiki = app.Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons);
// create imglinks
Xof_orig_wkr__img_links orig_wkr = new Xof_orig_wkr__img_links(wiki);
wiki.File__orig_mgr().Wkrs__set(orig_wkr);
Xof_orig_wkr__img_links_.Load_all(orig_wkr);
Xow_wiki_utl_.Clone_repos(wiki);
// init log_mgr / property_wkr
Xop_log_mgr log_mgr = ctx.App().Log_mgr();
log_mgr.Log_dir_(wiki.Fsys_mgr().Root_dir()); // put log in wiki dir, instead of user.temp
@ -140,7 +148,7 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
Xof_ext ext = Xof_ext_.new_by_ttl_(ttl);
double lnki_time = lnki.Time();
int lnki_page = lnki.Page();
byte[] ttl_commons = Xomp_file_logger.To_commons_ttl(ns_file_is_case_match_all, commons_wiki, ttl);
byte[] ttl_commons = Xomp_lnki_temp_wkr.To_commons_ttl(ns_file_is_case_match_all, commons_wiki, ttl);
if ( Xof_lnki_page.Null_n(lnki_page) // page set
&& Xof_lnki_time.Null_n(lnki_time)) // thumbtime set
usr_dlg.Warn_many("", "", "page and thumbtime both set; this may be an issue with fsdb: page=~{0} ttl=~{1}", ctx.Page().Ttl().Page_db_as_str(), String_.new_u8(ttl));

2
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__lnki_temp__create__tst.java
Unescape View File

@ -38,6 +38,6 @@ class Xobldr__lnki_temp__create__fxt {
return this;
}
public void Test__to_commons(String ttl, String expd) {
Tfds.Eq(expd, String_.new_u8(gplx.xowa.addons.bldrs.mass_parses.parses.Xomp_file_logger.To_commons_ttl(wiki_ns_file_is_case_match_all, commons_wiki, Bry_.new_u8(ttl))));
Tfds.Eq(expd, String_.new_u8(gplx.xowa.addons.bldrs.mass_parses.parses.utls.Xomp_lnki_temp_wkr.To_commons_ttl(wiki_ns_file_is_case_match_all, commons_wiki, Bry_.new_u8(ttl))));
}
}

1
400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_lnki_temp_tbl.java
Unescape View File

@ -42,6 +42,7 @@ public class Xob_lnki_temp_tbl implements Db_tbl {
public String Tbl_name() {return tbl_name;}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Insert_bgn() {conn.Txn_bgn("bldr__lnki_temp"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_stmt_make() {stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_commit() {conn.Txn_sav();}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public void Insert_cmd_by_batch(int tier_id, int page_id, byte[] ttl, byte[] ttl_commons, byte ext_id, byte img_type, byte lnki_src_tid, int w, int h, double upright, double time, int page) {

2
400_xowa/src/gplx/xowa/addons/bldrs/hdumps/diffs/Dumpdiff_mgr.java
Unescape View File

@ -54,7 +54,7 @@ class Dumpdiff_mgr {
if (diff != null) {
log_tbl.Insert_by_batch(page_id, diff[0], diff[1]);
++diff_count;
Gfo_usr_dlg_.Instance.Warn_many("", "", "hdump diff: pages=~{0} diffs=~{1} page_id=~{2} cur=~{3} prv=~{4}", page_count, diff_count, page_id, diff[0], diff[1]);
Gfo_usr_dlg_.Instance.Warn_many("", "", "hdump diff: pages=~{0} diffs=~{1} page_id=~{2} lhs=~{3} rhs=~{4}", page_count, diff_count, page_id, diff[0], diff[1]);
}
}
list.Clear();

7
400_xowa/src/gplx/xowa/addons/bldrs/infos/Xobc_info_doc.java
Unescape View File

@ -37,13 +37,14 @@ class Xobc_info_doc implements Mustache_doc_itm {
}
}
class Xobc_info_url implements Mustache_doc_itm {
private final String url; private final byte[] size;
public Xobc_info_url(String url, byte[] size) {
this.url = url; this.size = size;
private final String url, md5; private final byte[] size;
public Xobc_info_url(String url, byte[] size, String md5) {
this.url = url; this.size = size; this.md5 = md5;
}
public boolean Mustache__write(String key, Mustache_bfr bfr) {
if (String_.Eq(key, "url")) bfr.Add_str_u8(url);
else if (String_.Eq(key, "size")) bfr.Add_bry(size);
else if (String_.Eq(key, "md5")) bfr.Add_str_u8(md5);
else return false;
return true;
}

4
400_xowa/src/gplx/xowa/addons/bldrs/infos/Xobc_info_html.java
Unescape View File

@ -48,7 +48,7 @@ class Xobc_info_html extends Xow_special_wtr__base {
String src_fil = host_eval.Eval_src_fil(data_db, host_id, wiki_domain, step_itm.Import_name);
Io_size_.To_bfr_new(tmp_size_bfr, step_itm.Import_size_zip, 2);
total_size += step_itm.Import_size_raw;
Xobc_info_url step_url = new Xobc_info_url(src_fil, tmp_size_bfr.To_bry_and_clear());
Xobc_info_url step_url = new Xobc_info_url(src_fil, tmp_size_bfr.To_bry_and_clear(), step_itm.Import_md5);
step_urls[i] = step_url;
}
@ -66,7 +66,7 @@ class Xobc_info_html extends Xow_special_wtr__base {
String src_dir = host_eval.Eval_src_dir(data_db, host_id, wiki_domain);
String[] ary = String_.Split(key, "|");
ary[0] = String_.new_u8(wiki_domain.Abrv_wm());
torrent_fil = String_.Format("{0}/Xowa_{1}_{2}_{3}.torrent", src_dir, ary[0], ary[1], ary[2]); // EX: http://archive.org/download/Xowa_enwiki_latest/Xowa_enwiktionary_2016-06_core.zip
torrent_fil = String_.Format("{0}Xowa_{1}_latest_archive.torrent", src_dir, wiki_domain.Abrv_wm(), ary[1]); // EX: https://archive.org/download/Xowa_dewiki_latest/Xowa_dewiki_latest_archive.torrent
}
return new Xobc_info_doc

60
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_db_core.java
Unescape View File

@ -1,60 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
public class Xomp_db_core {
private final Object thread_lock = new Object();
private final Io_url root_dir;
Xomp_db_core(Io_url root_dir) {
this.root_dir = root_dir;
Io_url mgr_url = root_dir.GenSubFil("xomp.sqlite3");
this.mgr_db = new Xomp_mgr_db(mgr_url);
}
public Db_conn Conn() {return mgr_db.Conn();}
public Xomp_mgr_db Mgr_db() {return mgr_db;} private Xomp_mgr_db mgr_db;
public Xomp_wkr_db Wkr_db(boolean delete, int idx) {
Io_url wkr_url = root_dir.GenSubFil_nest("xomp_" + Int_.To_str_fmt(idx, "000"), "xomp_wkr.sqlite3");
if (delete) Io_mgr.Instance.DeleteFil(wkr_url);
return new Xomp_wkr_db(idx, wkr_url);
}
public int Wkr_count() {
Io_url[] wkr_dirs = Io_mgr.Instance.QueryDir_args(root_dir).DirOnly_().ExecAsUrlAry();
return wkr_dirs.length;
}
public void Update_wkr_id(int idx, Db_conn wkr_conn) {
synchronized (thread_lock) {
Db_attach_mgr attach_mgr = new Db_attach_mgr(mgr_db.Conn(), new Db_attach_itm("wkr_db", wkr_conn));
attach_mgr.Exec_sql_w_msg("updating page_regy: wkr_id=" + idx, String_.Concat_lines_nl_skip_last // ANSI.Y
( "UPDATE xomp_page"
, "SET xomp_wkr_id = " + Int_.To_str(idx)
, ", html_len = (SELECT length(body) FROM <wkr_db>html h WHERE h.page_id = xomp_page.page_id)"
, "WHERE page_id IN (SELECT page_id FROM <wkr_db>html h)"
));
}
}
public static Xomp_db_core New__make(Xowe_wiki wiki) {
Io_url root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
Io_mgr.Instance.DeleteDirDeep(root_dir);
return new Xomp_db_core(root_dir);
}
public static Xomp_db_core New__load(Xowe_wiki wiki) {
Io_url root_dir = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
return new Xomp_db_core(root_dir);
}
}

52
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_lock_req_tbl.java
Unescape View File

@ -0,0 +1,52 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
public class Xomp_lock_req_tbl implements Db_tbl {
private final String fld_machine_name, fld_req_time;
private final Db_conn conn;
public Xomp_lock_req_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "xomp_lock_req";
this.fld_machine_name = flds.Add_str("machine_name", 255); // EX: "MACHINE1"
this.fld_req_time = flds.Add_str("req_time", 32); // EX: 20160801 010203
conn.Rls_reg(this);
}
public String Tbl_name() {return tbl_name;} private final String tbl_name;
public Dbmeta_fld_list Flds() {return flds;} private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
public void Create_tbl() {
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds
, Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "req_time", fld_req_time)
));
}
public void Insert(String machine_name) {
conn.Stmt_insert(tbl_name, flds).Clear().Val_str(fld_machine_name, machine_name).Val_str(fld_req_time, Datetime_now.Get_force().XtoStr_fmt_yyyyMMdd_HHmmss()).Exec_insert();
}
public String Select_1st() {
String sql = String_.Format("SELECT * FROM {0} ORDER BY {1} DESC", tbl_name, fld_machine_name); // ANSI.y
Db_rdr rdr = conn.Stmt_sql(sql).Exec_select__rls_auto();
try {
if (!rdr.Move_next()) throw Err_.new_wo_type("xomp_lock_req has no rows");
return rdr.Read_str(fld_machine_name);}
finally {rdr.Rls();}
}
public void Delete(String machine_name) {
conn.Stmt_delete(tbl_name, fld_machine_name).Clear().Crt_str(fld_machine_name, machine_name).Exec_delete();
}
public void Rls() {}
}

46
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_lock_tbl.java
Unescape View File

@ -0,0 +1,46 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
public class Xomp_lock_tbl implements Db_tbl {
private final String fld_uid_prv;
private final Db_conn conn;
public Xomp_lock_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "xomp_lock";
this.fld_uid_prv = flds.Add_int("uid_prv"); // EX: -1
conn.Rls_reg(this);
}
public String Tbl_name() {return tbl_name;} private final String tbl_name;
public Dbmeta_fld_list Flds() {return flds;} private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
public void Create_tbl() {
conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));
conn.Stmt_insert(tbl_name, flds).Clear().Val_int(fld_uid_prv, -1).Exec_insert(); // always add default record when creating table
}
public int Select() {
Db_rdr rdr = conn.Stmt_select(tbl_name, flds).Exec_select__rls_auto();
try {
if (!rdr.Move_next()) throw Err_.new_wo_type("xomp_lock has no rows");
return rdr.Read_int(fld_uid_prv);}
finally {rdr.Rls();}
}
public void Update(int uid_prv) {
conn.Stmt_update(tbl_name, String_.Ary_empty, fld_uid_prv).Clear().Val_int(fld_uid_prv, uid_prv).Exec_update();
}
public void Rls() {}
}

29
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_mgr_db.java
Unescape View File

@ -17,13 +17,36 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*; import gplx.xowa.addons.bldrs.mass_parses.parses.locks.*;
public class Xomp_mgr_db {
public Xomp_mgr_db(Io_url url) {
this.url = url;
this.conn = Db_conn_bldr.Instance.Get_or_autocreate(true, url);
this.page_tbl = new Xomp_page_tbl(conn);
this.wkr_tbl = new Xomp_wkr_tbl(conn);
// this.lock_mgr = new Xomp_lock_mgr__db(conn, 5000);
this.lock_mgr = new Xomp_lock_mgr__fsys(5000, this.Dir());
}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public Io_url Url() {return url;} private final Io_url url;
public Io_url Dir() {return url.OwnerDir();}
public Xomp_page_tbl Page_tbl() {return page_tbl;} private final Xomp_page_tbl page_tbl;
public Xomp_wkr_tbl Wkr_tbl() {return wkr_tbl;} private final Xomp_wkr_tbl wkr_tbl;
public Xomp_lock_mgr Lock_mgr() {return lock_mgr;} private final Xomp_lock_mgr lock_mgr;
public void Remake() {
conn.Meta_tbl_remake_many(page_tbl, wkr_tbl);
lock_mgr.Remake();
}
public static Io_url New__url(Xowe_wiki wiki) {return wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");}
public static Xomp_mgr_db New__make(Xowe_wiki wiki) {
Io_url root_dir = New__url(wiki);
Io_mgr.Instance.DeleteDirDeep(root_dir);
return new Xomp_mgr_db(root_dir.GenSubFil("xomp.sqlite3"));
}
public static Xomp_mgr_db New__load(Xowe_wiki wiki) {return New__load(wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp"));}
public static Xomp_mgr_db New__load(Io_url root_dir) {
return new Xomp_mgr_db(root_dir.GenSubFil("xomp.sqlite3"));
}
public Io_url Url() {return url;} private Io_url url;
public Db_conn Conn() {return conn;} private Db_conn conn;
public Xomp_page_tbl Page_tbl() {return page_tbl;} private Xomp_page_tbl page_tbl;
}

15
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_page_tbl.java
Unescape View File

@ -18,8 +18,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
public class Xomp_page_tbl implements Db_tbl {
// private final String fld_page_id, fld_page_status, fld_page_mgr_id;
private final Object thread_lock = new Object();
private final Db_conn conn;
// private final String fld_page_id, fld_page_status, fld_page_mgr_id;
public Xomp_page_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "xomp_page";
@ -33,6 +34,7 @@ public class Xomp_page_tbl implements Db_tbl {
}
public String Tbl_name() {return tbl_name;} private final String tbl_name;
public Dbmeta_fld_list Flds() {return flds;} private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
public void Create_tbl() {
conn.Meta_tbl_create
( Dbmeta_tbl_itm.New(tbl_name, flds
@ -40,5 +42,16 @@ public class Xomp_page_tbl implements Db_tbl {
, Dbmeta_idx_itm.new_normal_by_tbl("xomp_page", "page_ns__page_id" , "page_ns", "page_id") // for make
));
}
public void Update_wkr_uid(int wkr_uid, Db_conn wkr_conn) {
synchronized (thread_lock) { // LOCK:used by multiple threads
Db_attach_mgr attach_mgr = new Db_attach_mgr(conn, new Db_attach_itm("wkr_db", wkr_conn));
attach_mgr.Exec_sql_w_msg("updating page_regy: wkr_id=" + wkr_uid, String_.Concat_lines_nl_skip_last // ANSI.Y
( "UPDATE xomp_page"
, "SET xomp_wkr_id = " + Int_.To_str(wkr_uid)
, ", html_len = (SELECT length(body) FROM <wkr_db>html h WHERE h.page_id = xomp_page.page_id)"
, "WHERE page_id IN (SELECT page_id FROM <wkr_db>html h)"
));
}
}
public void Rls() {}
}

5
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_wkr_db.java
Unescape View File

@ -30,4 +30,9 @@ public class Xomp_wkr_db {
public Io_url Url() {return url;} private Io_url url;
public Db_conn Conn() {return conn;} private Db_conn conn;
public Xowd_html_tbl Html_tbl() {return html_tbl;} private final Xowd_html_tbl html_tbl;
public static Xomp_wkr_db New(Io_url root_dir, int uid) {
Io_url url = root_dir.GenSubFil_nest("xomp_" + Int_.To_str_fmt(uid, "000"), "xomp_wkr.sqlite3");
return new Xomp_wkr_db(uid, url);
}
}

99
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_wkr_tbl.java
Unescape View File

@ -0,0 +1,99 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
public class Xomp_wkr_tbl implements Db_tbl {
private final String fld_wkr_uid, fld_wkr_url, fld_wkr_status, fld_wkr_status_time, fld_wkr_exec_count, fld_wkr_exec_time;
private final Db_conn conn;
private final Object thread_lock = new Object();
public Xomp_wkr_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "xomp_wkr";
fld_wkr_uid = flds.Add_int_pkey("wkr_uid"); // EX: 0
fld_wkr_url = flds.Add_str("wkr_url", 255); // EX: //MACHINE/C:/xowa/wiki/en.wikipedia.org/tmp/xomp
fld_wkr_status = flds.Add_int("wkr_status"); // EX: running; waiting
fld_wkr_status_time = flds.Add_str("wkr_status_time", 255); // EX: 20160801 010203
fld_wkr_exec_count = flds.Add_int("wkr_exec_count"); // EX: 1000
fld_wkr_exec_time = flds.Add_int("wkr_exec_time"); // EX: 123
conn.Rls_reg(this);
}
public String Tbl_name() {return tbl_name;} private final String tbl_name;
public Dbmeta_fld_list Flds() {return flds;} private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public int Init_wkrs(String wkr_url, int wkr_len) {
// delete all with machine_name
conn.Stmt_delete(tbl_name, fld_wkr_url).Crt_str(fld_wkr_url, wkr_url).Exec_delete();
// get bgn_uid / end_uid
int bgn_uid = conn.Exec_select_max_as_int(tbl_name, fld_wkr_uid, -1) + 1;
int end_uid = bgn_uid + wkr_len;
// insert into tbl
Db_stmt insert_stmt = conn.Stmt_insert(tbl_name, flds);
for (int i = bgn_uid; i < end_uid; ++i)
Insert(insert_stmt, i, wkr_url);
insert_stmt.Rls();
return bgn_uid;
}
public int Select_count() {
return conn.Exec_select_count_as_int(tbl_name, 0);
}
private void Insert(Db_stmt stmt, int wkr_uid, String wkr_url) {
stmt.Clear()
.Val_int(fld_wkr_uid, wkr_uid)
.Val_str(fld_wkr_url, wkr_url).Val_int(fld_wkr_status, Status__running).Val_str(fld_wkr_status_time, Datetime_now.Get_force().XtoStr_fmt_yyyyMMdd_HHmmss())
.Val_int(fld_wkr_exec_count, 0).Val_int(fld_wkr_exec_time, 0)
.Exec_insert();
}
public void Update_exec(int wkr_uid, int wkr_exec_count, long wkr_exec_time) {
synchronized (thread_lock) { // LOCK:wkr_tbl is shared by multiple threads / machines
int attempts = 0;
while (true) {
if (++attempts > 10) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "failed to update status; try=~{0}", attempts);
break;
}
try {
conn.Stmt_update(tbl_name, String_.Ary(fld_wkr_uid), fld_wkr_status, fld_wkr_status_time, fld_wkr_exec_count, fld_wkr_exec_time).Clear()
.Val_int(fld_wkr_status, Status__running).Val_str(fld_wkr_status_time, Datetime_now.Get_force().XtoStr_fmt_yyyyMMdd_HHmmss())
.Val_int(fld_wkr_exec_count, wkr_exec_count).Val_int(fld_wkr_exec_time, (int)(wkr_exec_time / 1000))
.Crt_int(fld_wkr_uid, wkr_uid)
.Exec_update();
break; // exit loop
} catch (Exception e) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "unable to update status; try=~{0} err=~{1}", attempts, Err_.Message_gplx_log(e));
gplx.core.threads.Thread_adp_.Sleep(10000);
continue;
}
}
}
}
public void Update_status(int wkr_uid, int status) {
synchronized (thread_lock) { // LOCK:wkr_tbl is shared by multiple threads
conn.Stmt_update(tbl_name, String_.Ary(fld_wkr_uid), fld_wkr_status, fld_wkr_status_time).Clear()
.Val_int(fld_wkr_status, status).Val_str(fld_wkr_status_time, Datetime_now.Get_force().XtoStr_fmt_yyyyMMdd_HHmmss())
.Crt_int(fld_wkr_uid, wkr_uid)
.Exec_update();
}
}
public void Rls() {}
public static final int Status__running = 1, Status__sleeping = 2;
}

10
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_cmd.java
Unescape View File

@ -18,16 +18,14 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.mass_parses.inits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xomp_init_cmd extends Xob_cmd__base {
private final Xomp_init_mgr mgr;
public Xomp_init_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);
mgr = new Xomp_init_mgr(wiki);
}
private final Xomp_init_mgr mgr = new Xomp_init_mgr();
public Xomp_init_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
mgr.Exec();
mgr.Exec(wiki);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__cfg)) return this;
if (ctx.Match(k, Invk__cfg)) return mgr.Cfg();
else return super.Invk(ctx, ikey, k, m);
} private static final String Invk__cfg = "cfg";

26
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_mgr.java
Unescape View File

@ -20,31 +20,31 @@ import gplx.dbs.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
class Xomp_init_mgr {
private final Xowe_wiki wiki;
public Xomp_init_mgr(Xowe_wiki wiki) {this.wiki = wiki;}
public void Exec() {
// init
Xomp_db_core db_core = Xomp_db_core.New__make(wiki);
Xomp_page_tbl page_tbl = db_core.Mgr_db().Page_tbl();
public Xomp_init_mgr_cfg Cfg() {return cfg;} private final Xomp_init_mgr_cfg cfg = new Xomp_init_mgr_cfg();
public void Exec(Xowe_wiki wiki) {
// init vars
cfg.Init(wiki);
Xomp_mgr_db mgr_db = Xomp_mgr_db.New__make(wiki);
Db_conn mgr_conn = mgr_db.Conn();
// rebuild table
Db_conn mgr_conn = db_core.Mgr_db().Conn();
mgr_conn.Meta_tbl_remake(page_tbl);
// remake all
mgr_db.Remake();
// fill table
// fill page tbl
Db_attach_mgr attach_mgr = new Db_attach_mgr(mgr_conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
int[] ns_ary = new int[] {0, 4, 14};
int[] ns_ary = cfg.Ns_ids();
int len = ns_ary.length;
String sql = String_.Concat_lines_nl_skip_last
String sql = String_.Concat_lines_nl_skip_last // ANSI.Y
( "INSERT INTO xomp_page (page_id, page_ns, page_status, html_len, xomp_wkr_id)"
, "SELECT p.page_id, p.page_namespace, 0, 0, 0"
, "FROM <page_db>page p"
, "WHERE p.page_namespace = {0}"
, "AND p.page_is_redirect = 0"
, "ORDER BY p.page_id"
);
for (int i = 0; i < len; ++i) {
int ns_id = ns_ary[i];
attach_mgr.Exec_sql_w_msg("adding rows for xomp_page: ns=" + ns_id, sql, ns_id);// ANSI.Y
attach_mgr.Exec_sql_w_msg("adding rows for xomp_page: ns=" + ns_id, sql, ns_id);
}
}
}

40
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_mgr_cfg.java
Unescape View File

@ -0,0 +1,40 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.inits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.xowa.wikis.nss.*;
class Xomp_init_mgr_cfg implements Gfo_invk {
public int[] Ns_ids() {return ns_ids;} private int[] ns_ids = new int[] {0, 4, 14, 100};
public void Init(Xowe_wiki wiki) {
if (ns_ids == null) ns_ids = Ns_ids(wiki.Ns_mgr());
}
private static int[] Ns_ids(Xow_ns_mgr ns_mgr) {
int ns_ids_len = ns_mgr.Ids_len();
int[] rv = new int[ns_ids_len];
for (int i = 0; i < ns_ids_len; ++i)
rv[i] = ns_mgr.Ids_get_at(i).Id();
return rv;
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__ns_ids_)) ns_ids = Int_.Ary_parse(m.ReadStr("v"), "|");
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String
Invk__ns_ids_ = "ns_ids_"
;
}

8
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xob_lnki_temp_row.java
Unescape View File

@ -18,7 +18,6 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xob_lnki_temp_row implements CompareAble {
public int Xomp_uid() {return xomp_uid;} private int xomp_uid;
public int Lnki_id() {return lnki_id;} private int lnki_id;
public int Lnki_tier_id() {return lnki_tier_id;} private int lnki_tier_id;
public int Lnki_page_id() {return lnki_page_id;} private int lnki_page_id;
@ -32,9 +31,8 @@ class Xob_lnki_temp_row implements CompareAble {
public double Lnki_upright() {return lnki_upright;} private double lnki_upright;
public double Lnki_time() {return lnki_time;} private double lnki_time;
public int Lnki_page() {return lnki_page;} private int lnki_page;
public void Load(Db_rdr rdr, int xomp_uid) {
this.xomp_uid = xomp_uid;
this.lnki_id = rdr.Read_int("lnki_id");
public void Load(Db_rdr rdr, int lnki_id) {
this.lnki_id = lnki_id;
this.lnki_tier_id = rdr.Read_int("lnki_tier_id");
this.lnki_page_id = rdr.Read_int("lnki_page_id");
this.lnki_ttl = rdr.Read_bry_by_str("lnki_ttl");
@ -50,8 +48,6 @@ class Xob_lnki_temp_row implements CompareAble {
}
public int compareTo(Object obj) {
Xob_lnki_temp_row comp = (Xob_lnki_temp_row)obj;
int rv = Int_.Compare(xomp_uid, comp.xomp_uid);
if (rv != CompareAble_.Same) return rv;
return Int_.Compare(lnki_id, comp.lnki_id);
}
}

14
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_rdr.java
Unescape View File

@ -20,17 +20,17 @@ import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*;
import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
class Xomp_html_db_rdr {
private final Xowd_html_tbl[] src_tbls;
private final Xomp_db_core db;
private final Xomp_mgr_db mgr_db;
public Xomp_html_db_rdr(Xowe_wiki wiki) {
this.db = Xomp_db_core.New__load(wiki);
this.src_tbls = new Xowd_html_tbl[db.Wkr_count()];
this.mgr_db = Xomp_mgr_db.New__load(wiki);
this.src_tbls = new Xowd_html_tbl[mgr_db.Wkr_tbl().Select_count()];
}
public void Rows__get(Xowd_html_row rv, int wkr_id, int page_id) {
Xowd_html_tbl src_tbl = src_tbls[wkr_id];
public void Rows__get(Xowd_html_row rv, int wkr_uid, int page_id) {
Xowd_html_tbl src_tbl = src_tbls[wkr_uid];
if (src_tbl == null) {
Db_conn wkr_conn = db.Wkr_db(Bool_.N, wkr_id).Conn();
Db_conn wkr_conn = Xomp_wkr_db.New(mgr_db.Dir(), wkr_uid).Conn();
src_tbl = new Xowd_html_tbl(wkr_conn);
src_tbls[wkr_id] = src_tbl;
src_tbls[wkr_uid] = src_tbl;
}
src_tbl.Select_as_row(rv, page_id);
}

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java
Unescape View File

@ -21,8 +21,8 @@ public class Xomp_make_cmd extends Xob_cmd__base {
public Xomp_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
new Xomp_make_wkr(wiki).Exec();
new Xomp_make_lnki().Exec(wiki, 1000);
new Xomp_make_html().Exec(wiki);
new Xomp_make_lnki().Exec(wiki, 10000);
}
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "wiki.mass_parse.make";

67
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_wkr.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_html.java
Unescape View File

@ -18,44 +18,48 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.brys.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
class Xomp_make_wkr {
private final Db_conn mgr_conn;
private final Xomp_html_db_wtr html_db_wtr;
private final Xomp_html_db_rdr html_db_rdr;
class Xomp_make_html {
private final Int_flag_bldr src_body_flag_bldr = Xowd_html_tbl.Make_body_flag_bldr();
public Xomp_make_wkr(Xowe_wiki wiki) {
this.db = Xomp_db_core.New__load(wiki);
this.mgr_conn = db.Mgr_db().Conn();
this.html_db_rdr = new Xomp_html_db_rdr(wiki);
this.html_db_wtr = new Xomp_html_db_wtr(wiki);
}
public Xomp_db_core Db() {return db;} private Xomp_db_core db;
public void Exec() {
Xowd_html_row src_row = new Xowd_html_row();
public void Exec(Xowe_wiki wiki) {
// init
Xomp_mgr_db mgr_db = Xomp_mgr_db.New__load(wiki);
Db_conn mgr_conn = mgr_db.Conn();
int[] ns_ary = new int[] {0, 4, 14};
int ns_ary_len = ns_ary.length;
for (int i = 0; i < ns_ary_len; ++i) {
int ns_id = ns_ary[i];
String sql = String_.Format("SELECT * FROM xomp_page WHERE page_ns = {0} AND html_len != 0 ORDER BY page_id;", ns_id); // NOTE: html_len == 0 when page failed
int count = 0;
Db_rdr rdr = mgr_conn.Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
try {
while (rdr.Move_next()) {
Make_page(rdr, src_row, ns_id);
if (++count % 10000 == 0)
Gfo_usr_dlg_.Instance.Prog_many("", "", "xomp.html.insert: ns=~{0} db=~{1} count=~{2}", Int_.To_str_pad_bgn_space(ns_id, 3), Int_.To_str_pad_bgn_space(html_db_wtr.Cur_db_id(), 3), Int_.To_str_pad_bgn_space(count, 8));
}
} finally {rdr.Rls();}
// update wkr_uid; note that this cannot be done in parse_wkr, b/c multiple-writer-errors for xomp.db|page
int wkrs_len = mgr_db.Wkr_tbl().Select_count();
for (int i = 0; i < wkrs_len; ++i) {
Xomp_wkr_db wkr_db = Xomp_wkr_db.New(mgr_db.Dir(), i);
mgr_db.Page_tbl().Update_wkr_uid(i, wkr_db.Conn());
}
this.Rls();
// init more
Xomp_html_db_rdr html_db_rdr = new Xomp_html_db_rdr(wiki);
Xomp_html_db_wtr html_db_wtr = new Xomp_html_db_wtr(wiki);
Xowd_html_row src_row = new Xowd_html_row();
// loop xomp|page and generate html dbs
String sql = String_.Format("SELECT * FROM xomp_page WHERE html_len != 0 ORDER BY xomp_uid;"); // NOTE: html_len == 0 when page failed
int count = 0;
Db_rdr rdr = mgr_conn.Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
try {
while (rdr.Move_next()) {
Make_page(html_db_rdr, rdr, html_db_wtr, src_row);
if (++count % 10000 == 0)
Gfo_usr_dlg_.Instance.Prog_many("", "", "xomp.html.insert: db=~{0} count=~{1}", Int_.To_str_pad_bgn_space(html_db_wtr.Cur_db_id(), 3), Int_.To_str_pad_bgn_space(count, 8));
}
} finally {rdr.Rls();}
// cleanup
mgr_conn.Rls_conn();
html_db_rdr.Rls();
html_db_wtr.Rls();
}
private void Make_page(Db_rdr rdr, Xowd_html_row src_row, int ns_id) {
private void Make_page(Xomp_html_db_rdr html_db_rdr, Db_rdr rdr, Xomp_html_db_wtr html_db_wtr, Xowd_html_row src_row) {
// get src_row
int page_id = rdr.Read_int("page_id");
int html_len = rdr.Read_int("html_len");
int wkr_id = rdr.Read_int("xomp_wkr_id");
int ns_id = rdr.Read_int("page_ns");
html_db_rdr.Rows__get(src_row, wkr_id, page_id);
src_body_flag_bldr.Decode(src_row.Body_flag());
@ -63,9 +67,4 @@ class Xomp_make_wkr {
Xowd_html_tbl trg_tbl = html_db_wtr.Tbls__get_or_new(ns_id, html_len);
trg_tbl.Insert(src_row.Page_id(), src_row.Head_flag(), src_body_flag_bldr.Get_as_int(0), src_body_flag_bldr.Get_as_int(1), src_row.Display_ttl(), src_row.Content_sub(), src_row.Sidebar_div(), src_row.Body());
}
private void Rls() {
mgr_conn.Rls_conn();
html_db_rdr.Rls();
html_db_wtr.Rls();
}
}

43
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java
Unescape View File

@ -16,36 +16,45 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.brys.*;
import gplx.core.brys.*; import gplx.core.primitives.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*; import gplx.xowa.addons.bldrs.files.dbs.*;
import gplx.xowa.bldrs.*;
class Xomp_make_lnki {
public void Exec(Xowe_wiki wiki, int uid_count) {
// init
Xomp_db_core xomp_db = Xomp_db_core.New__load(wiki);
Xomp_mgr_db xomp_db = Xomp_mgr_db.New__load(wiki);
Xob_db_file make_db = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir());
Xob_lnki_temp_tbl lnki_temp_tbl = new Xob_lnki_temp_tbl(make_db.Conn());
make_db.Conn().Meta_tbl_remake(lnki_temp_tbl);
lnki_temp_tbl.Insert_bgn();
// add index
int wkr_count = xomp_db.Wkr_count();
// create ary; add index
int wkr_count = xomp_db.Wkr_tbl().Select_count();
Xomp_wkr_db[] db_ary = new Xomp_wkr_db[wkr_count];
for (int i = 0; i < wkr_count; ++i) {
Xomp_wkr_db wkr_db = xomp_db.Wkr_db(Bool_.N, i);
Xomp_wkr_db wkr_db = Xomp_wkr_db.New(xomp_db.Dir(), i);
db_ary[i] = wkr_db;
wkr_db.Conn().Meta_idx_assert("lnki_temp", "lnki_page_id", "lnki_page_id");
}
// loop
// get max xomp_uid; note that xomp_uid is (a) per page; (b) ordered by page_ns, page_id; (c) starts from 1
int max_xomp_uid = xomp_db.Conn().Exec_select_max_as_int("xomp_page", "xomp_uid", -1);
// loop over wkrs using range of xomp_uid
List_adp rows = List_adp_.New();
int uid_bgn = -1;
int cur_xomp_uid = -1; Int_obj_ref lnki_id_ref = Int_obj_ref.New_zero();
while (true) {
int count = 0;
int tmp_xomp_uid_max = -1; // maximum uid for a grp of wkrs; EX: looping over 8 wkrs with xomp_uid range of 1 - 1000; max xomp_uid may only be 990 b/c pages are missing / failed
for (int i = 0; i < wkr_count; ++i) {
Xomp_wkr_db wkr_db = xomp_db.Wkr_db(Bool_.N, i);
count += Load_rows(rows, xomp_db, wkr_db, uid_bgn, uid_bgn + uid_count);
Xomp_wkr_db wkr_db = db_ary[i];
int wkr_uid_max = Load_rows(rows, xomp_db, wkr_db, cur_xomp_uid, cur_xomp_uid + uid_count, lnki_id_ref);
if (wkr_uid_max > tmp_xomp_uid_max)
tmp_xomp_uid_max = wkr_uid_max;
}
if (count < 0) break;
uid_bgn += count;
if (tmp_xomp_uid_max >= max_xomp_uid || tmp_xomp_uid_max == -1) break; // if max_xomp_uid seen, break; note that ">" necessary because max_xomp_uid may not be in set of wkrs;
cur_xomp_uid += uid_count; // note that this sequentially counts up by uid_count (1000), so inevitable that cur_xomp_uid will exceed wkr_uid_max
Gfo_usr_dlg_.Instance.Prog_many("", "", "building lnki_temp; cur_xomp_uid=~{0}", cur_xomp_uid);
Save_rows(rows, lnki_temp_tbl);
}
@ -54,16 +63,17 @@ class Xomp_make_lnki {
xomp_db.Conn().Rls_conn();
make_db.Conn().Rls_conn();
}
private int Load_rows(List_adp rows, Xomp_db_core xomp_db, Xomp_wkr_db wkr_db, int uid_bgn, int uid_end) {
private int Load_rows(List_adp rows, Xomp_mgr_db xomp_db, Xomp_wkr_db wkr_db, int uid_bgn, int uid_end, Int_obj_ref lnki_id) {
// build sql
Db_attach_mgr attach_mgr = new Db_attach_mgr(xomp_db.Conn());
attach_mgr.Conn_links_(new Db_attach_itm("wkr_db", wkr_db.Conn()));
String sql = String_.Format(String_.Concat_lines_nl_skip_last
( "SELECT mgr.xomp_uid"
, ", wkr.*"
, ", wkr.lnki_tier_id, wkr.lnki_page_id, wkr.lnki_ttl, wkr.lnki_commons_ttl, wkr.lnki_ext, wkr.lnki_type, wkr.lnki_src_tid, wkr.lnki_w, wkr.lnki_h, wkr.lnki_upright, wkr.lnki_time, wkr.lnki_page"
, "FROM <wkr_db>lnki_temp wkr"
, " JOIN xomp_page mgr ON wkr.lnki_page_id = mgr.page_id"
, "WHERE mgr.xomp_uid > {0} AND mgr.xomp_uid <= {1}"
, "WHERE mgr.xomp_uid > {0} AND mgr.xomp_uid <= {1}" // mgr.xomp_uid will sort pages by ns_id, page_id
)
, uid_bgn
, uid_end
@ -77,7 +87,7 @@ class Xomp_make_lnki {
while (rdr.Move_next()) {
rv = rdr.Read_int("xomp_uid");
Xob_lnki_temp_row row = new Xob_lnki_temp_row();
row.Load(rdr, rv);
row.Load(rdr, lnki_id.Val_add_pre());
rows.Add(row);
}
} finally {rdr.Rls();}
@ -90,8 +100,9 @@ class Xomp_make_lnki {
for (int i = 0; i < len; ++i) {
Xob_lnki_temp_row row = (Xob_lnki_temp_row)rows.Get_at(i);
lnki_temp_tbl.Insert_cmd_by_batch(row.Lnki_tier_id(), row.Lnki_page_id(), row.Lnki_ttl(), row.Lnki_commons_ttl()
, row.Lnki_ext(), row.Lnki_src_tid(), row.Lnki_src_tid(), row.Lnki_w(), row.Lnki_h(), row.Lnki_upright()
, row.Lnki_ext(), row.Lnki_type(), row.Lnki_src_tid(), row.Lnki_w(), row.Lnki_h(), row.Lnki_upright()
, row.Lnki_time(), row.Lnki_page());
}
rows.Clear();
}
}

1
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_cmd.java
Unescape View File

@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.*;
public class Xomp_parse_cmd extends Xob_cmd__base {
private final Xomp_parse_mgr mgr = new Xomp_parse_mgr();
public Xomp_parse_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}

11
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xow_wiki_utl_.java
Unescape View File

@ -17,6 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.cases.*;
import gplx.xowa.files.*;
public class Xow_wiki_utl_ {
public static Xowe_wiki Clone_wiki(Xowe_wiki wiki, Io_url wiki_dir) {
Xol_lang_itm lang = new Xol_lang_itm(wiki.App().Lang_mgr(), wiki.Lang().Key_bry());
@ -25,6 +26,16 @@ public class Xow_wiki_utl_ {
rv.Init_by_wiki();
rv.File_mgr().Repo_mgr().Clone(wiki.File_mgr().Repo_mgr());
rv.File__fsdb_mode().Tid__v2__bld__y_();
Clone_repos(wiki);
return rv;
}
public static void Clone_repos(Xowe_wiki wiki) {
// force all repos to be lnx; will not convert characters like *,",? to _; also force long titles
Xoa_repo_mgr repo_mgr = wiki.Appe().File_mgr().Repo_mgr();
int len = repo_mgr.Count();
for (int i = 0; i < len; ++i)
repo_mgr.Get_at(i).Fsys_is_wnt_(Bool_.N).Shorten_ttl_(Bool_.N);
}
}

23
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/locks/Xomp_lock_mgr.java
Unescape View File

@ -0,0 +1,23 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses.locks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
public interface Xomp_lock_mgr {
void Remake();
int Uid_prv__get(String machine_name);
void Uid_prv__rls(String machine_name, int uid_prv);
}

57
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/locks/Xomp_lock_mgr__db.java
Unescape View File

@ -0,0 +1,57 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses.locks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
public class Xomp_lock_mgr__db implements Xomp_lock_mgr {
private final Db_conn conn;
private final Xomp_lock_tbl lock_tbl;
private final Xomp_lock_req_tbl req_tbl;
private final int wait_time;
public Xomp_lock_mgr__db(Db_conn conn, int wait_time) {
this.conn = conn;
this.lock_tbl = new Xomp_lock_tbl(conn);
this.req_tbl = new Xomp_lock_req_tbl(conn);
this.wait_time = wait_time;
}
public void Remake() {
conn.Meta_tbl_remake(lock_tbl);
conn.Meta_tbl_remake(req_tbl);
}
public int Uid_prv__get(String machine_name) {
// insert into req_tbl
req_tbl.Insert(machine_name);
// loop until req is 1st record in req_tbl
while (true) {
String machine_name_1st = req_tbl.Select_1st();
if (String_.Eq(machine_name, machine_name_1st))
break;
else {
Gfo_usr_dlg_.Instance.Note_many("", "", "waiting for lock: ~{0}", machine_name);
gplx.core.threads.Thread_adp_.Sleep(wait_time);
}
}
// get next uid and fill pages
return lock_tbl.Select();
}
public void Uid_prv__rls(String machine_name, int uid_prv) {
lock_tbl.Update(uid_prv);
req_tbl.Delete(machine_name);
}
}

91
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/locks/Xomp_lock_mgr__fsys.java
Unescape View File

@ -0,0 +1,91 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses.locks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
public class Xomp_lock_mgr__fsys implements Xomp_lock_mgr {
private final Io_url root_dir, uid_fil, active_fil;
private final int wait_time;
private Io_url stop_fil;
public Xomp_lock_mgr__fsys(int wait_time, Io_url root_dir) {
this.wait_time = wait_time;
this.root_dir = root_dir;
this.uid_fil = root_dir.GenSubFil("xomp.semaphore.uid.txt");
this.active_fil = root_dir.GenSubFil("xomp.semaphore.active.txt");
}
public void Remake() {
Io_url[] fils = Io_mgr.Instance.QueryDir_fils(root_dir);
for (Io_url fil : fils) {
if (String_.Has_at_end(fil.NameAndExt(), ".sempahore.txt"))
Io_mgr.Instance.DeleteFil(fil);
}
Io_mgr.Instance.SaveFilStr(uid_fil, Int_.To_str(Uid__bos));
}
public int Uid_prv__get(String machine_name) {
// return -1 if stop file exists; note that -1 will stop machine
if (stop_fil == null) this.stop_fil = root_dir.GenSubFil("xomp.semaphore.stop." + machine_name + ".txt");
if (Io_mgr.Instance.ExistsFil(stop_fil)) return Uid__eos;
// loop until permit is acquired
int tries = 0;
while (true) {
++tries;
if (tries > 10) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "failed to acquire permit");
return Uid__eos; // too many tries; just exit now
}
// if active_file exists, then assume another machine is reading;
if (Io_mgr.Instance.ExistsFil(active_fil)) {
Sleep(machine_name, "active file exists");
continue;
}
// write to the active_file
Io_mgr.Instance.SaveFilStr(active_fil, machine_name);
// now read it to make sure it's the same
String cur_active = String_.new_u8(Io_mgr.Instance.LoadFilBryOr(active_fil, Bry_.Empty));
if (!String_.Eq(cur_active, machine_name)) {
Sleep(machine_name, "active file differs: " + cur_active);
continue;
}
break;
}
// get next uid and fill pages
byte[] cur_uid_bry = Io_mgr.Instance.LoadFilBryOr(uid_fil, null);
if (cur_uid_bry == null) return 0; // file is empty; should only occur on 1st run; return 0, which will start from beginning;
int cur_uid = Int_.Min_value;
if (cur_uid_bry != null)
cur_uid = Bry_.To_int_or(cur_uid_bry, Int_.Min_value);
if (cur_uid == Int_.Min_value) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "uid fil has bad data: data:~{0}", cur_uid_bry);
return Uid__eos;
}
return cur_uid;
}
private void Sleep(String machine_name, String reason) {
Gfo_usr_dlg_.Instance.Note_many("", "", "waiting for permit: machine:~{0} reason:~{1}", machine_name, reason);
gplx.core.threads.Thread_adp_.Sleep(wait_time);
}
public void Uid_prv__rls(String machine_name, int uid_prv) {
Io_mgr.Instance.SaveFilStr(uid_fil, Int_.To_str(uid_prv));
Io_mgr.Instance.DeleteFil(active_fil);
}
public static final int Uid__bos = 0, Uid__eos = -1;
}

57
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_mgr.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
Unescape View File

@ -15,57 +15,64 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.threads.*; import gplx.core.threads.utils.*; import gplx.core.caches.*;
import gplx.xowa.langs.*; import gplx.xowa.langs.cases.*;
import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.wikis.caches.*;
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.core.threads.*; import gplx.core.threads.utils.*;
import gplx.core.caches.*; import gplx.xowa.wikis.caches.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.wkrs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*;
import gplx.xowa.addons.bldrs.wmdumps.imglinks.*;
class Xomp_parse_mgr {
public class Xomp_parse_mgr {
private Gfo_countdown_latch latch;
public Xomp_parse_mgr_cfg Cfg() {return cfg;} private final Xomp_parse_mgr_cfg cfg = new Xomp_parse_mgr_cfg();
public Xomp_db_core Db_core() {return db_core;} private Xomp_db_core db_core;
public Xomp_prog_mgr Prog_mgr() {return prog_mgr;} private final Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
public void Wkrs_done_add_1() {latch.Countdown();}
public void Run(Xowe_wiki wiki) {
// init db, pool_loader, pool, prog_mgr
// init db
cfg.Init(wiki);
this.db_core = Xomp_db_core.New__load(wiki);
Xomp_page_pool_loader pool_loader = new Xomp_page_pool_loader(wiki, db_core.Mgr_db().Conn(), cfg.Num_pages_in_pool());
Xomp_page_pool page_pool = new Xomp_page_pool(pool_loader, cfg.Num_pages_per_wkr());
prog_mgr.Init(pool_loader.Get_pending_count(), cfg.Progress_interval());
Xomp_mgr_db mgr_db = Xomp_mgr_db.New__load(cfg.Mgr_url());
// init page_pool
Xomp_page_pool_loader page_pool_loader = new Xomp_page_pool_loader(wiki, mgr_db.Conn(), cfg.Num_pages_in_pool());
Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr());
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
prog_mgr.Init(page_pool_loader.Get_pending_count(), cfg.Progress_interval());
// cache: disable general settings
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of cache
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
Gfo_cache_mgr ifexist_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
Xof_orig_wkr__img_links orig_wkr = Xof_orig_wkr__img_links_.Load_all(wiki);
// cache: preload tmpls and imglinks
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());
Xof_orig_wkr__img_links file_orig_wkr = new Xof_orig_wkr__img_links(wiki);
if (cfg.Load_all_imglnks()) Xof_orig_wkr__img_links_.Load_all(file_orig_wkr);
// load_wkr: init and start
// Xomp_load_wkr load_wkr = new Xomp_load_wkr(wiki, db_core.Mgr_db().Conn(), cfg.Num_pages_in_pool(), cfg.Num_wkrs());
// Xomp_load_wkr load_wkr = new Xomp_load_wkr(wiki, db_mgr.Mgr_db().Conn(), cfg.Num_pages_in_pool(), cfg.Num_wkrs());
// Thread_adp_.Start_by_key("xomp.load", Cancelable_.Never, load_wkr, Xomp_load_wkr.Invk__exec);
// init parse_wkrs
// assert wkr_tbl
int wkr_len = cfg.Num_wkrs();
int wkr_uid_bgn = mgr_db.Wkr_tbl().Init_wkrs(cfg.Wkr_machine_name(), wkr_len);
latch = new Gfo_countdown_latch(wkr_len);
Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len];
// init parse_wkrs
for (int i = 0; i < wkr_len; ++i) {
// make wiki
Xowe_wiki wkr_wiki = Xow_wiki_utl_.Clone_wiki(wiki, wiki.Fsys_mgr().Root_dir());
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, wkr_wiki, orig_wkr, page_pool, i, cfg, cfg.Cleanup_interval(), cfg.Progress_interval(), cfg.Log_file_lnkis());
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
// remove wmf wkr, else will try to download images during parsing
if (wkr_wiki.File__bin_mgr() != null)
wkr_wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf);
wkr.Hdump_bldr().Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled());
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
// make wkr
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, cfg, mgr_db, page_pool, prog_mgr, file_orig_wkr, wkr_wiki, i + wkr_uid_bgn);
wkrs[i] = wkr;
}
// start threads; done separately b/c thread issues when done right after init
// start threads; done separately from init b/c thread issues
for (int i = 0; i < wkr_len; ++i) {
Xomp_parse_wkr wkr = wkrs[i];
Thread_adp_.Start_by_key("xomp." + Int_.To_str_fmt(i, "000"), Cancelable_.Never, wkr, Xomp_parse_wkr.Invk__exec);
}
// wait until wkrs are wkrs_done
// wait until wkrs are done
latch.Await();
page_pool.Rls();

24
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_mgr_cfg.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
Unescape View File

@ -15,11 +15,11 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.core.ios.streams.*;
class Xomp_parse_mgr_cfg implements Gfo_invk {
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1; // use env.available_processors
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = 1000;
public class Xomp_parse_mgr_cfg implements Gfo_invk {
public int Num_wkrs() {return num_wkrs;} private int num_wkrs = -1;
public int Num_pages_in_pool() {return num_pages_in_pool;} private int num_pages_in_pool = -1;
public int Num_pages_per_wkr() {return num_pages_per_wkr;} private int num_pages_per_wkr = 1000;
public int Progress_interval() {return progress_interval;} private int progress_interval = 1000;
public int Commit_interval() {return commit_interval;} private int commit_interval = 10000;
@ -29,15 +29,21 @@ class Xomp_parse_mgr_cfg implements Gfo_invk {
public boolean Hdiff_enabled() {return hdiff_enabled;} private boolean hdiff_enabled = true;
public boolean Log_file_lnkis() {return log_file_lnkis;} private boolean log_file_lnkis = true;
public boolean Load_all_templates() {return load_all_templates;} private boolean load_all_templates = true;
public boolean Load_all_imglnks() {return load_all_imglnks;} private boolean load_all_imglnks = true;
public byte Zip_tid() {return zip_tid;} private byte zip_tid = Io_stream_.Tid_gzip;
public Io_url Mgr_url() {return mgr_url;} private Io_url mgr_url;
public String Wkr_machine_name() {return wkr_machine_name;} private String wkr_machine_name;
public void Init(Xowe_wiki wiki) {
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Env_.System_cpu_count();
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count();
if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000;
if (mgr_url == null) mgr_url = wiki.Fsys_mgr().Root_dir().GenSubDir_nest("tmp", "xomp");
if (wkr_machine_name == null) wkr_machine_name = gplx.core.envs.System_.Env__machine_name();
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__num_wkrs_)) num_wkrs = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_in_pool_)) num_pages_in_pool = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__num_pages_per_wkr_)) num_pages_per_wkr = m.ReadInt("v");
else if (ctx.Match(k, Invk__progress_interval_)) progress_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__commit_interval_)) commit_interval = m.ReadInt("v");
else if (ctx.Match(k, Invk__cleanup_interval_)) cleanup_interval = m.ReadInt("v");
@ -46,7 +52,10 @@ class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, Invk__hdiff_enabled_)) hdiff_enabled = m.ReadBool("v");
else if (ctx.Match(k, Invk__zip_tid_)) zip_tid = m.ReadByte("v");
else if (ctx.Match(k, Invk__load_all_templates_)) load_all_templates = m.ReadYn("v");
else if (ctx.Match(k, Invk__load_all_imglnks_)) load_all_imglnks = m.ReadYn("v");
else if (ctx.Match(k, Invk__manual_now_)) Datetime_now.Manual_and_freeze_(m.ReadDate("v"));
else if (ctx.Match(k, Invk__mgr_url_)) mgr_url = m.ReadIoUrl("v");
else if (ctx.Match(k, Invk__wkr_machine_name_)) wkr_machine_name = m.ReadStr("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}
@ -54,6 +63,7 @@ class Xomp_parse_mgr_cfg implements Gfo_invk {
Invk__num_wkrs_ = "num_wkrs_", Invk__num_pages_in_pool_ = "num_pages_in_pool_", Invk__num_pages_per_wkr_ = "num_pages_per_wkr_"
, Invk__progress_interval_ = "progress_interval_", Invk__commit_interval_ = "commit_interval_", Invk__cleanup_interval_ = "cleanup_interval_"
, Invk__hdump_enabled_ = "hdump_enabled_", Invk__hzip_enabled_ = "hzip_enabled_", Invk__hdiff_enabled_ = "hdiff_enabled_", Invk__zip_tid_ = "zip_tid_"
, Invk__load_all_templates_ = "load_all_templates_", Invk__manual_now_ = "manual_now_"
, Invk__load_all_templates_ = "load_all_templates_", Invk__load_all_imglnks_ = "load_all_imglnks_", Invk__manual_now_ = "manual_now_"
, Invk__mgr_url_ = "mgr_url_", Invk__wkr_machine_name_ = "wkr_machine_name_"
;
}

8
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_prog_mgr.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_prog_mgr.java
Unescape View File

@ -15,8 +15,8 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
class Xomp_prog_mgr {
package gplx.xowa.addons.bldrs.mass_parses.parses.mgrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
public class Xomp_prog_mgr {
private final Object thread_lock = new Object();
private int progress_interval;
private int pages_done, pages_total;
@ -24,14 +24,14 @@ class Xomp_prog_mgr {
private final Bry_bfr prog_bfr = Bry_bfr_.New();
public void Init(int pages_total, int progress_interval) {
this.progress_interval = progress_interval;
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
this.time_bgn = this.time_prv = gplx.core.envs.System_.Ticks();
this.pages_total = pages_total;
}
public void Mark_done(int id) {
synchronized (thread_lock) {
pages_done += 1;
if (pages_done % progress_interval == 0) {
long time_cur = gplx.core.envs.Env_.TickCount();
long time_cur = gplx.core.envs.System_.Ticks();
int pages_left = pages_total - pages_done;
time_done += (time_cur - time_prv);
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);

10
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_load_wkr.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/pools/Xomp_load_wkr.java
Unescape View File

@ -15,9 +15,9 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*; import gplx.core.threads.utils.*;
class Xomp_load_wkr implements Gfo_invk {
package gplx.xowa.addons.bldrs.mass_parses.parses.pools; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*; import gplx.core.threads.utils.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*;
public class Xomp_load_wkr implements Gfo_invk {
private final Object thread_lock = new Object();
private final Xow_wiki wiki;
private final Db_conn mgr_conn;
@ -34,7 +34,7 @@ class Xomp_load_wkr implements Gfo_invk {
this.attach_mgr = new Db_attach_mgr(mgr_conn);
this.queue = new Gfo_blocking_queue(num_pages_in_pool);
this.num_wkrs = num_wkrs;
this.time_bgn = this.time_prv = gplx.core.envs.Env_.TickCount();
this.time_bgn = this.time_prv = gplx.core.envs.System_.Ticks();
this.pages_total = this.Get_pending_count();
}
public int Get_pending_count() {
@ -103,7 +103,7 @@ class Xomp_load_wkr implements Gfo_invk {
synchronized (thread_lock) {
pages_done += 1;
if (pages_done % 1000 == 0) {
long time_cur = gplx.core.envs.Env_.TickCount();
long time_cur = gplx.core.envs.System_.Ticks();
int pages_left = pages_total - pages_done;
time_done += (time_cur - time_prv);
double rate_cur = pages_done / (time_done / Time_span_.Ratio_f_to_s);

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_page_itm.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/pools/Xomp_page_itm.java
Unescape View File

@ -15,9 +15,9 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses.pools; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.xowa.wikis.data.tbls.*;
class Xomp_page_itm implements Xowd_text_bry_owner {
public class Xomp_page_itm implements Xowd_text_bry_owner {
public Xomp_page_itm(int id) {this.id = id;}
public int Id() {return id;} private final int id;
public int Ns_id() {return ns_id;} private int ns_id;

14
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_page_pool.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/pools/Xomp_page_pool.java
Unescape View File

@ -15,9 +15,9 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xomp_page_pool {
package gplx.xowa.addons.bldrs.mass_parses.parses.pools; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
public class Xomp_page_pool {
private final Object thread_lock = new Object();
private final Xomp_page_pool_loader loader;
private final int num_pages_per_wkr;
@ -26,15 +26,15 @@ class Xomp_page_pool {
this.loader = loader; this.num_pages_per_wkr = num_pages_per_wkr;
}
public boolean Empty() {synchronized (thread_lock) {return empty;}} private boolean empty = false;
public void Get_next(List_adp wkr_list) {
synchronized (thread_lock) {
// pool already marked exhausted by another wkr; return;
public void Get_next(Xomp_mgr_db mgr_db, String machine_name, List_adp wkr_list) {
synchronized (thread_lock) { // LOCK:shared by multiple wkrs
// all pages read; "empty" marked done by another wkr; return;
if (empty) return;
int wkr_end = pool_idx + num_pages_per_wkr;
// need pages to fulfill request
if (wkr_end > pool_len) {
this.pool = loader.Load(pool, pool_idx, pool_len);
this.pool = loader.Load(mgr_db, machine_name, pool, pool_idx, pool_len);
this.pool_idx = 0;
this.pool_len = pool.Len();
if (pool_len == 0) { // no more pages; return;

27
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_page_pool_loader.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/pools/Xomp_page_pool_loader.java
Unescape View File

@ -15,13 +15,13 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
class Xomp_page_pool_loader {
package gplx.xowa.addons.bldrs.mass_parses.parses.pools; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.wkrs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*; import gplx.xowa.addons.bldrs.mass_parses.parses.locks.*;
public class Xomp_page_pool_loader {
private final Xow_wiki wiki;
private final int num_pages_per_load;
private final Db_attach_mgr attach_mgr;
private int prv_uid = -1;
public Xomp_page_pool_loader(Xow_wiki wiki, Db_conn make_conn, int num_pages_per_load) {
this.wiki = wiki;
this.make_conn = make_conn;
@ -35,18 +35,25 @@ class Xomp_page_pool_loader {
return rdr.Move_next() ? rdr.Read_int("Count_of") : 0;
} finally {rdr.Rls();}
}
public List_adp Load(List_adp list, int list_idx, int list_len) {
public List_adp Load(Xomp_mgr_db mgr_db, String machine_name, List_adp list, int list_idx, int list_len) {
List_adp rv = List_adp_.New();
// add remaining pages from old pool to new_pool;
for (int i = list_idx; i < list_len; ++i) {
rv.Add((Xomp_page_itm)list.Get_at(i));
}
// load pages into new pool
this.Load_from_db(rv);
Xomp_lock_mgr lock_mgr = mgr_db.Lock_mgr();
int uid_db = lock_mgr.Uid_prv__get(machine_name);
if (uid_db == Xomp_lock_mgr__fsys.Uid__eos) return rv; // assert that uids must be incrementally larger; handle one machine reaching end, and putting -1 in queue;
int uid_new = 0;
try {uid_new = this.Load_from_db(rv, uid_db);}
finally {lock_mgr.Uid_prv__rls(machine_name, uid_new);}
Gfo_usr_dlg_.Instance.Note_many("", "", "fetched new pool: old=~{0} new=~{1}", uid_db, uid_new);
return rv;
}
private void Load_from_db(List_adp list) {
private int Load_from_db(List_adp list, int uid_prv) {
// prepare for page_tbl
String sql = String_.Format(String_.Concat_lines_nl_skip_last // ANSI.Y
( "SELECT mp.xomp_uid"
@ -59,17 +66,18 @@ class Xomp_page_pool_loader {
, "WHERE mp.xomp_uid > {0}"
, "AND mp.page_status = 0"
, "LIMIT {1}"
), prv_uid, num_pages_per_load);
), uid_prv, num_pages_per_load);
this.attach_mgr.Conn_links_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn()));
sql = attach_mgr.Resolve_sql(sql);
// run page_tbl
int rv = -1;
Xomp_text_db_loader text_db_loader = new Xomp_text_db_loader(wiki);
attach_mgr.Attach();
Db_rdr rdr = make_conn.Stmt_sql(sql).Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
prv_uid = rdr.Read_int("xomp_uid");
rv = rdr.Read_int("xomp_uid");
int text_db_id = rdr.Read_int("page_text_db_id");
Xomp_page_itm ppg = new Xomp_page_itm(rdr.Read_int("page_id"));
ppg.Init_by_page
@ -84,5 +92,6 @@ class Xomp_page_pool_loader {
attach_mgr.Detach();
text_db_loader.Load();
return rv;
}
}

18
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_file_logger.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_lnki_temp_wkr.java
Unescape View File

@ -15,40 +15,36 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*;
import gplx.xowa.parsers.*; import gplx.xowa.parsers.lnkis.*; import gplx.xowa.parsers.lnkis.files.*;
import gplx.xowa.files.*; import gplx.xowa.addons.bldrs.files.cmds.*; import gplx.xowa.addons.bldrs.files.dbs.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.domains.*;
public class Xomp_file_logger implements Xop_file_logger {
public class Xomp_lnki_temp_wkr implements Xop_file_logger {
private final Xob_lnki_temp_tbl tbl;
private final Xowe_wiki commons_wiki;
private boolean ns_file_is_case_match_all = true;
public Xomp_file_logger(Xowe_wiki wiki, Db_conn wkr_conn) {
public Xomp_lnki_temp_wkr(Xowe_wiki wiki, Db_conn wkr_conn) {
this.tbl = new Xob_lnki_temp_tbl(wkr_conn); wkr_conn.Meta_tbl_assert(tbl);
this.commons_wiki = wiki.Appe().Wiki_mgr().Get_by_or_make(Xow_domain_itm_.Bry__commons);
this.ns_file_is_case_match_all = wiki.Init_assert().Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all; // NOTE: wiki must be init'd;
}
public void Bgn() {
tbl.Insert_bgn();
tbl.Insert_stmt_make();
}
public void Log_file(Xop_ctx ctx, Xop_lnki_tkn lnki, byte caller_tid) {
if (lnki.Ttl().ForceLiteralLink()) return; // ignore literal links which create a link to file, but do not show the image; EX: [[:File:A.png|thumb|120px]] creates a link to File:A.png, regardless of other display-oriented args
// get caller_tid / tttl
if (lnki.Ns_id() == Xow_ns_.Tid__media) caller_tid = Xop_file_logger_.Tid__media;
// get lnki_data
byte[] ttl = lnki.Ttl().Page_db();
Xof_ext ext = Xof_ext_.new_by_ttl_(ttl);
byte[] ttl_commons = Xomp_file_logger.To_commons_ttl(ns_file_is_case_match_all, commons_wiki, ttl);
byte[] ttl_commons = Xomp_lnki_temp_wkr.To_commons_ttl(ns_file_is_case_match_all, commons_wiki, ttl);
if (lnki.Ns_id() == Xow_ns_.Tid__media) caller_tid = Xop_file_logger_.Tid__media;
// do insert
tbl.Insert_cmd_by_batch(ctx.Page().Bldr__ns_ord(), ctx.Page().Db().Page().Id(), ttl, ttl_commons, Byte_.By_int(ext.Id()), lnki.Lnki_type(), caller_tid, lnki.W(), lnki.H(), lnki.Upright(), lnki.Time(), lnki.Page());
}
public void End() {
tbl.Insert_end();
}
public void End() {}
public static byte[] To_commons_ttl(boolean ns_file_is_case_match_all, Xowe_wiki commons_wiki, byte[] ttl_bry) { // handle case-sensitive wikis (en.d) vs case-insensitive commons
if (!ns_file_is_case_match_all) return null; // return "" if wiki matches common
Xoa_ttl ttl = Xoa_ttl.Parse(commons_wiki, Xow_ns_.Tid__file, ttl_bry);

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_text_db_loader.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_text_db_loader.java
Unescape View File

@ -15,11 +15,11 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*;
import gplx.core.ios.*;
import gplx.xowa.wikis.data.tbls.*;
class Xomp_text_db_loader {
public class Xomp_text_db_loader {
private final Xow_wiki wiki;
private final Ordered_hash text_db_hash = Ordered_hash_.New();
private final Io_stream_zip_mgr zip_mgr;

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_tmpl_cache_bldr.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java
Unescape View File

@ -15,10 +15,10 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*;
import gplx.xowa.wikis.caches.*;
class Xomp_tmpl_cache_bldr {
public class Xomp_tmpl_cache_bldr {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all) {
Xow_page_cache rv = new Xow_page_cache(wiki);
if (fill_all) Fill_all(rv, wiki);

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xob_hdump_tbl_retriever__xomp.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xob_hdump_tbl_retriever__xomp.java
Unescape View File

@ -15,7 +15,7 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.htmls.core.bldrs.*; import gplx.xowa.htmls.core.dbs.*;
class Xob_hdump_tbl_retriever__xomp implements Xob_hdump_tbl_retriever {

94
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_wkr.java → 400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
Unescape View File

@ -15,33 +15,48 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
package gplx.xowa.addons.bldrs.mass_parses.parses.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.files.origs.*;
import gplx.xowa.htmls.core.bldrs.*;
import gplx.xowa.parsers.*;
class Xomp_parse_wkr implements Gfo_invk {
private final Xomp_parse_mgr mgr;
import gplx.xowa.parsers.*;
import gplx.xowa.addons.bldrs.mass_parses.parses.mgrs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.utls.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*;
public class Xomp_parse_wkr implements Gfo_invk {
// mgr vars
private final Xomp_parse_mgr mgr;
private final Xomp_mgr_db mgr_db;
private final Xomp_prog_mgr prog_mgr;
private final Xomp_page_pool page_pool;
private final Xof_orig_wkr orig_wkr;
private final int idx;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private final Xof_orig_wkr file_orig_wkr;
// cfg vars
private final Xomp_parse_mgr_cfg cfg;
private int done_count; private long done_time;
private Xomp_wkr_db wkr_db; private int cleanup_interval, commit_interval;
private int cleanup_interval, commit_interval;
private boolean log_file_lnkis;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xof_orig_wkr orig_wkr, Xomp_page_pool page_pool, int idx, Xomp_parse_mgr_cfg cfg, int cleanup_interval, int commit_interval, boolean log_file_lnkis) {
this.mgr = mgr; this.wiki = wiki; this.orig_wkr = orig_wkr;
this.page_pool = page_pool;
this.idx = idx;
this.wkr_db = mgr.Db_core().Wkr_db(Bool_.Y, idx); // NOTE: must go in ctor, or else thread issues
// wkr vars
private final Xowe_wiki wiki;
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
private final int uid;
private Xomp_wkr_db wkr_db;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private int done_count; private long done_time;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg, Xomp_mgr_db mgr_db, Xomp_page_pool page_pool, Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xowe_wiki wiki, int uid) {
// mgr vars
this.mgr = mgr; this.mgr_db = mgr_db;
this.page_pool = page_pool; this.prog_mgr = prog_mgr; this.file_orig_wkr = file_orig_wkr;
// cfg vars
this.cfg = cfg;
this.cleanup_interval = cleanup_interval;
this.commit_interval = commit_interval;
this.log_file_lnkis = log_file_lnkis;
this.cleanup_interval = cfg.Cleanup_interval();
this.commit_interval = cfg.Commit_interval();
this.log_file_lnkis = cfg.Log_file_lnkis();
// wkr-specific vars
this.wiki = wiki; this.uid = uid;
this.wkr_db = Xomp_wkr_db.New(Xomp_mgr_db.New__url(wiki), uid);
}
public Xowe_wiki Wiki() {return wiki;} private final Xowe_wiki wiki;
public Xob_hdump_bldr Hdump_bldr() {return hdump_bldr;} private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
public void Exec() {
// init
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
@ -49,30 +64,41 @@ class Xomp_parse_wkr implements Gfo_invk {
// disable file download
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__orig_mgr().Wkrs__set(orig_wkr);
wiki.File__orig_mgr().Wkrs__set(file_orig_wkr);
wiki.File_mgr().Fsdb_mode().Tid__v2__mp__y_();
// disable categories else progress messages written (also for PERF)
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(false);
// enable lnki_temp
Xomp_file_logger logger = null;
Xomp_lnki_temp_wkr logger = null;
if (log_file_lnkis) {
logger = new Xomp_file_logger(wiki, wkr_db.Conn());
logger = new Xomp_lnki_temp_wkr(wiki, wkr_db.Conn());
parser_mgr.Ctx().Lnki().File_logger_(logger);
logger.Bgn();
}
// enable hdump
hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hzip_enabled()).Zip_tid_(cfg.Zip_tid()).Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid());
hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
wkr_db.Conn().Txn_bgn("xomp");
// set status to running
mgr_db.Wkr_tbl().Update_status(uid, Xomp_wkr_tbl.Status__running);
// main loop
while (true) {
Xomp_page_itm ppg = Get_next(); if (ppg == Xomp_page_itm.Null) break; // no more pages
// get page from page pool
Xomp_page_itm ppg = Get_next();
if (ppg == Xomp_page_itm.Null) {
mgr_db.Wkr_tbl().Update_status(uid, Xomp_wkr_tbl.Status__sleeping);
break; // no more pages
}
if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
try {
// init page
long done_bgn = gplx.core.envs.Env_.TickCount();
long done_bgn = gplx.core.envs.System_.Ticks();
Xoa_ttl ttl = wiki.Ttl_parse(ppg.Ns_id(), ppg.Ttl_bry());
Xoae_page wpg = Xoae_page.New(wiki, ttl);
wpg.Db().Text().Text_bry_(ppg.Text());
@ -87,10 +113,10 @@ class Xomp_parse_wkr implements Gfo_invk {
hdump_bldr.Insert(pctx, wpg);
// mark done for sake of progress
mgr.Prog_mgr().Mark_done(ppg.Id());
prog_mgr.Mark_done(ppg.Id());
// update stats
long time_cur = gplx.core.envs.Env_.TickCount();
long time_cur = gplx.core.envs.System_.Ticks();
done_time += time_cur - done_bgn;
done_bgn = time_cur;
++done_count;
@ -111,18 +137,15 @@ class Xomp_parse_wkr implements Gfo_invk {
}
}
if (logger != null)
logger.End();
// cleanup
wkr_db.Conn().Txn_end(); // NOTE: must end txn before running update wkr_id
mgr.Db_core().Update_wkr_id(idx, wkr_db.Conn());
mgr.Wkrs_done_add_1();
if (logger != null) logger.End();
wkr_db.Conn().Txn_end();
wkr_db.Conn().Rls_conn();
mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors
}
public void Bld_stats(Bry_bfr bfr) {
int done_time_in_sec = (int)(done_time / 1000); if (done_time_in_sec == 0) done_time_in_sec = 1;
bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, idx );
bfr.Add_int_pad_bgn(Byte_ascii.Space, 4, uid );
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, (int)(done_count / done_time_in_sec));
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_count);
bfr.Add_int_pad_bgn(Byte_ascii.Space, 8, done_time_in_sec);
@ -130,8 +153,9 @@ class Xomp_parse_wkr implements Gfo_invk {
}
private Xomp_page_itm Get_next() {
if (list_idx == list_len) {
mgr_db.Wkr_tbl().Update_exec(uid, done_count, done_time);
list.Clear();
page_pool.Get_next(list);
page_pool.Get_next(mgr_db, cfg.Wkr_machine_name(), list);
list_len = list.Len();
if (list_len == 0) return Xomp_page_itm.Null;
list_idx = 0;

15
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_reg_tbl.java
Unescape View File

@ -32,9 +32,15 @@ public class Imglnk_reg_tbl implements Db_tbl {
}
public Db_conn Conn() {return conn;}
public String Tbl_name() {return tbl_name;}
public String Fld__img_src() {return fld__img_src;}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx__src_ttl() {conn.Meta_idx_create(tbl_name, fld__img_src, fld__img_src, fld__img_repo);}
public void Create_idx__trg_ttl() {conn.Meta_idx_create(tbl_name, fld__img_trg, fld__img_trg, fld__img_repo);}
public Db_stmt Select_by_ttl_stmt() {
if (select_by_ttl_stmt == null)
select_by_ttl_stmt = conn.Stmt_select(tbl_name, flds, fld__img_src);
return select_by_ttl_stmt;
} private Db_stmt select_by_ttl_stmt;
public void Insert(Db_conn conn, byte repo_id, Xowe_wiki wiki) {
String repo_id_str = Byte_.To_str(repo_id);
Db_attach_mgr attach_mgr = new Db_attach_mgr(conn);
@ -53,17 +59,20 @@ public class Imglnk_reg_tbl implements Db_tbl {
);
attach_mgr.Exec_sql_w_msg("imglnk_reg.insert.redirect: repo=" + repo_id_str, sql);
attach_mgr.Conn_links_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Tbl__page().Conn()));
Xob_db_file image_db = Xob_db_file.New__wiki_image(wiki.Fsys_mgr().Root_dir());
attach_mgr.Conn_links_(new Db_attach_itm("image_db", image_db.Conn()));
sql = String_.Concat_lines_nl_skip_last // ANSI.Y
( "INSERT INTO imglnk_reg (img_src, img_trg, img_repo, img_count)"
, "SELECT ilt.img_name, ilt.img_name, " + repo_id_str + ", Count(ilt.img_name)"
, "FROM imglnk_tmp ilt"
, " JOIN <page_db>page p ON p.page_namespace = 6 AND p.page_title = ilt.img_name"
, " JOIN <image_db>image i ON i.img_name = ilt.img_name"
, " LEFT JOIN imglnk_reg il ON il.img_src = ilt.img_name"
, "WHERE il.img_src IS NULL"
, "GROUP BY ilt.img_name"
);
attach_mgr.Exec_sql_w_msg("imglnk_reg.insert.direct: repo=" + repo_id_str, sql);
}
public void Rls() {}
public void Rls() {
select_by_ttl_stmt = Db_stmt_.Rls(select_by_ttl_stmt);
}
}

Some files were not shown because too many files have changed in this diff Show More

Write Preview
Loading…
Cancel
Save