Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2024-10-27 20:34:16 +00:00
Code Issues Projects Releases Wiki Activity

Cldr: Add initial support for cldr

Browse Source
This commit is contained in:
gnosygnu 2018-08-05 21:21:40 -04:00
parent be3979c5af
commit e78382a8ac
27 changed files with 1052 additions and 99 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
100_core/src/gplx/Bry_bfr.java
View File

@ -372,8 +372,9 @@ public class Bry_bfr {
}
public Bry_bfr Add_float(float f) {Add_str_a7(Float_.To_str(f)); return this;}
public Bry_bfr Add_double(double v) {Add_str_a7(Double_.To_str(v)); return this;}
public Bry_bfr Add_dte(DateAdp val) {return Add_dte_segs(val.Year(), val.Month(),val.Day(), val.Hour(), val.Minute(), val.Second(), val.Frac());}
public Bry_bfr Add_dte_segs(int y, int M, int d, int H, int m, int s, int f) { // yyyyMMdd HHmmss.fff
public Bry_bfr Add_dte(DateAdp val) {return Add_dte_segs(Byte_ascii.Space , val.Year(), val.Month(),val.Day(), val.Hour(), val.Minute(), val.Second(), val.Frac());}
public Bry_bfr Add_dte_under(DateAdp val) {return Add_dte_segs(Byte_ascii.Underline, val.Year(), val.Month(),val.Day(), val.Hour(), val.Minute(), val.Second(), val.Frac());}
private Bry_bfr Add_dte_segs(byte spr, int y, int M, int d, int H, int m, int s, int f) { // yyyyMMdd HHmmss.fff
if (bfr_len + 19 > bfr_max) Resize((bfr_len + 19) * 2);
bfr[bfr_len + 0] = (byte)((y / 1000) + Bry_.Ascii_zero); y %= 1000;
bfr[bfr_len + 1] = (byte)((y / 100) + Bry_.Ascii_zero); y %= 100;
@ -383,7 +384,7 @@ public class Bry_bfr {
bfr[bfr_len + 5] = (byte)( M + Bry_.Ascii_zero);
bfr[bfr_len + 6] = (byte)((d / 10) + Bry_.Ascii_zero); d %= 10;
bfr[bfr_len + 7] = (byte)( d + Bry_.Ascii_zero);
bfr[bfr_len + 8] = Byte_ascii.Space;
bfr[bfr_len + 8] = spr;
bfr[bfr_len + 9] = (byte)((H / 10) + Bry_.Ascii_zero); H %= 10;
bfr[bfr_len + 10] = (byte)( H + Bry_.Ascii_zero);
bfr[bfr_len + 11] = (byte)((m / 10) + Bry_.Ascii_zero); m %= 10;

3
100_core/src/gplx/core/envs/System_.java
View File

@ -16,8 +16,9 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
package gplx.core.envs; import gplx.*; import gplx.core.*;
public class System_ {
// *** ticks
public static final int Ticks__per_second = 1000;
public static long Ticks() {return Ticks__test_val >= 0 ? Ticks__test_val : System.currentTimeMillis();}
public static int Ticks__elapsed_in_sec (long time_bgn) {return (int)(Ticks() - time_bgn) / 1000;}
public static int Ticks__elapsed_in_sec (long time_bgn) {return (int)(Ticks() - time_bgn) / Ticks__per_second;}
public static int Ticks__elapsed_in_frac (long time_bgn) {return (int)(Ticks() - time_bgn);}
public static void Ticks__test_set(long v) {Ticks__test_val = v;}
public static void Ticks__test_add(long v) {Ticks__test_val += v;}

58
100_core/src/gplx/core/lists/Sorted_hash.java Normal file
View File

@ -0,0 +1,58 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists; import gplx.*; import gplx.core.*;
public class Sorted_hash implements Hash_adp {
public Sorted_hash() {this.hash = new java.util.TreeMap();}
public Sorted_hash(ComparerAble comparer) {this.hash = new java.util.TreeMap(comparer);}
public boolean Has(Object key) {return Has_base(key);}
public Object Get_by(Object key) {return Fetch_base(key);}
public Object Get_by_or_fail(Object key) {return Get_by_or_fail_base(key);}
public void Add(Object key, Object val) {Add_base(key, val);}
public Hash_adp Add_and_more(Object key, Object val) {Add_base(key, val); return this;}
public void Add_as_key_and_val(Object val) {Add_base(val, val);}
public void Add_if_dupe_use_nth(Object key, Object val) {
Object existing = Fetch_base(key); if (existing != null) Del(key); // overwrite if exists
Add(key, val);
}
public boolean Add_if_dupe_use_1st(Object key, Object val) {
if (Has(key)) return false;
Add(key, val);
return true;
}
public void Del(Object key) {Del_base(key);}
protected Object Get_by_or_fail_base(Object key) {
if (key == null) throw Err_.new_wo_type("key cannot be null");
if (!Has_base(key)) throw Err_.new_wo_type("key not found", "key", key);
return Fetch_base(key);
}
public Object[] Values_array() {
return hash.values().toArray();
}
public Object Del_val_at_0() {
return hash.pollFirstEntry().getValue();
}
private final java.util.TreeMap hash;
public int Len() {return hash.size();}
public int Count() {return hash.size();}
public void Clear() {hash.clear();}
private void Add_base(Object key, Object val) {hash.put(key, val);}
private void Del_base(Object key) {hash.remove(key);}
private boolean Has_base(Object key) {return hash.containsKey(key);}
private Object Fetch_base(Object key) {return hash.get(key);}
public Object Get_val_at_0() {return hash.firstEntry().getValue();}
public java.util.Iterator iterator() {return hash.values().iterator();}
}

92
400_xowa/src/gplx/core/lists/Binary_heap.java Normal file
View File

@ -0,0 +1,92 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists; import gplx.*; import gplx.core.*;
public class Binary_heap {
private final ComparerAble comparer;
private boolean is_max;
private Object[] heap;
private int size;
public Binary_heap(ComparerAble comparer, boolean is_max, int capacity) {
this.comparer = comparer;
this.is_max = is_max;
this.heap = new Object[capacity];
this.size = 0;
}
public Object Get() {
if (size == 0) throw Err_.new_wo_type("heap is empty");
return heap[0];
}
public void Add(Object val) {
if (size == heap.length) {
int old_max = heap.length;
Object[] new_heap = new Object[old_max * 2];
for (int i = 0; i < old_max; i++)
new_heap[i] = heap[i];
heap = new_heap;
}
heap[size++] = val;
Heapify_up(size - 1);
}
public Object Pop() {return Pop(0);}
public Object Pop(int idx) {
if (size == 0) throw Err_.new_wo_type("heap is empty");
Object val = heap[idx];
heap[idx] = heap[size -1];
size--;
Heapify_down(idx);
return val;
}
private int Parent(int idx) {
return (idx - 1) / SLOT;
}
private int Kth_child(int idx, int k) {
return (idx * SLOT) + k;
}
private int Max_child(int idx) {
int lhs = Kth_child(idx, 1);
int rhs = Kth_child(idx, 2);
int comp = ComparerAble_.Compare(comparer, heap[lhs], heap[rhs]);
boolean diff = is_max ? comp == CompareAble_.More : comp == CompareAble_.Less;
return diff ? lhs : rhs;
}
private void Heapify_up(int idx) {
Object val = heap[idx];
while (idx > 0) {
int comp = ComparerAble_.Compare(comparer, val, heap[Parent(idx)]);
if (!(is_max ? comp == CompareAble_.More : comp == CompareAble_.Less))
break;
heap[idx] = heap[Parent(idx)];
idx = Parent(idx);
}
heap[idx] = val;
}
private void Heapify_down(int idx) {
int child;
Object val = heap[idx];
while (Kth_child(idx, 1) < size) {
child = Max_child(idx);
int comp = ComparerAble_.Compare(comparer, val, heap[child]);
if (is_max ? comp == CompareAble_.Less : comp == CompareAble_.More)
heap[idx] = heap[child];
else
break;
idx = child;
}
heap[idx] = val;
}
private static final int SLOT = 2;
}

51
400_xowa/src/gplx/core/lists/Binary_heap_tst.java Normal file
View File

@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists; import gplx.*; import gplx.core.*;
import org.junit.*; import gplx.core.tests.*;
import gplx.core.primitives.*;
public class Binary_heap_tst {
private final Binary_heap_fxt fxt = new Binary_heap_fxt();
@Test public void Max() {
fxt.Init(Bool_.Y);
fxt.Exec__add(4, 3, 5, 7, 1, 6, 9, 8, 2);
fxt.Test__pop(9, 8, 7, 6, 5, 4, 3, 2, 1);
}
@Test public void Min() {
fxt.Init(Bool_.N);
fxt.Exec__add(4, 3, 5, 7, 1, 6, 9, 8, 2);
fxt.Test__pop(1, 2, 3, 4, 5, 6, 7, 8, 9);
}
}
class Binary_heap_fxt implements ComparerAble {
private Binary_heap heap;
public int compare(Object lhsObj, Object rhsObj) {
return CompareAble_.Compare_obj(lhsObj, rhsObj);
}
public void Init(boolean is_max) {
heap = new Binary_heap(this, is_max, 2);
}
public void Exec__add(int... ary) {
for (int i : ary)
heap.Add(new Int_obj_val(i));
}
public void Test__pop(int... expd) {
int len = expd.length;
int[] actl = new int[len];
for (int i = 0; i < len; i++)
actl[i] = ((Int_obj_val)heap.Pop()).Val();
Gftest.Eq__ary(expd, actl, "heaps don't match");
}
}

71
400_xowa/src/gplx/core/lists/caches/Mru_cache_itm.java Normal file
View File

@ -0,0 +1,71 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
class Mru_cache_itm {
private final long time_bgn;
private long time_dirty;
private long used_dirty = -1;
public Mru_cache_itm(Object key, Object val, long size, long time) {
this.key = key;
this.val = val;
this.size = size;
this.time_bgn = this.time_cur = time;
}
public Object Key() {return key;} private final Object key;
public Object Val() {return val;} private final Object val;
public long Size() {return size;} private long size;
public long Time_cur() {return time_cur;} private long time_cur;
public long Time_dif() {return time_cur - time_bgn;}
public long Used() {return used;} private long used;
public void Dirty(long time) {
this.time_dirty = time;
if (used_dirty == -1)
used_dirty = used + 1;
else
used_dirty++;
}
public void Update() {
this.time_cur = time_dirty;
this.used = used_dirty;
used_dirty = -1;
}
public void Update(long time) {
this.time_cur = time;
used++;
}
}
class Mru_cache_itm_comparer implements ComparerAble {
private long used_weight;
private long time_bgn;
public Mru_cache_itm_comparer(long used_weight, long time_bgn) {
Init(used_weight, time_bgn);
}
public void Init(long used_weight, long time_bgn) {
this.used_weight = used_weight;
this.time_bgn = time_bgn;
}
public long Score(Mru_cache_itm itm) {
return (itm.Time_cur() - time_bgn) + (itm.Used() * used_weight);
}
public int compare(Object lhsObj, Object rhsObj) {
Mru_cache_itm lhs = (Mru_cache_itm)lhsObj;
Mru_cache_itm rhs = (Mru_cache_itm)rhsObj;
int rv = Long_.Compare(Score(lhs), Score(rhs));
return rv == 0
? CompareAble_.Compare_obj(lhs.Key(), rhs.Key())
: rv;
}
}

108
400_xowa/src/gplx/core/lists/caches/Mru_cache_mgr.java Normal file
View File

@ -0,0 +1,108 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
import gplx.core.logs.*;
public class Mru_cache_mgr {
private final Mru_cache_time_mgr time_mgr;
private final Gfo_log_wtr log_wtr;
private final Hash_adp key_hash = Hash_adp_.New();
private final Sorted_hash val_hash;
private final Bry_bfr log_bfr = Bry_bfr_.New_w_size(255);
private final Mru_cache_itm_comparer comparer;
private final Ordered_hash dirty = Ordered_hash_.New();
private long cache_max, cache_size, compress_size;
Mru_cache_mgr(Mru_cache_time_mgr time_mgr, Gfo_log_wtr log_wtr, long cache_max, long compress_size, long used_weight) {
this.time_mgr = time_mgr;
this.log_wtr = log_wtr;
this.cache_max = cache_max;
this.compress_size = compress_size;
this.comparer = new Mru_cache_itm_comparer(used_weight, time_mgr.Now());
this.val_hash = new Sorted_hash(comparer);
}
public long Cache_max() {return cache_max;} public void Cache_max_(long v) {this.cache_max = v;}
public Object Get_or_null(String key) {
Object itm_obj = key_hash.Get_by(key);
if (itm_obj == null) return null;
Mru_cache_itm itm = (Mru_cache_itm)itm_obj;
dirty.Add_if_dupe_use_1st(key, itm);
itm.Dirty(time_mgr.Now());
return itm.Val();
}
public void Add(String key, Object val, long val_size) {
if (cache_size + val_size > cache_max) {
Compress(val_size);
}
Mru_cache_itm itm = new Mru_cache_itm(key, val, val_size, time_mgr.Now());
key_hash.Add(key, itm);
val_hash.Add(itm, itm);
cache_size += val_size;
}
public void Compress(long val_size) {
int dirty_len = dirty.Len();
for (int i = 0; i < dirty_len; i++) {
Mru_cache_itm dirty_itm = (Mru_cache_itm)dirty.Get_at(i);
val_hash.Del(dirty_itm);
dirty_itm.Update();
val_hash.Add(dirty_itm, dirty_itm);
}
dirty.Clear();
while (cache_size + val_size > compress_size) {
Mru_cache_itm old = (Mru_cache_itm)val_hash.Del_val_at_0();
key_hash.Del(old.Key());
cache_size -= old.Size();
if (log_wtr != null) {
Write(Bool_.Y, old);
log_wtr.Write(log_bfr);
}
}
}
public void Flush() {log_wtr.Flush();}
public void Print() {
Object[] vals = val_hash.Values_array();
int vals_len = vals.length;
for (int i = vals_len - 1; i >= 0; i--) {
Mru_cache_itm val = (Mru_cache_itm)vals[i];
Write(Bool_.N, val);
}
Io_mgr.Instance.SaveFilBfr(log_wtr.Fil_dir().GenSubFil("cache_mru_final.csv"), log_bfr);
log_bfr.Clear();
}
private void Write(boolean write_time, Mru_cache_itm itm) {
// 20180625_112443.506|Q2|123uses|10ms|1234bytes\n
if (write_time) log_bfr.Add_dte_under(Datetime_now.Get_force()).Add_byte_pipe();
log_bfr.Add_str_u8(Object_.Xto_str_strict_or_null_mark(itm.Key()));
log_bfr.Add_byte_pipe().Add_long_fixed(comparer.Score(itm), 10);
log_bfr.Add_byte_pipe().Add_long_variable(itm.Size());
log_bfr.Add_byte_pipe().Add_long_variable(itm.Time_dif());
log_bfr.Add_byte_pipe().Add_long_variable(itm.Used());
log_bfr.Add_byte_nl();
}
public void Clear() {
key_hash.Clear();
val_hash.Clear();
log_bfr.Clear();
dirty.Clear();
cache_size = 0;
}
public static Mru_cache_mgr New_by_mb_secs(Gfo_log_wtr log_wtr, long cache_max_in_mb, long compress_size, long used_weight_in_secs) {
return new Mru_cache_mgr(new Mru_cache_time_mgr__clock(), log_wtr, Io_mgr.Len_mb * cache_max_in_mb, Io_mgr.Len_mb * compress_size, gplx.core.envs.System_.Ticks__per_second * used_weight_in_secs);
}
// TEST
public Object[] Values_array() {return val_hash.Values_array();}
@gplx.Internal protected static Mru_cache_mgr New_test(Mru_cache_time_mgr time_mgr, Gfo_log_wtr log_wtr, long cache_max, long compress_size, long used_weight) {return new Mru_cache_mgr(time_mgr, log_wtr, cache_max, compress_size, used_weight);}
}

90
400_xowa/src/gplx/core/lists/caches/Mru_cache_mgr_tst.java Normal file
View File

@ -0,0 +1,90 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
import org.junit.*; import gplx.core.tests.*;
public class Mru_cache_mgr_tst {
private final Mru_cache_mgr_fxt fxt = new Mru_cache_mgr_fxt();
@Before public void init() {
fxt.Init__New_cache_mgr(3, 3, 2);
}
@Test public void Basic() {
fxt.Exec__Add("a", "b", "c", "d");
fxt.Test__Print("b", "c", "d"); // adding "d" pushes out "a"
}
@Test public void Used() {
fxt.Exec__Add("a", "b", "c");
fxt.Exec__Get_and_compress("a");
fxt.Test__Print("b", "c", "a"); // getting "a" pushes to back
}
@Test public void Used__more_uses_at_back() {
fxt.Exec__Add("a", "b", "c");
fxt.Exec__Get_and_compress("a", "a", "a");
fxt.Test__Print("b", "c", "a");
fxt.Exec__Get_and_compress("b");
fxt.Test__Print("c", "b", "a"); // getting "a" multiple time still keeps towards back
}
@Test public void Time() {
fxt.Exec__Add("a", "b", "c");
fxt.Exec__Get_and_compress("a", "a", "a");
fxt.Exec__Wait(10);
fxt.Exec__Get_and_compress("b");
fxt.Test__Print("c", "a", "b"); // long wait puts "b" at back
}
@Test public void Compress() {
fxt.Init__New_cache_mgr(3, 2, 2);
fxt.Exec__Add("a", "b", "c", "d");
fxt.Test__Print("c", "d");
}
}
class Mru_cache_mgr_fxt {
private final Mru_cache_time_mgr__mock time_mgr = new Mru_cache_time_mgr__mock();
private Mru_cache_mgr cache_mgr;
public void Init__New_cache_mgr(long cache_max, long used_weight, long compress_size) {
cache_mgr = Mru_cache_mgr.New_test(time_mgr, null, cache_max, used_weight, compress_size);
}
public void Exec__Add(String... ary) {
for (String itm : ary)
cache_mgr.Add(itm, itm, 1);
}
public void Exec__Get(String... ary) {
for (String itm : ary)
cache_mgr.Get_or_null(itm);
}
public void Exec__Wait(int time) {
time_mgr.Add(time);
}
public void Exec__Compress(long val_size) {
cache_mgr.Compress(val_size);
}
public void Exec__Get_and_compress(String... ary) {
this.Exec__Get(ary);
this.Exec__Compress(0);
}
public void Test__Print(String... expd) {
Object[] actl_objs = cache_mgr.Values_array();
int actl_len = actl_objs.length;
String[] actl = new String[actl_len];
for (int i = 0; i < actl_len; i++) {
actl[i] = Object_.Xto_str_strict_or_null(((Mru_cache_itm)actl_objs[i]).Val());
}
Gftest.Eq__ary(expd, actl);
}
}
class Mru_cache_time_mgr__mock implements Mru_cache_time_mgr {
private long time;
public long Now() {return time++;}
public void Add(int v) {time += v;}
}

12
400_xowa/src/gplx/xowa/apps/caches/Xoa_cache_mgr.java → 400_xowa/src/gplx/core/lists/caches/Mru_cache_time_mgr.java
View File

@ -13,10 +13,10 @@ The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.apps.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
public class Xoa_cache_mgr {
public Wbase_doc_cache Doc_cache() {return doc_cache;} private final Wbase_doc_cache doc_cache = new Wbase_doc_cache();
public void Free_mem_all() {
doc_cache.Free_mem_all();
}
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
interface Mru_cache_time_mgr {
long Now();
}
class Mru_cache_time_mgr__clock implements Mru_cache_time_mgr {
public long Now() {return gplx.core.envs.System_.Ticks();}
}

54
400_xowa/src/gplx/core/logs/Gfo_log_wtr.java Normal file
View File

@ -0,0 +1,54 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.logs; import gplx.*; import gplx.core.*;
public class Gfo_log_wtr {
private final Bry_bfr bfr = Bry_bfr_.New();
private final int bfr_max;
private final int fil_max;
private final Io_url fil_dir;
private final String fil_fmt;
private final int fil_idx_places;
private Io_url fil_url;
private int fil_idx = -1, fil_len;
public Gfo_log_wtr(int bfr_max, int fil_max, Io_url fil_dir, String fil_fmt, int fil_idx_places) {
this.bfr_max = bfr_max;
this.fil_max = fil_max;
this.fil_dir = fil_dir;
this.fil_fmt = fil_fmt;
this.fil_idx_places = fil_idx_places;
}
public Io_url Fil_dir() {return fil_dir;}
public void Write(Bry_bfr add) {
int add_len = add.Len();
if (bfr.Len() + add_len > bfr_max)
Flush();
bfr.Add_bfr_and_clear(add);
}
public void Flush() {
if (fil_idx == -1 || bfr.Len() + fil_len > fil_max) {
fil_idx++;
fil_url = fil_dir.GenSubFil(String_.Format(fil_fmt, Int_.To_str_pad_bgn_zero(fil_idx, fil_idx_places)));
fil_len = 0;
}
fil_len += bfr.Len();
Io_mgr.Instance.AppendFilBfr(fil_url, bfr);
bfr.Clear();
}
public static Gfo_log_wtr New_dflt(String sub_dir, String file_fmt) {
return new Gfo_log_wtr(Io_mgr.Len_mb, 10 * Io_mgr.Len_mb, Gfo_usr_dlg_.Instance.Log_wkr().Session_dir().GenSubDir(sub_dir), file_fmt, 4);
}
}

1
400_xowa/src/gplx/langs/jsons/Json_doc_wtr.java
View File

@ -60,6 +60,7 @@ public class Json_doc_wtr {
bfr.Add_byte_nl();
return this;
}
public Json_doc_wtr Key(boolean comma, String key) {return Key(comma, Bry_.new_u8(key));}
public Json_doc_wtr Key(boolean comma, byte[] key) {
Key_internal(comma, key);
bfr.Add_byte_nl();

2
400_xowa/src/gplx/xowa/Xoa_app_.java
View File

@ -31,7 +31,7 @@ public class Xoa_app_ {
}
public static final String Name = "xowa";
public static final int Version_id = 542;
public static final String Version = "4.5.20.1801";
public static final String Version = "4.5.21.1808";
public static String Build_date = "2012-12-30 00:00:00";
public static String Build_date_fmt = "yyyy-MM-dd HH:mm:ss";
public static String Op_sys_str;

3
400_xowa/src/gplx/xowa/Xoae_app.java
View File

@ -15,7 +15,7 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa; import gplx.*;
import gplx.core.brys.*; import gplx.core.btries.*; import gplx.core.brys.fmtrs.*; import gplx.core.flds.*; import gplx.core.ios.*; import gplx.core.threads.*; import gplx.langs.jsons.*; import gplx.core.primitives.*; import gplx.core.net.*; import gplx.core.log_msgs.*; import gplx.core.envs.*;
import gplx.xowa.apps.*; import gplx.xowa.apps.fsys.*; import gplx.xowa.apps.site_cfgs.*; import gplx.xowa.apps.caches.*; import gplx.xowa.apps.apis.*; import gplx.xowa.apps.metas.*; import gplx.langs.htmls.encoders.*; import gplx.xowa.apps.progs.*; import gplx.xowa.apps.gfs.*;
import gplx.xowa.apps.*; import gplx.xowa.apps.fsys.*; import gplx.xowa.apps.site_cfgs.*; import gplx.xowa.apps.apis.*; import gplx.xowa.apps.metas.*; import gplx.langs.htmls.encoders.*; import gplx.xowa.apps.progs.*; import gplx.xowa.apps.gfs.*;
import gplx.xowa.langs.*; import gplx.xowa.specials.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.css.*; import gplx.xowa.bldrs.installs.*;
import gplx.xowa.files.*; import gplx.xowa.files.caches.*; import gplx.xowa.files.imgs.*;
@ -138,7 +138,6 @@ public class Xoae_app implements Xoa_app, Gfo_invk {
public Xoa_fsys_eval Url_cmd_eval() {return url_cmd_eval;} Xoa_fsys_eval url_cmd_eval;
public Xoa_cur Cur_redirect() {return cur_redirect;} private Xoa_cur cur_redirect;
public Io_stream_zip_mgr Zip_mgr() {return zip_mgr;} Io_stream_zip_mgr zip_mgr = new Io_stream_zip_mgr();
public Xoa_cache_mgr Cache_mgr() {return cache_mgr;} private final Xoa_cache_mgr cache_mgr = new Xoa_cache_mgr();
public Xosrv_server Tcp_server() {return tcp_server;} private Xosrv_server tcp_server = new Xosrv_server();
public Http_server_mgr Http_server() {return http_server;} private Http_server_mgr http_server;

3
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
View File

@ -45,6 +45,8 @@ public class Xomp_parse_mgr {
Xof_orig_wkr__img_links file_orig_wkr = new Xof_orig_wkr__img_links(wiki);
if (cfg.Load_all_imglinks()) Xof_orig_wkr__img_links_.Load_all(file_orig_wkr);
wiki.Appe().Wiki_mgr().Wdata_mgr().Doc_mgr.Cache__init(cfg.Wbase_cache_mru_type(), cfg.Wbase_cache_mru_size(), cfg.Wbase_cache_mru_compress_size(), cfg.Wbase_cache_mru_weight());
// load_wkr: init and start
// Xomp_load_wkr load_wkr = new Xomp_load_wkr(wiki, db_mgr.Mgr_db().Conn(), cfg.Num_pages_in_pool(), cfg.Num_wkrs());
// Thread_adp_.Start_by_key("xomp.load", Cancelable_.Never, load_wkr, Xomp_load_wkr.Invk__exec);
@ -84,6 +86,7 @@ public class Xomp_parse_mgr {
latch.Await();
page_pool.Rls();
if (indexer != null) indexer.Term();
wiki.Appe().Wiki_mgr().Wdata_mgr().Doc_mgr.Cleanup();
// print stats
Bry_bfr bfr = Bry_bfr_.New();

8
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
View File

@ -38,6 +38,10 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
public boolean Show_msg__fetched_pool() {return show_msg__fetched_pool;} private boolean show_msg__fetched_pool;
public boolean Indexer_enabled() {return indexer_enabled;} private boolean indexer_enabled;
public String Indexer_opt() {return indexer_opt;} private String indexer_opt = gplx.gflucene.indexers.Gflucene_idx_opt.Docs_and_freqs.Key();
public String Wbase_cache_mru_type() {return wbase_cache_mru_type;} private String wbase_cache_mru_type = "mru";
public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100;
public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10;
public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70;
public void Init(Xowe_wiki wiki) {
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count();
if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000;
@ -68,6 +72,10 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, Invk__log_math_)) log_math = m.ReadYn("v");
else if (ctx.Match(k, "indexer_enabled_")) indexer_enabled = m.ReadYn("v");
else if (ctx.Match(k, "indexer_opt_")) indexer_opt = m.ReadStr("v");
else if (ctx.Match(k, "wbase_cache_mru_type_")) wbase_cache_mru_type = m.ReadStr("v");
else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v");
else return Gfo_invk_.Rv_unhandled;
return this;
}

24
400_xowa/src/gplx/xowa/apps/caches/Wbase_doc_cache.java
View File

@ -1,24 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.apps.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*;
import gplx.xowa.xtns.wbases.*;
public class Wbase_doc_cache {
private final Hash_adp_bry hash = Hash_adp_bry.cs();
public void Add(byte[] qid, Wdata_doc doc) {hash.Add(qid, doc);}
public Wdata_doc Get_or_null(byte[] qid) {return (Wdata_doc)hash.Get_by_bry(qid);}
public void Free_mem_all() {this.Clear();}
public void Clear() {hash.Clear();}
}

1
400_xowa/src/gplx/xowa/bldrs/cmds/Xob_dump_mgr_base.java
View File

@ -66,6 +66,7 @@ public abstract class Xob_dump_mgr_base extends Xob_itm_basic_base implements Xo
Init_reset(conn);
}
bmk_mgr.Load(wiki.Appe(), this);
Cmd_bgn_end();
}
protected abstract void Cmd_bgn_end();

45
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_lang_tbl.java
View File

@ -1,45 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
import gplx.dbs.*;
class Cldr_lang_tbl implements Rls_able {
private final String tbl_name = "cldr_lang"; private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private final String fld_cldr_code, fld_lang_code, fld_lang_name;
private final Db_conn conn; private Db_stmt stmt_select, stmt_insert;
public Cldr_lang_tbl(Db_conn conn) {
this.conn = conn;
this.fld_cldr_code = flds.Add_str("cldr_code", 32);
this.fld_lang_code = flds.Add_str("lang_code", 32);
this.fld_lang_name = flds.Add_str("lang_name", 2048);
conn.Rls_reg(this);
}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "main", fld_cldr_code, fld_lang_code)));}
public void Insert_bgn() {conn.Txn_bgn("cldr_lang__inser"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
public void Insert_cmd_by_batch(byte[] cldr_code, byte[] lang_code, byte[] lang_name) {
stmt_insert.Clear().Val_bry_as_str(fld_cldr_code, cldr_code).Val_bry_as_str(fld_lang_code, lang_code).Val_bry_as_str(fld_lang_name, lang_name).Exec_insert();
}
public byte[] Select(byte[] cldr_code, byte[] lang_code) {
if (stmt_select == null) stmt_select = conn.Stmt_select(tbl_name, flds, fld_cldr_code, fld_lang_code);
Db_rdr rdr = stmt_select.Clear().Val_bry_as_str(fld_cldr_code, cldr_code).Val_bry_as_str(fld_lang_code, lang_code).Exec_select__rls_manual();
try {return (byte[])rdr.Read_bry(fld_lang_name);}
finally {rdr.Rls();}
}
public void Rls() {
stmt_select = Db_stmt_.Rls(stmt_select);
stmt_insert = Db_stmt_.Rls(stmt_insert);
}
}

113
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_converter.java Normal file
View File

@ -0,0 +1,113 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
import gplx.core.primitives.*;
import gplx.langs.phps.*;
import gplx.langs.jsons.*;
class Cldr_name_converter {
private final Php_parser parser = new Php_parser();
private final Php_evaluator eval = new Php_evaluator(new gplx.core.log_msgs.Gfo_msg_log("test"));
private final Php_text_itm_parser text_itm_parser = new Php_text_itm_parser().Quote_is_single_(true);
private final Json_doc_wtr doc_wtr = new Json_doc_wtr();
private final List_adp tmp_list = List_adp_.New();
private final Byte_obj_ref tmp_result = Byte_obj_ref.zero_();
private final Bry_bfr tmp_bfr = Bry_bfr_.New();
public void Convert(Io_url src, Io_url trg) {
Cldr_name_file[] fils = Parse_dir(src);
for (Cldr_name_file fil : fils) {
Io_mgr.Instance.SaveFilStr(trg.GenSubDir(File_CldrNames).GenSubFil_ary(File_CldrNames, fil.Key(), ".json"), To_json(fil));
Gfo_usr_dlg_.Instance.Note_many("", "", "converted file; key={0}", fil.Key());
}
}
public Cldr_name_file[] Parse_dir(Io_url dir) {
List_adp rv = List_adp_.New();
Io_url[] fils = Io_mgr.Instance.QueryDir_fils(dir);
for (Io_url fil : fils) {
String key = Extract_key_or_fail(fil.NameAndExt());
byte[] src = Io_mgr.Instance.LoadFilBry(fil);
rv.Add(Parse_fil(key, src));
}
return (Cldr_name_file[])rv.To_ary_and_clear(Cldr_name_file.class);
}
public String Extract_key_or_fail(String fil_name) {
if (!String_.Has_at_bgn(fil_name, File_CldrNames) || !String_.Has_at_end(fil_name, ".php")) throw Err_.new_wo_type("file name must have a format of CldrNamesLANG.php", "fil_name", fil_name);
return String_.Mid(fil_name, String_.Len(File_CldrNames), String_.Len(fil_name) - String_.Len(".php"));
}
public Cldr_name_file Parse_fil(String key, byte[] src) {
Cldr_name_file file = new Cldr_name_file(key);
parser.Parse_tkns(src, eval);
Php_line[] lines = (Php_line[])eval.List().To_ary(Php_line.class);
int lines_len = lines.length;
for (int i = 0; i < lines_len; i++) {
Php_line line = lines[i];
Php_line_assign assign_line = (Php_line_assign)line;
String assign_key = String_.new_u8(assign_line.Key().Val_obj_bry());
Ordered_hash hash = null;
if (String_.Eq(assign_key, Node_languageNames)) hash = file.Language_names();
else if (String_.Eq(assign_key, Node_currencyNames)) hash = file.Currency_names();
else if (String_.Eq(assign_key, Node_currencySymbols)) hash = file.Currency_symbols();
else if (String_.Eq(assign_key, Node_countryNames)) hash = file.Country_names();
else if (String_.Eq(assign_key, Node_timeUnits)) hash = file.Time_units();
else throw Err_.new_unhandled_default(assign_key);
Parse_assign_line(key, assign_key, assign_line, hash);
}
eval.Clear();
return file;
}
private void Parse_assign_line(String lang_key, String assign_key, Php_line_assign assign, Ordered_hash hash) {
Php_itm_ary ary = (Php_itm_ary)assign.Val();
int ary_len = ary.Subs_len();
for (int i = 0; i < ary_len; i++) {
Php_itm_kv kv = (Php_itm_kv)ary.Subs_get(i);
String key_str = String_.new_u8(kv.Key().Val_obj_bry());
String val_str = String_.new_u8(text_itm_parser.Parse_as_bry(tmp_list, kv.Val().Val_obj_bry(), tmp_result, tmp_bfr));
if (hash.Has(key_str)) throw Err_.new_wo_type("key already exists", "lang", lang_key, "type", assign_key, "key", key_str);
hash.Add(key_str, Keyval_.new_(key_str, val_str));
}
}
public String To_json(Cldr_name_file name_file) {
Int_obj_ref written = Int_obj_ref.New_zero();
doc_wtr.Nde_bgn();
To_json(written, Node_languageNames , name_file.Language_names());
To_json(written, Node_currencyNames , name_file.Currency_names());
To_json(written, Node_currencySymbols , name_file.Currency_symbols());
To_json(written, Node_countryNames , name_file.Country_names());
To_json(written, Node_timeUnits , name_file.Time_units());
doc_wtr.Nde_end();
return doc_wtr.Bld_as_str();
}
private void To_json(Int_obj_ref written, String node_name, Ordered_hash hash) {
int len = hash.Count();
if (len == 0) return;
doc_wtr.Key(written.Val() != 0, node_name);
doc_wtr.Nde_bgn();
for (int i = 0; i < len; i++) {
Keyval kv = (Keyval)hash.Get_at(i);
doc_wtr.Kv(i != 0, Bry_.new_u8(kv.Key()), Bry_.new_u8(kv.Val_to_str_or_null()));
}
doc_wtr.Nde_end();
written.Val_add_post();
}
public static final String
File_CldrNames = "CldrNames"
, Node_languageNames = "languageNames"
, Node_currencyNames = "currencyNames"
, Node_currencySymbols = "currencySymbols"
, Node_countryNames = "countryNames"
, Node_timeUnits = "timeUnits"
;
}

166
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_converter_tst.java Normal file
View File

@ -0,0 +1,166 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
import org.junit.*; import gplx.core.tests.*;
import gplx.langs.phps.*;
public class Cldr_name_converter_tst {
private final Cldr_name_converter_fxt fxt = new Cldr_name_converter_fxt();
@Test public void Extract_key_or_fail() {
fxt.Test__Extract_key_or_fail("CldrNamesEn.php" , "En");
fxt.Test__Extract_key_or_fail("CldrNameEn.php" , null);
fxt.Test__Extract_key_or_fail("CldrNamesEn.txt" , null);
}
@Test public void Parse_fil() {
Cldr_name_file file = fxt.Exec__Parse_fil("En", String_.Concat_lines_nl
( "$languageNames = ["
, " 'aa' => 'Afar',"
, " 'mic' => 'Mi\\'kmaq',"
, " 'zza' => 'Zaza',"
, "];"
, ""
, "$currencyNames = ["
, " 'ADP' => 'Andorran Peseta',"
, " 'ZWR' => 'Zimbabwean Dollar (2008)',"
, "];"
, ""
, "$currencySymbols = ["
, " 'JPY' => '¥',"
, " 'USD' => '$',"
, "];"
, ""
, "$countryNames = ["
, " 'AC' => 'Ascension Island',"
, " 'ZW' => 'Zimbabwe',"
, "];"
, ""
, "$timeUnits = ["
, " 'century-one' => '{0} century',"
, " 'year-short-past-other' => '{0} yr. ago',"
, "];"
));
Assert__parse_fil(file);
String expd = String_.Concat_lines_nl
( "{"
, " \"languageNames\":"
, " {"
, " \"aa\":\"Afar\""
, " , \"mic\":\"Mi'kmaq\""
, " , \"zza\":\"Zaza\""
, " }"
, ", \"currencyNames\":"
, " {"
, " \"ADP\":\"Andorran Peseta\""
, " , \"ZWR\":\"Zimbabwean Dollar (2008)\""
, " }"
, ", \"currencySymbols\":"
, " {"
, " \"JPY\":\"¥\""
, " , \"USD\":\"$\""
, " }"
, ", \"countryNames\":"
, " {"
, " \"AC\":\"Ascension Island\""
, " , \"ZW\":\"Zimbabwe\""
, " }"
, ", \"timeUnits\":"
, " {"
, " \"century-one\":\"{0} century\""
, " , \"year-short-past-other\":\"{0} yr. ago\""
, " }"
, "}"
);
fxt.Test__To_json(file, expd);
file = fxt.Exec__To_file("En", expd);
Assert__parse_fil(file);
}
private void Assert__parse_fil(Cldr_name_file file) {
fxt.Test__node(file.Language_names()
, Keyval_.new_("aa", "Afar")
, Keyval_.new_("mic", "Mi'kmaq")
, Keyval_.new_("zza", "Zaza")
);
fxt.Test__node(file.Currency_names()
, Keyval_.new_("ADP", "Andorran Peseta")
, Keyval_.new_("ZWR", "Zimbabwean Dollar (2008)")
);
fxt.Test__node(file.Currency_symbols()
, Keyval_.new_("JPY", "¥")
, Keyval_.new_("USD", "$")
);
fxt.Test__node(file.Country_names()
, Keyval_.new_("AC", "Ascension Island")
, Keyval_.new_("ZW", "Zimbabwe")
);
fxt.Test__node(file.Time_units()
, Keyval_.new_("century-one", "{0} century")
, Keyval_.new_("year-short-past-other", "{0} yr. ago")
);
}
@Test public void Smoke() {
Cldr_name_converter bldr = new Cldr_name_converter();
bldr.Convert(Io_url_.new_dir_("C:\\000\\100_bin\\200_server\\200_http\\100_apache\\100_v2.4\\htdocs\\mediawiki\\v1.29.1\\extensions\\cldr\\CldrNames\\"), Io_url_.new_dir_("C:\\xowa\\bin\\any\\xowa\\xtns\\cldr\\"));
}
}
class Cldr_name_converter_fxt {
private final Cldr_name_converter bldr = new Cldr_name_converter();
private final String dir_name = "mem/CldrNames/";
public void Init__file(String fil_name, String txt) {
Io_mgr.Instance.SaveFilStr(Io_url_.new_fil_(dir_name + fil_name), txt);
}
public void Test__Extract_key_or_fail(String fil_name, String expd) {
String actl = null;
try {
actl = bldr.Extract_key_or_fail(fil_name);
}
catch (Exception exc) {
Err_.Noop(exc);
actl = null;
}
if (expd == null && actl == null) {
}
else if (expd == null) {
Tfds.Fail("expecting null; got " + actl);
}
else if (actl == null) {
Tfds.Fail("got null; expected " + expd);
}
else {
Gftest.Eq__str(expd, actl);
}
}
public Cldr_name_file[] Exec__Parse_dir() {
return bldr.Parse_dir(Io_url_.new_fil_(dir_name));
}
public Cldr_name_file Exec__Parse_fil(String key, String src) {
return bldr.Parse_fil(key, Bry_.new_u8(src));
}
public Cldr_name_file Exec__To_file(String key, String json) {
Cldr_name_loader json_parser = new Cldr_name_loader(Io_url_.mem_dir_("mem/Cldr"));
return json_parser.Load(key, Bry_.new_u8(json));
}
public void Test__node(Ordered_hash hash, Keyval... expd) {
Keyval[] actl = (Keyval[])hash.To_ary(Keyval.class);
Gftest.Eq__ary__lines(Keyval_.Ary_to_str(expd), Keyval_.Ary_to_str(actl), "cldr_names_comp_failed");
}
public void Test__To_json(Cldr_name_file file, String expd) {
String actl = bldr.To_json(file);
Gftest.Eq__ary__lines(expd, actl, "json_failed");
}
}

27
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_file.java Normal file
View File

@ -0,0 +1,27 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
public class Cldr_name_file {
public Cldr_name_file(String key) {
this.key = key;
}
public String Key() {return key;} private final String key;
public Ordered_hash Language_names() {return language_names;} private final Ordered_hash language_names = Ordered_hash_.New();
public Ordered_hash Currency_names() {return currency_names;} private final Ordered_hash currency_names = Ordered_hash_.New();
public Ordered_hash Currency_symbols() {return currency_symbols;} private final Ordered_hash currency_symbols = Ordered_hash_.New();
public Ordered_hash Country_names() {return country_names;} private final Ordered_hash country_names = Ordered_hash_.New();
public Ordered_hash Time_units() {return time_units;} private final Ordered_hash time_units = Ordered_hash_.New();
}

70
400_xowa/src/gplx/xowa/xtns/cldrs/Cldr_name_loader.java Normal file
View File

@ -0,0 +1,70 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.cldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*;
import gplx.langs.jsons.*;
public class Cldr_name_loader {
private final Json_parser parser = new Json_parser();
private final Io_url cldr_dir;
private final Hash_adp hash = Hash_adp_.New();
public Cldr_name_loader(Io_url xowa_xtn_dir) {
cldr_dir = xowa_xtn_dir.GenSubDir_nest("cldr", "CldrNames");
}
public Cldr_name_file Load(String lang_key) {
Cldr_name_file file = (Cldr_name_file)hash.Get_by(lang_key);
if (file != null) return file;
byte[] json = Io_mgr.Instance.LoadFilBry(cldr_dir.GenSubFil_ary("CldrNames", lang_key, ".json"));
if (json == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "no cldrName file exists for lang; lang=~{lang}", lang_key);
return null;
}
file = Load(lang_key, json);
hash.Add(lang_key, file);
return file;
}
public Cldr_name_file Load(String lang_key, byte[] json) {
Cldr_name_file file = new Cldr_name_file(lang_key);
Json_doc jdoc = parser.Parse(json);
Json_nde root = jdoc.Root_nde();
int nodes_len = root.Len();
for (int i = 0; i < nodes_len; i++) {
Json_kv node = root.Get_at_as_kv(i);
String key = node.Key_as_str();
Json_nde val = node.Val_as_nde();
Ordered_hash hash = null;
if (String_.Eq(key, Cldr_name_converter.Node_languageNames)) hash = file.Language_names();
else if (String_.Eq(key, Cldr_name_converter.Node_currencyNames)) hash = file.Currency_names();
else if (String_.Eq(key, Cldr_name_converter.Node_currencySymbols)) hash = file.Currency_symbols();
else if (String_.Eq(key, Cldr_name_converter.Node_countryNames)) hash = file.Country_names();
else if (String_.Eq(key, Cldr_name_converter.Node_timeUnits)) hash = file.Time_units();
else throw Err_.new_unhandled_default(key);
Load_ary(file, hash, val);
}
return file;
}
private void Load_ary(Cldr_name_file file, Ordered_hash hash, Json_nde nde) {
int len = nde.Len();
for (int i = 0; i < len; i++) {
Json_kv kv = (Json_kv)nde.Get_at(i);
String key = kv.Key_as_str();
hash.Add(key, Keyval_.new_(key, String_.new_u8(kv.Val_as_bry())));
}
}
}

1
400_xowa/src/gplx/xowa/xtns/wbases/Wdata_doc.java
View File

@ -26,6 +26,7 @@ public class Wdata_doc {
this.slink_list = slink_list; this.label_list = label_list; this.descr_list = descr_list; this.alias_list = alias_list; this.claim_list = claim_list;
}
public Json_doc Jdoc() {return jdoc;} private Json_doc jdoc;
public int Jdoc_size() {return jdoc == null ? 1 : jdoc.Src().length;}
public byte[] Qid() {return qid;} private byte[] qid;
public byte[][] Sort_langs() {return sort_langs;} public void Sort_langs_(byte[][] v) {sort_langs = v;} private byte[][] sort_langs = Bry_.Ary_empty;
public Ordered_hash Slink_list() {if (slink_list == null) slink_list = mgr.Wdoc_parser(jdoc).Parse_sitelinks(qid, jdoc); return slink_list;} private Ordered_hash slink_list;

4
400_xowa/src/gplx/xowa/xtns/wbases/Wdata_wiki_mgr.java
View File

@ -31,8 +31,8 @@ public class Wdata_wiki_mgr implements Gfo_evt_itm, Gfo_invk {
this.app = app;
this.evt_mgr = new Gfo_evt_mgr(this);
this.Qid_mgr = new Wbase_qid_mgr(this);
this.Pid_mgr = new Wbase_pid_mgr(this);
this.Doc_mgr = new Wbase_doc_mgr(this, this.Qid_mgr, app.Cache_mgr().Doc_cache());
this.Pid_mgr = new Wbase_pid_mgr(this);
this.Doc_mgr = new Wbase_doc_mgr(this, this.Qid_mgr);
this.prop_mgr = new Wbase_prop_mgr(Wbase_prop_mgr_loader_.New_db(this));
this.prop_val_visitor = new Wdata_prop_val_visitor(app, this);
this.Enabled_(true);

50
400_xowa/src/gplx/xowa/xtns/wbases/stores/Wbase_doc_cache.java Normal file
View File

@ -0,0 +1,50 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.stores; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.core.lists.caches.*;
import gplx.core.logs.*;
public interface Wbase_doc_cache {
void Add(byte[] qid, Wdata_doc doc);
Wdata_doc Get_or_null(byte[] qid);
void Clear();
void Term();
}
class Wbase_doc_cache__null implements Wbase_doc_cache {
public void Add(byte[] qid, Wdata_doc doc) {}
public Wdata_doc Get_or_null(byte[] qid) {return null;}
public void Clear() {}
public void Term() {}
}
class Wbase_doc_cache__hash implements Wbase_doc_cache {
private final Hash_adp_bry hash = Hash_adp_bry.cs();
public void Add(byte[] qid, Wdata_doc doc) {hash.Add(qid, doc);}
public Wdata_doc Get_or_null(byte[] qid) {return (Wdata_doc)hash.Get_by_bry(qid);}
public void Clear() {hash.Clear();}
public void Term() {hash.Clear();}
}
class Wbase_doc_cache__mru implements Wbase_doc_cache {
private final Mru_cache_mgr cache;
public Wbase_doc_cache__mru(long cache_max, long compress_size, long used_weight) {
this.cache = Mru_cache_mgr.New_by_mb_secs(Gfo_log_wtr.New_dflt("wbase", "cache_mru_{0}.csv"), cache_max, compress_size, used_weight);
}
public void Add(byte[] qid, Wdata_doc doc) {cache.Add(String_.new_a7(qid), doc, doc.Jdoc_size());}
public Wdata_doc Get_or_null(byte[] qid) {return (Wdata_doc)cache.Get_or_null(String_.new_a7(qid));}
public void Clear() {}
public void Term() {
cache.Flush();
cache.Print();
}
}

85
400_xowa/src/gplx/xowa/xtns/wbases/stores/Wbase_doc_mgr.java
View File

@ -14,22 +14,49 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.xtns.wbases.stores; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.wbases.*;
import gplx.core.logs.*; import gplx.core.primitives.*;
import gplx.langs.jsons.*;
import gplx.xowa.wikis.pages.*;
import gplx.xowa.apps.caches.*;
import gplx.xowa.xtns.wbases.core.*;
public class Wbase_doc_mgr {
private final Wdata_wiki_mgr wbase_mgr;
private final Wbase_qid_mgr qid_mgr;
private final Wbase_doc_cache doc_cache;
public Wbase_doc_mgr(Wdata_wiki_mgr wbase_mgr, Wbase_qid_mgr qid_mgr, Wbase_doc_cache doc_cache) {
private Wbase_doc_cache doc_cache;
private final Object thread_lock = new Object();
private final Ordered_hash wbase_db_hash = Ordered_hash_.New_bry();
private final Gfo_log_wtr wbase_db_log;
public Wbase_doc_mgr(Wdata_wiki_mgr wbase_mgr, Wbase_qid_mgr qid_mgr) {
this.wbase_mgr = wbase_mgr;
this.qid_mgr = qid_mgr;
this.doc_cache = doc_cache;
this.doc_cache = new Wbase_doc_cache__hash();
this.wbase_db_log = Gfo_log_wtr.New_dflt("wbase", "db_log_{0}.csv");
}
public void Enabled_(boolean v) {this.enabled = v;} private boolean enabled;
public void Cache__init(String cache_type, long cache_max, long compress_size, long used_weight) {
if (String_.Eq(cache_type, "null")) doc_cache = new Wbase_doc_cache__null();
else if (String_.Eq(cache_type, "hash")) doc_cache = new Wbase_doc_cache__hash();
else if (String_.Eq(cache_type, "mru" )) doc_cache = new Wbase_doc_cache__mru(cache_max, compress_size, used_weight);
else throw Err_.new_unhandled_default(cache_type);
}
public void Cleanup() {
doc_cache.Term();
wbase_db_log__flush();
}
private void wbase_db_log__flush() {
int len = wbase_db_hash.Len();
Bry_bfr tmp_bfr = Bry_bfr_.New();
for (int i = 0; i < len; i++) {
Wbase_db_log_itm itm = (Wbase_db_log_itm)wbase_db_hash.Get_at(i);
tmp_bfr.Add(itm.Ttl());
tmp_bfr.Add_byte_pipe().Add_int_variable(itm.Count());
tmp_bfr.Add_byte_pipe().Add_int_variable(itm.Elapsed());
tmp_bfr.Add_byte_nl();
wbase_db_log.Write(tmp_bfr);
}
wbase_db_log.Flush();
}
public void Clear() {
synchronized (doc_cache) { // LOCK:app-level
synchronized (thread_lock) { // LOCK:app-level
doc_cache.Clear();
}
}
@ -43,7 +70,10 @@ public class Wbase_doc_mgr {
}
public Wdata_doc Get_by_exact_id_or_null(byte[] ttl_bry) {// must correct case and ns; EX:"Q2" or "Property:P1"; not "q2" or "P2"
// load from cache
Wdata_doc rv = doc_cache.Get_or_null(ttl_bry);
Wdata_doc rv = null;
synchronized (thread_lock) {
rv = doc_cache.Get_or_null(ttl_bry);
}
if (rv == null) {
// load from db
rv = Load_wdoc_or_null(ttl_bry);
@ -54,7 +84,17 @@ public class Wbase_doc_mgr {
}
private Wdata_doc Load_wdoc_or_null(byte[] ttl_bry) { // EX:"Q2" or "Property:P1"
if (!enabled) return null;
synchronized (doc_cache) { // LOCK:app-level; jdoc_parser; moved synchronized higher up; DATE:2016-09-03
// loggging
Wbase_db_log_itm wbase_db_itm = (Wbase_db_log_itm)wbase_db_hash.Get_by(ttl_bry);
if (wbase_db_itm == null) {
wbase_db_itm = new Wbase_db_log_itm(ttl_bry);
wbase_db_hash.Add(ttl_bry, wbase_db_itm);
}
long time_bgn = gplx.core.envs.System_.Ticks();
Wdata_doc rv = null;
synchronized (thread_lock) { // LOCK:app-level; jdoc_parser; moved synchronized higher up; DATE:2016-09-03
byte[] cur_ttl_bry = ttl_bry;
int load_count = -1;
while (load_count < 2) { // limit to 2 tries (i.e.: 1 redirect)
@ -62,18 +102,18 @@ public class Wbase_doc_mgr {
Xoa_ttl cur_ttl = wbase_mgr.Wdata_wiki().Ttl_parse(cur_ttl_bry);
if (cur_ttl == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "invalid wbase ttl: orig=~{0} cur=~{1}", ttl_bry, cur_ttl_bry);
return null;
break;
}
// get page
Xoae_page page = wbase_mgr.Wdata_wiki().Data_mgr().Load_page_by_ttl(cur_ttl);
if (!page.Db().Page().Exists()) return null;
if (!page.Db().Page().Exists()) break;
// parse jdoc
Json_doc jdoc = wbase_mgr.Jdoc_parser().Parse(page.Db().Text().Text_bry());
if (jdoc == null) {
Gfo_usr_dlg_.Instance.Warn_many("", "", "invalid jdoc for ttl: orig=~{0} cur=~{1}", ttl_bry, cur_ttl_bry);
return null;
break;
}
// check for redirect; EX: {"entity":"Q22350516","redirect":"Q21006972"}; PAGE:fr.w:Tour_du_Táchira_2016; DATE:2016-08-13
@ -81,22 +121,39 @@ public class Wbase_doc_mgr {
byte[] redirect_ttl = jdoc_root.Get_as_bry_or(Bry__redirect, null);
if (redirect_ttl != null) {
cur_ttl_bry = redirect_ttl;
load_count++;
continue;
}
// is json doc, and not a redirect; return
return new Wdata_doc(cur_ttl_bry, wbase_mgr, jdoc);
rv = new Wdata_doc(cur_ttl_bry, wbase_mgr, jdoc);
break;
}
Gfo_usr_dlg_.Instance.Warn_many("", "", "too many redirects for ttl: orig=~{0} cur=~{1}", ttl_bry, cur_ttl_bry);
if (rv == null && load_count >= 2)
Gfo_usr_dlg_.Instance.Warn_many("", "", "too many redirects for ttl: orig=~{0} cur=~{1}", ttl_bry, cur_ttl_bry);
}
return null;
wbase_db_itm.Update(gplx.core.envs.System_.Ticks__elapsed_in_frac(time_bgn));
return rv;
}
private static final byte[] Bry__redirect = Bry_.new_a7("redirect");
public void Add(byte[] full_db, Wdata_doc page) { // TEST:
synchronized (doc_cache) { // LOCK:app-level
synchronized (thread_lock) { // LOCK:app-level
if (doc_cache.Get_or_null(full_db) == null)
doc_cache.Add(full_db, page);
}
}
}
class Wbase_db_log_itm {
public Wbase_db_log_itm(byte[] ttl) {
this.ttl = ttl;
}
public byte[] Ttl() {return ttl;} private final byte[] ttl;
public int Count() {return count;} private int count;
public int Elapsed() {return elapsed;} private int elapsed;
public void Update(int elapsed_diff) {
count++;
this.elapsed += elapsed_diff;
}
}

2
400_xowa/src/gplx/xowa/xtns/wbases/stores/Wbase_pid_mgr.java
View File

@ -42,7 +42,7 @@ public class Wbase_pid_mgr { // EX: "en|road_map" -> 15 ("Property:P15")
if (rv == -1) {
// get from db
rv = wbase_mgr.Wdata_wiki().Db_mgr().Load_mgr().Load_pid(lang_key, pid_name);
if (rv == Wbase_pid.Id_null) return Wbase_pid.Id_null;
if (rv == Wbase_pid.Id_null) rv = Wbase_pid.Id_null;
Add(pid_key, rv);
}
return rv;