Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Cldr: Add initial support for cldr

Browse Source
This commit is contained in:
gnosygnu
2018-08-05 21:21:40 -04:00
parent be3979c5af
commit e78382a8ac
27 changed files with 1052 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
400_xowa/src/gplx/core/lists/Binary_heap.java Normal file
View File

@@ -0,0 +1,92 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists; import gplx.*; import gplx.core.*;
public class Binary_heap {
private final ComparerAble comparer;
private boolean is_max;
private Object[] heap;
private int size;
public Binary_heap(ComparerAble comparer, boolean is_max, int capacity) {
this.comparer = comparer;
this.is_max = is_max;
this.heap = new Object[capacity];
this.size = 0;
}
public Object Get() {
if (size == 0) throw Err_.new_wo_type("heap is empty");
return heap[0];
}
public void Add(Object val) {
if (size == heap.length) {
int old_max = heap.length;
Object[] new_heap = new Object[old_max * 2];
for (int i = 0; i < old_max; i++)
new_heap[i] = heap[i];
heap = new_heap;
}
heap[size++] = val;
Heapify_up(size - 1);
}
public Object Pop() {return Pop(0);}
public Object Pop(int idx) {
if (size == 0) throw Err_.new_wo_type("heap is empty");
Object val = heap[idx];
heap[idx] = heap[size -1];
size--;
Heapify_down(idx);
return val;
}
private int Parent(int idx) {
return (idx - 1) / SLOT;
}
private int Kth_child(int idx, int k) {
return (idx * SLOT) + k;
}
private int Max_child(int idx) {
int lhs = Kth_child(idx, 1);
int rhs = Kth_child(idx, 2);
int comp = ComparerAble_.Compare(comparer, heap[lhs], heap[rhs]);
boolean diff = is_max ? comp == CompareAble_.More : comp == CompareAble_.Less;
return diff ? lhs : rhs;
}
private void Heapify_up(int idx) {
Object val = heap[idx];
while (idx > 0) {
int comp = ComparerAble_.Compare(comparer, val, heap[Parent(idx)]);
if (!(is_max ? comp == CompareAble_.More : comp == CompareAble_.Less))
break;
heap[idx] = heap[Parent(idx)];
idx = Parent(idx);
}
heap[idx] = val;
}
private void Heapify_down(int idx) {
int child;
Object val = heap[idx];
while (Kth_child(idx, 1) < size) {
child = Max_child(idx);
int comp = ComparerAble_.Compare(comparer, val, heap[child]);
if (is_max ? comp == CompareAble_.Less : comp == CompareAble_.More)
heap[idx] = heap[child];
else
break;
idx = child;
}
heap[idx] = val;
}
private static final int SLOT = 2;
}

51
400_xowa/src/gplx/core/lists/Binary_heap_tst.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists; import gplx.*; import gplx.core.*;
import org.junit.*; import gplx.core.tests.*;
import gplx.core.primitives.*;
public class Binary_heap_tst {
private final Binary_heap_fxt fxt = new Binary_heap_fxt();
@Test public void Max() {
fxt.Init(Bool_.Y);
fxt.Exec__add(4, 3, 5, 7, 1, 6, 9, 8, 2);
fxt.Test__pop(9, 8, 7, 6, 5, 4, 3, 2, 1);
}
@Test public void Min() {
fxt.Init(Bool_.N);
fxt.Exec__add(4, 3, 5, 7, 1, 6, 9, 8, 2);
fxt.Test__pop(1, 2, 3, 4, 5, 6, 7, 8, 9);
}
}
class Binary_heap_fxt implements ComparerAble {
private Binary_heap heap;
public int compare(Object lhsObj, Object rhsObj) {
return CompareAble_.Compare_obj(lhsObj, rhsObj);
}
public void Init(boolean is_max) {
heap = new Binary_heap(this, is_max, 2);
}
public void Exec__add(int... ary) {
for (int i : ary)
heap.Add(new Int_obj_val(i));
}
public void Test__pop(int... expd) {
int len = expd.length;
int[] actl = new int[len];
for (int i = 0; i < len; i++)
actl[i] = ((Int_obj_val)heap.Pop()).Val();
Gftest.Eq__ary(expd, actl, "heaps don't match");
}
}

71
400_xowa/src/gplx/core/lists/caches/Mru_cache_itm.java Normal file
View File

@@ -0,0 +1,71 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
class Mru_cache_itm {
private final long time_bgn;
private long time_dirty;
private long used_dirty = -1;
public Mru_cache_itm(Object key, Object val, long size, long time) {
this.key = key;
this.val = val;
this.size = size;
this.time_bgn = this.time_cur = time;
}
public Object Key() {return key;} private final Object key;
public Object Val() {return val;} private final Object val;
public long Size() {return size;} private long size;
public long Time_cur() {return time_cur;} private long time_cur;
public long Time_dif() {return time_cur - time_bgn;}
public long Used() {return used;} private long used;
public void Dirty(long time) {
this.time_dirty = time;
if (used_dirty == -1)
used_dirty = used + 1;
else
used_dirty++;
}
public void Update() {
this.time_cur = time_dirty;
this.used = used_dirty;
used_dirty = -1;
}
public void Update(long time) {
this.time_cur = time;
used++;
}
}
class Mru_cache_itm_comparer implements ComparerAble {
private long used_weight;
private long time_bgn;
public Mru_cache_itm_comparer(long used_weight, long time_bgn) {
Init(used_weight, time_bgn);
}
public void Init(long used_weight, long time_bgn) {
this.used_weight = used_weight;
this.time_bgn = time_bgn;
}
public long Score(Mru_cache_itm itm) {
return (itm.Time_cur() - time_bgn) + (itm.Used() * used_weight);
}
public int compare(Object lhsObj, Object rhsObj) {
Mru_cache_itm lhs = (Mru_cache_itm)lhsObj;
Mru_cache_itm rhs = (Mru_cache_itm)rhsObj;
int rv = Long_.Compare(Score(lhs), Score(rhs));
return rv == 0
? CompareAble_.Compare_obj(lhs.Key(), rhs.Key())
: rv;
}
}

108
400_xowa/src/gplx/core/lists/caches/Mru_cache_mgr.java Normal file
View File

@@ -0,0 +1,108 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
import gplx.core.logs.*;
public class Mru_cache_mgr {
private final Mru_cache_time_mgr time_mgr;
private final Gfo_log_wtr log_wtr;
private final Hash_adp key_hash = Hash_adp_.New();
private final Sorted_hash val_hash;
private final Bry_bfr log_bfr = Bry_bfr_.New_w_size(255);
private final Mru_cache_itm_comparer comparer;
private final Ordered_hash dirty = Ordered_hash_.New();
private long cache_max, cache_size, compress_size;
Mru_cache_mgr(Mru_cache_time_mgr time_mgr, Gfo_log_wtr log_wtr, long cache_max, long compress_size, long used_weight) {
this.time_mgr = time_mgr;
this.log_wtr = log_wtr;
this.cache_max = cache_max;
this.compress_size = compress_size;
this.comparer = new Mru_cache_itm_comparer(used_weight, time_mgr.Now());
this.val_hash = new Sorted_hash(comparer);
}
public long Cache_max() {return cache_max;} public void Cache_max_(long v) {this.cache_max = v;}
public Object Get_or_null(String key) {
Object itm_obj = key_hash.Get_by(key);
if (itm_obj == null) return null;
Mru_cache_itm itm = (Mru_cache_itm)itm_obj;
dirty.Add_if_dupe_use_1st(key, itm);
itm.Dirty(time_mgr.Now());
return itm.Val();
}
public void Add(String key, Object val, long val_size) {
if (cache_size + val_size > cache_max) {
Compress(val_size);
}
Mru_cache_itm itm = new Mru_cache_itm(key, val, val_size, time_mgr.Now());
key_hash.Add(key, itm);
val_hash.Add(itm, itm);
cache_size += val_size;
}
public void Compress(long val_size) {
int dirty_len = dirty.Len();
for (int i = 0; i < dirty_len; i++) {
Mru_cache_itm dirty_itm = (Mru_cache_itm)dirty.Get_at(i);
val_hash.Del(dirty_itm);
dirty_itm.Update();
val_hash.Add(dirty_itm, dirty_itm);
}
dirty.Clear();
while (cache_size + val_size > compress_size) {
Mru_cache_itm old = (Mru_cache_itm)val_hash.Del_val_at_0();
key_hash.Del(old.Key());
cache_size -= old.Size();
if (log_wtr != null) {
Write(Bool_.Y, old);
log_wtr.Write(log_bfr);
}
}
}
public void Flush() {log_wtr.Flush();}
public void Print() {
Object[] vals = val_hash.Values_array();
int vals_len = vals.length;
for (int i = vals_len - 1; i >= 0; i--) {
Mru_cache_itm val = (Mru_cache_itm)vals[i];
Write(Bool_.N, val);
}
Io_mgr.Instance.SaveFilBfr(log_wtr.Fil_dir().GenSubFil("cache_mru_final.csv"), log_bfr);
log_bfr.Clear();
}
private void Write(boolean write_time, Mru_cache_itm itm) {
// 20180625_112443.506|Q2|123uses|10ms|1234bytes\n
if (write_time) log_bfr.Add_dte_under(Datetime_now.Get_force()).Add_byte_pipe();
log_bfr.Add_str_u8(Object_.Xto_str_strict_or_null_mark(itm.Key()));
log_bfr.Add_byte_pipe().Add_long_fixed(comparer.Score(itm), 10);
log_bfr.Add_byte_pipe().Add_long_variable(itm.Size());
log_bfr.Add_byte_pipe().Add_long_variable(itm.Time_dif());
log_bfr.Add_byte_pipe().Add_long_variable(itm.Used());
log_bfr.Add_byte_nl();
}
public void Clear() {
key_hash.Clear();
val_hash.Clear();
log_bfr.Clear();
dirty.Clear();
cache_size = 0;
}
public static Mru_cache_mgr New_by_mb_secs(Gfo_log_wtr log_wtr, long cache_max_in_mb, long compress_size, long used_weight_in_secs) {
return new Mru_cache_mgr(new Mru_cache_time_mgr__clock(), log_wtr, Io_mgr.Len_mb * cache_max_in_mb, Io_mgr.Len_mb * compress_size, gplx.core.envs.System_.Ticks__per_second * used_weight_in_secs);
}
// TEST
public Object[] Values_array() {return val_hash.Values_array();}
@gplx.Internal protected static Mru_cache_mgr New_test(Mru_cache_time_mgr time_mgr, Gfo_log_wtr log_wtr, long cache_max, long compress_size, long used_weight) {return new Mru_cache_mgr(time_mgr, log_wtr, cache_max, compress_size, used_weight);}
}

90
400_xowa/src/gplx/core/lists/caches/Mru_cache_mgr_tst.java Normal file
View File

@@ -0,0 +1,90 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
import org.junit.*; import gplx.core.tests.*;
public class Mru_cache_mgr_tst {
private final Mru_cache_mgr_fxt fxt = new Mru_cache_mgr_fxt();
@Before public void init() {
fxt.Init__New_cache_mgr(3, 3, 2);
}
@Test public void Basic() {
fxt.Exec__Add("a", "b", "c", "d");
fxt.Test__Print("b", "c", "d"); // adding "d" pushes out "a"
}
@Test public void Used() {
fxt.Exec__Add("a", "b", "c");
fxt.Exec__Get_and_compress("a");
fxt.Test__Print("b", "c", "a"); // getting "a" pushes to back
}
@Test public void Used__more_uses_at_back() {
fxt.Exec__Add("a", "b", "c");
fxt.Exec__Get_and_compress("a", "a", "a");
fxt.Test__Print("b", "c", "a");
fxt.Exec__Get_and_compress("b");
fxt.Test__Print("c", "b", "a"); // getting "a" multiple time still keeps towards back
}
@Test public void Time() {
fxt.Exec__Add("a", "b", "c");
fxt.Exec__Get_and_compress("a", "a", "a");
fxt.Exec__Wait(10);
fxt.Exec__Get_and_compress("b");
fxt.Test__Print("c", "a", "b"); // long wait puts "b" at back
}
@Test public void Compress() {
fxt.Init__New_cache_mgr(3, 2, 2);
fxt.Exec__Add("a", "b", "c", "d");
fxt.Test__Print("c", "d");
}
}
class Mru_cache_mgr_fxt {
private final Mru_cache_time_mgr__mock time_mgr = new Mru_cache_time_mgr__mock();
private Mru_cache_mgr cache_mgr;
public void Init__New_cache_mgr(long cache_max, long used_weight, long compress_size) {
cache_mgr = Mru_cache_mgr.New_test(time_mgr, null, cache_max, used_weight, compress_size);
}
public void Exec__Add(String... ary) {
for (String itm : ary)
cache_mgr.Add(itm, itm, 1);
}
public void Exec__Get(String... ary) {
for (String itm : ary)
cache_mgr.Get_or_null(itm);
}
public void Exec__Wait(int time) {
time_mgr.Add(time);
}
public void Exec__Compress(long val_size) {
cache_mgr.Compress(val_size);
}
public void Exec__Get_and_compress(String... ary) {
this.Exec__Get(ary);
this.Exec__Compress(0);
}
public void Test__Print(String... expd) {
Object[] actl_objs = cache_mgr.Values_array();
int actl_len = actl_objs.length;
String[] actl = new String[actl_len];
for (int i = 0; i < actl_len; i++) {
actl[i] = Object_.Xto_str_strict_or_null(((Mru_cache_itm)actl_objs[i]).Val());
}
Gftest.Eq__ary(expd, actl);
}
}
class Mru_cache_time_mgr__mock implements Mru_cache_time_mgr {
private long time;
public long Now() {return time++;}
public void Add(int v) {time += v;}
}

22
400_xowa/src/gplx/core/lists/caches/Mru_cache_time_mgr.java Normal file
View File

@@ -0,0 +1,22 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
interface Mru_cache_time_mgr {
long Now();
}
class Mru_cache_time_mgr__clock implements Mru_cache_time_mgr {
public long Now() {return gplx.core.envs.System_.Ticks();}
}

54
400_xowa/src/gplx/core/logs/Gfo_log_wtr.java Normal file
View File

@@ -0,0 +1,54 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.logs; import gplx.*; import gplx.core.*;
public class Gfo_log_wtr {
private final Bry_bfr bfr = Bry_bfr_.New();
private final int bfr_max;
private final int fil_max;
private final Io_url fil_dir;
private final String fil_fmt;
private final int fil_idx_places;
private Io_url fil_url;
private int fil_idx = -1, fil_len;
public Gfo_log_wtr(int bfr_max, int fil_max, Io_url fil_dir, String fil_fmt, int fil_idx_places) {
this.bfr_max = bfr_max;
this.fil_max = fil_max;
this.fil_dir = fil_dir;
this.fil_fmt = fil_fmt;
this.fil_idx_places = fil_idx_places;
}
public Io_url Fil_dir() {return fil_dir;}
public void Write(Bry_bfr add) {
int add_len = add.Len();
if (bfr.Len() + add_len > bfr_max)
Flush();
bfr.Add_bfr_and_clear(add);
}
public void Flush() {
if (fil_idx == -1 || bfr.Len() + fil_len > fil_max) {
fil_idx++;
fil_url = fil_dir.GenSubFil(String_.Format(fil_fmt, Int_.To_str_pad_bgn_zero(fil_idx, fil_idx_places)));
fil_len = 0;
}
fil_len += bfr.Len();
Io_mgr.Instance.AppendFilBfr(fil_url, bfr);
bfr.Clear();
}
public static Gfo_log_wtr New_dflt(String sub_dir, String file_fmt) {
return new Gfo_log_wtr(Io_mgr.Len_mb, 10 * Io_mgr.Len_mb, Gfo_usr_dlg_.Instance.Log_wkr().Session_dir().GenSubDir(sub_dir), file_fmt, 4);
}
}