Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.5.4.1

Browse Source
This commit is contained in:
gnosygnu
2015-05-24 22:43:55 -04:00
parent 6eec99a713
commit 51e6188c1e
1577 changed files with 11555 additions and 10080 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
400_xowa/src_040_io/gplx/ios/BinaryHeap_Io_line_rdr_tst.java
View File

@@ -31,7 +31,7 @@ class BinaryHeap_Io_line_rdr_fxt {
file_total = ary.length;
for (int i = 0; i < file_total; i++) {
Io_url url = Io_url_.mem_fil_("mem/fil_" + ary[i] + ".txt");
Io_mgr._.SaveFilStr(url, ary[i]);
Io_mgr.I.SaveFilStr(url, ary[i]);
Io_line_rdr stream = new Io_line_rdr(Gfo_usr_dlg_.Test(), url);
stream.Read_next();
heap.Add(stream);
@@ -42,7 +42,7 @@ class BinaryHeap_Io_line_rdr_fxt {
String[] actl = new String[file_total];
for (int i = 0; i < actl.length; i++) {
Io_line_rdr bfr = heap.Pop();
actl[i] = String_.new_utf8_(bfr.Bfr(), 0, bfr.Bfr_len());
actl[i] = String_.new_u8(bfr.Bfr(), 0, bfr.Bfr_len());
}
Tfds.Eq_ary_str(expd, actl);
return this;

6
400_xowa/src_040_io/gplx/ios/Io_buffer_rdr.java
View File

@@ -22,7 +22,7 @@ public class Io_buffer_rdr implements RlsAble { //_20120115
if (bfr_len <= 0) throw Err_.new_("bfr_len must be > 0").Add("bfr_len", bfr_len);
bfr = new byte[bfr_len]; this.bfr_len = bfr_len;
this.rdr = rdr;
IoItmFil fil = Io_mgr._.QueryFil(url); if (!fil.Exists()) throw Err_.new_("fil does not exist").Add("url", url);
IoItmFil fil = Io_mgr.I.QueryFil(url); if (!fil.Exists()) throw Err_.new_("fil does not exist").Add("url", url);
fil_len = fil.Size();
fil_pos = 0;
fil_eof = false;
@@ -58,8 +58,8 @@ public class Io_buffer_rdr implements RlsAble { //_20120115
if (rdr != null) rdr.Rls();
}
@gplx.Internal protected void DumpToFil(int bgn, int len, String urlStr, String msg) {
String text = String_.new_utf8_len_safe_(bfr, bgn, len);
Io_mgr._.AppendFilStr(Io_url_.new_any_(urlStr), msg + text + "\n");
String text = String_.new_u8_by_len(bfr, bgn, len);
Io_mgr.I.AppendFilStr(Io_url_.new_any_(urlStr), msg + text + "\n");
}
public static Io_buffer_rdr new_(Io_stream_rdr rdr, int bfr_len) {
Io_buffer_rdr rv = new Io_buffer_rdr(rdr, rdr.Url(), bfr_len);

6
400_xowa/src_040_io/gplx/ios/Io_buffer_rdr_tst.java
View File

@@ -19,7 +19,7 @@ package gplx.ios; import gplx.*;
import org.junit.*; import gplx.ios.*;
public class Io_buffer_rdr_tst {
@Before public void init() {
Io_mgr._.InitEngine_mem();
Io_mgr.I.InitEngine_mem();
fil = Io_url_.mem_fil_("mem/byteStreamRdr.txt");
ini_Write("0123456789");
rdr = Io_buffer_rdr.new_(Io_stream_rdr_.file_(fil), 4);
@@ -52,11 +52,11 @@ public class Io_buffer_rdr_tst {
rdr.Bfr_load_all(); // NOTE: change to zip_rdrs make eof detection difficult; force another load to ensure that file_pos goes past file_len
tst_Bfr("8", "9").tst_ReadDone(true);
}
private void ini_Write(String s) {Io_mgr._.SaveFilStr(fil, s);}
private void ini_Write(String s) {Io_mgr.I.SaveFilStr(fil, s);}
Io_buffer_rdr_tst tst_Bfr(String... expdAry) {
String[] actlAry = new String[rdr.Bfr_len()];
for (int i = 0; i < actlAry.length; i++)
actlAry[i] = String_.new_utf8_(rdr.Bfr(), i, i + 1);
actlAry[i] = String_.new_u8(rdr.Bfr(), i, i + 1);
Tfds.Eq_ary(expdAry, actlAry);
return this;
}

6
400_xowa/src_040_io/gplx/ios/Io_line_rdr.java
View File

@@ -88,7 +88,7 @@ public class Io_line_rdr {
}
while (true) {
int compare = Bry_.Compare(ttl, 0, ttl.length, bfr, key_pos_bgn, key_pos_end);
// if (String_.new_utf8_(bfr, key_pos_bgn, key_pos_end) == "US Naval Jack.svg") {
// if (String_.new_u8(bfr, key_pos_bgn, key_pos_end) == "US Naval Jack.svg") {
// Tfds.Write();
// }
if (compare == CompareAble_.Same) { // eq; return true and move fwd; EX: "BA" and "BA"
@@ -137,7 +137,7 @@ public class Io_line_rdr {
Io_url url = urls[url_idx];
usr_dlg.Prog_many(GRP_KEY, "load", "loading dump file: ~{0}", url.NameAndExt());
if (file_skip_line0) {
byte[] stream_bry = Io_mgr._.LoadFilBry(url);
byte[] stream_bry = Io_mgr.I.LoadFilBry(url);
int stream_bry_len = stream_bry.length;
int nl_pos = Bry_finder.Find_fwd(stream_bry, Byte_ascii.NewLine, 0, stream_bry_len);
if (nl_pos == Bry_.NotFound)
@@ -147,7 +147,7 @@ public class Io_line_rdr {
stream = gplx.ios.IoStream_.ary_(stream_bry);
}
else {
stream = Io_mgr._.OpenStreamRead(url);
stream = Io_mgr.I.OpenStreamRead(url);
}
file_pos = 0; file_len = stream.Len();
file_done = false;

16
400_xowa/src_040_io/gplx/ios/Io_line_rdr_tst.java
View File

@@ -48,35 +48,35 @@ public class Io_line_rdr_tst {
}
class Io_line_rdr_fxt {
Io_line_rdr rdr;
ListAdp lines = ListAdp_.new_(); Bry_bfr tmp = Bry_bfr.new_();
List_adp lines = List_adp_.new_(); Bry_bfr tmp = Bry_bfr.new_();
public Io_line_rdr_fxt(Io_url... urls) {rdr = new Io_line_rdr(Gfo_usr_dlg_.Test(), urls);}
public Io_line_rdr_fxt Load_len_lines_(int v) {return Load_len_(v * 3);} // 3: 2=##, 1=\n
public Io_line_rdr_fxt Load_len_(int v) {rdr.Load_len_(v); return this;}
public Io_line_rdr_fxt File_lines_(int count) {
for (int i = 0; i < count; i++)
tmp.Add_int_fixed(i, 2).Add_byte_nl();
Io_mgr._.SaveFilBry(rdr.Urls()[0], tmp.Xto_bry_and_clear());
Io_mgr.I.SaveFilBry(rdr.Urls()[0], tmp.Xto_bry_and_clear());
return this;
}
// public Io_url[] Src_fils() {return src_fils;} public Io_line_rdr_fxt Src_fils_(Io_url[] v) {src_fils = v; return this;} Io_url[] src_fils;
public Io_line_rdr_fxt tst_Match(String match, String expd) {
rdr.Key_gen_(Io_line_rdr_key_gen_.first_pipe);
boolean match_v = rdr.Match(Bry_.new_utf8_(match));
String actl = match_v ? String_.new_utf8_(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end()) : "";
boolean match_v = rdr.Match(Bry_.new_u8(match));
String actl = match_v ? String_.new_u8(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end()) : "";
Tfds.Eq(expd, actl);
return this;
}
public Io_line_rdr_fxt File_lines_pipe_(int count) {
for (int i = 0; i < count; i++)
tmp.Add_int_fixed(i, 2).Add_byte(Byte_ascii.Pipe).Add_byte_nl();
Io_mgr._.SaveFilBry(rdr.Urls()[0], tmp.Xto_bry_and_clear());
Io_mgr.I.SaveFilBry(rdr.Urls()[0], tmp.Xto_bry_and_clear());
return this;
}
public Io_line_rdr_fxt File_lines_(int fil_idx, int bgn, int end) {
for (int i = bgn; i < end; i++)
tmp.Add_int_fixed(i, 2).Add_byte_nl();
Io_mgr._.SaveFilBry(rdr.Urls()[fil_idx], tmp.Xto_bry_and_clear());
Io_mgr.I.SaveFilBry(rdr.Urls()[fil_idx], tmp.Xto_bry_and_clear());
return this;
}
public Io_line_rdr_fxt Clear() {rdr.Clear(); return this;}
@@ -86,11 +86,11 @@ class Io_line_rdr_fxt {
expd[i] = expd[i] + Op_sys.Lnx.Nl_str();
for (int i = 0; i < count; i++) {
if (rdr.Read_next())
lines.Add(String_.new_utf8_(rdr.Bfr(), rdr.Itm_pos_bgn(), rdr.Itm_pos_end()));
lines.Add(String_.new_u8(rdr.Bfr(), rdr.Itm_pos_bgn(), rdr.Itm_pos_end()));
else
break;
}
Tfds.Eq_ary_str(expd, lines.XtoStrAry());
Tfds.Eq_ary_str(expd, lines.To_str_ary());
return this;
}
}

20
400_xowa/src_040_io/gplx/ios/Io_sort.java
View File

@@ -22,20 +22,20 @@ public class Io_sort {
public Io_url[] Split(Gfo_usr_dlg usr_dlg, Io_url_gen src_fil_gen, Io_url_gen trg_fil_gen, Io_line_rdr_key_gen key_gen) {return Split(usr_dlg, src_fil_gen, trg_fil_gen, Io_sort_split_itm_sorter._, key_gen);}
public Io_url[] Split(Gfo_usr_dlg usr_dlg, Io_url_gen src_fil_gen, Io_url_gen trg_fil_gen, ComparerAble row_comparer, Io_line_rdr_key_gen key_gen) {
Io_line_rdr rdr = new Io_line_rdr(usr_dlg, src_fil_gen.Prv_urls()).Load_len_(4 * Io_mgr.Len_kb).Key_gen_(key_gen); // NOTE: do not set load_len to memory_max; only want to load in increments
ListAdp rv = ListAdp_.new_();
List_adp rv = List_adp_.new_();
Bry_bfr bfr = Bry_bfr.reset_(Const_bfr_max); int size_cur = 0;
ListAdp row_list = ListAdp_.new_();
List_adp row_list = List_adp_.new_();
while (true) {
boolean reading = rdr.Read_next();
int size_row = rdr.Itm_pos_end() - rdr.Itm_pos_bgn();
int size_new = size_cur + size_row;
if (size_new > memory_max || !reading) {
usr_dlg.Prog_none(GRP_KEY, "sort", "sorting chunk");
row_list.SortBy(row_comparer);
row_list.Sort_by(row_comparer);
Io_url trg_url = trg_fil_gen.Nxt_url();
usr_dlg.Prog_one(GRP_KEY, "write", "writing chunk: ~{0}", trg_url.Raw());
Split_flush(trg_url, row_list, memory_max, bfr, rv);
row_list.ResizeBounds(16); // MEM: resize bounds manually; note that each Flush-set may have widely disparately #of rows (EX: 1 row with a million pages vs. 1 million rows with 1 page)
row_list.Resize_bounds(16); // MEM: resize bounds manually; note that each Flush-set may have widely disparately #of rows (EX: 1 row with a million pages vs. 1 million rows with 1 page)
size_new = size_row; Env_.GarbageCollect();
if (!reading) break;
}
@@ -43,7 +43,7 @@ public class Io_sort {
size_cur = size_new;
}
rdr.Rls(); bfr.Rls(); Env_.GarbageCollect();
return (Io_url[])rv.Xto_ary(Io_url.class);
return (Io_url[])rv.To_ary(Io_url.class);
}
public void Merge(Gfo_usr_dlg usr_dlg, Io_url[] src_ary, ComparerAble comparer, Io_line_rdr_key_gen key_gen, Io_sort_cmd cmd) {
BinaryHeap_Io_line_rdr heap = load_(usr_dlg, src_ary, comparer, key_gen, memory_max); if (heap.Len() == 0) return;//throw Err_.new_(Array_.XtoStr(src_ary));
@@ -65,16 +65,16 @@ public class Io_sort {
cmd.Sort_end();
heap.Rls();
}
private static void Split_flush(Io_url url, ListAdp list, int max, Bry_bfr tmp, ListAdp url_list) {
private static void Split_flush(Io_url url, List_adp list, int max, Bry_bfr tmp, List_adp url_list) {
int len = list.Count();
for (int i = 0; i < len; i++) {
Io_sort_split_itm itm = (Io_sort_split_itm)list.FetchAt(i);
Io_sort_split_itm itm = (Io_sort_split_itm)list.Get_at(i);
int add_len = itm.Row_end() - itm.Row_bgn();
if ((tmp.Len() + add_len) > Const_bfr_max) Io_mgr._.AppendFilBfr(url, tmp);
if ((tmp.Len() + add_len) > Const_bfr_max) Io_mgr.I.AppendFilBfr(url, tmp);
tmp.Add_mid(itm.Bfr(), itm.Row_bgn(), itm.Row_end());
itm.Rls();
}
Io_mgr._.AppendFilBfr(url, tmp);
Io_mgr.I.AppendFilBfr(url, tmp);
list.Clear();
url_list.Add(url);
}
@@ -84,7 +84,7 @@ public class Io_sort {
int default_load_len = memory_max / urls_len + 1;
for (int i = 0; i < urls_len; i++) {
Io_url url = urls[i];
int file_len = (int)Io_mgr._.QueryFil(url).Size();
int file_len = (int)Io_mgr.I.QueryFil(url).Size();
int load_len = file_len < default_load_len ? file_len : default_load_len; // PERF.NOTE: 32 MB is default, but if file is 1 MB (or else) only create a bfr for 1 MB; using 32 MB will throw OutOfMemory on -Xmx 64m; DATE:20130112
Io_line_rdr stream_bfr = new Io_line_rdr(usr_dlg, url).Key_gen_(key_gen).Load_len_(load_len);
boolean read = stream_bfr.Read_next();

2
400_xowa/src_040_io/gplx/ios/Io_sort_fil_basic.java
View File

@@ -31,7 +31,7 @@ public class Io_sort_fil_basic implements Io_sort_cmd { // 123|bgn|end|1
private void Flush() {
Io_url url = url_gen.Nxt_url();
usr_dlg.Prog_one(GRP_KEY, "make", "making: ~{0}", url.NameAndExt());
Io_mgr._.SaveFilBry(url, bfr.Bfr(), bfr.Len());
Io_mgr.I.SaveFilBry(url, bfr.Bfr(), bfr.Len());
bfr.Clear();
}
static final String GRP_KEY = "xowa.bldr.io_sort";

8
400_xowa/src_040_io/gplx/ios/Io_sort_misc_tst.java
View File

@@ -25,11 +25,11 @@ public class Io_sort_misc_tst {
}
private void tst_Io_url_gen_dir(String dir_str, String fmt, int digits, int calls, String... expd) {
Io_url dir = Io_url_.mem_dir_(dir_str);
ListAdp actl_list = ListAdp_.new_();
List_adp actl_list = List_adp_.new_();
Io_url_gen wkr = Io_url_gen_.dir_(dir, fmt, digits);
for (int i = 0; i < calls; i++)
actl_list.Add(wkr.Nxt_url().Raw());
String[] actl = actl_list.XtoStrAry();
String[] actl = actl_list.To_str_ary();
for (int i = 0; i < expd.length; i++)
expd[i] = dir_str + expd[i];
Tfds.Eq_ary_str(expd, actl);
@@ -42,12 +42,12 @@ public class Io_sort_misc_tst {
tst_Io_line_rdr_fld_comparer( 1, "ab", "a");
}
private void tst_Io_line_rdr_fld_comparer(int expd, String lhs_str, String rhs_str) {
byte[] lhs = Bry_.new_utf8_(lhs_str), rhs = Bry_.new_utf8_(rhs_str);
byte[] lhs = Bry_.new_u8(lhs_str), rhs = Bry_.new_u8(rhs_str);
Tfds.Eq(expd, Bry_.Compare(lhs, 0, lhs.length, rhs, 0, rhs.length));
}
Io_line_rdr new_Io_line_rdr(String url_str, String text) {
Io_url url = Io_url_.mem_fil_(url_str);
Io_mgr._.SaveFilStr(url, text);
Io_mgr.I.SaveFilStr(url, text);
Io_line_rdr rv = new Io_line_rdr(Gfo_usr_dlg_.Test(), url);
rv.Read_next();
return rv;

2
400_xowa/src_040_io/gplx/ios/Io_sort_split_itm_sorter.java
View File

@@ -19,7 +19,7 @@ package gplx.ios; import gplx.*;
public class Io_sort_split_itm_sorter implements gplx.lists.ComparerAble {
public int compare(Object lhsObj, Object rhsObj) {
Io_sort_split_itm lhs = (Io_sort_split_itm)lhsObj, rhs = (Io_sort_split_itm)rhsObj;
// Tfds.Write(String_.new_utf8_(lhs.Bfr(), lhs.Key_bgn(), lhs.Key_end()), String_.new_utf8_(rhs.Bfr(), rhs.Key_bgn(), rhs.Key_end()));
// Tfds.Write(String_.new_u8(lhs.Bfr(), lhs.Key_bgn(), lhs.Key_end()), String_.new_u8(rhs.Bfr(), rhs.Key_bgn(), rhs.Key_end()));
return Bry_.Compare(lhs.Bfr(), lhs.Key_bgn(), lhs.Key_end(), rhs.Bfr(), rhs.Key_bgn(), rhs.Key_end());
}
public static final Io_sort_split_itm_sorter _ = new Io_sort_split_itm_sorter(); Io_sort_split_itm_sorter() {}

12
400_xowa/src_040_io/gplx/ios/Io_sort_tst.java
View File

@@ -27,16 +27,16 @@ public class Io_sort_tst {
class Io_sort_fxt {
Io_sort externalSort = new Io_sort().Memory_max_(Io_mgr.Len_kb);
String_bldr sb = String_bldr_.new_();
public Io_sort_fxt Clear() {Io_mgr._.InitEngine_mem(); return this;}
public Io_sort_fxt Clear() {Io_mgr.I.InitEngine_mem(); return this;}
public Io_sort_fxt Memory_max_(int v) {externalSort.Memory_max_(v); return this;}
public Io_sort_fxt Src_(String v) {src = v; return this;} private String src;
public Io_sort_fxt Sorted_(String v) {sorted = v; return this;} private String sorted;
public void tst() {
Io_url src_url = Io_url_.mem_fil_("mem/src.txt");
Io_url trg_url = Io_url_.mem_fil_("mem/trg.txt");
Io_mgr._.DeleteFil(src_url); Io_mgr._.DeleteFil(trg_url);
Io_mgr.I.DeleteFil(src_url); Io_mgr.I.DeleteFil(trg_url);
Io_mgr._.SaveFilStr(src_url, src);
Io_mgr.I.SaveFilStr(src_url, src);
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Test();
Io_url_gen src_fil_gen = Io_url_gen_.fil_(src_url);
@@ -44,16 +44,16 @@ class Io_sort_fxt {
Io_sort_fil_basic cmd = new Io_sort_fil_basic(usr_dlg, Io_url_gen_.fil_(trg_url), Io_mgr.Len_kb);
externalSort.Merge(usr_dlg, tmp_url_ary, Io_sort_split_itm_sorter._, Io_line_rdr_key_gen_.first_pipe, cmd);
String actl = Io_mgr._.LoadFilStr(trg_url);
String actl = Io_mgr.I.LoadFilStr(trg_url);
Tfds.Eq_ary_str(String_.SplitLines_nl(sorted), String_.SplitLines_nl(actl));
}
public String GenRandom(int rows, int pad) {
ListAdp list = ListAdp_.new_();
List_adp list = List_adp_.new_();
for (int i = 0; i < rows; i++)
list.Add(Int_.Xto_str_pad_bgn_zero(i, pad) + "|");
list.Shuffle();
for (int i = 0; i < rows; i++) {
String itm = (String)list.FetchAt(i);
String itm = (String)list.Get_at(i);
sb.Add(itm).Add_char_nl();
}
return sb.Xto_str_and_clear();

4
400_xowa/src_040_io/gplx/ios/Io_url_gen_.java
View File

@@ -33,13 +33,13 @@ class Io_url_gen_dir implements Io_url_gen {
}
return rv;
}
public void Del_all() {if (Io_mgr._.ExistsDir(dir)) Io_mgr._.DeleteDirDeep(dir);}
public void Del_all() {if (Io_mgr.I.ExistsDir(dir)) Io_mgr.I.DeleteDirDeep(dir);}
public Io_url_gen_dir(Io_url dir) {this.dir = dir;} Io_url dir;
}
class Io_url_gen_fil implements Io_url_gen {
public Io_url Cur_url() {return cur_url;} Io_url cur_url;
public Io_url Nxt_url() {return cur_url;}
public Io_url[] Prv_urls() {return new Io_url[]{cur_url};}
public void Del_all() {Io_mgr._.DeleteFil_args(cur_url).MissingFails_off().Exec();}
public void Del_all() {Io_mgr.I.DeleteFil_args(cur_url).MissingFails_off().Exec();}
public Io_url_gen_fil(Io_url fil) {this.cur_url = fil;}
}