Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.7.2.1

Browse Source
This commit is contained in:
gnosygnu
2015-07-12 21:10:02 -04:00
commit 794b5a232f
3099 changed files with 238212 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
400_xowa/src_040_io/gplx/ios/BinaryHeap_Io_line_rdr.java Normal file
View File

@@ -0,0 +1,84 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
import gplx.lists.*;
class BinaryHeap_Io_line_rdr {
public BinaryHeap_Io_line_rdr(ComparerAble comparer) {this.comparer = comparer;} ComparerAble comparer;
Io_line_rdr[] ary = Ary_empty; int ary_len = 0, ary_max = 0;
public int Len() {return ary_len;}
public void Add(Io_line_rdr itm) {
int new_len = ary_len + 1;
if (new_len > ary_max) {
ary_max = new_len * 2;
ary = (Io_line_rdr[])Array_.Resize(ary, ary_max);
}
ary[ary_len] = itm;
ary_len = new_len;
Add_move_up(ary_len - 1);
}
public Io_line_rdr Pop() {
if (ary_len == 0) return null;
Io_line_rdr rv = ary[0];
--ary_len;
if (ary_len > 0) {
ary[0] = ary[ary_len];
Pop_move_down(0);
}
return rv;
}
public void Rls() {
for (int i = 0; i < ary_len; i++) {
Io_line_rdr rdr = ary[i];
if (rdr != null) rdr.Rls();
ary[i] = null;
}
ary = null;
ary_len = 0;
}
private void Add_move_up(int pos) {
while (pos > 0) {
int owner = (pos - 1) / 2;
if (Compare(pos, owner) > CompareAble_.Less) break;
Swap(pos, owner);
pos = owner;
}
}
private void Pop_move_down(int pos) {
int idx_last = ary_len - 1;
while (pos < ary_len / 2) {
int sub = 2 * pos + 1;
if (sub < idx_last && Compare(sub, sub + 1) > CompareAble_.Same)
++sub;
if (Compare(pos, sub) < CompareAble_.More) break;
Swap(pos, sub);
pos = sub;
}
}
int Compare(int lhs_idx, int rhs_idx) {
Io_line_rdr lhs = ary[lhs_idx], rhs = ary[rhs_idx];
lhs_itm.Set(lhs); rhs_itm.Set(rhs);
return comparer.compare(lhs_itm, rhs_itm);
// return Bry_.Compare(lhs.Bfr(), lhs.Key_pos_bgn(), lhs.Key_pos_end(), rhs.Bfr(), rhs.Key_pos_bgn(), rhs.Key_pos_end());
} Io_sort_split_itm lhs_itm = new Io_sort_split_itm(), rhs_itm = new Io_sort_split_itm();
private void Swap(int lhs_idx, int rhs_idx) {
Io_line_rdr tmp = ary[lhs_idx];
ary[lhs_idx] = ary[rhs_idx];
ary[rhs_idx] = tmp;
}
private static final Io_line_rdr[] Ary_empty = new Io_line_rdr[0];
}

50
400_xowa/src_040_io/gplx/ios/BinaryHeap_Io_line_rdr_tst.java Normal file
View File

@@ -0,0 +1,50 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
import org.junit.*;
public class BinaryHeap_Io_line_rdr_tst {
BinaryHeap_Io_line_rdr_fxt fxt = new BinaryHeap_Io_line_rdr_fxt();
@Test public void Add() {
fxt.Add("c", "a", "b").tst("a", "b", "c");
fxt.Add("b", "a", "a").tst("a", "a", "b");
fxt.Add("f", "b", "d", "c", "e", "a").tst("a", "b", "c", "d", "e", "f");
}
}
class BinaryHeap_Io_line_rdr_fxt {
BinaryHeap_Io_line_rdr heap = new BinaryHeap_Io_line_rdr(Io_sort_split_itm_sorter._); int file_total;
public BinaryHeap_Io_line_rdr_fxt Add(String... ary) {
file_total = ary.length;
for (int i = 0; i < file_total; i++) {
Io_url url = Io_url_.mem_fil_("mem/fil_" + ary[i] + ".txt");
Io_mgr.I.SaveFilStr(url, ary[i]);
Io_line_rdr stream = new Io_line_rdr(Gfo_usr_dlg_.Test(), url);
stream.Read_next();
heap.Add(stream);
}
return this;
}
public BinaryHeap_Io_line_rdr_fxt tst(String... expd) {
String[] actl = new String[file_total];
for (int i = 0; i < actl.length; i++) {
Io_line_rdr bfr = heap.Pop();
actl[i] = String_.new_u8(bfr.Bfr(), 0, bfr.Bfr_len());
}
Tfds.Eq_ary_str(expd, actl);
return this;
}
}

73
400_xowa/src_040_io/gplx/ios/Io_buffer_rdr.java Normal file
View File

@@ -0,0 +1,73 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
import gplx.ios.*;/*IoStream*/
public class Io_buffer_rdr implements RlsAble {
private Io_stream_rdr rdr;
Io_buffer_rdr(Io_stream_rdr rdr, Io_url url, int bfr_len) {
this.rdr = rdr; this.url = url;
if (bfr_len <= 0) throw Exc_.new_("bfr_len must be > 0", "bfr_len", bfr_len);
bfr = new byte[bfr_len]; this.bfr_len = bfr_len;
IoItmFil fil = Io_mgr.I.QueryFil(url); if (!fil.Exists()) throw Exc_.new_("fil does not exist", "url", url);
fil_len = fil.Size();
fil_pos = 0;
fil_eof = false;
}
public Io_url Url() {return url;} private Io_url url;
public byte[] Bfr() {return bfr;} private byte[] bfr;
public int Bfr_len() {return bfr_len;} private int bfr_len;
public long Fil_len() {return fil_len;} long fil_len;
public long Fil_pos() {return fil_pos;} long fil_pos;
public boolean Fil_eof() {return fil_eof;} private boolean fil_eof;
public boolean Bfr_load_all() {return Bfr_load(0, bfr_len);}
public boolean Bfr_load_from(int bfr_pos) {
if (bfr_pos < 0 || bfr_pos > bfr_len) throw Exc_.new_("invalid bfr_pos", "bfr_pos", bfr_pos, "bfr_len", bfr_len);
for (int i = bfr_pos; i < bfr_len; i++) // shift end of bfr to bgn; EX: bfr[10] and load_from(8); [8] -> [0]; [9] -> [1];
bfr[i - bfr_pos] = bfr[i];
return Bfr_load(bfr_len - bfr_pos, bfr_pos); // fill rest of bfr; EX: [2]... will come from file
}
private boolean Bfr_load(int bgn, int len) {
int read = rdr.Read(bfr, bgn, len);
if (read == gplx.ios.Io_stream_rdr_.Read_done) {fil_eof = true; return false;}
fil_pos += read;
bfr_len = bgn + read;
if (read < len) fil_eof = true;
return true;
}
public void Seek(long fil_pos) {
this.fil_pos = fil_pos;
rdr.Skip(fil_pos);
this.Bfr_load_all();
}
public void Rls() {
bfr = null;
bfr_len = -1;
if (rdr != null) rdr.Rls();
}
@gplx.Internal protected void Dump_to_file(int bgn, int len, String url_str, String msg) { // DBG:
String text = String_.new_u8_by_len(bfr, bgn, len);
Io_mgr.I.AppendFilStr(Io_url_.new_any_(url_str), msg + text + "\n");
}
public static Io_buffer_rdr new_(Io_stream_rdr rdr, int bfr_len) {
Io_buffer_rdr rv = new Io_buffer_rdr(rdr, rdr.Url(), bfr_len);
rdr.Open();
rv.Bfr_load(0, bfr_len);
return rv;
}
public static final Io_buffer_rdr Null = new Io_buffer_rdr(); Io_buffer_rdr() {}
}

64
400_xowa/src_040_io/gplx/ios/Io_buffer_rdr_tst.java Normal file
View File

@@ -0,0 +1,64 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
import org.junit.*; import gplx.ios.*;
public class Io_buffer_rdr_tst {
@Before public void init() {
Io_mgr.I.InitEngine_mem();
fil = Io_url_.mem_fil_("mem/byteStreamRdr.txt");
ini_Write("0123456789");
rdr = Io_buffer_rdr.new_(Io_stream_rdr_.file_(fil), 4);
} Io_buffer_rdr rdr; Io_url fil;
@After public void teardown() {rdr.Rls();}
@Test public void Bfr_load_all() {
tst_Bfr("0", "1", "2", "3").tst_ReadDone(false);
rdr.Bfr_load_all();
tst_Bfr("4", "5", "6", "7").tst_ReadDone(false);
rdr.Bfr_load_all();
tst_Bfr("8", "9");
rdr.Bfr_load_all(); // NOTE: change to zip_rdrs make eof detection difficult; force another load to ensure that file_pos goes past file_len
tst_ReadDone(true); // NOTE: bfr truncated from 4 to 2
}
@Test public void Bfr_load_from() {
tst_Bfr("0", "1", "2", "3").tst_ReadDone(false);
rdr.Bfr_load_from(3); // read from pos 3
tst_Bfr("3", "4", "5", "6").tst_ReadDone(false);
rdr.Bfr_load_from(1); // read from pos 1
tst_Bfr("4", "5", "6", "7").tst_ReadDone(false);
rdr.Bfr_load_from(1);
tst_Bfr("5", "6", "7", "8").tst_ReadDone(false);
rdr.Bfr_load_from(3);
rdr.Bfr_load_all(); // NOTE: change to zip_rdrs make eof detection difficult; force another load to ensure that file_pos goes past file_len
tst_Bfr("8", "9").tst_ReadDone(true);
}
private void ini_Write(String s) {Io_mgr.I.SaveFilStr(fil, s);}
Io_buffer_rdr_tst tst_Bfr(String... expdAry) {
String[] actlAry = new String[rdr.Bfr_len()];
for (int i = 0; i < actlAry.length; i++)
actlAry[i] = String_.new_u8(rdr.Bfr(), i, i + 1);
Tfds.Eq_ary(expdAry, actlAry);
return this;
}
Io_buffer_rdr_tst tst_ReadDone(boolean expd) {Tfds.Eq(expd, rdr.Fil_eof()); return this;}
}

31
400_xowa/src_040_io/gplx/ios/Io_fil_chkr.java Normal file
View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public class Io_fil_chkr implements Tst_chkr {
public Io_fil_chkr(Io_url url, String data) {this.expd_url = url; this.expd_data = data;}
public Io_url Expd_url() {return expd_url;} public Io_fil_chkr Expd_url_(Io_url v) {expd_url = v; return this;} Io_url expd_url;
public String Expd_data() {return expd_data;} public Io_fil_chkr Expd_data_(String v) {expd_data = v; return this;} private String expd_data;
public Class<?> TypeOf() {return gplx.ios.Io_fil.class;}
public int Chk(Tst_mgr mgr, String path, Object actl) {
gplx.ios.Io_fil fil = (gplx.ios.Io_fil)actl;
int rv = 0;
rv += mgr.Tst_val(expd_url == null, path, "url", expd_url, fil.Url());
rv += mgr.Tst_val(expd_data == null, path, "data", expd_data, fil.Data());
return rv;
}
}

165
400_xowa/src_040_io/gplx/ios/Io_line_rdr.java Normal file
View File

@@ -0,0 +1,165 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public class Io_line_rdr {
public Io_line_rdr (Gfo_usr_dlg usr_dlg, Io_url... urls) {this.usr_dlg = usr_dlg; this.urls = urls; if (urls.length == 0) bfr_state = Bfr_state_end;} Gfo_usr_dlg usr_dlg;
public int Url_idx() {return url_idx;} private int url_idx;
public Io_url[] Urls() {return urls;} Io_url[] urls;
public void Reset_one(Io_url url) {
this.Clear();
urls[0] = url;
url_idx = 0;
}
public byte Line_dlm() {return line_dlm;} public Io_line_rdr Line_dlm_(byte v) {line_dlm = v; return this;} private byte line_dlm = Byte_ascii.Nl;
public byte[] Bfr() {return bfr;} private byte[] bfr;
public int Bfr_len() {return bfr_len;} private int bfr_len;
public byte Bfr_state() {return bfr_state;} private byte bfr_state = Bfr_state_bgn; static final byte Bfr_state_bgn = 0, Bfr_state_mid = 1, Bfr_state_end = 2;
public int Bfr_last_read() {return bfr_last_read;} private int bfr_last_read;
public void Bfr_last_read_add(int v) {bfr_last_read += v;}
public int Load_len() {return load_len;} public Io_line_rdr Load_len_(int v) {load_len = v; return this;} private int load_len = 4096;
long File_len() {return file_len;} long file_len = 0;
long File_pos() {return file_pos;} long file_pos = 0;
public boolean File_skip_line0() {return file_skip_line0;} public Io_line_rdr File_skip_line0_(boolean v) {file_skip_line0 = v; return this;} private boolean file_skip_line0;
public int Itm_pos_bgn() {return itm_pos_bgn;} private int itm_pos_bgn = 0;
public int Itm_pos_end() {return itm_pos_end;} private int itm_pos_end = 0;
public int Key_pos_bgn() {return key_pos_bgn;} public Io_line_rdr Key_pos_bgn_(int v) {key_pos_bgn = v; return this;} private int key_pos_bgn = -1;
public int Key_pos_end() {return key_pos_end;} public Io_line_rdr Key_pos_end_(int v) {key_pos_end = v; return this;} private int key_pos_end = -1;
public Io_line_rdr_key_gen Key_gen() {return key_gen;} public Io_line_rdr Key_gen_(Io_line_rdr_key_gen v) {key_gen = v; return this;} Io_line_rdr_key_gen key_gen = Io_line_rdr_key_gen_.first_pipe;
public void Truncate(int pos) {
this.Read_next();
int end = Bry_finder.Find_fwd(bfr, Byte_ascii.Nil); if (end == -1) end = bfr.length;
bfr = Bry_.Mid(bfr, pos, end);
bfr_len = bfr.length;
bfr_last_read = 0;
}
public boolean Read_next() {
switch (bfr_state) {
case Bfr_state_bgn:
bfr_state = Bfr_state_mid; // do not place after Load
Open_fil();
if (!Load()) return false;
break;
case Bfr_state_end:
return false;
}
itm_pos_bgn = bfr_last_read; itm_pos_end = bfr_len; key_pos_bgn = key_pos_end = -1;
while (true) {
for (int i = bfr_last_read; i < bfr_len; i++) {
if (bfr[i] == line_dlm) {
itm_pos_end = i + 1; // +1: include find
bfr_last_read = itm_pos_end;
key_gen.Gen(this);
return true;
}
}
if (Load()) // line_dlm not found; load more
itm_pos_bgn = 0;
else { // nothing loaded; return;
itm_pos_end = bfr_len;
key_gen.Gen(this); // call key_gen b/c there may be a stray line at end; EX: "\nb|c"; call key_gen to get "b"
bfr_state = Bfr_state_end;
return bfr_last_read < bfr_len; // true if stray line at end; otherwise bfr_last_read == bfr_len and return false;
}
}
}
public boolean Match(byte[] ttl) {
switch (bfr_state) {
case Bfr_state_bgn:
if (!Read_next()) return false;
bfr_state = Bfr_state_mid; // NOTE: must set back to mid; possible for 1st read to read entire buffer; EX: 8 MB bfr, but only 1 MB file;
break;
case Bfr_state_end:
return false;
}
while (true) {
int compare = Bry_.Compare(ttl, 0, ttl.length, bfr, key_pos_bgn, key_pos_end);
// if (String_.new_u8(bfr, key_pos_bgn, key_pos_end) == "US Naval Jack.svg") {
// Tfds.Write();
// }
if (compare == CompareAble_.Same) { // eq; return true and move fwd; EX: "BA" and "BA"
return true;
}
else if (compare < CompareAble_.Same) { // lt; return false; EX: ttl is "BA" but rdr is "BC"
return false;
}
else { // gt; keep reading; EX: ttl is "BC" but rdr is "BA"
if (!this.Read_next()) return false;
}
}
}
boolean Load() {
int old_bfr_len = bfr_len - bfr_last_read; // NOTE: preserve bytes between bfr_last_read and bfr_len; EX: "ab\nc"; preserve "c" for next ary
if (file_done) {++url_idx; if (url_idx == urls.length) {bfr_state = Bfr_state_end; return false;} Open_fil(); file_done = false;}
byte[] load_ary = new byte[old_bfr_len + load_len]; int load_ary_len = load_ary.length; // NOTE: must go after file_done chk; otherwise small wikis will allocate another 32 MB bry for sort_memory_len and cause an OutOfMemory error; DATE:20130112
int read_len = stream.Read(load_ary, old_bfr_len, load_ary_len - old_bfr_len);
if (read_len == 0) { // nothing read; return;
++url_idx;
if (url_idx < urls.length) {
stream.Rls();
Open_fil();
return true;
}
else {
stream.Rls();
bfr_state = Bfr_state_end;
return false;
}
}
if (read_len == file_len) {
stream.Rls();
file_done = true;
// ++url_idx;
// bfr_state = Bfr_state_end;
}
if (old_bfr_len > 0) Array_.CopyTo(bfr, bfr_last_read, load_ary, 0, old_bfr_len); // copy old_bfr over
file_pos += read_len;
bfr = load_ary;
bfr_last_read = 0;
bfr_len = read_len == load_len ? load_ary_len : old_bfr_len + read_len; // stream.Read() may return less bytes than load_ary at EOF; if so, don't shrink bfr; just mark bfr_len less
return true;
} IoStream stream;
private void Open_fil() {
Io_url url = urls[url_idx];
usr_dlg.Prog_many(GRP_KEY, "load", "loading dump file: ~{0}", url.NameAndExt());
if (file_skip_line0) {
byte[] stream_bry = Io_mgr.I.LoadFilBry(url);
int stream_bry_len = stream_bry.length;
int nl_pos = Bry_finder.Find_fwd(stream_bry, Byte_ascii.Nl, 0, stream_bry_len);
if (nl_pos == Bry_.NotFound)
stream_bry = Bry_.Empty;
else
stream_bry = Bry_.Mid(stream_bry, nl_pos + 1, stream_bry_len);
stream = gplx.ios.IoStream_.ary_(stream_bry);
}
else {
stream = Io_mgr.I.OpenStreamRead(url);
}
file_pos = 0; file_len = stream.Len();
file_done = false;
} boolean file_done = false;
public void Clear() {
bfr_state = Bfr_state_bgn;
if (stream != null) stream.Rls();
bfr = null;
}
public void Rls() {
this.Clear();
bfr = null;
}
static final String GRP_KEY = "xowa.bldr.line_rdr";
}

21
400_xowa/src_040_io/gplx/ios/Io_line_rdr_key_gen.java Normal file
View File

@@ -0,0 +1,21 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public interface Io_line_rdr_key_gen {
void Gen(Io_line_rdr bfr);
}

56
400_xowa/src_040_io/gplx/ios/Io_line_rdr_key_gen_.java Normal file
View File

@@ -0,0 +1,56 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public class Io_line_rdr_key_gen_ {
public static final Io_line_rdr_key_gen first_pipe = new Io_line_rdr_key_gen_first(Byte_ascii.Pipe);
public static final Io_line_rdr_key_gen last_pipe = new Io_line_rdr_key_gen_last(Byte_ascii.Pipe);
public static final Io_line_rdr_key_gen noop = new Io_line_rdr_key_gen_noop();
}
class Io_line_rdr_key_gen_last implements Io_line_rdr_key_gen {
public Io_line_rdr_key_gen_last(byte fld_dlm) {this.fld_dlm = fld_dlm;} private byte fld_dlm;
public void Gen(Io_line_rdr bfr) {
int bgn = bfr.Itm_pos_bgn(), end = bfr.Itm_pos_end() - 1; // -1: ignore row_dlm
bfr.Key_pos_bgn_(end).Key_pos_end_(end);
byte[] bry = bfr.Bfr();
for (int i = end; i >= bgn; i--) {
if (bry[i] == fld_dlm) {
bfr.Key_pos_bgn_(i + 1); // +1 to position after fldDlm
return;
}
}
bfr.Key_pos_bgn_(0); // nothing found; position at bgn
}
}
class Io_line_rdr_key_gen_first implements Io_line_rdr_key_gen {
public Io_line_rdr_key_gen_first(byte fld_dlm) {this.fld_dlm = fld_dlm;} private byte fld_dlm;
public void Gen(Io_line_rdr bfr) {
int bgn = bfr.Itm_pos_bgn(), end = bfr.Itm_pos_end();
bfr.Key_pos_bgn_(bgn).Key_pos_end_(bgn);
byte[] bry = bfr.Bfr();
for (int i = bgn; i < end; i++) {
if (bry[i] == fld_dlm) {
bfr.Key_pos_end_(i);
return;
}
}
bfr.Key_pos_end_(end); // nothing found; position at end
}
}
class Io_line_rdr_key_gen_noop implements Io_line_rdr_key_gen {
public void Gen(Io_line_rdr bfr) {}
}

96
400_xowa/src_040_io/gplx/ios/Io_line_rdr_tst.java Normal file
View File

@@ -0,0 +1,96 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
import org.junit.*;
public class Io_line_rdr_tst {
Io_line_rdr_fxt fxt;
@Before public void init() {
fxt = new Io_line_rdr_fxt(Io_url_.new_fil_("mem/test.txt"));
fxt.Clear();
}
@Test public void Basic() {
fxt.File_lines_(3).tst_Read_til_lines(3, "00", "01", "02");
fxt.tst_Read_til_lines(1); // make sure nothing more is read
}
@Test public void Load_3x() {
fxt.File_lines_(9).Load_len_lines_(3).tst_Read_til_lines(9, "00", "01", "02", "03", "04", "05", "06", "07", "08");
}
@Test public void Load_irregular() {
fxt.File_lines_(9).Load_len_(4).tst_Read_til_lines(9, "00", "01", "02", "03", "04", "05", "06", "07", "08");
}
@Test public void Load_multiple_files() {
fxt = new Io_line_rdr_fxt(Io_url_.new_fil_("mem/test0.txt"), Io_url_.new_fil_("mem/test1.txt"), Io_url_.new_fil_("mem/test2.txt"));
fxt.File_lines_(0, 0, 3).File_lines_(1, 3, 5).File_lines_(2, 5, 9).Load_len_(4).tst_Read_til_lines(9, "00", "01", "02", "03", "04", "05", "06", "07", "08");
}
@Test public void Match() {
fxt.File_lines_pipe_(9).Load_len_(6);
fxt.tst_Match("00", "00");
fxt.tst_Match("01", "01");
fxt.tst_Match("03", "03");
fxt.tst_Match("08", "08");
fxt.tst_Match("12", "");
}
}
class Io_line_rdr_fxt {
Io_line_rdr rdr;
List_adp lines = List_adp_.new_(); Bry_bfr tmp = Bry_bfr.new_();
public Io_line_rdr_fxt(Io_url... urls) {rdr = new Io_line_rdr(Gfo_usr_dlg_.Test(), urls);}
public Io_line_rdr_fxt Load_len_lines_(int v) {return Load_len_(v * 3);} // 3: 2=##, 1=\n
public Io_line_rdr_fxt Load_len_(int v) {rdr.Load_len_(v); return this;}
public Io_line_rdr_fxt File_lines_(int count) {
for (int i = 0; i < count; i++)
tmp.Add_int_fixed(i, 2).Add_byte_nl();
Io_mgr.I.SaveFilBry(rdr.Urls()[0], tmp.Xto_bry_and_clear());
return this;
}
// public Io_url[] Src_fils() {return src_fils;} public Io_line_rdr_fxt Src_fils_(Io_url[] v) {src_fils = v; return this;} Io_url[] src_fils;
public Io_line_rdr_fxt tst_Match(String match, String expd) {
rdr.Key_gen_(Io_line_rdr_key_gen_.first_pipe);
boolean match_v = rdr.Match(Bry_.new_u8(match));
String actl = match_v ? String_.new_u8(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end()) : "";
Tfds.Eq(expd, actl);
return this;
}
public Io_line_rdr_fxt File_lines_pipe_(int count) {
for (int i = 0; i < count; i++)
tmp.Add_int_fixed(i, 2).Add_byte(Byte_ascii.Pipe).Add_byte_nl();
Io_mgr.I.SaveFilBry(rdr.Urls()[0], tmp.Xto_bry_and_clear());
return this;
}
public Io_line_rdr_fxt File_lines_(int fil_idx, int bgn, int end) {
for (int i = bgn; i < end; i++)
tmp.Add_int_fixed(i, 2).Add_byte_nl();
Io_mgr.I.SaveFilBry(rdr.Urls()[fil_idx], tmp.Xto_bry_and_clear());
return this;
}
public Io_line_rdr_fxt Clear() {rdr.Clear(); return this;}
public Io_line_rdr_fxt tst_Read_til_lines(int count, String... expd) {
lines.Clear();
for (int i = 0; i < expd.length; i++)
expd[i] = expd[i] + Op_sys.Lnx.Nl_str();
for (int i = 0; i < count; i++) {
if (rdr.Read_next())
lines.Add(String_.new_u8(rdr.Bfr(), rdr.Itm_pos_bgn(), rdr.Itm_pos_end()));
else
break;
}
Tfds.Eq_ary_str(expd, lines.To_str_ary());
return this;
}
}

21
400_xowa/src_040_io/gplx/ios/Io_make_cmd.java Normal file
View File

@@ -0,0 +1,21 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public interface Io_make_cmd extends Io_sort_cmd {
Io_sort_cmd Make_dir_(Io_url v);
}

98
400_xowa/src_040_io/gplx/ios/Io_sort.java Normal file
View File

@@ -0,0 +1,98 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
import gplx.lists.*;
public class Io_sort {
public Io_sort Memory_max_(int v) {memory_max = v; return this;} private int memory_max = Io_mgr.Len_kb;
public Io_url[] Split(Gfo_usr_dlg usr_dlg, Io_url_gen src_fil_gen, Io_url_gen trg_fil_gen, Io_line_rdr_key_gen key_gen) {return Split(usr_dlg, src_fil_gen, trg_fil_gen, Io_sort_split_itm_sorter._, key_gen);}
public Io_url[] Split(Gfo_usr_dlg usr_dlg, Io_url_gen src_fil_gen, Io_url_gen trg_fil_gen, ComparerAble row_comparer, Io_line_rdr_key_gen key_gen) {
Io_line_rdr rdr = new Io_line_rdr(usr_dlg, src_fil_gen.Prv_urls()).Load_len_(4 * Io_mgr.Len_kb).Key_gen_(key_gen); // NOTE: do not set load_len to memory_max; only want to load in increments
List_adp rv = List_adp_.new_();
Bry_bfr bfr = Bry_bfr.reset_(Const_bfr_max); int size_cur = 0;
List_adp row_list = List_adp_.new_();
while (true) {
boolean reading = rdr.Read_next();
int size_row = rdr.Itm_pos_end() - rdr.Itm_pos_bgn();
int size_new = size_cur + size_row;
if (size_new > memory_max || !reading) {
usr_dlg.Prog_none(GRP_KEY, "sort", "sorting chunk");
row_list.Sort_by(row_comparer);
Io_url trg_url = trg_fil_gen.Nxt_url();
usr_dlg.Prog_one(GRP_KEY, "write", "writing chunk: ~{0}", trg_url.Raw());
Split_flush(trg_url, row_list, memory_max, bfr, rv);
row_list.Resize_bounds(16); // MEM: resize bounds manually; note that each Flush-set may have widely disparately #of rows (EX: 1 row with a million pages vs. 1 million rows with 1 page)
size_new = size_row; Env_.GarbageCollect();
if (!reading) break;
}
row_list.Add(new Io_sort_split_itm(rdr));
size_cur = size_new;
}
rdr.Rls(); bfr.Rls(); Env_.GarbageCollect();
return (Io_url[])rv.To_ary(Io_url.class);
}
public void Merge(Gfo_usr_dlg usr_dlg, Io_url[] src_ary, ComparerAble comparer, Io_line_rdr_key_gen key_gen, Io_sort_cmd cmd) {
BinaryHeap_Io_line_rdr heap = load_(usr_dlg, src_ary, comparer, key_gen, memory_max); if (heap.Len() == 0) return;//throw Err_.new_(Array_.XtoStr(src_ary));
Io_line_rdr stream = null;
cmd.Sort_bgn();
while (true) {
if (stream != null) { // stream found on previous iteration; try to put it back on heap
boolean read = stream.Read_next();
if (read) // stream has line; add to heap
heap.Add(stream);
else { // stream is empty; rls
stream.Rls();
if (heap.Len() == 0) break; // heap is empty; stop;
}
}
stream = heap.Pop(); // note that .Pop removes stream from heap
cmd.Sort_do(stream);
}
cmd.Sort_end();
heap.Rls();
}
private static void Split_flush(Io_url url, List_adp list, int max, Bry_bfr tmp, List_adp url_list) {
int len = list.Count();
for (int i = 0; i < len; i++) {
Io_sort_split_itm itm = (Io_sort_split_itm)list.Get_at(i);
int add_len = itm.Row_end() - itm.Row_bgn();
if ((tmp.Len() + add_len) > Const_bfr_max) Io_mgr.I.AppendFilBfr(url, tmp);
tmp.Add_mid(itm.Bfr(), itm.Row_bgn(), itm.Row_end());
itm.Rls();
}
Io_mgr.I.AppendFilBfr(url, tmp);
list.Clear();
url_list.Add(url);
}
private static BinaryHeap_Io_line_rdr load_(Gfo_usr_dlg usr_dlg, Io_url[] urls, ComparerAble comparer, Io_line_rdr_key_gen key_gen, int memory_max) {
BinaryHeap_Io_line_rdr rv = new BinaryHeap_Io_line_rdr(comparer);
int urls_len = urls.length;
int default_load_len = memory_max / urls_len + 1;
for (int i = 0; i < urls_len; i++) {
Io_url url = urls[i];
int file_len = (int)Io_mgr.I.QueryFil(url).Size();
int load_len = file_len < default_load_len ? file_len : default_load_len; // PERF.NOTE: 32 MB is default, but if file is 1 MB (or else) only create a bfr for 1 MB; using 32 MB will throw OutOfMemory on -Xmx 64m; DATE:20130112
Io_line_rdr stream_bfr = new Io_line_rdr(usr_dlg, url).Key_gen_(key_gen).Load_len_(load_len);
boolean read = stream_bfr.Read_next();
if (read) // guard against empty files
rv.Add(stream_bfr);
}
return rv;
}
static final int Const_bfr_max = Io_mgr.Len_mb;
static final String GRP_KEY = "xowa.bldr.io_sort";
}

23
400_xowa/src_040_io/gplx/ios/Io_sort_cmd.java Normal file
View File

@@ -0,0 +1,23 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public interface Io_sort_cmd {
void Sort_bgn();
void Sort_do(Io_line_rdr rdr);
void Sort_end();
}

29
400_xowa/src_040_io/gplx/ios/Io_sort_filCmd.java Normal file
View File

@@ -0,0 +1,29 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public interface Io_sort_filCmd {
void Bfr_add(Io_line_rdr stream);
void Fil_bgn(Io_line_rdr stream);
void Fil_end();
}
class Io_sort_filCmd_null implements Io_sort_filCmd {
public void Bfr_add(Io_line_rdr stream) {}
public void Fil_bgn(Io_line_rdr stream) {}
public void Fil_end() {}
public static final Io_sort_filCmd_null _ = new Io_sort_filCmd_null(); Io_sort_filCmd_null() {}
}

38
400_xowa/src_040_io/gplx/ios/Io_sort_fil_basic.java Normal file
View File

@@ -0,0 +1,38 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public class Io_sort_fil_basic implements Io_sort_cmd { // 123|bgn|end|1
public Io_sort_fil_basic(Gfo_usr_dlg usr_dlg, Io_url_gen url_gen, int flush_len) {this.usr_dlg = usr_dlg; this.url_gen = url_gen; this.flush_len = flush_len;} Io_url_gen url_gen; Bry_bfr bfr = Bry_bfr.new_(); int flush_len; Gfo_usr_dlg usr_dlg;
public void Sort_bgn() {}
public void Sort_do(Io_line_rdr rdr) {
int bgn = rdr.Itm_pos_bgn(), end = rdr.Itm_pos_end();
if (bfr.Len() + (end - bgn) > flush_len) Flush();
bfr.Add_mid(rdr.Bfr(), bgn, end);
}
public void Sort_end() {
Flush();
bfr.Rls();
}
private void Flush() {
Io_url url = url_gen.Nxt_url();
usr_dlg.Prog_one(GRP_KEY, "make", "making: ~{0}", url.NameAndExt());
Io_mgr.I.SaveFilBry(url, bfr.Bfr(), bfr.Len());
bfr.Clear();
}
static final String GRP_KEY = "xowa.bldr.io_sort";
}

58
400_xowa/src_040_io/gplx/ios/Io_sort_misc_tst.java Normal file
View File

@@ -0,0 +1,58 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
import org.junit.*;
public class Io_sort_misc_tst {
@Before public void init() {
}
@Test public void Io_url_gen_dir() {
tst_Io_url_gen_dir("mem/dir/", "{0}.xdat", 4, 3, "0000.xdat", "0001.xdat", "0002.xdat");
}
private void tst_Io_url_gen_dir(String dir_str, String fmt, int digits, int calls, String... expd) {
Io_url dir = Io_url_.mem_dir_(dir_str);
List_adp actl_list = List_adp_.new_();
Io_url_gen wkr = Io_url_gen_.dir_(dir, fmt, digits);
for (int i = 0; i < calls; i++)
actl_list.Add(wkr.Nxt_url().Raw());
String[] actl = actl_list.To_str_ary();
for (int i = 0; i < expd.length; i++)
expd[i] = dir_str + expd[i];
Tfds.Eq_ary_str(expd, actl);
}
@Test public void Io_line_rdr_comparer_all() {
tst_Io_line_rdr_fld_comparer(-1, "a", "b");
tst_Io_line_rdr_fld_comparer( 0, "a", "a");
tst_Io_line_rdr_fld_comparer( 1, "b", "a");
tst_Io_line_rdr_fld_comparer(-1, "a", "ab");
tst_Io_line_rdr_fld_comparer( 1, "ab", "a");
}
private void tst_Io_line_rdr_fld_comparer(int expd, String lhs_str, String rhs_str) {
byte[] lhs = Bry_.new_u8(lhs_str), rhs = Bry_.new_u8(rhs_str);
Tfds.Eq(expd, Bry_.Compare(lhs, 0, lhs.length, rhs, 0, rhs.length));
}
Io_line_rdr new_Io_line_rdr(String url_str, String text) {
Io_url url = Io_url_.mem_fil_(url_str);
Io_mgr.I.SaveFilStr(url, text);
Io_line_rdr rv = new Io_line_rdr(Gfo_usr_dlg_.Test(), url);
rv.Read_next();
return rv;
}
@Test public void ExternalSort() {
// fxt("c", "a", "b")
}
}

35
400_xowa/src_040_io/gplx/ios/Io_sort_split_itm.java Normal file
View File

@@ -0,0 +1,35 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public class Io_sort_split_itm {
public Io_sort_split_itm() {}
public Io_sort_split_itm(Io_line_rdr rdr) {Set(rdr);}
public int Row_bgn() {return row_bgn;} private int row_bgn;
public int Row_end() {return row_end;} private int row_end;
public int Key_bgn() {return key_bgn;} private int key_bgn;
public int Key_end() {return key_end;} private int key_end;
public byte[] Bfr() {return bfr;} private byte[] bfr;
public void Set(Io_line_rdr rdr) {
bfr = rdr.Bfr();
row_bgn = rdr.Itm_pos_bgn();
row_end = rdr.Itm_pos_end();
key_bgn = rdr.Key_pos_bgn();
key_end = rdr.Key_pos_end();
}
public void Rls() {bfr = null;}
}

26
400_xowa/src_040_io/gplx/ios/Io_sort_split_itm_sorter.java Normal file
View File

@@ -0,0 +1,26 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public class Io_sort_split_itm_sorter implements gplx.lists.ComparerAble {
public int compare(Object lhsObj, Object rhsObj) {
Io_sort_split_itm lhs = (Io_sort_split_itm)lhsObj, rhs = (Io_sort_split_itm)rhsObj;
// Tfds.Write(String_.new_u8(lhs.Bfr(), lhs.Key_bgn(), lhs.Key_end()), String_.new_u8(rhs.Bfr(), rhs.Key_bgn(), rhs.Key_end()));
return Bry_.Compare(lhs.Bfr(), lhs.Key_bgn(), lhs.Key_end(), rhs.Bfr(), rhs.Key_bgn(), rhs.Key_end());
}
public static final Io_sort_split_itm_sorter _ = new Io_sort_split_itm_sorter(); Io_sort_split_itm_sorter() {}
}

66
400_xowa/src_040_io/gplx/ios/Io_sort_tst.java Normal file
View File

@@ -0,0 +1,66 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
import org.junit.*; import gplx.core.strings.*;
public class Io_sort_tst {
Io_sort_fxt fxt = new Io_sort_fxt();
@Test public void ExternalSort() {
fxt.Clear().Memory_max_(12).Src_(fxt.GenRandom(6, 4)).Sorted_(fxt.GenOrdered(6, 4)).tst();
fxt.Clear().Memory_max_(64).Src_(fxt.GenRandom(50, 4)).Sorted_(fxt.GenOrdered(50, 4)).tst();
}
}
class Io_sort_fxt {
Io_sort externalSort = new Io_sort().Memory_max_(Io_mgr.Len_kb);
String_bldr sb = String_bldr_.new_();
public Io_sort_fxt Clear() {Io_mgr.I.InitEngine_mem(); return this;}
public Io_sort_fxt Memory_max_(int v) {externalSort.Memory_max_(v); return this;}
public Io_sort_fxt Src_(String v) {src = v; return this;} private String src;
public Io_sort_fxt Sorted_(String v) {sorted = v; return this;} private String sorted;
public void tst() {
Io_url src_url = Io_url_.mem_fil_("mem/src.txt");
Io_url trg_url = Io_url_.mem_fil_("mem/trg.txt");
Io_mgr.I.DeleteFil(src_url); Io_mgr.I.DeleteFil(trg_url);
Io_mgr.I.SaveFilStr(src_url, src);
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Test();
Io_url_gen src_fil_gen = Io_url_gen_.fil_(src_url);
Io_url[] tmp_url_ary = externalSort.Split(usr_dlg, src_fil_gen, Io_url_gen_.dir_(src_url.OwnerDir()), Io_line_rdr_key_gen_.first_pipe);
Io_sort_fil_basic cmd = new Io_sort_fil_basic(usr_dlg, Io_url_gen_.fil_(trg_url), Io_mgr.Len_kb);
externalSort.Merge(usr_dlg, tmp_url_ary, Io_sort_split_itm_sorter._, Io_line_rdr_key_gen_.first_pipe, cmd);
String actl = Io_mgr.I.LoadFilStr(trg_url);
Tfds.Eq_ary_str(String_.SplitLines_nl(sorted), String_.SplitLines_nl(actl));
}
public String GenRandom(int rows, int pad) {
List_adp list = List_adp_.new_();
for (int i = 0; i < rows; i++)
list.Add(Int_.Xto_str_pad_bgn_zero(i, pad) + "|");
list.Shuffle();
for (int i = 0; i < rows; i++) {
String itm = (String)list.Get_at(i);
sb.Add(itm).Add_char_nl();
}
return sb.Xto_str_and_clear();
}
public String GenOrdered(int rows, int pad) {
for (int i = 0; i < rows; i++)
sb.Add(Int_.Xto_str_pad_bgn_zero(i, pad) + "|" + "\n");
return sb.Xto_str_and_clear();
}
}

24
400_xowa/src_040_io/gplx/ios/Io_url_gen.java Normal file
View File

@@ -0,0 +1,24 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public interface Io_url_gen {
Io_url Cur_url();
Io_url Nxt_url();
Io_url[] Prv_urls();
void Del_all();
}

45
400_xowa/src_040_io/gplx/ios/Io_url_gen_.java Normal file
View File

@@ -0,0 +1,45 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.ios; import gplx.*;
public class Io_url_gen_ {
public static Io_url_gen dir_(Io_url v) {return new Io_url_gen_dir(v);}
public static Io_url_gen dir_(Io_url v, String fmt, int digits) {return new Io_url_gen_dir(v).Fmt_(fmt).Fmt_digits_(digits);}
public static Io_url_gen fil_(Io_url v) {return new Io_url_gen_fil(v);}
}
class Io_url_gen_dir implements Io_url_gen {
public String Fmt() {return fmt;} public Io_url_gen_dir Fmt_(String v) {fmt = v; return this;} private String fmt = "{0}.csv";
public int Fmt_digits() {return fmt_digits;} public Io_url_gen_dir Fmt_digits_(int v) {fmt_digits = v; return this;} private int fmt_digits = 10;
public Io_url Cur_url() {return cur_url;} Io_url cur_url;
public Io_url Nxt_url() {cur_url = dir.GenSubFil(String_.Format(fmt, Int_.Xto_str_pad_bgn_zero(idx++, fmt_digits))); return cur_url;} private int idx = 0;
public Io_url[] Prv_urls() {
Io_url[] rv = new Io_url[idx];
for (int i = 0; i < idx; i++) {
rv[i] = dir.GenSubFil(String_.Format(fmt, Int_.Xto_str_pad_bgn_zero(i, fmt_digits)));
}
return rv;
}
public void Del_all() {if (Io_mgr.I.ExistsDir(dir)) Io_mgr.I.DeleteDirDeep(dir);}
public Io_url_gen_dir(Io_url dir) {this.dir = dir;} Io_url dir;
}
class Io_url_gen_fil implements Io_url_gen {
public Io_url Cur_url() {return cur_url;} Io_url cur_url;
public Io_url Nxt_url() {return cur_url;}
public Io_url[] Prv_urls() {return new Io_url[]{cur_url};}
public void Del_all() {Io_mgr.I.DeleteFil_args(cur_url).MissingFails_off().Exec();}
public Io_url_gen_fil(Io_url fil) {this.cur_url = fil;}
}