Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.11.1.1

Browse Source
This commit is contained in:
gnosygnu
2015-11-01 20:50:05 -05:00
parent 4f43f51b18
commit b990ec409f
858 changed files with 6758 additions and 4187 deletions
Download Patch File Download Diff File Expand all files Collapse all files

130
400_xowa/src/gplx/core/brys/Bry_parser.java Normal file
View File

@@ -0,0 +1,130 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.brys; import gplx.*; import gplx.core.*;
public class Bry_parser {
private final gplx.core.primitives.Int_obj_ref pos_ref = gplx.core.primitives.Int_obj_ref.neg1_();
private byte[] page; private String wkr_name; private int hook_bgn;
public byte[] Src() {return src;} private byte[] src;
public int Src_len() {return src_len;} private int src_len;
public int Pos() {return pos;} private int pos;
public void Init_src(byte[] page, byte[] src, int src_len, int pos) {
this.page = page; this.src = src; this.src_len = src_len; this.pos = pos;
}
public void Init_hook(String wkr_name, int hook_bgn, int hook_end) {
this.wkr_name = wkr_name; this.hook_bgn = hook_bgn; this.pos = hook_end;
}
public int Pos_(int v) {this.pos = v; return pos;}
public int Pos_add_one() {return Pos_add(1);}
public int Pos_add(int adj) {
this.pos += adj;
return pos;
}
public byte Read_byte() {
byte rv = src[pos];
++pos;
return rv;
}
public int Read_int_to(byte to_char) {
int bgn = pos;
int rv = 0;
int negative = 1;
while (pos < src_len) {
byte b = src[pos++];
switch (b) {
case Byte_ascii.Num_0: case Byte_ascii.Num_1: case Byte_ascii.Num_2: case Byte_ascii.Num_3: case Byte_ascii.Num_4:
case Byte_ascii.Num_5: case Byte_ascii.Num_6: case Byte_ascii.Num_7: case Byte_ascii.Num_8: case Byte_ascii.Num_9:
rv = (rv * 10) + (b - Byte_ascii.Num_0);
break;
case Byte_ascii.Dash:
if (negative == -1) // 2nd negative
throw Fail("invalid int", String_.new_u8(src, bgn, pos));
else // 1st negative
negative = -1; // flag negative
break;
default: {
boolean match = b == to_char;
if (to_char == Byte_ascii.Null) {// hack for Read_int_to_non_num
--pos;
match = true;
}
if (!match) throw Fail("invalid int", String_.new_u8(src, bgn, pos));
return rv * negative;
}
}
}
if (bgn == pos) throw Fail("int is empty", String_.Empty);
return rv * negative;
}
public byte Read_byte_as_a7_int() {
byte rv = Byte_ascii.To_a7_int(src[pos]);
++pos;
return rv;
}
public int Read_int_by_base85(int reqd) {
int rv = gplx.xowa.htmls.core.hzips.Xoh_hzip_int_.Decode(reqd, src, src_len, pos, pos_ref);
pos = pos_ref.Val();
return rv;
}
public boolean Is(byte find) {
boolean rv = src[pos] == find;
if (rv) ++pos; // only advance if match;
return rv;
}
public int Chk(byte[] find) {
int find_end = pos + find.length;
if (!Bry_.Match(src, pos, find_end, find)) throw Fail("failed check", String_.new_u8(find));
pos = find_end;
return pos;
}
public int Chk(byte find) {
if (src[pos] != find) throw Fail("failed check", Byte_ascii.To_str(find));
++pos;
return pos;
}
public int Fwd_bgn(byte[] find) {
int find_pos = Bry_find_.Find_fwd(src, find, pos, src_len);
if (find_pos == Bry_find_.Not_found) throw Fail("missing", String_.new_u8(find));
pos = find_pos + find.length;
return find_pos;
}
public int Fwd_bgn(byte find) {
int find_pos = Bry_find_.Find_fwd(src, find, pos, src_len);
if (find_pos == Bry_find_.Not_found) throw Fail("missing", Byte_ascii.To_str(find));
pos = find_pos + 1;
return find_pos;
}
public int Fwd_end(byte[] find) {
int find_pos = Bry_find_.Find_fwd(src, find, pos, src_len);
if (find_pos == Bry_find_.Not_found) throw Fail("missing", String_.new_u8(find));
pos = find_pos + find.length;
return pos;
}
public int Fwd_end(byte find) {
int find_pos = Bry_find_.Find_fwd(src, find, pos, src_len);
if (find_pos == Bry_find_.Not_found) throw Fail("missing", Byte_ascii.To_str(find));
pos = find_pos + 1;
return pos;
}
public int Fwd_while(byte find) {
this.pos = Bry_find_.Find_fwd_while(src, pos, src_len, find);
return pos;
}
public Err Fail(String msg, String arg) {
return Err_.new_("Bry_parser", msg, "arg", arg, "page", page, "wkr", wkr_name, "excerpt", Bry_.Mid_by_len_safe(src, hook_bgn, 255));
}
}

5
400_xowa/src/gplx/core/flds/Gfo_fld_rdr.java
View File

@@ -16,6 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.flds; import gplx.*; import gplx.core.*;
import gplx.core.encoders.*;
public class Gfo_fld_rdr extends Gfo_fld_base {
private Bry_bfr bfr = Bry_bfr.new_(); private static final byte[] Bry_nil = Bry_.new_a7("\\0");
public byte[] Data() {return data;} public Gfo_fld_rdr Data_(byte[] v) {data = v; data_len = v.length; pos = 0; return this;} private byte[] data; int data_len;
@@ -28,8 +29,8 @@ public class Gfo_fld_rdr extends Gfo_fld_base {
public String Read_str_simple() {Move_next_simple(); return String_.new_u8(data, fld_bgn, fld_end);}
public byte[] Read_bry_simple() {Move_next_simple(); return Bry_.Mid(data, fld_bgn, fld_end);} // was Mid_by_len???; 20120915
public int Read_int_base85_lenN(int len) {fld_bgn = pos; fld_end = pos + len - 1 ; pos = pos + len + 1 ; return Base85_utl.XtoIntByAry(data, fld_bgn, fld_end);}
public int Read_int_base85_len5() {fld_bgn = pos; fld_end = pos + 4 ; pos = pos + 6 ; return Base85_utl.XtoIntByAry(data, fld_bgn, fld_end);}
public int Read_int_base85_lenN(int len) {fld_bgn = pos; fld_end = pos + len - 1 ; pos = pos + len + 1 ; return Base85_.To_int_by_bry(data, fld_bgn, fld_end);}
public int Read_int_base85_len5() {fld_bgn = pos; fld_end = pos + 4 ; pos = pos + 6 ; return Base85_.To_int_by_bry(data, fld_bgn, fld_end);}
public int Read_int() {Move_next_simple(); return Bry_.To_int_or(data, fld_bgn, fld_end, -1);}
public byte Read_int_as_byte() {Move_next_simple(); return (byte)Bry_.To_int_or(data, fld_bgn, fld_end, -1);}
public byte Read_byte() {Move_next_simple(); return data[fld_bgn];}

2
400_xowa/src/gplx/core/flds/Gfo_fld_rdr_tst.java
View File

@@ -17,7 +17,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.flds; import gplx.*; import gplx.core.*;
import org.junit.*;
import gplx.ios.*;
import gplx.core.ios.*;
public class Gfo_fld_rdr_tst {
Gfo_fld_rdr_fxt fxt = new Gfo_fld_rdr_fxt();
@Test public void Read_int() {fxt.ini_xdat().Raw_("123|") .tst_Read_int(123);}

6
400_xowa/src/gplx/core/flds/Gfo_fld_wtr.java
View File

@@ -16,11 +16,11 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.flds; import gplx.*; import gplx.core.*;
import gplx.ios.*;
import gplx.core.ios.*; import gplx.core.encoders.*;
public class Gfo_fld_wtr extends Gfo_fld_base {
public Bry_bfr Bfr() {return bfr;} public Gfo_fld_wtr Bfr_(Bry_bfr v) {bfr = v; return this;} Bry_bfr bfr;
public Gfo_fld_wtr() {this.bfr = Bry_bfr.new_();}
public Gfo_fld_wtr Write_int_base85_len5_fld(int v) {bfr.Add_base85(v, Base85_utl.Len_int); bfr.Add_byte(fld_dlm); return this;}
public Gfo_fld_wtr Write_int_base85_len5_fld(int v) {bfr.Add_base85(v, Base85_.Len_int); bfr.Add_byte(fld_dlm); return this;}
public Gfo_fld_wtr Write_int_base85_lenN_fld(int v, int len) {bfr.Add_base85(v, len); bfr.Add_byte(fld_dlm); return this;}
public Gfo_fld_wtr Write_int_variable_fld(int v) {bfr.Add_int_variable(v); bfr.Add_byte(fld_dlm); return this;}
public Gfo_fld_wtr Write_int_fixed_fld(int v, int len) {bfr.Add_int_fixed(v, len); bfr.Add_byte(fld_dlm); return this;}
@@ -31,7 +31,7 @@ public class Gfo_fld_wtr extends Gfo_fld_base {
public Gfo_fld_wtr Write_dlm_row() { bfr.Add_byte(row_dlm); return this;}
public Gfo_fld_wtr Write_dlm_fld() { bfr.Add_byte(fld_dlm); return this;}
public Gfo_fld_wtr Write_int_base85_lenN_row(int v, int len) {bfr.Add_base85(v, len); bfr.Add_byte(row_dlm); return this;}
public Gfo_fld_wtr Write_int_base85_len5_row(int v) {bfr.Add_base85(v, Base85_utl.Len_int); bfr.Add_byte(row_dlm); return this;}
public Gfo_fld_wtr Write_int_base85_len5_row(int v) {bfr.Add_base85(v, Base85_.Len_int); bfr.Add_byte(row_dlm); return this;}
public Gfo_fld_wtr Write_bry_escape_row(byte[] val) {Write_bry_escape(val, 0, val.length); bfr.Add_byte(row_dlm); return this;}
public Gfo_fld_wtr Write_bry_escape_row(byte[] val, int bgn, int end) {Write_bry_escape(val, bgn, end); bfr.Add_byte(row_dlm); return this;}
public Gfo_fld_wtr Write_double_row(double v) {bfr.Add_double(v); bfr.Add_byte(row_dlm); return this;}

84
400_xowa/src/gplx/core/ios/BinaryHeap_Io_line_rdr.java Normal file
View File

@@ -0,0 +1,84 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import gplx.lists.*;
class BinaryHeap_Io_line_rdr {
public BinaryHeap_Io_line_rdr(ComparerAble comparer) {this.comparer = comparer;} ComparerAble comparer;
Io_line_rdr[] ary = Ary_empty; int ary_len = 0, ary_max = 0;
public int Len() {return ary_len;}
public void Add(Io_line_rdr itm) {
int new_len = ary_len + 1;
if (new_len > ary_max) {
ary_max = new_len * 2;
ary = (Io_line_rdr[])Array_.Resize(ary, ary_max);
}
ary[ary_len] = itm;
ary_len = new_len;
Add_move_up(ary_len - 1);
}
public Io_line_rdr Pop() {
if (ary_len == 0) return null;
Io_line_rdr rv = ary[0];
--ary_len;
if (ary_len > 0) {
ary[0] = ary[ary_len];
Pop_move_down(0);
}
return rv;
}
public void Rls() {
for (int i = 0; i < ary_len; i++) {
Io_line_rdr rdr = ary[i];
if (rdr != null) rdr.Rls();
ary[i] = null;
}
ary = null;
ary_len = 0;
}
private void Add_move_up(int pos) {
while (pos > 0) {
int owner = (pos - 1) / 2;
if (Compare(pos, owner) > CompareAble_.Less) break;
Swap(pos, owner);
pos = owner;
}
}
private void Pop_move_down(int pos) {
int idx_last = ary_len - 1;
while (pos < ary_len / 2) {
int sub = 2 * pos + 1;
if (sub < idx_last && Compare(sub, sub + 1) > CompareAble_.Same)
++sub;
if (Compare(pos, sub) < CompareAble_.More) break;
Swap(pos, sub);
pos = sub;
}
}
int Compare(int lhs_idx, int rhs_idx) {
Io_line_rdr lhs = ary[lhs_idx], rhs = ary[rhs_idx];
lhs_itm.Set(lhs); rhs_itm.Set(rhs);
return comparer.compare(lhs_itm, rhs_itm);
// return Bry_.Compare(lhs.Bfr(), lhs.Key_pos_bgn(), lhs.Key_pos_end(), rhs.Bfr(), rhs.Key_pos_bgn(), rhs.Key_pos_end());
} Io_sort_split_itm lhs_itm = new Io_sort_split_itm(), rhs_itm = new Io_sort_split_itm();
private void Swap(int lhs_idx, int rhs_idx) {
Io_line_rdr tmp = ary[lhs_idx];
ary[lhs_idx] = ary[rhs_idx];
ary[rhs_idx] = tmp;
}
private static final Io_line_rdr[] Ary_empty = new Io_line_rdr[0];
}

50
400_xowa/src/gplx/core/ios/BinaryHeap_Io_line_rdr_tst.java Normal file
View File

@@ -0,0 +1,50 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import org.junit.*;
public class BinaryHeap_Io_line_rdr_tst {
BinaryHeap_Io_line_rdr_fxt fxt = new BinaryHeap_Io_line_rdr_fxt();
@Test public void Add() {
fxt.Add("c", "a", "b").tst("a", "b", "c");
fxt.Add("b", "a", "a").tst("a", "a", "b");
fxt.Add("f", "b", "d", "c", "e", "a").tst("a", "b", "c", "d", "e", "f");
}
}
class BinaryHeap_Io_line_rdr_fxt {
BinaryHeap_Io_line_rdr heap = new BinaryHeap_Io_line_rdr(Io_sort_split_itm_sorter.Instance); int file_total;
public BinaryHeap_Io_line_rdr_fxt Add(String... ary) {
file_total = ary.length;
for (int i = 0; i < file_total; i++) {
Io_url url = Io_url_.mem_fil_("mem/fil_" + ary[i] + ".txt");
Io_mgr.Instance.SaveFilStr(url, ary[i]);
Io_line_rdr stream = new Io_line_rdr(Gfo_usr_dlg_.Test(), url);
stream.Read_next();
heap.Add(stream);
}
return this;
}
public BinaryHeap_Io_line_rdr_fxt tst(String... expd) {
String[] actl = new String[file_total];
for (int i = 0; i < actl.length; i++) {
Io_line_rdr bfr = heap.Pop();
actl[i] = String_.new_u8(bfr.Bfr(), 0, bfr.Bfr_len());
}
Tfds.Eq_ary_str(expd, actl);
return this;
}
}

73
400_xowa/src/gplx/core/ios/Io_buffer_rdr.java Normal file
View File

@@ -0,0 +1,73 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import gplx.core.ios.*;/*IoStream*/
public class Io_buffer_rdr implements RlsAble {
private Io_stream_rdr rdr;
Io_buffer_rdr(Io_stream_rdr rdr, Io_url url, int bfr_len) {
this.rdr = rdr; this.url = url;
if (bfr_len <= 0) throw Err_.new_wo_type("bfr_len must be > 0", "bfr_len", bfr_len);
bfr = new byte[bfr_len]; this.bfr_len = bfr_len;
IoItmFil fil = Io_mgr.Instance.QueryFil(url); if (!fil.Exists()) throw Err_.new_wo_type("fil does not exist", "url", url);
fil_len = fil.Size();
fil_pos = 0;
fil_eof = false;
}
public Io_url Url() {return url;} private Io_url url;
public byte[] Bfr() {return bfr;} private byte[] bfr;
public int Bfr_len() {return bfr_len;} private int bfr_len;
public long Fil_len() {return fil_len;} long fil_len;
public long Fil_pos() {return fil_pos;} long fil_pos;
public boolean Fil_eof() {return fil_eof;} private boolean fil_eof;
public boolean Bfr_load_all() {return Bfr_load(0, bfr_len);}
public boolean Bfr_load_from(int bfr_pos) {
if (bfr_pos < 0 || bfr_pos > bfr_len) throw Err_.new_wo_type("invalid bfr_pos", "bfr_pos", bfr_pos, "bfr_len", bfr_len);
for (int i = bfr_pos; i < bfr_len; i++) // shift end of bfr to bgn; EX: bfr[10] and load_from(8); [8] -> [0]; [9] -> [1];
bfr[i - bfr_pos] = bfr[i];
return Bfr_load(bfr_len - bfr_pos, bfr_pos); // fill rest of bfr; EX: [2]... will come from file
}
private boolean Bfr_load(int bgn, int len) {
int read = rdr.Read(bfr, bgn, len);
if (read == gplx.core.ios.Io_stream_rdr_.Read_done) {fil_eof = true; return false;}
fil_pos += read;
bfr_len = bgn + read;
if (read < len) fil_eof = true;
return true;
}
public void Seek(long fil_pos) {
this.fil_pos = fil_pos;
rdr.Skip(fil_pos);
this.Bfr_load_all();
}
public void Rls() {
bfr = null;
bfr_len = -1;
if (rdr != null) rdr.Rls();
}
@gplx.Internal protected void Dump_to_file(int bgn, int len, String url_str, String msg) { // DBG:
String text = String_.new_u8__by_len(bfr, bgn, len);
Io_mgr.Instance.AppendFilStr(Io_url_.new_any_(url_str), msg + text + "\n");
}
public static Io_buffer_rdr new_(Io_stream_rdr rdr, int bfr_len) {
Io_buffer_rdr rv = new Io_buffer_rdr(rdr, rdr.Url(), bfr_len);
rdr.Open();
rv.Bfr_load(0, bfr_len);
return rv;
}
public static final Io_buffer_rdr Null = new Io_buffer_rdr(); Io_buffer_rdr() {}
}

64
400_xowa/src/gplx/core/ios/Io_buffer_rdr_tst.java Normal file
View File

@@ -0,0 +1,64 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import org.junit.*; import gplx.core.ios.*;
public class Io_buffer_rdr_tst {
@Before public void init() {
Io_mgr.Instance.InitEngine_mem();
fil = Io_url_.mem_fil_("mem/byteStreamRdr.txt");
ini_Write("0123456789");
rdr = Io_buffer_rdr.new_(Io_stream_rdr_.file_(fil), 4);
} Io_buffer_rdr rdr; Io_url fil;
@After public void teardown() {rdr.Rls();}
@Test public void Bfr_load_all() {
tst_Bfr("0", "1", "2", "3").tst_ReadDone(false);
rdr.Bfr_load_all();
tst_Bfr("4", "5", "6", "7").tst_ReadDone(false);
rdr.Bfr_load_all();
tst_Bfr("8", "9");
rdr.Bfr_load_all(); // NOTE: change to zip_rdrs make eof detection difficult; force another load to ensure that file_pos goes past file_len
tst_ReadDone(true); // NOTE: bfr truncated from 4 to 2
}
@Test public void Bfr_load_from() {
tst_Bfr("0", "1", "2", "3").tst_ReadDone(false);
rdr.Bfr_load_from(3); // read from pos 3
tst_Bfr("3", "4", "5", "6").tst_ReadDone(false);
rdr.Bfr_load_from(1); // read from pos 1
tst_Bfr("4", "5", "6", "7").tst_ReadDone(false);
rdr.Bfr_load_from(1);
tst_Bfr("5", "6", "7", "8").tst_ReadDone(false);
rdr.Bfr_load_from(3);
rdr.Bfr_load_all(); // NOTE: change to zip_rdrs make eof detection difficult; force another load to ensure that file_pos goes past file_len
tst_Bfr("8", "9").tst_ReadDone(true);
}
private void ini_Write(String s) {Io_mgr.Instance.SaveFilStr(fil, s);}
Io_buffer_rdr_tst tst_Bfr(String... expdAry) {
String[] actlAry = new String[rdr.Bfr_len()];
for (int i = 0; i < actlAry.length; i++)
actlAry[i] = String_.new_u8(rdr.Bfr(), i, i + 1);
Tfds.Eq_ary(expdAry, actlAry);
return this;
}
Io_buffer_rdr_tst tst_ReadDone(boolean expd) {Tfds.Eq(expd, rdr.Fil_eof()); return this;}
}

32
400_xowa/src/gplx/core/ios/Io_fil_chkr.java Normal file
View File

@@ -0,0 +1,32 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import gplx.core.tests.*;
public class Io_fil_chkr implements Tst_chkr {
public Io_fil_chkr(Io_url url, String data) {this.expd_url = url; this.expd_data = data;}
public Io_url Expd_url() {return expd_url;} public Io_fil_chkr Expd_url_(Io_url v) {expd_url = v; return this;} Io_url expd_url;
public String Expd_data() {return expd_data;} public Io_fil_chkr Expd_data_(String v) {expd_data = v; return this;} private String expd_data;
public Class<?> TypeOf() {return gplx.core.ios.Io_fil.class;}
public int Chk(Tst_mgr mgr, String path, Object actl) {
gplx.core.ios.Io_fil fil = (gplx.core.ios.Io_fil)actl;
int rv = 0;
rv += mgr.Tst_val(expd_url == null, path, "url", expd_url, fil.Url());
rv += mgr.Tst_val(expd_data == null, path, "data", expd_data, fil.Data());
return rv;
}
}

165
400_xowa/src/gplx/core/ios/Io_line_rdr.java Normal file
View File

@@ -0,0 +1,165 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public class Io_line_rdr {
public Io_line_rdr (Gfo_usr_dlg usr_dlg, Io_url... urls) {this.usr_dlg = usr_dlg; this.urls = urls; if (urls.length == 0) bfr_state = Bfr_state_end;} Gfo_usr_dlg usr_dlg;
public int Url_idx() {return url_idx;} private int url_idx;
public Io_url[] Urls() {return urls;} Io_url[] urls;
public void Reset_one(Io_url url) {
this.Clear();
urls[0] = url;
url_idx = 0;
}
public byte Line_dlm() {return line_dlm;} public Io_line_rdr Line_dlm_(byte v) {line_dlm = v; return this;} private byte line_dlm = Byte_ascii.Nl;
public byte[] Bfr() {return bfr;} private byte[] bfr;
public int Bfr_len() {return bfr_len;} private int bfr_len;
public byte Bfr_state() {return bfr_state;} private byte bfr_state = Bfr_state_bgn; static final byte Bfr_state_bgn = 0, Bfr_state_mid = 1, Bfr_state_end = 2;
public int Bfr_last_read() {return bfr_last_read;} private int bfr_last_read;
public void Bfr_last_read_add(int v) {bfr_last_read += v;}
public int Load_len() {return load_len;} public Io_line_rdr Load_len_(int v) {load_len = v; return this;} private int load_len = 4096;
long File_len() {return file_len;} long file_len = 0;
long File_pos() {return file_pos;} long file_pos = 0;
public boolean File_skip_line0() {return file_skip_line0;} public Io_line_rdr File_skip_line0_(boolean v) {file_skip_line0 = v; return this;} private boolean file_skip_line0;
public int Itm_pos_bgn() {return itm_pos_bgn;} private int itm_pos_bgn = 0;
public int Itm_pos_end() {return itm_pos_end;} private int itm_pos_end = 0;
public int Key_pos_bgn() {return key_pos_bgn;} public Io_line_rdr Key_pos_bgn_(int v) {key_pos_bgn = v; return this;} private int key_pos_bgn = -1;
public int Key_pos_end() {return key_pos_end;} public Io_line_rdr Key_pos_end_(int v) {key_pos_end = v; return this;} private int key_pos_end = -1;
public Io_line_rdr_key_gen Key_gen() {return key_gen;} public Io_line_rdr Key_gen_(Io_line_rdr_key_gen v) {key_gen = v; return this;} Io_line_rdr_key_gen key_gen = Io_line_rdr_key_gen_.first_pipe;
public void Truncate(int pos) {
this.Read_next();
int end = Bry_find_.Find_fwd(bfr, Byte_ascii.Null); if (end == -1) end = bfr.length;
bfr = Bry_.Mid(bfr, pos, end);
bfr_len = bfr.length;
bfr_last_read = 0;
}
public boolean Read_next() {
switch (bfr_state) {
case Bfr_state_bgn:
bfr_state = Bfr_state_mid; // do not place after Load
Open_fil();
if (!Load()) return false;
break;
case Bfr_state_end:
return false;
}
itm_pos_bgn = bfr_last_read; itm_pos_end = bfr_len; key_pos_bgn = key_pos_end = -1;
while (true) {
for (int i = bfr_last_read; i < bfr_len; i++) {
if (bfr[i] == line_dlm) {
itm_pos_end = i + 1; // +1: include find
bfr_last_read = itm_pos_end;
key_gen.Gen(this);
return true;
}
}
if (Load()) // line_dlm not found; load more
itm_pos_bgn = 0;
else { // nothing loaded; return;
itm_pos_end = bfr_len;
key_gen.Gen(this); // call key_gen b/c there may be a stray line at end; EX: "\nb|c"; call key_gen to get "b"
bfr_state = Bfr_state_end;
return bfr_last_read < bfr_len; // true if stray line at end; otherwise bfr_last_read == bfr_len and return false;
}
}
}
public boolean Match(byte[] ttl) {
switch (bfr_state) {
case Bfr_state_bgn:
if (!Read_next()) return false;
bfr_state = Bfr_state_mid; // NOTE: must set back to mid; possible for 1st read to read entire buffer; EX: 8 MB bfr, but only 1 MB file;
break;
case Bfr_state_end:
return false;
}
while (true) {
int compare = Bry_.Compare(ttl, 0, ttl.length, bfr, key_pos_bgn, key_pos_end);
// if (String_.new_u8(bfr, key_pos_bgn, key_pos_end) == "US Naval Jack.svg") {
// Tfds.Write();
// }
if (compare == CompareAble_.Same) { // eq; return true and move fwd; EX: "BA" and "BA"
return true;
}
else if (compare < CompareAble_.Same) { // lt; return false; EX: ttl is "BA" but rdr is "BC"
return false;
}
else { // gt; keep reading; EX: ttl is "BC" but rdr is "BA"
if (!this.Read_next()) return false;
}
}
}
boolean Load() {
int old_bfr_len = bfr_len - bfr_last_read; // NOTE: preserve bytes between bfr_last_read and bfr_len; EX: "ab\nc"; preserve "c" for next ary
if (file_done) {++url_idx; if (url_idx == urls.length) {bfr_state = Bfr_state_end; return false;} Open_fil(); file_done = false;}
byte[] load_ary = new byte[old_bfr_len + load_len]; int load_ary_len = load_ary.length; // NOTE: must go after file_done chk; otherwise small wikis will allocate another 32 MB bry for sort_memory_len and cause an OutOfMemory error; DATE:20130112
int read_len = stream.Read(load_ary, old_bfr_len, load_ary_len - old_bfr_len);
if (read_len == 0) { // nothing read; return;
++url_idx;
if (url_idx < urls.length) {
stream.Rls();
Open_fil();
return true;
}
else {
stream.Rls();
bfr_state = Bfr_state_end;
return false;
}
}
if (read_len == file_len) {
stream.Rls();
file_done = true;
// ++url_idx;
// bfr_state = Bfr_state_end;
}
if (old_bfr_len > 0) Array_.Copy_to(bfr, bfr_last_read, load_ary, 0, old_bfr_len); // copy old_bfr over
file_pos += read_len;
bfr = load_ary;
bfr_last_read = 0;
bfr_len = read_len == load_len ? load_ary_len : old_bfr_len + read_len; // stream.Read() may return less bytes than load_ary at EOF; if so, don't shrink bfr; just mark bfr_len less
return true;
} IoStream stream;
private void Open_fil() {
Io_url url = urls[url_idx];
usr_dlg.Prog_many(GRP_KEY, "load", "loading dump file: ~{0}", url.NameAndExt());
if (file_skip_line0) {
byte[] stream_bry = Io_mgr.Instance.LoadFilBry(url);
int stream_bry_len = stream_bry.length;
int nl_pos = Bry_find_.Find_fwd(stream_bry, Byte_ascii.Nl, 0, stream_bry_len);
if (nl_pos == Bry_find_.Not_found)
stream_bry = Bry_.Empty;
else
stream_bry = Bry_.Mid(stream_bry, nl_pos + 1, stream_bry_len);
stream = gplx.core.ios.IoStream_.ary_(stream_bry);
}
else {
stream = Io_mgr.Instance.OpenStreamRead(url);
}
file_pos = 0; file_len = stream.Len();
file_done = false;
} boolean file_done = false;
public void Clear() {
bfr_state = Bfr_state_bgn;
if (stream != null) stream.Rls();
bfr = null;
}
public void Rls() {
this.Clear();
bfr = null;
}
static final String GRP_KEY = "xowa.bldr.line_rdr";
}

21
400_xowa/src/gplx/core/ios/Io_line_rdr_key_gen.java Normal file
View File

@@ -0,0 +1,21 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public interface Io_line_rdr_key_gen {
void Gen(Io_line_rdr bfr);
}

56
400_xowa/src/gplx/core/ios/Io_line_rdr_key_gen_.java Normal file
View File

@@ -0,0 +1,56 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public class Io_line_rdr_key_gen_ {
public static final Io_line_rdr_key_gen first_pipe = new Io_line_rdr_key_gen_first(Byte_ascii.Pipe);
public static final Io_line_rdr_key_gen last_pipe = new Io_line_rdr_key_gen_last(Byte_ascii.Pipe);
public static final Io_line_rdr_key_gen noop = new Io_line_rdr_key_gen_noop();
}
class Io_line_rdr_key_gen_last implements Io_line_rdr_key_gen {
public Io_line_rdr_key_gen_last(byte fld_dlm) {this.fld_dlm = fld_dlm;} private byte fld_dlm;
public void Gen(Io_line_rdr bfr) {
int bgn = bfr.Itm_pos_bgn(), end = bfr.Itm_pos_end() - 1; // -1: ignore row_dlm
bfr.Key_pos_bgn_(end).Key_pos_end_(end);
byte[] bry = bfr.Bfr();
for (int i = end; i >= bgn; i--) {
if (bry[i] == fld_dlm) {
bfr.Key_pos_bgn_(i + 1); // +1 to position after fldDlm
return;
}
}
bfr.Key_pos_bgn_(0); // nothing found; position at bgn
}
}
class Io_line_rdr_key_gen_first implements Io_line_rdr_key_gen {
public Io_line_rdr_key_gen_first(byte fld_dlm) {this.fld_dlm = fld_dlm;} private byte fld_dlm;
public void Gen(Io_line_rdr bfr) {
int bgn = bfr.Itm_pos_bgn(), end = bfr.Itm_pos_end();
bfr.Key_pos_bgn_(bgn).Key_pos_end_(bgn);
byte[] bry = bfr.Bfr();
for (int i = bgn; i < end; i++) {
if (bry[i] == fld_dlm) {
bfr.Key_pos_end_(i);
return;
}
}
bfr.Key_pos_end_(end); // nothing found; position at end
}
}
class Io_line_rdr_key_gen_noop implements Io_line_rdr_key_gen {
public void Gen(Io_line_rdr bfr) {}
}

96
400_xowa/src/gplx/core/ios/Io_line_rdr_tst.java Normal file
View File

@@ -0,0 +1,96 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import org.junit.*;
public class Io_line_rdr_tst {
Io_line_rdr_fxt fxt;
@Before public void init() {
fxt = new Io_line_rdr_fxt(Io_url_.new_fil_("mem/test.txt"));
fxt.Clear();
}
@Test public void Basic() {
fxt.File_lines_(3).tst_Read_til_lines(3, "00", "01", "02");
fxt.tst_Read_til_lines(1); // make sure nothing more is read
}
@Test public void Load_3x() {
fxt.File_lines_(9).Load_len_lines_(3).tst_Read_til_lines(9, "00", "01", "02", "03", "04", "05", "06", "07", "08");
}
@Test public void Load_irregular() {
fxt.File_lines_(9).Load_len_(4).tst_Read_til_lines(9, "00", "01", "02", "03", "04", "05", "06", "07", "08");
}
@Test public void Load_multiple_files() {
fxt = new Io_line_rdr_fxt(Io_url_.new_fil_("mem/test0.txt"), Io_url_.new_fil_("mem/test1.txt"), Io_url_.new_fil_("mem/test2.txt"));
fxt.File_lines_(0, 0, 3).File_lines_(1, 3, 5).File_lines_(2, 5, 9).Load_len_(4).tst_Read_til_lines(9, "00", "01", "02", "03", "04", "05", "06", "07", "08");
}
@Test public void Match() {
fxt.File_lines_pipe_(9).Load_len_(6);
fxt.tst_Match("00", "00");
fxt.tst_Match("01", "01");
fxt.tst_Match("03", "03");
fxt.tst_Match("08", "08");
fxt.tst_Match("12", "");
}
}
class Io_line_rdr_fxt {
Io_line_rdr rdr;
List_adp lines = List_adp_.new_(); Bry_bfr tmp = Bry_bfr.new_();
public Io_line_rdr_fxt(Io_url... urls) {rdr = new Io_line_rdr(Gfo_usr_dlg_.Test(), urls);}
public Io_line_rdr_fxt Load_len_lines_(int v) {return Load_len_(v * 3);} // 3: 2=##, 1=\n
public Io_line_rdr_fxt Load_len_(int v) {rdr.Load_len_(v); return this;}
public Io_line_rdr_fxt File_lines_(int count) {
for (int i = 0; i < count; i++)
tmp.Add_int_fixed(i, 2).Add_byte_nl();
Io_mgr.Instance.SaveFilBry(rdr.Urls()[0], tmp.To_bry_and_clear());
return this;
}
// public Io_url[] Src_fils() {return src_fils;} public Io_line_rdr_fxt Src_fils_(Io_url[] v) {src_fils = v; return this;} Io_url[] src_fils;
public Io_line_rdr_fxt tst_Match(String match, String expd) {
rdr.Key_gen_(Io_line_rdr_key_gen_.first_pipe);
boolean match_v = rdr.Match(Bry_.new_u8(match));
String actl = match_v ? String_.new_u8(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end()) : "";
Tfds.Eq(expd, actl);
return this;
}
public Io_line_rdr_fxt File_lines_pipe_(int count) {
for (int i = 0; i < count; i++)
tmp.Add_int_fixed(i, 2).Add_byte(Byte_ascii.Pipe).Add_byte_nl();
Io_mgr.Instance.SaveFilBry(rdr.Urls()[0], tmp.To_bry_and_clear());
return this;
}
public Io_line_rdr_fxt File_lines_(int fil_idx, int bgn, int end) {
for (int i = bgn; i < end; i++)
tmp.Add_int_fixed(i, 2).Add_byte_nl();
Io_mgr.Instance.SaveFilBry(rdr.Urls()[fil_idx], tmp.To_bry_and_clear());
return this;
}
public Io_line_rdr_fxt Clear() {rdr.Clear(); return this;}
public Io_line_rdr_fxt tst_Read_til_lines(int count, String... expd) {
lines.Clear();
for (int i = 0; i < expd.length; i++)
expd[i] = expd[i] + Op_sys.Lnx.Nl_str();
for (int i = 0; i < count; i++) {
if (rdr.Read_next())
lines.Add(String_.new_u8(rdr.Bfr(), rdr.Itm_pos_bgn(), rdr.Itm_pos_end()));
else
break;
}
Tfds.Eq_ary_str(expd, lines.To_str_ary());
return this;
}
}

21
400_xowa/src/gplx/core/ios/Io_make_cmd.java Normal file
View File

@@ -0,0 +1,21 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public interface Io_make_cmd extends Io_sort_cmd {
Io_sort_cmd Make_dir_(Io_url v);
}

98
400_xowa/src/gplx/core/ios/Io_sort.java Normal file
View File

@@ -0,0 +1,98 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import gplx.lists.*;
public class Io_sort {
public Io_sort Memory_max_(int v) {memory_max = v; return this;} private int memory_max = Io_mgr.Len_kb;
public Io_url[] Split(Gfo_usr_dlg usr_dlg, Io_url_gen src_fil_gen, Io_url_gen trg_fil_gen, Io_line_rdr_key_gen key_gen) {return Split(usr_dlg, src_fil_gen, trg_fil_gen, Io_sort_split_itm_sorter.Instance, key_gen);}
public Io_url[] Split(Gfo_usr_dlg usr_dlg, Io_url_gen src_fil_gen, Io_url_gen trg_fil_gen, ComparerAble row_comparer, Io_line_rdr_key_gen key_gen) {
Io_line_rdr rdr = new Io_line_rdr(usr_dlg, src_fil_gen.Prv_urls()).Load_len_(4 * Io_mgr.Len_kb).Key_gen_(key_gen); // NOTE: do not set load_len to memory_max; only want to load in increments
List_adp rv = List_adp_.new_();
Bry_bfr bfr = Bry_bfr.reset_(Const_bfr_max); int size_cur = 0;
List_adp row_list = List_adp_.new_();
while (true) {
boolean reading = rdr.Read_next();
int size_row = rdr.Itm_pos_end() - rdr.Itm_pos_bgn();
int size_new = size_cur + size_row;
if (size_new > memory_max || !reading) {
usr_dlg.Prog_none(GRP_KEY, "sort", "sorting chunk");
row_list.Sort_by(row_comparer);
Io_url trg_url = trg_fil_gen.Nxt_url();
usr_dlg.Prog_one(GRP_KEY, "write", "writing chunk: ~{0}", trg_url.Raw());
Split_flush(trg_url, row_list, memory_max, bfr, rv);
row_list.Resize_bounds(16); // MEM: resize bounds manually; note that each Flush-set may have widely disparately #of rows (EX: 1 row with a million pages vs. 1 million rows with 1 page)
size_new = size_row; Env_.GarbageCollect();
if (!reading) break;
}
row_list.Add(new Io_sort_split_itm(rdr));
size_cur = size_new;
}
rdr.Rls(); bfr.Rls(); Env_.GarbageCollect();
return (Io_url[])rv.To_ary(Io_url.class);
}
public void Merge(Gfo_usr_dlg usr_dlg, Io_url[] src_ary, ComparerAble comparer, Io_line_rdr_key_gen key_gen, Io_sort_cmd cmd) {
BinaryHeap_Io_line_rdr heap = load_(usr_dlg, src_ary, comparer, key_gen, memory_max); if (heap.Len() == 0) return;//throw Err_.new_wo_type(Array_.To_str(src_ary));
Io_line_rdr stream = null;
cmd.Sort_bgn();
while (true) {
if (stream != null) { // stream found on previous iteration; try to put it back on heap
boolean read = stream.Read_next();
if (read) // stream has line; add to heap
heap.Add(stream);
else { // stream is empty; rls
stream.Rls();
if (heap.Len() == 0) break; // heap is empty; stop;
}
}
stream = heap.Pop(); // note that .Pop removes stream from heap
cmd.Sort_do(stream);
}
cmd.Sort_end();
heap.Rls();
}
private static void Split_flush(Io_url url, List_adp list, int max, Bry_bfr tmp, List_adp url_list) {
int len = list.Count();
for (int i = 0; i < len; i++) {
Io_sort_split_itm itm = (Io_sort_split_itm)list.Get_at(i);
int add_len = itm.Row_end() - itm.Row_bgn();
if ((tmp.Len() + add_len) > Const_bfr_max) Io_mgr.Instance.AppendFilBfr(url, tmp);
tmp.Add_mid(itm.Bfr(), itm.Row_bgn(), itm.Row_end());
itm.Rls();
}
Io_mgr.Instance.AppendFilBfr(url, tmp);
list.Clear();
url_list.Add(url);
}
private static BinaryHeap_Io_line_rdr load_(Gfo_usr_dlg usr_dlg, Io_url[] urls, ComparerAble comparer, Io_line_rdr_key_gen key_gen, int memory_max) {
BinaryHeap_Io_line_rdr rv = new BinaryHeap_Io_line_rdr(comparer);
int urls_len = urls.length;
int default_load_len = memory_max / urls_len + 1;
for (int i = 0; i < urls_len; i++) {
Io_url url = urls[i];
int file_len = (int)Io_mgr.Instance.QueryFil(url).Size();
int load_len = file_len < default_load_len ? file_len : default_load_len; // PERF.NOTE: 32 MB is default, but if file is 1 MB (or else) only create a bfr for 1 MB; using 32 MB will throw OutOfMemory on -Xmx 64m; DATE:20130112
Io_line_rdr stream_bfr = new Io_line_rdr(usr_dlg, url).Key_gen_(key_gen).Load_len_(load_len);
boolean read = stream_bfr.Read_next();
if (read) // guard against empty files
rv.Add(stream_bfr);
}
return rv;
}
static final int Const_bfr_max = Io_mgr.Len_mb;
static final String GRP_KEY = "xowa.bldr.io_sort";
}

23
400_xowa/src/gplx/core/ios/Io_sort_cmd.java Normal file
View File

@@ -0,0 +1,23 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public interface Io_sort_cmd {
void Sort_bgn();
void Sort_do(Io_line_rdr rdr);
void Sort_end();
}

29
400_xowa/src/gplx/core/ios/Io_sort_filCmd.java Normal file
View File

@@ -0,0 +1,29 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public interface Io_sort_filCmd {
void Bfr_add(Io_line_rdr stream);
void Fil_bgn(Io_line_rdr stream);
void Fil_end();
}
class Io_sort_filCmd_null implements Io_sort_filCmd {
public void Bfr_add(Io_line_rdr stream) {}
public void Fil_bgn(Io_line_rdr stream) {}
public void Fil_end() {}
public static final Io_sort_filCmd_null Instance = new Io_sort_filCmd_null(); Io_sort_filCmd_null() {}
}

38
400_xowa/src/gplx/core/ios/Io_sort_fil_basic.java Normal file
View File

@@ -0,0 +1,38 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public class Io_sort_fil_basic implements Io_sort_cmd { // 123|bgn|end|1
public Io_sort_fil_basic(Gfo_usr_dlg usr_dlg, Io_url_gen url_gen, int flush_len) {this.usr_dlg = usr_dlg; this.url_gen = url_gen; this.flush_len = flush_len;} Io_url_gen url_gen; Bry_bfr bfr = Bry_bfr.new_(); int flush_len; Gfo_usr_dlg usr_dlg;
public void Sort_bgn() {}
public void Sort_do(Io_line_rdr rdr) {
int bgn = rdr.Itm_pos_bgn(), end = rdr.Itm_pos_end();
if (bfr.Len() + (end - bgn) > flush_len) Flush();
bfr.Add_mid(rdr.Bfr(), bgn, end);
}
public void Sort_end() {
Flush();
bfr.Rls();
}
private void Flush() {
Io_url url = url_gen.Nxt_url();
usr_dlg.Prog_one(GRP_KEY, "make", "making: ~{0}", url.NameAndExt());
Io_mgr.Instance.SaveFilBry(url, bfr.Bfr(), bfr.Len());
bfr.Clear();
}
static final String GRP_KEY = "xowa.bldr.io_sort";
}

58
400_xowa/src/gplx/core/ios/Io_sort_misc_tst.java Normal file
View File

@@ -0,0 +1,58 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import org.junit.*;
public class Io_sort_misc_tst {
@Before public void init() {
}
@Test public void Io_url_gen_dir() {
tst_Io_url_gen_dir("mem/dir/", "{0}.xdat", 4, 3, "0000.xdat", "0001.xdat", "0002.xdat");
}
private void tst_Io_url_gen_dir(String dir_str, String fmt, int digits, int calls, String... expd) {
Io_url dir = Io_url_.mem_dir_(dir_str);
List_adp actl_list = List_adp_.new_();
Io_url_gen wkr = Io_url_gen_.dir_(dir, fmt, digits);
for (int i = 0; i < calls; i++)
actl_list.Add(wkr.Nxt_url().Raw());
String[] actl = actl_list.To_str_ary();
for (int i = 0; i < expd.length; i++)
expd[i] = dir_str + expd[i];
Tfds.Eq_ary_str(expd, actl);
}
@Test public void Io_line_rdr_comparer_all() {
tst_Io_line_rdr_fld_comparer(-1, "a", "b");
tst_Io_line_rdr_fld_comparer( 0, "a", "a");
tst_Io_line_rdr_fld_comparer( 1, "b", "a");
tst_Io_line_rdr_fld_comparer(-1, "a", "ab");
tst_Io_line_rdr_fld_comparer( 1, "ab", "a");
}
private void tst_Io_line_rdr_fld_comparer(int expd, String lhs_str, String rhs_str) {
byte[] lhs = Bry_.new_u8(lhs_str), rhs = Bry_.new_u8(rhs_str);
Tfds.Eq(expd, Bry_.Compare(lhs, 0, lhs.length, rhs, 0, rhs.length));
}
Io_line_rdr new_Io_line_rdr(String url_str, String text) {
Io_url url = Io_url_.mem_fil_(url_str);
Io_mgr.Instance.SaveFilStr(url, text);
Io_line_rdr rv = new Io_line_rdr(Gfo_usr_dlg_.Test(), url);
rv.Read_next();
return rv;
}
@Test public void ExternalSort() {
// fxt("c", "a", "b")
}
}

35
400_xowa/src/gplx/core/ios/Io_sort_split_itm.java Normal file
View File

@@ -0,0 +1,35 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public class Io_sort_split_itm {
public Io_sort_split_itm() {}
public Io_sort_split_itm(Io_line_rdr rdr) {Set(rdr);}
public int Row_bgn() {return row_bgn;} private int row_bgn;
public int Row_end() {return row_end;} private int row_end;
public int Key_bgn() {return key_bgn;} private int key_bgn;
public int Key_end() {return key_end;} private int key_end;
public byte[] Bfr() {return bfr;} private byte[] bfr;
public void Set(Io_line_rdr rdr) {
bfr = rdr.Bfr();
row_bgn = rdr.Itm_pos_bgn();
row_end = rdr.Itm_pos_end();
key_bgn = rdr.Key_pos_bgn();
key_end = rdr.Key_pos_end();
}
public void Rls() {bfr = null;}
}

26
400_xowa/src/gplx/core/ios/Io_sort_split_itm_sorter.java Normal file
View File

@@ -0,0 +1,26 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public class Io_sort_split_itm_sorter implements gplx.lists.ComparerAble {
public int compare(Object lhsObj, Object rhsObj) {
Io_sort_split_itm lhs = (Io_sort_split_itm)lhsObj, rhs = (Io_sort_split_itm)rhsObj;
// Tfds.Write(String_.new_u8(lhs.Bfr(), lhs.Key_bgn(), lhs.Key_end()), String_.new_u8(rhs.Bfr(), rhs.Key_bgn(), rhs.Key_end()));
return Bry_.Compare(lhs.Bfr(), lhs.Key_bgn(), lhs.Key_end(), rhs.Bfr(), rhs.Key_bgn(), rhs.Key_end());
}
public static final Io_sort_split_itm_sorter Instance = new Io_sort_split_itm_sorter(); Io_sort_split_itm_sorter() {}
}

66
400_xowa/src/gplx/core/ios/Io_sort_tst.java Normal file
View File

@@ -0,0 +1,66 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import org.junit.*; import gplx.core.strings.*;
public class Io_sort_tst {
Io_sort_fxt fxt = new Io_sort_fxt();
@Test public void ExternalSort() {
fxt.Clear().Memory_max_(12).Src_(fxt.GenRandom(6, 4)).Sorted_(fxt.GenOrdered(6, 4)).tst();
fxt.Clear().Memory_max_(64).Src_(fxt.GenRandom(50, 4)).Sorted_(fxt.GenOrdered(50, 4)).tst();
}
}
class Io_sort_fxt {
Io_sort externalSort = new Io_sort().Memory_max_(Io_mgr.Len_kb);
String_bldr sb = String_bldr_.new_();
public Io_sort_fxt Clear() {Io_mgr.Instance.InitEngine_mem(); return this;}
public Io_sort_fxt Memory_max_(int v) {externalSort.Memory_max_(v); return this;}
public Io_sort_fxt Src_(String v) {src = v; return this;} private String src;
public Io_sort_fxt Sorted_(String v) {sorted = v; return this;} private String sorted;
public void tst() {
Io_url src_url = Io_url_.mem_fil_("mem/src.txt");
Io_url trg_url = Io_url_.mem_fil_("mem/trg.txt");
Io_mgr.Instance.DeleteFil(src_url); Io_mgr.Instance.DeleteFil(trg_url);
Io_mgr.Instance.SaveFilStr(src_url, src);
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Test();
Io_url_gen src_fil_gen = Io_url_gen_.fil_(src_url);
Io_url[] tmp_url_ary = externalSort.Split(usr_dlg, src_fil_gen, Io_url_gen_.dir_(src_url.OwnerDir()), Io_line_rdr_key_gen_.first_pipe);
Io_sort_fil_basic cmd = new Io_sort_fil_basic(usr_dlg, Io_url_gen_.fil_(trg_url), Io_mgr.Len_kb);
externalSort.Merge(usr_dlg, tmp_url_ary, Io_sort_split_itm_sorter.Instance, Io_line_rdr_key_gen_.first_pipe, cmd);
String actl = Io_mgr.Instance.LoadFilStr(trg_url);
Tfds.Eq_ary_str(String_.SplitLines_nl(sorted), String_.SplitLines_nl(actl));
}
public String GenRandom(int rows, int pad) {
List_adp list = List_adp_.new_();
for (int i = 0; i < rows; i++)
list.Add(Int_.To_str_pad_bgn_zero(i, pad) + "|");
list.Shuffle();
for (int i = 0; i < rows; i++) {
String itm = (String)list.Get_at(i);
sb.Add(itm).Add_char_nl();
}
return sb.To_str_and_clear();
}
public String GenOrdered(int rows, int pad) {
for (int i = 0; i < rows; i++)
sb.Add(Int_.To_str_pad_bgn_zero(i, pad) + "|" + "\n");
return sb.To_str_and_clear();
}
}

67
400_xowa/src/gplx/core/ios/Io_stream_rdr_process.java Normal file
View File

@@ -0,0 +1,67 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import java.io.InputStream;
public class Io_stream_rdr_process implements Io_stream_rdr {
private Process process;
private InputStream stream_read;
private String[] process_args;
Io_stream_rdr_process(Io_url process_exe, Io_url stream_url, String[] process_args) {this.process_exe = process_exe; this.url = stream_url; this.process_args = process_args;}
public byte Tid() {return Io_stream_.Tid_bzip2;} // for now, classify as bzip2; not sure if separate tid is necessary
public boolean Exists() {return this.Len() > 0;}
public Io_url Url() {return url;} public Io_stream_rdr Url_(Io_url v) {url = v; return this;} private Io_url url;
public long Len() {return len;} public Io_stream_rdr Len_(long v) {len = v; return this;} private long len;
public Io_url Process_exe() {return process_exe;} private Io_url process_exe;
public Io_stream_rdr Open() {
ProcessBuilder pb = new ProcessBuilder(process_args);
pb.redirectErrorStream(false);
try {process = pb.start();}
catch (Exception e) {throw Err_.new_exc(e, "core", "process init failed", "args", String_.AryXtoStr(process_args));}
stream_read = process.getInputStream();
return this;
}
public void Open_mem(byte[] v) {throw Err_.new_unimplemented();}
public Object Under() {throw Err_.new_unimplemented();}
public int Read(byte[] bry, int bgn, int len) {
try {
int rv = 0;
int cur_pos = bgn;
int cur_len = len;
while (true) {
int read = stream_read.read(bry, cur_pos, cur_len);
if (read <= 0) break;
rv += read;
cur_pos += read;
cur_len -= read;
if (rv >= len) break;
}
return rv;
} catch (Exception e) {throw Err_.new_exc(e, "io", "process read failed", "bgn", bgn, "len", len);}
}
public long Skip(long len) {
try {return stream_read.skip(len);}
catch (Exception e) {throw Err_.new_exc(e, "io", "process skip failed", "len", len);}
}
public void Rls() {
try {stream_read.close();}
catch (Exception e) {throw Err_.new_exc(e, "io", "process rls failed");}
process.destroy();
}
public static Io_stream_rdr_process new_(Io_url process_exe, Io_url stream_url, String... process_args) {return new Io_stream_rdr_process(process_exe, stream_url, process_args);}
}

54
400_xowa/src/gplx/core/ios/Io_stream_zip_mgr.java Normal file
View File

@@ -0,0 +1,54 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public class Io_stream_zip_mgr {
private final Bry_bfr bfr = Bry_bfr.reset_(256);
private Io_stream_wtr wtr_gzip, wtr_zip, wtr_bzip2;
private Io_stream_rdr rdr_gzip, rdr_zip, rdr_bzip2;
public byte[] Zip(byte type, byte[] val) {
if (type == Io_stream_.Tid_raw) return val;
Io_stream_wtr wtr = Wtr(type);
wtr.Write(val, 0, val.length);
wtr.Flush();
return wtr.To_ary_and_clear();
}
public byte[] Unzip(byte type, byte[] val) {
if (type == Io_stream_.Tid_raw) return val;
Io_stream_rdr rdr = Rdr(type);
rdr.Open_mem(val);
return Io_stream_rdr_.Load_all_as_bry(bfr, rdr);
}
private Io_stream_wtr Wtr(byte type) {
switch (type) {
case Io_stream_.Tid_gzip : if (wtr_gzip == null) wtr_gzip = Io_stream_wtr_.new_by_mem(bfr, Io_stream_.Tid_gzip) ; return wtr_gzip.Open();
case Io_stream_.Tid_zip : if (wtr_zip == null) wtr_zip = Io_stream_wtr_.new_by_mem(bfr, Io_stream_.Tid_zip) ; return wtr_zip.Open();
case Io_stream_.Tid_bzip2 : if (wtr_bzip2 == null) wtr_bzip2 = Io_stream_wtr_.new_by_mem(bfr, Io_stream_.Tid_bzip2) ; return wtr_bzip2.Open();
case Io_stream_.Tid_raw :
default : throw Err_.new_unhandled(type);
}
}
private Io_stream_rdr Rdr(byte type) {
switch (type) {
case Io_stream_.Tid_gzip : if (rdr_gzip == null) rdr_gzip = Io_stream_rdr_.new_by_tid_(Io_stream_.Tid_gzip) ; return rdr_gzip;
case Io_stream_.Tid_zip : if (rdr_zip == null) rdr_zip = Io_stream_rdr_.new_by_tid_(Io_stream_.Tid_zip) ; return rdr_zip;
case Io_stream_.Tid_bzip2 : if (rdr_bzip2 == null) rdr_bzip2 = Io_stream_rdr_.new_by_tid_(Io_stream_.Tid_bzip2) ; return rdr_bzip2;
case Io_stream_.Tid_raw :
default : throw Err_.new_unhandled(type);
}
}
}

24
400_xowa/src/gplx/core/ios/Io_url_gen.java Normal file
View File

@@ -0,0 +1,24 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public interface Io_url_gen {
Io_url Cur_url();
Io_url Nxt_url();
Io_url[] Prv_urls();
void Del_all();
}

45
400_xowa/src/gplx/core/ios/Io_url_gen_.java Normal file
View File

@@ -0,0 +1,45 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public class Io_url_gen_ {
public static Io_url_gen dir_(Io_url v) {return new Io_url_gen_dir(v);}
public static Io_url_gen dir_(Io_url v, String fmt, int digits) {return new Io_url_gen_dir(v).Fmt_(fmt).Fmt_digits_(digits);}
public static Io_url_gen fil_(Io_url v) {return new Io_url_gen_fil(v);}
}
class Io_url_gen_dir implements Io_url_gen {
public String Fmt() {return fmt;} public Io_url_gen_dir Fmt_(String v) {fmt = v; return this;} private String fmt = "{0}.csv";
public int Fmt_digits() {return fmt_digits;} public Io_url_gen_dir Fmt_digits_(int v) {fmt_digits = v; return this;} private int fmt_digits = 10;
public Io_url Cur_url() {return cur_url;} Io_url cur_url;
public Io_url Nxt_url() {cur_url = dir.GenSubFil(String_.Format(fmt, Int_.To_str_pad_bgn_zero(idx++, fmt_digits))); return cur_url;} private int idx = 0;
public Io_url[] Prv_urls() {
Io_url[] rv = new Io_url[idx];
for (int i = 0; i < idx; i++) {
rv[i] = dir.GenSubFil(String_.Format(fmt, Int_.To_str_pad_bgn_zero(i, fmt_digits)));
}
return rv;
}
public void Del_all() {if (Io_mgr.Instance.ExistsDir(dir)) Io_mgr.Instance.DeleteDirDeep(dir);}
public Io_url_gen_dir(Io_url dir) {this.dir = dir;} Io_url dir;
}
class Io_url_gen_fil implements Io_url_gen {
public Io_url Cur_url() {return cur_url;} Io_url cur_url;
public Io_url Nxt_url() {return cur_url;}
public Io_url[] Prv_urls() {return new Io_url[]{cur_url};}
public void Del_all() {Io_mgr.Instance.DeleteFil_args(cur_url).MissingFails_off().Exec();}
public Io_url_gen_fil(Io_url fil) {this.cur_url = fil;}
}

25
400_xowa/src/gplx/core/ios/Io_zip_mgr.java Normal file
View File

@@ -0,0 +1,25 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public interface Io_zip_mgr {
void Zip_fil(Io_url src_fil, Io_url trg_fil);
byte[] Zip_bry(byte[] src, int bgn, int len);
byte[] Unzip_bry(byte[] src, int bgn, int len);
void Unzip_to_dir(Io_url src_fil, Io_url trg_dir);
void Zip_dir(Io_url src_dir, Io_url trg_fil);
}

119
400_xowa/src/gplx/core/ios/Io_zip_mgr_base.java Normal file
View File

@@ -0,0 +1,119 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import java.io.*;
import java.util.zip.*;
public class Io_zip_mgr_base implements Io_zip_mgr {
public void Zip_fil(Io_url src_fil, Io_url trg_fil) {
byte[] src_bry = Io_mgr.Instance.LoadFilBry(src_fil);
byte[] trg_bry = Zip_bry(src_bry, 0, src_bry.length);
Io_mgr.Instance.SaveFilBry(trg_fil, trg_bry);
}
public void Zip_dir(Io_url src_dir, Io_url trg_fil) {
try {
byte[] bry = new byte[4096];
FileOutputStream fil_strm = new FileOutputStream(trg_fil.Raw());
ZipOutputStream zip_strm = new ZipOutputStream(fil_strm);
Zip_dir__add_dir(zip_strm, bry, "", src_dir, Zip_dir__get_subs(src_dir));
zip_strm.flush();
zip_strm.close();
} catch(IOException e) {Err_.new_exc(e, "io", "error duing zip", "src", src_dir.Raw(), "trg", trg_fil.Raw());}
}
private void Zip_dir__add_dir(ZipOutputStream zip_strm, byte[] bry, String zip_path, Io_url owner_dir, Io_url[] subs) {
int len = subs.length;
for (int i = 0; i < len; i++) {
Io_url sub = subs[i];
String sub_path = zip_path + sub.NameAndExt_noDirSpr();
if (sub.Type_dir())
Zip_dir__add_dir(zip_strm, bry, sub_path + "/", sub, Zip_dir__get_subs(sub));
else
Zip_dir__add_fil(zip_strm, bry, sub_path, sub);
}
}
private void Zip_dir__add_fil(ZipOutputStream zip_strm, byte[] bry, String zip_path, Io_url fil_url) {
try {
int len;
FileInputStream fil_strm = new FileInputStream(fil_url.Raw());
zip_strm.putNextEntry(new ZipEntry(zip_path));
while ((len = fil_strm.read(bry)) > 0)
zip_strm.write(bry, 0, len);
fil_strm.close();
} catch(IOException e) {throw Err_.new_exc(e, "io", "error duing zip", "src", zip_path);}
}
private Io_url[] Zip_dir__get_subs(Io_url url) {
return Io_mgr.Instance.QueryDir_args(url).DirInclude_().ExecAsUrlAry();
}
public byte[] Zip_bry(byte[] src, int bgn, int len) {
ByteArrayInputStream src_stream = new ByteArrayInputStream(src, bgn, len);
ByteArrayOutputStream trg_stream = new ByteArrayOutputStream(len);
try {
ZipOutputStream trgZip = new ZipOutputStream(trg_stream);
ZipEntry entry = new ZipEntry("file");
trgZip.putNextEntry(entry);
int count;
while((count = src_stream.read(tmp, 0, tmpLen)) != -1) {
trgZip.write(tmp, 0, count);
}
trgZip.close();
} catch(Exception e) {throw Err_.new_wo_type("failed to zip", "err", e.getMessage());}
return trg_stream.toByteArray();
}
public byte[] Unzip_bry(byte[] src, int bgn, int len) {
ByteArrayInputStream src_stream = new ByteArrayInputStream(src, bgn, len);
ByteArrayOutputStream trg_stream = new ByteArrayOutputStream(len);
try {
ZipInputStream srcZip = new ZipInputStream(src_stream);
int count;
while(srcZip.getNextEntry() != null) {
while ((count = srcZip.read(tmp, 0, tmpLen)) != -1) {
trg_stream.write(tmp, 0, count);
}
}
} catch(Exception e) {throw Err_.new_wo_type("failed to unzip", "err", e.getMessage());}
return trg_stream.toByteArray();
}
public void Unzip_to_dir(Io_url src_fil, Io_url trg_dir) {
byte[] buffer = new byte[4096];
try{
Io_mgr.Instance.CreateDirIfAbsent(trg_dir);
ZipInputStream zip_strm = new ZipInputStream(new FileInputStream(src_fil.Raw()));
ZipEntry zip_eny = zip_strm.getNextEntry();
while (zip_eny != null) {
String itm_name = zip_eny.getName();
if (Op_sys.Cur().Tid_is_wnt()) itm_name = String_.Replace(itm_name, "/", "\\");
Io_url itm_url = Io_url_.new_any_(trg_dir.GenSubFil(itm_name).Raw());
Io_mgr.Instance.CreateDirIfAbsent(itm_url.OwnerDir()); // make sure owner dir exists
if (itm_url.Type_fil()) {
Io_mgr.Instance.SaveFilStr_args(itm_url, "").Exec();
File itm_file = new File(itm_url.Raw());
FileOutputStream itm_strm = new FileOutputStream(itm_file);
int len;
while ((len = zip_strm.read(buffer)) > 0)
itm_strm.write(buffer, 0, len);
itm_strm.close();
}
zip_eny = zip_strm.getNextEntry();
}
zip_strm.closeEntry();
zip_strm.close();
} catch(IOException e) {throw Err_.new_exc(e, "io", "error duing unzip", "src", src_fil.Raw(), "trg", trg_dir.Raw());}
}
byte[] tmp = new byte[4096]; int tmpLen = 4096;
public static final Io_zip_mgr Instance = new Io_zip_mgr_base();
}

36
400_xowa/src/gplx/core/ios/Io_zip_mgr_mok.java Normal file
View File

@@ -0,0 +1,36 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
public class Io_zip_mgr_mok implements Io_zip_mgr {
public void Zip_fil(Io_url src_fil, Io_url trg_fil) {
byte[] src_bry = Io_mgr.Instance.LoadFilBry(src_fil);
byte[] zip_bry = Zip_bry(src_bry, 0, src_bry.length);
Io_mgr.Instance.SaveFilBry(trg_fil, zip_bry);
}
public void Zip_dir(Io_url src_dir, Io_url trg_fil) {}
public byte[] Zip_bry(byte[] src, int bgn, int len) {return Bry_.Add(Bry_zipped, Bry_.Mid(src, bgn, len));}
public byte[] Unzip_bry(byte[] src, int bgn, int len) {
if (src == Bry_.Empty) return src;
byte[] section = Bry_.Mid(src, bgn, bgn + len);
if (!Bry_.Has_at_bgn(section, Bry_zipped, 0, section.length)) throw Err_.new_wo_type("src not zipped", "section", String_.new_u8(section));
return Bry_.Mid(section, Bry_zipped.length, section.length);
}
public void Unzip_to_dir(Io_url src_fil, Io_url trg_dir) {}
private static final byte[] Bry_zipped = Bry_.new_a7("zipped:");
public static final Io_zip_mgr_mok Instance = new Io_zip_mgr_mok(); Io_zip_mgr_mok() {}
}

31
400_xowa/src/gplx/core/ios/Io_zip_mgr_tst.java Normal file
View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.ios; import gplx.*; import gplx.core.*;
import org.junit.*;
public class Io_zip_mgr_tst {
@Test public void Zip_unzip() {
Zip_unzip_tst("abcdefghijklmnopqrstuvwxyz");
}
private void Zip_unzip_tst(String s) {
Io_zip_mgr zip_mgr = Io_zip_mgr_base.Instance;
byte[] src = Bry_.new_a7(s);
byte[] zip = zip_mgr.Zip_bry(src, 0, src.length);
byte[] unz = zip_mgr.Unzip_bry(zip, 0, zip.length);
Tfds.Eq_ary(src, unz);
}
}

2
400_xowa/src/gplx/core/net/Gfo_inet_conn__http.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.net; import gplx.*; import gplx.core.*;
import gplx.ios.*;
import gplx.core.ios.*;
class Gfo_inet_conn__http implements Gfo_inet_conn {
private final IoEngine_xrg_downloadFil downloader = IoEngine_xrg_downloadFil.new_("", Io_url_.Empty);
public int Tid() {return Gfo_inet_conn_.Tid__http;}

4
400_xowa/src/gplx/core/net/Gfo_url_parser.java
View File

@@ -47,7 +47,7 @@ public class Gfo_url_parser {
}
if (!rel) { // search for ":"; NOTE: only search if not rel; i.e.: "//"
int colon_pos = Bry_find_.Find_fwd(src, Byte_ascii.Colon, pos, src_end); // no colon found; EX: "//a.org/b"; "a.org/b"
if (colon_pos != Bry_.NotFound) // colon found; EX: "http://" or "https://"
if (colon_pos != Bry_find_.Not_found) // colon found; EX: "http://" or "https://"
pos = colon_pos + Int_.Const_dlm_len;
if (pos < src_end && src[pos] == Byte_ascii.Slash) { // skip slash after colon
pos += 1;
@@ -56,7 +56,7 @@ public class Gfo_url_parser {
}
}
int slash_pos = Bry_find_.Find_fwd(src, Byte_ascii.Slash, pos, src_end);
if (slash_pos == Bry_.NotFound) // no terminating slash; EX: http://a.org
if (slash_pos == Bry_find_.Not_found) // no terminating slash; EX: http://a.org
slash_pos = src_end;
slash_pos = Bry_.Trim_end_pos(src, slash_pos);
site_data.Atrs_set(rel, pos, slash_pos);

2
400_xowa/src/gplx/core/net/Http_client_wtr.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.net; import gplx.*; import gplx.core.*;
import gplx.ios.*;
import gplx.core.ios.*;
public interface Http_client_wtr {
void Stream_(Object o);
void Write_bry(byte[] bry);

2
400_xowa/src/gplx/core/net/Http_client_wtr__stream.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.core.net; import gplx.*; import gplx.core.*;
import gplx.ios.*;
import gplx.core.ios.*;
import java.io.*;
class Http_client_wtr__stream implements Http_client_wtr {
private final byte[] tmp_stream_bry = new byte[1024];

2
400_xowa/src/gplx/core/threads/Gfo_thread_cmd_download.java
View File

@@ -59,7 +59,7 @@ public class Gfo_thread_cmd_download implements Gfo_thread_cmd {
kit.Ask_ok(GRP_KEY, "download.fail", "download failed. Please select 'read from file' if you've already downloaded a dump: url=~{0} error=~{1}", src, xrg.Rslt_err_str());
}
} boolean download_pass = true;
protected gplx.ios.IoEngine_xrg_downloadFil xrg = Io_mgr.Instance.DownloadFil_args("", Io_url_.Empty);
protected gplx.core.ios.IoEngine_xrg_downloadFil xrg = Io_mgr.Instance.DownloadFil_args("", Io_url_.Empty);
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_async_bgn)) Download();
else if (ctx.Match(k, Invk_owner)) return owner;

2
400_xowa/src/gplx/core/threads/Gfo_thread_cmd_unzip.java
View File

@@ -32,7 +32,7 @@ public class Gfo_thread_cmd_unzip implements Gfo_thread_cmd {
public boolean Async_prog_enabled() {return true;}
public void Async_prog_run(int async_sleep_sum) {
String size_str = " please wait...";
if (trg.Type_fil()) size_str = gplx.ios.Io_size_.To_str(Io_mgr.Instance.QueryFil(trg).Size());
if (trg.Type_fil()) size_str = gplx.core.ios.Io_size_.To_str(Io_mgr.Instance.QueryFil(trg).Size());
usr_dlg.Prog_many(GRP_KEY, "unzip", "unzipping: ~{0}", size_str);
}
@gplx.Virtual public byte Async_init() {

80
400_xowa/src/gplx/core/threads/poolables/Gfo_poolable_mgr.java
View File

@@ -19,10 +19,10 @@ package gplx.core.threads.poolables; import gplx.*; import gplx.core.*; import g
public class Gfo_poolable_mgr {
private final Object thread_lock = new Object();
private final Gfo_poolable_itm prototype; private final Object[] make_args, clear_args;
private Gfo_poolable_itm[] pool; private int pool_next, pool_len; private final int pool_max;
public Gfo_poolable_mgr(Gfo_poolable_itm prototype, Object[] make_args, Object[] clear_args, int init_pool_len, int pool_max) {// NOTE: random IndexOutOfBounds errors in Get around free[--free_len] with free_len being -1; put member variable initialization within thread_lock to try to avoid; DATE:2014-09-21
private Gfo_poolable_itm[] pool; private int pool_nxt, pool_len;
public Gfo_poolable_mgr(Gfo_poolable_itm prototype, Object[] make_args, Object[] clear_args, int init_pool_len, int pool_max) {// NOTE: random IndexOutOfBounds errors in Get around free_ary[--free_len] with free_len being -1; put member variable initialization within thread_lock to try to avoid; DATE:2014-09-21
this.prototype = prototype; this.make_args = make_args; this.clear_args = clear_args;
this.pool_len = init_pool_len; this.pool_max = pool_max;
this.pool_len = init_pool_len;
this.Clear_fast();
}
public void Clear_safe() {synchronized (thread_lock) {Clear_fast();}}
@@ -30,21 +30,21 @@ public class Gfo_poolable_mgr {
this.pool = new Gfo_poolable_itm[pool_len];
for (int i = 0; i < pool_len; ++i)
pool[i] = prototype.Pool__make(i, make_args);
this.free = new int[pool_len];
pool_next = free_len = 0;
this.free_ary = new int[pool_len];
pool_nxt = free_len = 0;
}
public Gfo_poolable_itm Get_safe() {synchronized (thread_lock) {return Get_fast();}}
public Gfo_poolable_itm Get_fast() {
Gfo_poolable_itm rv = null;
int pool_idx = -1;
if (free_len > 0) { // free_itms in pool; use it
pool_idx = free[--free_len];
pool_idx = free_ary[--free_len];
rv = pool[pool_idx];
}
else { // nothing in pool; take next
if (pool_next == pool_len)
if (pool_nxt == pool_len)
Expand_pool();
pool_idx = pool_next++;
pool_idx = pool_nxt++;
rv = pool[pool_idx];
if (rv == null) {
rv = prototype.Pool__make(pool_idx, make_args);
@@ -57,23 +57,18 @@ public class Gfo_poolable_mgr {
public void Rls_safe(int idx) {synchronized (thread_lock) {Rls_safe(idx);}}
public void Rls_fast(int idx) {
if (idx == -1) throw Err_.new_wo_type("rls called on poolable that was not created by pool_mgr");
int pool_idx = pool_next - 1;
int pool_idx = pool_nxt - 1;
if (idx == pool_idx) // in-sequence; decrement count
this.pool_next = pool_idx;
else { // out-of-sequence
if (free_len == pool_max) { // all used; assume entire pool released, but out of order
// Array_.Sort((Object[])free);
// for (int i = 0; i < pool_max; ++i) {
// if (i != free[i]) throw Err_.new_("pool", "available_list out of order", "contents", "");
// }
if (free_len == 0)
this.pool_nxt = pool_idx;
else { // idx == pool_idx; assume entire pool released, but out of order
for (int i = 0; i < free_len; ++i)
free[i] = 0;
free_ary[i] = 0;
free_len = 0;
}
else { // add to free pool
free[free_len] = idx;
++free_len;
}
else { // out-of-sequence
free_ary[free_len] = idx;
++free_len;
}
}
private void Expand_pool() {
@@ -84,44 +79,13 @@ public class Gfo_poolable_mgr {
this.pool = new_pool;
this.pool_len = new_pool_len;
// expand free to same len
// expand free_ary to same len
int[] new_free = new int[pool_len];
Array_.Copy_to(free, 0, new_free, 0, free_len);
this.free = new_free;
Array_.Copy_to(free_ary, 0, new_free, 0, free_len);
this.free_ary = new_free;
}
@gplx.Internal protected int[] Free() {return free;} private int[] free; private int free_len;
@gplx.Internal protected int[] Free_ary() {return free_ary;} private int[] free_ary; private int free_len;
@gplx.Internal protected int Free_len() {return free_len;}
@gplx.Internal protected int Pool_len() {return pool_len;}
// public int Mgr_id() {return mgr_id;} private int mgr_id;
// public void Reset_if_gt(int v) {
// this.Clear(); // TODO: for now, just call clear
// }
// public void Clear_fail_check() {
// synchronized (thread_lock) {
// for (int i = 0; i < ary_max; i++) {
// Object itm = ary[i];
// if (itm != null) {
// if (!itm.Mkr_idx_is_null()) throw Err_.new_wo_type("failed to clear bfr", "idx", Int_.To_str(i));
// itm.Clear();
// }
// ary[i] = null;
// }
// ary = Ary_empty;
// free = Int_.Ary_empty;
// free_len = 0;
// nxt_idx = ary_max = 0;
// }
// }
// public void Clear() {
// synchronized (thread_lock) {
// for (int i = 0; i < ary_max; i++) {
// Object itm = ary[i];
// if (itm != null) itm.Clear();
// ary[i] = null;
// }
// ary = Ary_empty;
// free = Int_.Ary_empty;
// free_len = 0;
// nxt_idx = ary_max = 0;
// }
// }
@gplx.Internal protected int Pool_nxt() {return pool_nxt;}
}

47
400_xowa/src/gplx/core/threads/poolables/Gfo_poolable_mgr_tst.java
View File

@@ -22,9 +22,43 @@ public class Gfo_poolable_mgr_tst {
@Before public void init() {tstr.Clear();}
@Test public void Get__one() {
tstr.Test__get(0);
tstr.Test__mgr__free(0, 0);
tstr.Test__free__len(0);
tstr.Test__pool__len(2);
}
@Test public void Get__many__expand() {
tstr.Test__get(0);
tstr.Test__get(1);
tstr.Test__get(2);
tstr.Test__free__len(0);
tstr.Test__pool__len(4);
}
@Test public void Rls__lifo() {
tstr.Test__get(0);
tstr.Test__get(1);
tstr.Test__get(2);
tstr.Exec__rls(2);
tstr.Exec__rls(1);
tstr.Exec__rls(0);
tstr.Test__pool__nxt(0);
tstr.Test__free__len(0);
}
@Test public void Rls__fifo() {
tstr.Test__get(0);
tstr.Test__get(1);
tstr.Test__get(2);
tstr.Exec__rls(0);
tstr.Exec__rls(1);
tstr.Test__pool__nxt(3);
tstr.Test__free__len(2); // 2 items in free_ary
tstr.Test__free__ary(0, 1, 0, 0);
tstr.Test__get(1);
tstr.Exec__rls(1);
tstr.Exec__rls(2);
tstr.Test__free__len(0); // 0 items in free_ary
tstr.Test__free__ary(0, 0, 0, 0);
}
}
class Gfo_poolable_mgr_tstr {
private final Gfo_poolable_mgr mgr = new Gfo_poolable_mgr(new Sample_poolable_itm(-1, Object_.Ary_empty), Object_.Ary("make"), Object_.Ary("clear"), 2, 8);
@@ -33,12 +67,11 @@ class Gfo_poolable_mgr_tstr {
Sample_poolable_itm actl_itm = (Sample_poolable_itm)mgr.Get_fast();
Tfds.Eq(expd_idx, actl_itm.Pool__idx(), "pool_idx");
}
public void Test__mgr__free(int... expd) {
Tfds.Eq_ary(expd, mgr.Free(), "mgr.Free()");
}
public void Test__pool__len(int expd) {
Tfds.Eq(expd, mgr.Pool_len(), "mgr.Pool_len()");
}
public void Test__free__ary(int... expd) {Tfds.Eq_ary(expd, mgr.Free_ary(), "mgr.Free_ary()");}
public void Test__free__len(int expd) {Tfds.Eq(expd, mgr.Free_len(), "mgr.Free_len()");}
public void Test__pool__len(int expd) {Tfds.Eq(expd, mgr.Pool_len(), "mgr.Pool_len()");}
public void Test__pool__nxt(int expd) {Tfds.Eq(expd, mgr.Pool_nxt(), "mgr.Pool_nxt()");}
public void Exec__rls(int idx) {mgr.Rls_fast(idx);}
}
class Sample_poolable_itm implements Gfo_poolable_itm {
public Sample_poolable_itm(int pool_idx, Object[] make_args) {this.pool_idx = pool_idx; this.pool__make_args = make_args;}