diff --git a/100_core/src/gplx/Bry_.java b/100_core/src/gplx/Bry_.java index 8bff1f3cc..864d8652f 100644 --- a/100_core/src/gplx/Bry_.java +++ b/100_core/src/gplx/Bry_.java @@ -603,9 +603,10 @@ public class Bry_ { } } public static byte To_int_as_byte(byte[] ary, int bgn, int end, byte or) {return (byte)To_int_or(ary, bgn, end, or);} - public static int To_int(byte[] ary) { - int rv = To_int_or(ary, 0, ary.length, Int_.Min_value, Bool_.Y, null); - if (rv == Int_.Min_value) throw Err_.new_wo_type("could not parse to int", "val", String_.new_u8(ary)); + public static int To_int(byte[] ary) {return To_int_or_fail(ary, 0, ary.length);} + public static int To_int_or_fail(byte[] ary, int bgn, int end) { + int rv = To_int_or(ary, bgn, end, Int_.Min_value, Bool_.Y, null); + if (rv == Int_.Min_value) throw Err_.new_wo_type("could not parse to int", "val", String_.new_u8(ary, bgn, end)); return rv; } public static int To_int_or_neg1(byte[] ary) {return To_int_or(ary, 0 , ary.length, -1, Bool_.Y, null);} diff --git a/100_core/src/gplx/Err_.java b/100_core/src/gplx/Err_.java index 1510c580d..9f5482ae2 100644 --- a/100_core/src/gplx/Err_.java +++ b/100_core/src/gplx/Err_.java @@ -20,6 +20,7 @@ public class Err_ { private static String Type__gplx = "gplx"; @gplx.Internal protected static String Trace_null = null; public static void Noop(Exception e) {} public static Err as_(Object obj) {return obj instanceof Err ? (Err)obj : null;} + public static Err New(String msg, Object... args) {return new Err(Bool_.Y, Trace_null, "", String_.Format(msg, args));} public static Err new_(String type, String msg, Object... args) {return new Err(Bool_.Y, Trace_null, type, msg, args);} public static Err new_wo_type(String msg, Object... args) {return new Err(Bool_.Y, Trace_null, Type__gplx, msg, args);} public static Err new_exc(Exception e, String type, String msg, Object... args) { @@ -71,5 +72,5 @@ public class Err_ { public static String Message_gplx_full(Exception e) {return cast_or_make(e).To_str__full();} public static String Message_gplx_log(Exception e) {return cast_or_make(e).To_str__log();} public static Err cast_or_make(Throwable e) {return Type_adp_.Eq_typeSafe(e, Err.class) ? (Err)e : new Err(Bool_.N, Err_.Trace_lang(e), Type_adp_.NameOf_obj(e), Err_.Message_lang(e));} - public static final String Type__op_canceled = "gplx.op_canceled"; + public static final String Type__op_canceled = "gplx.op_canceled"; } diff --git a/100_core/src/gplx/core/brys/Bry_rdr.java b/100_core/src/gplx/core/brys/Bry_rdr.java index 5608d2ed7..3e6786031 100644 --- a/100_core/src/gplx/core/brys/Bry_rdr.java +++ b/100_core/src/gplx/core/brys/Bry_rdr.java @@ -173,6 +173,10 @@ public class Bry_rdr { Object rv = trie.Match_at(trv, src, pos, src_end); if (rv == null) err_wkr.Fail("failed trie check", "mid", String_.new_u8(Bry_.Mid_by_len_safe(src, pos, 16))); return rv; } + public byte Chk_or(Btrie_rv trv, Btrie_slim_mgr trie, byte or) { + Object rv_obj = trie.Match_at(trv, src, pos, src_end); + return rv_obj == null ? or : ((gplx.core.primitives.Byte_obj_val)rv_obj).Val(); + } public byte Chk_or(Btrie_slim_mgr trie, byte or) {return Chk_or(trie, pos, src_end, or);} public byte Chk(Btrie_slim_mgr trie, int itm_bgn, int itm_end) { byte rv = Chk_or(trie, itm_bgn, itm_end, Byte_.Max_value_127); diff --git a/100_core/src/gplx/core/btries/Btrie_slim_mgr.java b/100_core/src/gplx/core/btries/Btrie_slim_mgr.java index 41e1c0283..f6a57241f 100644 --- a/100_core/src/gplx/core/btries/Btrie_slim_mgr.java +++ b/100_core/src/gplx/core/btries/Btrie_slim_mgr.java @@ -87,6 +87,10 @@ public class Btrie_slim_mgr implements Btrie_mgr { Object rv_obj = Match_at(trv, src, bgn, end); return rv_obj == null ? or : ((Byte_obj_val)rv_obj).Val(); } + public byte Match_byte_or(Btrie_rv trv, byte[] src, byte or) { + Object rv_obj = Match_at(trv, src, 0, src.length); + return rv_obj == null ? or : ((Byte_obj_val)rv_obj).Val(); + } public Btrie_slim_mgr Add_bry_tid(byte[] bry, byte tid) {return (Btrie_slim_mgr)Add_obj(bry, Byte_obj_val.new_(tid));} public Btrie_slim_mgr Add_bry_int(byte[] key, int val) {return (Btrie_slim_mgr)Add_obj(key, new Int_obj_val(val));} public Btrie_slim_mgr Add_str_byte(String key, byte val) {return (Btrie_slim_mgr)Add_obj(Bry_.new_u8(key), Byte_obj_val.new_(val));} diff --git a/100_core/src/gplx/core/envs/Op_sys.java b/100_core/src/gplx/core/envs/Op_sys.java index 8584ad934..a253f6e40 100644 --- a/100_core/src/gplx/core/envs/Op_sys.java +++ b/100_core/src/gplx/core/envs/Op_sys.java @@ -37,7 +37,7 @@ public class Op_sys { public boolean Tid_is_drd() {return tid == Tid_drd;} public String To_str() {return os_name + Bitness_str();} - public static final byte Tid_nil = 0, Tid_wnt = 1, Tid_lnx = 2, Tid_osx = 3, Tid_drd = 4; + public static final byte Tid_nil = 0, Tid_wnt = 1, Tid_lnx = 2, Tid_osx = 3, Tid_drd = 4, Tid_arm = 5; public static final byte Sub_tid_unknown = 0, Sub_tid_win_xp = 1, Sub_tid_win_7 = 2, Sub_tid_win_8 = 3; public static final byte Bitness_32 = 1, Bitness_64 = 2; public static final char Nl_char_lnx = '\n'; @@ -77,6 +77,8 @@ public class Op_sys { else throw Err_.new_wo_type("unknown bitness; expecting 32 or 64; System.getProperty(\"bit.level\")", "val", bitness_str); os_name = System.getProperty("os.name").toLowerCase(); + String os_arch = System.getProperty("os.arch").toLowerCase(); + if (String_.Eq(os_arch, "arm")) return new_unx_flavor_(Tid_arm, os_name, bitness_byte); // EX:arm; DATE:2016-09-23;"arm" and "32" if (String_.Has_at_bgn(os_name, "win")) { String os_version = System.getProperty("os.version").toLowerCase();// "Windows 7".equals(osName) && "6.1".equals(osVersion); byte sub_tid = Sub_tid_unknown; @@ -86,7 +88,7 @@ public class Op_sys { return new_wnt_(bitness_byte, sub_tid); } else if (String_.Eq(os_name, "linux")) return new_unx_flavor_(Tid_lnx, os_name, bitness_byte); - else if (String_.Has_at_bgn(os_name, "mac")) return new_unx_flavor_(Tid_osx, os_name, bitness_byte); // EX:Mac OS X + else if (String_.Has_at_bgn(os_name, "mac")) return new_unx_flavor_(Tid_osx, os_name, bitness_byte); // EX:Mac OS X else throw Err_.new_wo_type("unknown os_name; expecting windows, linux, mac; System.getProperty(\"os.name\")", "val", os_name); } catch (Exception exc) {Drd.os_name = os_name; return Drd;} } diff --git a/100_core/src/gplx/core/tests/Gftest.java b/100_core/src/gplx/core/tests/Gftest.java index 67cf86560..8d8099295 100644 --- a/100_core/src/gplx/core/tests/Gftest.java +++ b/100_core/src/gplx/core/tests/Gftest.java @@ -19,6 +19,7 @@ package gplx.core.tests; import gplx.*; import gplx.core.*; import gplx.core.brys.*; public class Gftest { private static final Bry_bfr bfr = Bry_bfr_.New(); + public static void Eq__ary(boolean[] expd, boolean[] actl, String msg_fmt, Object... msg_args) {Eq__array(Type_adp_.Tid__bool, expd, actl, msg_fmt, msg_args);} public static void Eq__ary(int[] expd, int[] actl, String msg_fmt, Object... msg_args) {Eq__array(Type_adp_.Tid__int, expd, actl, msg_fmt, msg_args);} public static void Eq__ary(long[] expd, long[] actl, String msg_fmt, Object... msg_args) {Eq__array(Type_adp_.Tid__long, expd, actl, msg_fmt, msg_args);} public static void Eq__ary__lines(String expd, byte[] actl, String msg_fmt, Object... msg_args) {Eq__ary__lines(expd, String_.new_u8(actl), msg_fmt, msg_args);} @@ -150,10 +151,11 @@ public class Gftest { private static void Write__itm(Bry_bfr bfr, int type_id, Object ary, int len, int idx) { if (idx < len) { switch (type_id) { - case Type_adp_.Tid__bry: bfr.Add((byte[])Array_.Get_at(ary, idx)); break; - case Type_adp_.Tid__long: bfr.Add_long_variable(Long_.cast(Array_.Get_at(ary, idx))); break; - case Type_adp_.Tid__int: bfr.Add_int_variable(Int_.cast(Array_.Get_at(ary, idx))); break; - default: throw Err_.new_unhandled_default(type_id); + case Type_adp_.Tid__bool: bfr.Add_yn(Bool_.cast(Array_.Get_at(ary, idx))); break; + case Type_adp_.Tid__bry: bfr.Add((byte[])Array_.Get_at(ary, idx)); break; + case Type_adp_.Tid__long: bfr.Add_long_variable(Long_.cast(Array_.Get_at(ary, idx))); break; + case Type_adp_.Tid__int: bfr.Add_int_variable(Int_.cast(Array_.Get_at(ary, idx))); break; + default: throw Err_.new_unhandled_default(type_id); } } else @@ -173,6 +175,7 @@ public class Gftest { else if (expd_obj == null || actl_obj == null) eq = false; else { switch (tid) { + case Type_adp_.Tid__bool: eq = Bool_.cast(expd_obj) == Bool_.cast(actl_obj); break; case Type_adp_.Tid__bry: eq = Bry_.Eq((byte[])expd_obj, (byte[])actl_obj); break; case Type_adp_.Tid__long: eq = Long_.cast(expd_obj) == Long_.cast(actl_obj); break; case Type_adp_.Tid__int: eq = Int_.cast(expd_obj) == Int_.cast(actl_obj); break; diff --git a/140_dbs/src/gplx/dbs/Db_conn.java b/140_dbs/src/gplx/dbs/Db_conn.java index 85759dd33..7716dac24 100644 --- a/140_dbs/src/gplx/dbs/Db_conn.java +++ b/140_dbs/src/gplx/dbs/Db_conn.java @@ -68,6 +68,7 @@ public class Db_conn { } public String Meta_fld_append_if_missing(String tbl_name, Dbmeta_fld_list flds, Dbmeta_fld_itm fld) { String fld_name = fld.Name(); + // if fld doesn't exist, add it; NOTE: need to check if tbl exists first else meta_fld not available if ( this.Meta_tbl_exists(tbl_name) && !this.Meta_fld_exists(tbl_name, fld_name)) { try {this.Meta_fld_append(tbl_name, fld);} @@ -76,8 +77,11 @@ public class Db_conn { fld_name = Dbmeta_fld_itm.Key_null; } } - else - fld_name = flds.Add(fld); + // if fld does exist, or tbl doesn't exist, just add fld to collection + else { + if (!flds.Has(fld.Name())) // NOTE: need to check if it already exists; DATE:2016-09-22 + fld_name = flds.Add(fld); + } return fld_name; } public Dbmeta_tbl_mgr Meta_mgr() {return engine.Meta_mgr();} diff --git a/140_dbs/src/gplx/dbs/Db_conn_bldr.java b/140_dbs/src/gplx/dbs/Db_conn_bldr.java index 1afd9c611..8fab6fbce 100644 --- a/140_dbs/src/gplx/dbs/Db_conn_bldr.java +++ b/140_dbs/src/gplx/dbs/Db_conn_bldr.java @@ -45,5 +45,10 @@ public class Db_conn_bldr { else throw Err_.new_("dbs", "db does not exist", "url", url.Raw()); } } + public Db_conn Get_or_fail(Io_url url) { + Db_conn rv = Get(url); + if (rv == Db_conn_.Noop) throw Err_.new_wo_type("connection is null; file does not exist: file={0}", "file", url.Raw()); + return rv; + } public static final Db_conn_bldr Instance = new Db_conn_bldr(); Db_conn_bldr() {} } diff --git a/400_xowa/src/gplx/core/lists/Binary_search_.java b/400_xowa/src/gplx/core/lists/Binary_search_.java deleted file mode 100644 index 81193c791..000000000 --- a/400_xowa/src/gplx/core/lists/Binary_search_.java +++ /dev/null @@ -1,84 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.core.lists; import gplx.*; import gplx.core.*; -public class Binary_search_ { - public static int Search(CompareAble[] ary, int ary_len, CompareAble val) { - if (ary_len == 1) return 0; - int interval = ary_len / 2; - int pos = interval - List_adp_.Base1; - int pos_last = ary_len - 1; - int pos_prv = -1; - int loop_count = 0; - while (loop_count++ < 32) { // 32 bit integer - CompareAble lo = ary[pos]; - CompareAble hi = pos + 1 == ary_len ? null : ary[pos + 1]; - int adj = 0; - int lo_comp = val.compareTo(lo); - if (lo_comp == CompareAble_.Less) // val is < lo; search slots below - adj = -1; - else { - if (hi == null) return pos; // hi is null when at last slot in ary - int hi_comp = val.compareTo(hi); - if (hi_comp == CompareAble_.More) // val is > hi; search slots above - adj = 1; - else - return pos; // val is > lo and < hi; return slot - } - interval /= 2; - if (interval == 0) interval = 1; // do not allow 0 intervals; pos must always change; - pos += (interval * adj); - if (pos == 0 && pos_prv == 0) break; // NOTE: this will only happen when 1st member is not "" - if (pos < 0) pos = 0; - else if (pos > pos_last) pos = pos_last; - pos_prv = pos; - } - return Int_.Min_value; // should only occur if (a) ary's 0th slot is not ""; or (b) some unknown error - } - public static int Search(List_adp__getable list, int list_len, CompareAble val) { - if (list_len == 1) return 0; - int interval = list_len / 2; - int pos = interval - List_adp_.Base1; - int pos_last = list_len - 1; - int pos_prv = -1; - int loop_count = 0; - while (loop_count++ < 32) { // 32 bit integer - CompareAble lo = (CompareAble)list.Get_at(pos); - CompareAble hi = pos + 1 == list_len ? null : (CompareAble)list.Get_at(pos + 1); - int adj = 0; - int lo_comp = val.compareTo(lo); - if (lo_comp == CompareAble_.Less) // val is < lo; search slots below - adj = -1; - else { - if (hi == null) return pos; // hi is null when at last slot in ary - int hi_comp = val.compareTo(hi); - if (hi_comp == CompareAble_.More) // val is > hi; search slots above - adj = 1; - else - return pos; // val is > lo and < hi; return slot - } - interval /= 2; - if (interval == 0) interval = 1; // do not allow 0 intervals; pos must always change; - pos += (interval * adj); - if (pos == 0 && pos_prv == 0) break; // NOTE: this will only happen when 1st member is not "" - if (pos < 0) pos = 0; - else if (pos > pos_last) pos = pos_last; - pos_prv = pos; - } - return Int_.Min_value; // should only occur if (a) ary's 0th slot is not ""; or (b) some unknown error - } -} diff --git a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_data.java b/400_xowa/src/gplx/core/lists/binary_searches/Binary_comparer.java similarity index 65% rename from 400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_data.java rename to 400_xowa/src/gplx/core/lists/binary_searches/Binary_comparer.java index 53f56243c..3127fe579 100644 --- a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_data.java +++ b/400_xowa/src/gplx/core/lists/binary_searches/Binary_comparer.java @@ -15,9 +15,7 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; -public class Sql_file_parser_data { - public boolean Cancel_row() {return cancel_row;} - public Sql_file_parser_data Cancel_row_n_() {cancel_row = false; return this;} - public Sql_file_parser_data Cancel_row_y_() {cancel_row = true; return this;} private boolean cancel_row; -} \ No newline at end of file +package gplx.core.lists.binary_searches; import gplx.*; import gplx.core.*; import gplx.core.lists.*; +public interface Binary_comparer { + int Compare_val_to_obj(Object val, Object obj); +} diff --git a/400_xowa/src/gplx/core/lists/binary_searches/Binary_search_.java b/400_xowa/src/gplx/core/lists/binary_searches/Binary_search_.java new file mode 100644 index 000000000..93017f0b7 --- /dev/null +++ b/400_xowa/src/gplx/core/lists/binary_searches/Binary_search_.java @@ -0,0 +1,63 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.core.lists.binary_searches; import gplx.*; import gplx.core.*; import gplx.core.lists.*; +public class Binary_search_ { + public static int Search(CompareAble[] ary , CompareAble val) {return Search(new Binary_search_grp__ary(ary) , new Binary_search_cmp__comparable(val));} + public static int Search(Object[] ary , Binary_comparer comparer, Object val) {return Search(new Binary_search_grp__ary(ary), new Binary_search_cmp__comparer(comparer, val));} + private static int Search(Binary_search_grp grp, Binary_search_cmp cmp) { + int grp_len = grp.Len(); + if (grp_len == 1) return 0; // if 1 item, return 0; + if (grp_len == 0) return Bry_find_.Not_found; // if 0 item, return -1 + + // init + int interval = grp_len / 2; + int pos = interval - List_adp_.Base1; + int pos_last = grp_len - 1; + int pos_prv = -1; + int loop_count = 0; + while (loop_count++ < 32) { // 32=32-bit integer + Object lo = grp.Get_at(pos); + Object hi = pos + 1 == grp_len ? null : grp.Get_at(pos + 1); + int adj = 0; + int lo_comp = cmp.Compare(lo); + if (lo_comp == CompareAble_.Less) // val is < lo; search slots below + adj = -1; + else { + if (hi == null) return pos; // hi is null when at last slot in ary + int hi_comp = cmp.Compare(hi); + switch (hi_comp) { + case CompareAble_.More: // val is > hi; search slots above + adj = 1; + break; + case CompareAble_.Same: // val is > hi; search slots above + return pos + 1; + case CompareAble_.Less: // val is > lo and < hi; return slot + return pos; + } + } + interval /= 2; + if (interval == 0) interval = 1; // do not allow 0 intervals; pos must always change; + pos += (interval * adj); + if (pos == 0 && pos_prv == 0) break; // NOTE: this will only happen when 1st member is not "" + if (pos < 0) pos = 0; + else if (pos > pos_last) pos = pos_last; + pos_prv = pos; + } + return Bry_find_.Not_found; // should only occur if (a) ary's 0th slot is not ""; or (b) some unknown error + } +} diff --git a/400_xowa/src/gplx/core/lists/Binary_search__tst.java b/400_xowa/src/gplx/core/lists/binary_searches/Binary_search__tst.java similarity index 53% rename from 400_xowa/src/gplx/core/lists/Binary_search__tst.java rename to 400_xowa/src/gplx/core/lists/binary_searches/Binary_search__tst.java index 5165d3c87..216dd91fb 100644 --- a/400_xowa/src/gplx/core/lists/Binary_search__tst.java +++ b/400_xowa/src/gplx/core/lists/binary_searches/Binary_search__tst.java @@ -15,33 +15,41 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package gplx.core.lists; import gplx.*; import gplx.core.*; +package gplx.core.lists.binary_searches; import gplx.*; import gplx.core.*; import gplx.core.lists.*; import org.junit.*; import gplx.core.primitives.*; public class Binary_search__tst { - private Binary_search__fxt fxt = new Binary_search__fxt(); + private final Binary_search__fxt fxt = new Binary_search__fxt(); @Test public void Basic() { - fxt.Init_ary("", "e", "j", "o", "t", "y"); - fxt.Test_binary_search("a", 0); - fxt.Test_binary_search("f", 1); - fxt.Test_binary_search("k", 2); - fxt.Test_binary_search("p", 3); - fxt.Test_binary_search("u", 4); - fxt.Test_binary_search("z", 5); + fxt.Init__ary("", "e", "j", "o", "t", "y"); + fxt.Test__binary_search("a", 0); + fxt.Test__binary_search("f", 1); + fxt.Test__binary_search("k", 2); + fxt.Test__binary_search("p", 3); + fxt.Test__binary_search("u", 4); + fxt.Test__binary_search("z", 5); } @Test public void One() { - fxt.Init_ary(""); - fxt.Test_binary_search("a", 0); + fxt.Init__ary(""); + fxt.Test__binary_search("a", 0); + } + @Test public void Catpage() { + String[] ary = new String[25]; + for (int i = 0; i < 25; ++i) + ary[i] = Int_.To_str_pad_bgn_zero(i, 2); + fxt.Init__ary(ary); + fxt.Test__binary_search("10", 10); // was 9 } } class Binary_search__fxt { - public void Init_ary(String... v) { + private String_obj_val[] ary; + public void Init__ary(String... v) { int ary_len = v.length; ary = new String_obj_val[ary_len]; for (int i = 0; i < ary_len; i++) ary[i] = String_obj_val.new_(v[i]); - } private String_obj_val[] ary; - public void Test_binary_search(String val, int expd) { - int actl = Binary_search_.Search(ary, ary.length, String_obj_val.new_(val)); + } + public void Test__binary_search(String val, int expd) { + int actl = Binary_search_.Search(ary, String_obj_val.new_(val)); Tfds.Eq(expd, actl, val); } } diff --git a/400_xowa/src/gplx/core/lists/binary_searches/Binary_search_cmp.java b/400_xowa/src/gplx/core/lists/binary_searches/Binary_search_cmp.java new file mode 100644 index 000000000..38c7169b3 --- /dev/null +++ b/400_xowa/src/gplx/core/lists/binary_searches/Binary_search_cmp.java @@ -0,0 +1,36 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.core.lists.binary_searches; import gplx.*; import gplx.core.*; import gplx.core.lists.*; +import gplx.core.lists.*; +interface Binary_search_cmp { + int Compare(Object comp); +} +class Binary_search_cmp__comparable implements Binary_search_cmp { + private final CompareAble val; + public Binary_search_cmp__comparable(CompareAble val) {this.val = val;} + public int Compare(Object comp) { + return val.compareTo((CompareAble)comp); + } +} +class Binary_search_cmp__comparer implements Binary_search_cmp { + private final Binary_comparer comparer; private final Object val; + public Binary_search_cmp__comparer(Binary_comparer comparer, Object val) {this.comparer = comparer; this.val = val;} + public int Compare(Object comp) { + return comparer.Compare_val_to_obj(val, comp); + } +} diff --git a/400_xowa/src/gplx/core/lists/binary_searches/Binary_search_grp.java b/400_xowa/src/gplx/core/lists/binary_searches/Binary_search_grp.java new file mode 100644 index 000000000..fac66bb08 --- /dev/null +++ b/400_xowa/src/gplx/core/lists/binary_searches/Binary_search_grp.java @@ -0,0 +1,34 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.core.lists.binary_searches; import gplx.*; import gplx.core.*; import gplx.core.lists.*; +interface Binary_search_grp { + int Len(); + Object Get_at(int i); +} +class Binary_search_grp__ary implements Binary_search_grp { + private final Object[] ary; + public Binary_search_grp__ary(Object[] ary) {this.ary = ary;} + public int Len() {return ary.length;} + public Object Get_at(int i) {return ary[i];} +} +class Binary_search_grp__list implements Binary_search_grp { + private final List_adp list; + public Binary_search_grp__list(List_adp list) {this.list = list;} + public int Len() {return list.Len();} + public Object Get_at(int i) {return list.Get_at(i);} +} diff --git a/400_xowa/src/gplx/core/lists/hashs/Hash_adp__int.java b/400_xowa/src/gplx/core/lists/hashs/Hash_adp__int.java index 30579df35..98d83285f 100644 --- a/400_xowa/src/gplx/core/lists/hashs/Hash_adp__int.java +++ b/400_xowa/src/gplx/core/lists/hashs/Hash_adp__int.java @@ -22,7 +22,7 @@ public class Hash_adp__int { private final Int_obj_ref tmp_key = Int_obj_ref.New_neg1(); public void Clear() {hash.Clear();} public int Len() {return hash.Count();} - public Object Get_by(int key) {return hash.Get_by_or_fail(tmp_key.Val_(key));} + public Object Get_by_or_fail(int key) {return hash.Get_by_or_fail(tmp_key.Val_(key));} public Object Get_by_or_null(int key) {return hash.Get_by(tmp_key.Val_(key));} public void Add(int key, Object obj) {hash.Add(Int_obj_ref.New(key), obj);} public void Add(Int_obj_ref key, Object obj) {hash.Add(key, obj);} diff --git a/400_xowa/src/gplx/core/net/downloads/Http_download_wkr__base.java b/400_xowa/src/gplx/core/net/downloads/Http_download_wkr__base.java index 3cb54d84b..5aa5174a5 100644 --- a/400_xowa/src/gplx/core/net/downloads/Http_download_wkr__base.java +++ b/400_xowa/src/gplx/core/net/downloads/Http_download_wkr__base.java @@ -22,10 +22,13 @@ public abstract class Http_download_wkr__base implements Http_download_wkr { private Io_url tmp_url, checkpoint_url; private long downloaded; private long checkpoint_interval = 1024 * 1024, checkpoint_nxt = 0; + protected Io_url Checkpoint_url() {return checkpoint_url;} + protected Io_url Trg_url() {return trg_url;} private Io_url trg_url; public Io_url Tmp_url() {return tmp_url;} public String Fail_msg() {return fail_msg;} private String fail_msg; public abstract Http_download_wkr Make_new(); public byte Exec(gplx.core.progs.Gfo_prog_ui prog_ui, String src_str, Io_url trg_url, long expd_size_val) { + this.trg_url = trg_url; this.downloaded = this.Checkpoint__load_by_trg_fil(trg_url); this.checkpoint_nxt = downloaded + checkpoint_interval; this.expd_size = expd_size_val; diff --git a/400_xowa/src/gplx/core/net/downloads/Http_download_wkr__jre.java b/400_xowa/src/gplx/core/net/downloads/Http_download_wkr__jre.java index 4d98de179..31f902a2e 100644 --- a/400_xowa/src/gplx/core/net/downloads/Http_download_wkr__jre.java +++ b/400_xowa/src/gplx/core/net/downloads/Http_download_wkr__jre.java @@ -32,7 +32,7 @@ public class Http_download_wkr__jre extends Http_download_wkr__base { File trg_fil = new File(trg_url.Xto_api()); FileOutputStream trg_stream = null; try {trg_stream = new FileOutputStream(trg_fil.getPath(), prog_resumed);} // pass true for append - catch (FileNotFoundException e) {throw Err_.new_("download_file", "write failed; permission error?", "trg", trg_url, "err", e.toString());} + catch (FileNotFoundException e) {throw Err_.new_wo_type("write failed; permission error?", "trg", trg_url, "err", e.toString());} // open src stream InputStream src_stream = null; @@ -41,7 +41,7 @@ public class Http_download_wkr__jre extends Http_download_wkr__base { catch (MalformedURLException e) { try {if (trg_stream != null) trg_stream.close();} catch (IOException e1) {} - throw Err_.new_("download_file", "bad url", "src", src_url, "err" + e.toString()); + throw Err_.new_wo_type("bad url", "src", src_url, "err" + e.toString()); } HttpURLConnection src_conn = null; try { @@ -58,21 +58,25 @@ public class Http_download_wkr__jre extends Http_download_wkr__base { if (response_code != HttpURLConnection.HTTP_PARTIAL) { try {if (trg_stream != null) trg_stream.close();} catch (IOException e1) {} - throw Err_.new_("download_file", "server returned non-partial response code", "src", src_url, "code", src_conn.getResponseCode(), "msg", src_conn.getResponseMessage()); + if (response_code == 416 && prog_data_cur > 0) { // 416=Requested Range not satisfiable; if resuming at position > max_len, assume critical failure; delete files to start over from scratch; DATE:2016-09-24 + Io_mgr.Instance.DeleteFil(this.Trg_url()); + Io_mgr.Instance.DeleteFil(this.Checkpoint_url()); + } + throw Err_.new_wo_type("server returned non-partial response code", "src", src_url, "code", src_conn.getResponseCode(), "msg", src_conn.getResponseMessage()); } } else { if (response_code != HttpURLConnection.HTTP_OK) { try {if (trg_stream != null) trg_stream.close();} catch (IOException e1) {} - throw Err_.new_("download_file", "server returned non-OK response code", "src", src_url, "code", src_conn.getResponseCode(), "msg", src_conn.getResponseMessage()); + throw Err_.new_wo_type("server returned non-OK response code", "src", src_url, "code", src_conn.getResponseCode(), "msg", src_conn.getResponseMessage()); } } src_stream = src_conn.getInputStream(); } catch (Exception e) { try {if (trg_stream != null) trg_stream.close();} catch (IOException e1) {} - throw Err_.new_("download_file", "src connection failed", "src", src_url, "err",e.toString()); + throw Err_.new_wo_type(Err__server_connection_failed, "src", src_url, "err", e.toString()); } // do downloading @@ -89,7 +93,7 @@ public class Http_download_wkr__jre extends Http_download_wkr__base { if (prog_ui.Prog_notify_and_chk_if_suspended(prog_data_cur, prog_data_end)) return Gfo_prog_ui_.Status__suspended; } } catch (Exception e) { - throw Err_.new_("download_file", "downloading failed", "src", src_url, "trg_url", trg_url, "err", e.toString()); + throw Err_.new_wo_type(Err__server_download_failed, "src", src_url, "trg_url", trg_url, "err", e.toString()); } finally { try { @@ -100,4 +104,8 @@ public class Http_download_wkr__jre extends Http_download_wkr__base { } return Gfo_prog_ui_.Status__done; } - } + public static final String + Err__server_connection_failed = "server connection failed" + , Err__server_download_failed = "server download failed" + ; +} diff --git a/400_xowa/src/gplx/dbs/cfgs/Db_cfg_tbl.java b/400_xowa/src/gplx/dbs/cfgs/Db_cfg_tbl.java index f406289b0..64b6f8e56 100644 --- a/400_xowa/src/gplx/dbs/cfgs/Db_cfg_tbl.java +++ b/400_xowa/src/gplx/dbs/cfgs/Db_cfg_tbl.java @@ -17,7 +17,7 @@ along with this program. If not, see . */ package gplx.dbs.cfgs; import gplx.*; import gplx.dbs.*; import gplx.core.primitives.*; -public class Db_cfg_tbl implements Rls_able { +public class Db_cfg_tbl implements Db_tbl { private final String tbl_name; private final Dbmeta_fld_list flds = new Dbmeta_fld_list(); private final String fld_grp, fld_key, fld_val; private Db_stmt stmt_insert, stmt_update, stmt_select; @@ -34,7 +34,8 @@ public class Db_cfg_tbl implements Rls_able { stmt_update = Db_stmt_.Rls(stmt_update); stmt_select = Db_stmt_.Rls(stmt_select); } - public Db_cfg_tbl Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "main", fld_grp, fld_key, fld_val))); return this;} + public String Tbl_name() {return tbl_name;} + public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "main", fld_grp, fld_key, fld_val)));} public void Delete_val(String grp, String key) {conn.Stmt_delete(tbl_name, fld_grp, fld_key).Crt_str(fld_grp, grp).Crt_str(fld_key, key).Exec_delete();} public void Delete_grp(String grp) {conn.Stmt_delete(tbl_name, fld_grp).Crt_str(fld_grp, grp).Exec_delete();} public void Delete_all() {conn.Stmt_delete(tbl_name, Dbmeta_fld_itm.Str_ary_empty).Exec_delete();} diff --git a/400_xowa/src/gplx/fsdb/data/Fsd_bin_itm.java b/400_xowa/src/gplx/fsdb/data/Fsd_bin_itm.java index f0908a8dd..7e1ae0bc6 100644 --- a/400_xowa/src/gplx/fsdb/data/Fsd_bin_itm.java +++ b/400_xowa/src/gplx/fsdb/data/Fsd_bin_itm.java @@ -24,11 +24,11 @@ public class Fsd_bin_itm { this.bin_data_url = bin_data_url; this.bin_data = bin_data; } - public int Bin_owner_id() {return bin_owner_id;} private final int bin_owner_id; - public byte Bin_owner_tid() {return bin_owner_tid;} private final byte bin_owner_tid; - public int Bin_part_id() {return bin_part_id;} private final int bin_part_id; - public String Bin_data_url() {return bin_data_url;} private final String bin_data_url; - public byte[] Bin_data() {return bin_data;} private final byte[] bin_data; + public int Bin_owner_id() {return bin_owner_id;} private final int bin_owner_id; + public byte Bin_owner_tid() {return bin_owner_tid;} private final byte bin_owner_tid; + public int Bin_part_id() {return bin_part_id;} private final int bin_part_id; + public String Bin_data_url() {return bin_data_url;} private final String bin_data_url; + public byte[] Bin_data() {return bin_data;} private final byte[] bin_data; public static final int Db_row_size_fixed = (3 * 4); // bin_owner_id, bin_part_id, bin_owner_tid (assume byte saved as int in SQLITE) } diff --git a/400_xowa/src/gplx/fsdb/data/Fsd_dir_itm.java b/400_xowa/src/gplx/fsdb/data/Fsd_dir_itm.java index 20d4643f1..b6f41e4df 100644 --- a/400_xowa/src/gplx/fsdb/data/Fsd_dir_itm.java +++ b/400_xowa/src/gplx/fsdb/data/Fsd_dir_itm.java @@ -17,10 +17,15 @@ along with this program. If not, see . */ package gplx.fsdb.data; import gplx.*; import gplx.fsdb.*; public class Fsd_dir_itm { - public Fsd_dir_itm(int dir_id, int owner, byte[] name) {this.dir_id = dir_id; this.owner = owner; this.name = name;} - public int Dir_id() {return dir_id;} private final int dir_id; - public int Owner() {return owner;} private final int owner; - public byte[] Name() {return name;} private final byte[] name; + public Fsd_dir_itm(int dir_id, int owner, byte[] name) { + this.dir_id = dir_id; + this.owner = owner; + this.name = name; + } + public int Dir_id() {return dir_id;} private final int dir_id; + public int Owner() {return owner;} private final int owner; + public byte[] Name() {return name;} private final byte[] name; + public static final int Owner_root = 0; public static final Fsd_dir_itm Null = null; } diff --git a/400_xowa/src/gplx/fsdb/data/Fsd_dir_tbl.java b/400_xowa/src/gplx/fsdb/data/Fsd_dir_tbl.java index cedbcd047..3b6e4b3e5 100644 --- a/400_xowa/src/gplx/fsdb/data/Fsd_dir_tbl.java +++ b/400_xowa/src/gplx/fsdb/data/Fsd_dir_tbl.java @@ -20,7 +20,7 @@ import gplx.dbs.*; public class Fsd_dir_tbl implements Rls_able { private final String tbl_name = "fsdb_dir"; private final Dbmeta_fld_list flds = new Dbmeta_fld_list(); private final String fld_id, fld_owner_id, fld_name; - private final Db_conn conn; private Db_stmt stmt_insert, stmt_update, stmt_select_by_name; + private final Db_conn conn; private Db_stmt stmt_select_by_name; public Fsd_dir_tbl(Db_conn conn, boolean schema_is_1) { this.conn = conn; this.fld_id = flds.Add_int_pkey ("dir_id"); @@ -29,8 +29,6 @@ public class Fsd_dir_tbl implements Rls_able { conn.Rls_reg(this); } public void Rls() { - stmt_insert = Db_stmt_.Rls(stmt_insert); - stmt_update = Db_stmt_.Rls(stmt_update); stmt_select_by_name = Db_stmt_.Rls(stmt_select_by_name); } public void Create_tbl() { @@ -39,20 +37,22 @@ public class Fsd_dir_tbl implements Rls_able { , Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "name", fld_name, fld_owner_id, fld_id))); } public void Insert(int id, byte[] name, int owner_id) { - if (stmt_insert == null) stmt_insert = conn.Stmt_insert(tbl_name, flds); + Db_stmt stmt_insert = conn.Stmt_insert(tbl_name, flds); stmt_insert.Clear() .Val_int(fld_id, id) .Val_int(fld_owner_id, owner_id) .Val_bry_as_str(fld_name, name) .Exec_insert(); + stmt_insert.Rls(); } public void Update(int id, byte[] name, int owner_id) { - if (stmt_update == null) stmt_update = conn.Stmt_update_exclude(tbl_name, flds, fld_id); + Db_stmt stmt_update = conn.Stmt_update_exclude(tbl_name, flds, fld_id); stmt_update.Clear() .Val_int(fld_owner_id, owner_id) .Val_bry_as_str(fld_name, name) .Crt_int(fld_id, id) .Exec_update(); + stmt_update.Rls(); } public Fsd_dir_itm Select_or_null(byte[] name) { if (stmt_select_by_name == null) stmt_select_by_name = conn.Stmt_select(tbl_name, flds, fld_name); diff --git a/400_xowa/src/gplx/fsdb/data/Fsd_fil_itm.java b/400_xowa/src/gplx/fsdb/data/Fsd_fil_itm.java index 6b8d45bef..0eb5139ed 100644 --- a/400_xowa/src/gplx/fsdb/data/Fsd_fil_itm.java +++ b/400_xowa/src/gplx/fsdb/data/Fsd_fil_itm.java @@ -17,26 +17,21 @@ along with this program. If not, see . */ package gplx.fsdb.data; import gplx.*; import gplx.fsdb.*; public class Fsd_fil_itm { - public int Mnt_id() {return mnt_id;} private int mnt_id; - public int Fil_id() {return fil_id;} private int fil_id; - public int Dir_id() {return dir_id;} private int dir_id; - public int Xtn_id() {return xtn_id;} private int xtn_id; - public int Ext_id() {return ext_id;} private int ext_id; - public byte[] Name() {return name;} private byte[] name; - public int Bin_db_id() {return bin_db_id;} private int bin_db_id; - public long Size() {return size;} private long size; - public String Modified_on() {return modified_on;} private String modified_on; - public String Hash_md5() {return hash_md5;} private String hash_md5; - - public Fsd_fil_itm Ctor(int mnt_id, int dir_id, int fil_id, int bin_db_id, byte[] name, int ext_id) { - this.mnt_id = mnt_id; this.dir_id = dir_id; this.fil_id = fil_id; this.bin_db_id = bin_db_id; this.name = name; this.ext_id = ext_id; - return this; - } - public Fsd_fil_itm Load_by_rdr__full(int mnt_id, int dir_id, int fil_id, int xtn_id, int ext_id, byte[] name, long size, String modified_on, String hash_md5, int bin_db_id) { + public Fsd_fil_itm (int mnt_id, int dir_id, int fil_id, int xtn_id, int ext_id, byte[] name, long size, String modified_on, String hash_md5, int bin_db_id) { this.mnt_id = mnt_id; this.dir_id = dir_id; this.fil_id = fil_id; this.xtn_id = xtn_id; this.ext_id = ext_id; this.name = name; this.size = size; this.modified_on = modified_on; this.hash_md5 = hash_md5; this.bin_db_id = bin_db_id; - return this; } + public int Mnt_id() {return mnt_id;} private final int mnt_id; + public int Dir_id() {return dir_id;} private final int dir_id; + public int Fil_id() {return fil_id;} private final int fil_id; + public int Xtn_id() {return xtn_id;} private final int xtn_id; + public int Ext_id() {return ext_id;} private final int ext_id; + public byte[] Name() {return name;} private final byte[] name; + public long Size() {return size;} private final long size; + public String Modified_on() {return modified_on;} private final String modified_on; + public String Hash_md5() {return hash_md5;} private final String hash_md5; + public int Bin_db_id() {return bin_db_id;} private final int bin_db_id; + public int Db_row_size() {return Db_row_size_fixed + name.length;} private static final int Db_row_size_fixed = (7 * 4) // 6 int fields + 1 byte field diff --git a/400_xowa/src/gplx/fsdb/data/Fsd_fil_tbl.java b/400_xowa/src/gplx/fsdb/data/Fsd_fil_tbl.java index 88d20f386..bf2aacbdd 100644 --- a/400_xowa/src/gplx/fsdb/data/Fsd_fil_tbl.java +++ b/400_xowa/src/gplx/fsdb/data/Fsd_fil_tbl.java @@ -87,7 +87,7 @@ public class Fsd_fil_tbl implements Rls_able { .Crt_bry_as_str(fld_name, fil_name) .Exec_select__rls_manual(); try { - return rdr.Move_next() ? Load_by_rdr(mnt_id, rdr) : Fsd_fil_itm.Null; + return rdr.Move_next() ? New_by_rdr(mnt_id, rdr) : Fsd_fil_itm.Null; } finally {rdr.Rls();} } @@ -95,28 +95,25 @@ public class Fsd_fil_tbl implements Rls_able { Db_rdr rdr = conn.Stmt_select(tbl_name, flds, Dbmeta_fld_itm.Str_ary_empty).Exec_select__rls_auto(); try { while (rdr.Move_next()) { - Fsd_fil_itm fil = Load_by_rdr(mnt_id, rdr); + Fsd_fil_itm fil = New_by_rdr(mnt_id, rdr); byte[] cache_key = Fsd_fil_itm.Gen_cache_key(key_bfr, fil.Dir_id(), fil.Name()); cache.Add(cache_key, fil); } } finally {rdr.Rls();} } - public Fsd_fil_itm Load_by_rdr(int mnt_id, Db_rdr rdr) { - return new Fsd_fil_itm().Ctor(mnt_id, rdr.Read_int(fld_owner_id), rdr.Read_int(fld_id), rdr.Read_int(fld_bin_db_id), rdr.Read_bry_by_str(fld_name), rdr.Read_int(fld_ext_id)); - } - public Fsd_fil_itm Load_by_rdr__full(int mnt_id, Db_rdr rdr) { - return new Fsd_fil_itm() - .Load_by_rdr__full(mnt_id - , rdr.Read_int(fld_owner_id) - , rdr.Read_int(fld_id) - , rdr.Read_int(fld_xtn_id) - , rdr.Read_int(fld_ext_id) - , rdr.Read_bry_by_str(fld_name) - , rdr.Read_long(fld_size) - , rdr.Read_str(fld_modified) - , rdr.Read_str(fld_hash) - , rdr.Read_int(fld_bin_db_id) - ); + public Fsd_fil_itm New_by_rdr(int mnt_id, Db_rdr rdr) { + return new Fsd_fil_itm + ( mnt_id + , rdr.Read_int(fld_owner_id) + , rdr.Read_int(fld_id) + , rdr.Read_int(fld_xtn_id) + , rdr.Read_int(fld_ext_id) + , rdr.Read_bry_by_str(fld_name) + , rdr.Read_long(fld_size) + , rdr.Read_str(fld_modified) + , rdr.Read_str(fld_hash) + , rdr.Read_int(fld_bin_db_id) + ); } } diff --git a/400_xowa/src/gplx/fsdb/data/Fsd_img_itm.java b/400_xowa/src/gplx/fsdb/data/Fsd_img_itm.java index 3140d2f45..24e72ffd3 100644 --- a/400_xowa/src/gplx/fsdb/data/Fsd_img_itm.java +++ b/400_xowa/src/gplx/fsdb/data/Fsd_img_itm.java @@ -17,11 +17,11 @@ along with this program. If not, see . */ package gplx.fsdb.data; import gplx.*; import gplx.fsdb.*; public class Fsd_img_itm { - public void Ctor(int mnt_id, int dir_id, int fil_id, int bin_db_id) { + public Fsd_img_itm(int mnt_id, int dir_id, int fil_id, int bin_db_id) { this.mnt_id = mnt_id; this.dir_id = dir_id; this.fil_id = fil_id; this.bin_db_id = bin_db_id; } - public int Mnt_id() {return mnt_id;} private int mnt_id; - public int Fil_id() {return fil_id;} private int fil_id; - public int Dir_id() {return dir_id;} private int dir_id; - public int Bin_db_id() {return bin_db_id;} private int bin_db_id; + public int Mnt_id() {return mnt_id;} private final int mnt_id; + public int Dir_id() {return dir_id;} private final int dir_id; + public int Fil_id() {return fil_id;} private final int fil_id; + public int Bin_db_id() {return bin_db_id;} private final int bin_db_id; } diff --git a/400_xowa/src/gplx/fsdb/data/Fsd_thm_itm.java b/400_xowa/src/gplx/fsdb/data/Fsd_thm_itm.java index 8707ed19b..3c33adac6 100644 --- a/400_xowa/src/gplx/fsdb/data/Fsd_thm_itm.java +++ b/400_xowa/src/gplx/fsdb/data/Fsd_thm_itm.java @@ -22,21 +22,21 @@ public class Fsd_thm_itm { this.w = w; this.h = h; this.time = time; this.page = page; this.size = size; this.modified = modified; this.hash = hash; } - public int Mnt_id() {return mnt_id;} private int mnt_id; - public int Dir_id() {return dir_id;} private int dir_id; - public int Fil_id() {return fil_id;} private int fil_id; - public int Thm_id() {return thm_id;} private int thm_id; - public int Bin_db_id() {return bin_db_id;} private int bin_db_id; - public int W() {return w;} private int w; - public int H() {return h;} private int h; - public double Time() {return time;} private double time; - public int Page() {return page;} private int page; - public long Size() {return size;} private long size; - public String Modified() {return modified;} private String modified; - public String Hash() {return hash;} private String hash; - public int Req_w() {return req_w;} private int req_w; - public double Req_time() {return req_time;} private double req_time; - public int Req_page() {return req_page;} private int req_page; + public int Mnt_id() {return mnt_id;} private int mnt_id; + public int Dir_id() {return dir_id;} private int dir_id; + public int Fil_id() {return fil_id;} private int fil_id; + public int Thm_id() {return thm_id;} private int thm_id; + public int Bin_db_id() {return bin_db_id;} private int bin_db_id; + public int W() {return w;} private int w; + public int H() {return h;} private int h; + public double Time() {return time;} private double time; + public int Page() {return page;} private int page; + public long Size() {return size;} private long size; + public String Modified() {return modified;} private String modified; + public String Hash() {return hash;} private String hash; + public int Req_w() {return req_w;} private int req_w; + public double Req_time() {return req_time;} private double req_time; + public int Req_page() {return req_page;} private int req_page; public void Init_by_req(int w, double time, int page) {this.w = w; this.time = time; this.page = page;} public void Init_by_match(Fsd_thm_itm comp) { this.req_w = w; this.req_time = time; this.req_page = page; diff --git a/400_xowa/src/gplx/fsdb/meta/Fsm_atr_fil.java b/400_xowa/src/gplx/fsdb/meta/Fsm_atr_fil.java index a9577d2ff..b0ce83e2f 100644 --- a/400_xowa/src/gplx/fsdb/meta/Fsm_atr_fil.java +++ b/400_xowa/src/gplx/fsdb/meta/Fsm_atr_fil.java @@ -39,17 +39,15 @@ public class Fsm_atr_fil { public boolean Select_thm(boolean exact, Fsd_thm_itm rv, int dir_id, int fil_id) { return exact ? tbl_thm.Select_itm_by_w_exact(dir_id, fil_id, rv) : tbl_thm.Select_itm_by_w_near(dir_id, fil_id, rv); } - public int Insert_fil(Fsd_fil_itm rv, byte[] dir, byte[] fil, int ext_id, int bin_db_id, long bin_len, Io_stream_rdr bin_rdr) { + public Fsd_fil_itm Insert_fil(byte[] dir, byte[] fil, int ext_id, int bin_db_id, long bin_len, Io_stream_rdr bin_rdr) { int dir_id = Get_dir_id_or_make(dir); int fil_id = Get_fil_id_or_make(Tid_none, dir_id, fil, ext_id, bin_db_id, bin_len); - rv.Ctor(mnt_id, fil_id, dir_id, bin_db_id, fil, ext_id); - return fil_id; + return new Fsd_fil_itm(mnt_id, dir_id, fil_id, 0, ext_id, fil, bin_len, null, null, bin_db_id); } - public int Insert_img(Fsd_img_itm rv, byte[] dir, byte[] fil, int ext_id, int img_w, int img_h, int bin_db_id, long bin_len, Io_stream_rdr bin_rdr) { + public Fsd_img_itm Insert_img(byte[] dir, byte[] fil, int ext_id, int img_w, int img_h, int bin_db_id, long bin_len, Io_stream_rdr bin_rdr) { int dir_id = Get_dir_id_or_make(dir); int fil_id = Get_fil_id_or_make(Tid_img, dir_id, fil, ext_id, bin_db_id, bin_len); - rv.Ctor(mnt_id, dir_id, fil_id, bin_db_id); - return fil_id; + return new Fsd_img_itm(mnt_id, dir_id, fil_id, bin_db_id); } public int Insert_thm(Fsd_thm_itm rv, byte[] dir, byte[] fil, int ext_id, int w, int h, double time, int page, int bin_db_id, long bin_len, Io_stream_rdr bin_rdr) { int dir_id = Get_dir_id_or_make(dir); diff --git a/400_xowa/src/gplx/fsdb/meta/Fsm_mnt_itm.java b/400_xowa/src/gplx/fsdb/meta/Fsm_mnt_itm.java index b5879ef40..63a56ff8d 100644 --- a/400_xowa/src/gplx/fsdb/meta/Fsm_mnt_itm.java +++ b/400_xowa/src/gplx/fsdb/meta/Fsm_mnt_itm.java @@ -43,13 +43,15 @@ public class Fsm_mnt_itm { Fsd_fil_itm fil_itm = atr_mgr.Select_fil_or_null(dir, fil); return fil_itm == Fsd_fil_itm.Null ? Bool_.N : atr_mgr.Select_thm(exact, rv, fil_itm.Dir_id(), fil_itm.Fil_id()); } - public void Insert_img(Fsd_img_itm rv, Fsm_atr_fil atr_fil, Fsm_bin_fil bin_fil, byte[] dir, byte[] fil, int ext_id, int img_w, int img_h, long bin_len, Io_stream_rdr bin_rdr) { - int fil_id = atr_fil.Insert_img(rv, dir, fil, ext_id, img_w, img_h, bin_fil.Id(), bin_len, bin_rdr); - bin_fil.Insert(fil_id, Fsd_bin_tbl.Owner_tid_fil, bin_len, bin_rdr); + public Fsd_img_itm Insert_img(Fsm_atr_fil atr_fil, Fsm_bin_fil bin_fil, byte[] dir, byte[] fil, int ext_id, int img_w, int img_h, long bin_len, Io_stream_rdr bin_rdr) { + Fsd_img_itm rv = atr_fil.Insert_img(dir, fil, ext_id, img_w, img_h, bin_fil.Id(), bin_len, bin_rdr); + bin_fil.Insert(rv.Fil_id(), Fsd_bin_tbl.Owner_tid_fil, bin_len, bin_rdr); + return rv; } - public void Insert_fil(Fsd_fil_itm rv, Fsm_atr_fil atr_fil, Fsm_bin_fil bin_fil, byte[] dir, byte[] fil, int ext_id, long bin_len, gplx.core.ios.streams.Io_stream_rdr bin_rdr) { - int fil_id = atr_fil.Insert_fil(rv, dir, fil, ext_id, bin_fil.Id(), bin_len, bin_rdr); - bin_fil.Insert(fil_id, Fsd_bin_tbl.Owner_tid_fil, bin_len, bin_rdr); + public Fsd_fil_itm Insert_fil(Fsm_atr_fil atr_fil, Fsm_bin_fil bin_fil, byte[] dir, byte[] fil, int ext_id, long bin_len, gplx.core.ios.streams.Io_stream_rdr bin_rdr) { + Fsd_fil_itm rv = atr_fil.Insert_fil(dir, fil, ext_id, bin_fil.Id(), bin_len, bin_rdr); + bin_fil.Insert(rv.Fil_id(), Fsd_bin_tbl.Owner_tid_fil, bin_len, bin_rdr); + return rv; } public void Insert_thm(Fsd_thm_itm rv, Fsm_atr_fil atr_fil, Fsm_bin_fil bin_fil, byte[] dir, byte[] fil, int ext_id, int w, int h, double time, int page, long bin_len, Io_stream_rdr bin_rdr) { int thm_id = atr_fil.Insert_thm(rv, dir, fil, ext_id, w, h, time, page, bin_fil.Id(), bin_len, bin_rdr); diff --git a/400_xowa/src/gplx/langs/htmls/Gfh_tag_.java b/400_xowa/src/gplx/langs/htmls/Gfh_tag_.java index 2e610d699..fdf7824f1 100644 --- a/400_xowa/src/gplx/langs/htmls/Gfh_tag_.java +++ b/400_xowa/src/gplx/langs/htmls/Gfh_tag_.java @@ -76,6 +76,7 @@ public class Gfh_tag_ { // NOTE: not serialized; used by tag_rdr , Id__blockquote = 52 , Id__map = 53 , Id__bdo = 54 + , Id__time = 55 ; // private static final int Id__ary_max = 54; public static final byte[] @@ -156,6 +157,7 @@ public class Gfh_tag_ { // NOTE: not serialized; used by tag_rdr .Add_str_int("blockquote" , Id__blockquote) .Add_str_int("map" , Id__map) .Add_str_int("bdo" , Id__bdo) + .Add_str_int("time" , Id__time) ; public static String To_str(int tid) { switch (tid) { @@ -217,6 +219,7 @@ public class Gfh_tag_ { // NOTE: not serialized; used by tag_rdr case Id__blockquote: return "blockquote"; case Id__map: return "map"; case Id__bdo: return "bdo"; + case Id__time: return "time"; default: throw Err_.new_unhandled(tid); } } diff --git a/400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder_.java b/400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder_.java index 8710a503c..d330465da 100644 --- a/400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder_.java +++ b/400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder_.java @@ -85,6 +85,13 @@ public class Gfo_url_encoder_ { return new Gfo_url_encoder_mkr().Init(Byte_ascii.Percent).Init_common(Bool_.Y) .Init__same__many(Byte_ascii.Paren_bgn, Byte_ascii.Paren_end, Byte_ascii.Apos, Byte_ascii.Semic); } + public static Gfo_url_encoder_mkr New__mw_ttl() { + return new Gfo_url_encoder_mkr().Init(Byte_ascii.Percent) + .Init__same__rng(0, 255) + .Init__diff__many(Byte_ascii.Percent, Byte_ascii.Amp, Byte_ascii.Apos, Byte_ascii.Eq, Byte_ascii.Plus) + .Init__diff__one(Byte_ascii.Space, Byte_ascii.Underline) + ; + } public static final Gfo_url_encoder Id = Gfo_url_encoder_.New__html_id().Make() , Href = Gfo_url_encoder_.New__html_href_mw(Bool_.Y).Make() @@ -95,5 +102,6 @@ public class Gfo_url_encoder_ { , Xourl = Gfo_url_encoder_.New__html_href_mw(Bool_.Y).Init__same__many(Byte_ascii.Underline).Make() , Http_url = Gfo_url_encoder_.New__http_url().Make() , Http_url_ttl = Gfo_url_encoder_.New__http_url_ttl().Make() + , Mw_ttl = Gfo_url_encoder_.New__mw_ttl().Make() ; } diff --git a/400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder_tst.java b/400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder_tst.java index f6f15615b..7ed992980 100644 --- a/400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder_tst.java +++ b/400_xowa/src/gplx/langs/htmls/encoders/Gfo_url_encoder_tst.java @@ -32,6 +32,8 @@ public class Gfo_url_encoder_tst { fxt.Encoder_id().Encoder().Decode(tmp_bfr, Bool_.N, raw, 0, raw.length); Tfds.Eq("0%.jpg", tmp_bfr.To_str_and_clear()); } + @Test public void Ttl__syms__diff() {fxt.Encoder_ttl().Test__encode(" &'=+", "_%26%27%3D%2B");} + @Test public void Ttl__syms__same() {fxt.Encoder_ttl().Test__encode("!\"#$%()*,-./:;<>?@[\\]^_`{|}~", "!\"#$%()*,-./:;<>?@[\\]^_`{|}~");} @Test public void Url__syms() {fxt.Encoder_url().Test__bicode("!?^~", "%21%3F%5E%7E");} @Test public void Url__foreign() {fxt.Encoder_url().Test__bicode("aéb", "a%C3%A9b");} @Test public void Url__space() {fxt.Encoder_url().Test__bicode("a b", "a+b");} @@ -54,6 +56,7 @@ class Gfo_url_encoder_fxt { public Gfo_url_encoder_fxt Encoder_id() {encoder = Gfo_url_encoder_.Id; return this;} public Gfo_url_encoder_fxt Encoder_href() {encoder = Gfo_url_encoder_.Href; return this;} public Gfo_url_encoder_fxt Encoder_url() {encoder = Gfo_url_encoder_.Http_url; return this;} + public Gfo_url_encoder_fxt Encoder_ttl() {encoder = Gfo_url_encoder_.Mw_ttl; return this;} public Gfo_url_encoder_fxt Encoder_fsys_safe() {encoder = Gfo_url_encoder_.New__fsys_wnt().Make(); return this;} public void Test__bicode(String raw, String encoded) { Test__encode(raw, encoded); diff --git a/400_xowa/src/gplx/xowa/Xoa_app_.java b/400_xowa/src/gplx/xowa/Xoa_app_.java index 2e8a3ca37..22708aa20 100644 --- a/400_xowa/src/gplx/xowa/Xoa_app_.java +++ b/400_xowa/src/gplx/xowa/Xoa_app_.java @@ -32,7 +32,7 @@ public class Xoa_app_ { } } public static final String Name = "xowa"; - public static final String Version = "3.9.2.1"; + public static final String Version = "3.9.4.1"; public static String Build_date = "2012-12-30 00:00:00"; public static String Op_sys_str; public static String User_agent = ""; diff --git a/400_xowa/src/gplx/xowa/Xoa_ttl.java b/400_xowa/src/gplx/xowa/Xoa_ttl.java index dbebb121d..d8d0a6fc3 100644 --- a/400_xowa/src/gplx/xowa/Xoa_ttl.java +++ b/400_xowa/src/gplx/xowa/Xoa_ttl.java @@ -20,7 +20,7 @@ import gplx.core.primitives.*; import gplx.core.btries.*; import gplx.langs.htmls.entitys.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.xwikis.*; import gplx.xowa.parsers.amps.*; import gplx.xowa.parsers.miscs.*; -import gplx.xowa.apps.urls.*; import gplx.langs.htmls.encoders.*; import gplx.xowa.langs.cases.*; +import gplx.xowa.apps.utls.*; import gplx.langs.htmls.encoders.*; import gplx.xowa.langs.cases.*; public class Xoa_ttl { // PAGE:en.w:http://en.wikipedia.org/wiki/Help:Link; REF.MW: Ttl.php|secureAndSplit; private int wik_bgn = -1, ns_bgn = -1, page_bgn = 0, leaf_bgn = -1, anch_bgn = -1, root_bgn = -1; private byte[] tors_txt; diff --git a/400_xowa/src/gplx/xowa/Xop_fxt.java b/400_xowa/src/gplx/xowa/Xop_fxt.java index 86ac8c735..1dc2fab5a 100644 --- a/400_xowa/src/gplx/xowa/Xop_fxt.java +++ b/400_xowa/src/gplx/xowa/Xop_fxt.java @@ -171,7 +171,6 @@ public class Xop_fxt { } public void Init_defn_clear() {wiki.Cache_mgr().Defn_cache().Free_mem_all();} public Xop_fxt Init_id_create(int id, int fil_idx, int row_idx, boolean type_redirect, int itm_len, int ns_id, String ttl) {Xow_hive_mgr_fxt.Create_id(app, wiki.Hive_mgr(), id, fil_idx, row_idx, type_redirect, itm_len, ns_id, ttl); return this;} - public Xop_fxt Init_ctg_create(String ctg, int... pages) {Xow_hive_mgr_fxt.Create_ctg(app, wiki.Hive_mgr(), ctg, pages); return this;} public Xop_fxt Init_page_create(String ttl) {return Init_page_create(wiki, ttl, "");} public Xop_fxt Init_page_create(String ttl, String txt) {return Init_page_create(wiki, ttl, txt);} public Xop_fxt Init_page_create(Xowe_wiki wiki, String ttl, String txt) {Init_page_create_static(wiki, ttl, txt);return this;} diff --git a/400_xowa/src/gplx/xowa/addons/Xoax_addon_mgr.java b/400_xowa/src/gplx/xowa/addons/Xoax_addon_mgr.java index ea1c63b89..ecd8664ff 100644 --- a/400_xowa/src/gplx/xowa/addons/Xoax_addon_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/Xoax_addon_mgr.java @@ -41,7 +41,7 @@ public class Xoax_addon_mgr { , new gplx.xowa.addons.bldrs.wmdumps.imglinks .Imglnk_addon() , new gplx.xowa.addons.bldrs.utils_rankings .Xoax_builds_utils_rankings_addon() , new gplx.xowa.addons.wikis.searchs .Xoax_builds_search_addon() - , new gplx.xowa.addons.bldrs.updates.files .Xoax_updates_files_addon() + , new gplx.xowa.addons.bldrs.updates.files .Xodel_addon() , new gplx.xowa.addons.bldrs.htmls .Html__dump_to_fsys__addon() , new gplx.xowa.addons.bldrs.exports .Export_addon() , new gplx.xowa.addons.wikis.pages.randoms .Rndm_addon() diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__download.java b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__download.java index 35d66e67f..7e1a03422 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__download.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__download.java @@ -34,25 +34,35 @@ public class Xobc_cmd__download extends Xobc_cmd__base { @Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) { int error_wait = 10000, error_tries_max = 6, error_tries_cur = 0; // retry every 10 seconds for a total of 6 tries (1 min) + + // loop while "server ... " error while (true) { - long trg_size_bgn = Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size(); + long trg_size_bgn = Tmp_url_size(); byte status = wkr.Exec(this, src_url, trg_url, expd_size); if (status == Gfo_prog_ui_.Status__fail) { - // check if anything more downloaded; if so, then reset to 0; DATE:2016-09-03 - long trg_size_cur = Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size(); - if (trg_size_cur > trg_size_bgn) { - error_tries_cur = 0; - trg_size_bgn = trg_size_cur; + + // check if server error; note: must not loop if bad size; DATE:2016-09-24 + String fail_msg = wkr.Fail_msg(); + if (String_.Has_at_bgn(fail_msg, Http_download_wkr__jre.Err__server_download_failed)) { + + // check if anything more downloaded; if so, then reset to 0; DATE:2016-09-03 + long trg_size_cur = Tmp_url_size(); + if (trg_size_cur > trg_size_bgn) { + error_tries_cur = 0; + trg_size_bgn = trg_size_cur; + } + + // retry + if (error_tries_cur++ < error_tries_max) { + task_mgr.Work_mgr().On_stat(this.Task_id(), String_.Format("connection interrupted: retrying in {0} seconds; attempt {1} of {2}", error_wait / 1000, error_tries_cur, error_tries_max)); + Gfo_usr_dlg_.Instance.Log_many("", "", "xobc_cmd task download interrupted; ~{0} ~{1} ~{2} ~{3}", this.Task_id(), this.Step_id(), trg_url, Io_mgr.Instance.QueryFil(trg_url).Size()); + gplx.core.threads.Thread_adp_.Sleep(error_wait); + continue; + } } - // retry - if (error_tries_cur++ < error_tries_max) { - task_mgr.Work_mgr().On_stat(this.Task_id(), String_.Format("connection interrupted: retrying in {0} seconds; attempt {1} of {2}", error_wait / 1000, error_tries_cur, error_tries_max)); - Gfo_usr_dlg_.Instance.Log_many("", "", "xobc_cmd task download interrupted; ~{0} ~{1} ~{2} ~{3}", this.Task_id(), this.Step_id(), trg_url, Io_mgr.Instance.QueryFil(trg_url).Size()); - gplx.core.threads.Thread_adp_.Sleep(error_wait); - continue; - } - this.Cmd_exec_err_(wkr.Fail_msg()); + // otherewise exit loop + this.Cmd_exec_err_(fail_msg); break; } else @@ -60,6 +70,9 @@ public class Xobc_cmd__download extends Xobc_cmd__base { } Gfo_log_.Instance.Info("xobc_cmd task download", "task_id", this.Task_id(), "step_id", this.Step_id(), "trg_url", trg_url, "trg_len", Io_mgr.Instance.QueryFil(trg_url).Size()); } + private long Tmp_url_size() { + return wkr.Tmp_url() == null ? 0 : Io_mgr.Instance.QueryFil(wkr.Tmp_url()).Size(); // NOTE: wkr.Tmp_url is null in some extreme exceptions; DATE:2016-09-24 + } @Override public void Cmd_cleanup() { wkr.Exec_cleanup(); } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__fsdb_delete.java b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__fsdb_delete.java new file mode 100644 index 000000000..b108797a7 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__fsdb_delete.java @@ -0,0 +1,46 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.centrals.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; +import gplx.dbs.*; +import gplx.xowa.wikis.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; +import gplx.xowa.addons.bldrs.updates.files.*; +public class Xobc_cmd__fsdb_delete extends Xobc_cmd__base { + private final Io_url deletion_db_url; + public Xobc_cmd__fsdb_delete(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url deletion_db_url) {super(task_mgr, task_id, step_id, cmd_idx); + this.deletion_db_url = deletion_db_url; + } + @Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.fsdb.delete"; + @Override public String Cmd_name() {return "deleting old files";} + @Override public boolean Cmd_suspendable() {return true;} + @Override protected void Cmd_exec_hook(Xobc_cmd_ctx ctx) { + if (!Io_mgr.Instance.ExistsFil(deletion_db_url)) throw Err_.New("deletion db does not exist; file={0}", deletion_db_url.Raw()); + boolean pass = false; + try { + new Xodel_exec_mgr().Exec_delete(this, ctx.App().Bldr(), deletion_db_url); + pass = true; + } + catch (Exception e) { + this.Cmd_exec_err_(Err_.Message_gplx_log(e)); + } + Gfo_log_.Instance.Info("xobc_cmd task delete", "task_id", this.Task_id(), "step_id", this.Step_id(), "delete_url", deletion_db_url.Raw(), "pass", pass); + } + @Override public void Cmd_cleanup() { + if (Io_mgr.Instance.ExistsFil(deletion_db_url)) + new Xodel_exec_mgr().Exec_cleanup(deletion_db_url); + } +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__verify_dir.java b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__verify_dir.java index 7fb41ff69..5fda9e111 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__verify_dir.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/cmds/Xobc_cmd__verify_dir.java @@ -20,9 +20,9 @@ import gplx.core.progs.*; import gplx.core.security.*; import gplx.core.security.files.*; public class Xobc_cmd__verify_dir extends Xobc_cmd__base { private final Io_url delete_fil, checksum_fil; - public Xobc_cmd__verify_dir(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url dir_url, String checksum_fil_name, Io_url delete_fil) {super(task_mgr, task_id, step_id, cmd_idx); + public Xobc_cmd__verify_dir(Xobc_task_mgr task_mgr, int task_id, int step_id, int cmd_idx, Io_url checksum_fil, Io_url delete_fil) {super(task_mgr, task_id, step_id, cmd_idx); + this.checksum_fil = checksum_fil; this.delete_fil = delete_fil; - this.checksum_fil = dir_url.GenSubFil(checksum_fil_name); } @Override public String Cmd_type() {return CMD_TYPE;} public static final String CMD_TYPE = "xowa.core.hash_dir"; @Override public String Cmd_name() {return "verify";} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java index eb7d92eab..8aefafaa2 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java @@ -18,13 +18,15 @@ along with this program. If not, see . package gplx.xowa.addons.bldrs.centrals.dbs.datas.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; public class Xobc_import_type { public static final int // SERIALIZED: bc_db; import_step - Tid__ignore = 0 - , Tid__pack = 1 - , Tid__wiki__core = 2 - , Tid__wiki__srch = 3 - , Tid__wiki__html = 4 - , Tid__wiki__text = 5 - , Tid__file__core = 6 - , Tid__file__data = 7 + Tid__ignore = 0 + , Tid__pack = 1 + , Tid__wiki__core = 2 + , Tid__wiki__srch = 3 + , Tid__wiki__html = 4 + , Tid__wiki__text = 5 + , Tid__file__core = 6 + , Tid__file__data = 7 + , Tid__fsdb__delete = 8 + , Tid__wiki__ctg = 9 ; } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/steps/Xobc_step_factory.java b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/steps/Xobc_step_factory.java index d06862d0e..8114f4249 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/steps/Xobc_step_factory.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/steps/Xobc_step_factory.java @@ -20,6 +20,9 @@ import gplx.core.brys.evals.*; import gplx.core.primitives.*; import gplx.xowa.addons.bldrs.centrals.tasks.*; import gplx.xowa.addons.bldrs.centrals.cmds.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.utils.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.hosts.*; import gplx.xowa.addons.bldrs.exports.merges.*; +import gplx.xowa.addons.bldrs.updates.files.*; +import gplx.xowa.addons.bldrs.exports.packs.files.*; +import gplx.xowa.bldrs.*; import gplx.xowa.wikis.domains.*; public class Xobc_step_factory { private final Xobc_task_mgr task_mgr; @@ -60,16 +63,24 @@ public class Xobc_step_factory { Io_url zip_file_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__download_fil, wiki_domain, file_name)); Io_url unzip_dir_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__unzip_dir, wiki_domain, file_name)); Io_url wiki_dir_url = Eval_url(Bry_eval_wkr__builder_central.Make_str(Bry_eval_wkr__builder_central.Type__wiki_dir, wiki_domain, file_name)); - list.Add(new Xobc_cmd__download (task_mgr, task_id, step_id, 0, src_http_url, zip_file_url, import_itm.Import_size_zip)); - list.Add(new Xobc_cmd__verify_fil (task_mgr, task_id, step_id, 1, zip_file_url, import_itm.Import_md5, import_itm.Import_size_zip)); - list.Add(new Xobc_cmd__unzip (task_mgr, task_id, step_id, 2, zip_file_url, unzip_dir_url, import_itm.Import_size_raw)); - list.Add(new Xobc_cmd__verify_dir (task_mgr, task_id, step_id, 3, unzip_dir_url, String_.Replace(file_name, ".zip", ".md5"), zip_file_url)); - // list.Add(new Xobc_cmd__wiki_merge (task_mgr, task_id, step_id, 4, merge_mgr, wiki_domain, unzip_dir_url, import_itm.Import_prog_data_max, import_itm.Import_prog_row_max, step_seqn)); - list.Add(new Xobc_cmd__move_fils (task_mgr, task_id, step_id, 4, unzip_dir_url, wiki_dir_url)); - if (import_itm.Import_type == Xobc_import_type.Tid__wiki__core) { - list.Add(new Xobc_cmd__wiki_reg (task_mgr, task_id, step_id, 5, wiki_dir_url, wiki_domain)); + Io_url checksum_url = unzip_dir_url.GenSubFil(file_name + ".md5"); + int cmd_idx = 0; + list.Add(new Xobc_cmd__download (task_mgr, task_id, step_id, cmd_idx++, src_http_url, zip_file_url, import_itm.Import_size_zip)); + list.Add(new Xobc_cmd__verify_fil (task_mgr, task_id, step_id, cmd_idx++, zip_file_url, import_itm.Import_md5, import_itm.Import_size_zip)); + list.Add(new Xobc_cmd__unzip (task_mgr, task_id, step_id, cmd_idx++, zip_file_url, unzip_dir_url, import_itm.Import_size_raw)); + list.Add(new Xobc_cmd__verify_dir (task_mgr, task_id, step_id, cmd_idx++, checksum_url, zip_file_url)); + // list.Add(new Xobc_cmd__wiki_merge (task_mgr, task_id, step_id, cmd_idx++, merge_mgr, wiki_domain, unzip_dir_url, import_itm.Import_prog_data_max, import_itm.Import_prog_row_max, step_seqn)); + list.Add(new Xobc_cmd__move_fils (task_mgr, task_id, step_id, cmd_idx++, unzip_dir_url, wiki_dir_url)); + + switch (import_itm.Import_type) { + case Xobc_import_type.Tid__wiki__core: list.Add(new Xobc_cmd__wiki_reg (task_mgr, task_id, step_id, cmd_idx++, wiki_dir_url, wiki_domain)); break; + case Xobc_import_type.Tid__fsdb__delete: list.Add(new Xobc_cmd__fsdb_delete (task_mgr, task_id, step_id, cmd_idx++, Pack_zip_name_bldr.To_wiki_url(wiki_dir_url, zip_file_url.OwnerDir()))); break; } return (Xobc_cmd_itm[])list.To_ary_and_clear(Xobc_cmd_itm.class); } private Io_url Eval_url(String src) {return Io_url_.new_any_(String_.new_u8(eval_mgr.Eval(Bry_.new_u8(src))));} + public static Xow_wiki Get_wiki_by_abrv(Xoa_app app, byte[] wiki_abrv) { + Xow_domain_itm domain_itm = Xow_abrv_xo_.To_itm(wiki_abrv); + return app.Wiki_mgri().Get_by_or_make_init_y(domain_itm.Domain_bry()); + } } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/tasks/Xobc_task_regy__work.java b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/tasks/Xobc_task_regy__work.java index 3cf22f0c7..93b052448 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/tasks/Xobc_task_regy__work.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/tasks/Xobc_task_regy__work.java @@ -79,8 +79,8 @@ public class Xobc_task_regy__work extends Xobc_task_regy__base { thread_mgr.Halt(cmd_uid, Thread_halt_cbk_.Noop); } private void Redo_task(Xobc_task_itm task, Xobc_step_itm step, Xobc_cmd_itm cmd) { - cmd.Cmd_clear(); cmd = step.Step_fallback_to(cmd.Cmd_fallback()); + cmd.Cmd_clear(); task.Task_status_(gplx.core.progs.Gfo_prog_ui_.Status__working); task_mgr.Send_json("xo.bldr.work.prog__start__recv", task.Save_to(Gfobj_nde.New())); thread_mgr.Add(cmd.Cmd_uid(), Thread_adp_.Start_by_val("xobc: " + cmd.Cmd_name(), cmd, cmd, Xobc_cmd__base.Invk__exec, ctx)); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/utils/Bry_eval_wkr__builder_central.java b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/utils/Bry_eval_wkr__builder_central.java index a762d54ea..88e892a50 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/utils/Bry_eval_wkr__builder_central.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/utils/Bry_eval_wkr__builder_central.java @@ -22,7 +22,7 @@ public class Bry_eval_wkr__builder_central implements Bry_eval_wkr { public Bry_eval_wkr__builder_central(Io_url wiki_dir) {this.wiki_dir = wiki_dir.RawBry();} public String Key() {return "builder_central";} public void Resolve(Bry_bfr rv, byte[] src, int args_bgn, int args_end) { - // EX: "~{builder_central|download_fil|en.wikipedia.org|en.wikipedia.org-2016.05-html-ns.000-db.001.zip}" -> "/xowa/wiki/en.wikipedia.org/tmp/bldr/en.wikipedia.org-2016.05-html-ns.000-db.001.zip/download.zip" + // EX: "~{builder_central|download_fil|en.wikipedia.org|Xowa_enwiki_2016-05_html_ns.000_db.001.zip}" -> "/xowa/wiki/en.wikipedia.org/tmp/bldr/Xowa_enwiki_2016-05_html_ns.000_db.001.zip/download.zip" byte[][] args = Bry_split_.Split(src, args_bgn, args_end, Byte_ascii.Pipe, Bool_.N); int type = hash.Get_as_byte_or(args[0], Byte_.Max_value_127); if (type == Byte_.Max_value_127) throw Err_.new_wo_type("unknown eval type", "src", src); @@ -32,7 +32,7 @@ public class Bry_eval_wkr__builder_central implements Bry_eval_wkr { if (type == Type__wiki_dir) return; rv.Add_str_a7("tmp").Add_byte(dir_spr); // "tmp/" rv.Add_str_a7("bldr").Add_byte(dir_spr); // "bldr/" - rv.Add(args[2]).Add_byte(dir_spr); // "en.wikipedia.org-2016.05-html-ns.000-db.001.zip/" + rv.Add(args[2]).Add_byte(dir_spr); // "Xowa_enwiki_2016-05_html_ns.000_db.001.zip/" switch (type) { case Type__download_fil: rv.Add_str_a7("download.zip"); break; case Type__unzip_dir: rv.Add_str_a7("unzip").Add_byte(dir_spr); break; diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java index 636d31a6c..fee0124e1 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java @@ -21,6 +21,7 @@ public class Pack_file_bldr_cfg implements Gfo_invk { public Io_url Deploy_dir() {return deploy_dir;} private Io_url deploy_dir; public boolean Pack_html() {return pack_html;} private boolean pack_html = true; public boolean Pack_file() {return pack_file;} private boolean pack_file = true; + public boolean Pack_fsdb_delete() {return pack_fsdb_delete;} private boolean pack_fsdb_delete; public DateAdp Pack_file_cutoff() {return pack_file_cutoff;} private DateAdp pack_file_cutoff = null; public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { @@ -28,10 +29,12 @@ public class Pack_file_bldr_cfg implements Gfo_invk { else if (ctx.Match(k, Invk__pack_html_)) pack_html = m.ReadYn("v"); else if (ctx.Match(k, Invk__pack_file_)) pack_file = m.ReadYn("v"); else if (ctx.Match(k, Invk__pack_file_cutoff_)) pack_file_cutoff = m.ReadDate("v"); + else if (ctx.Match(k, Invk__pack_fsdb_delete_)) pack_fsdb_delete = m.ReadYn("v"); else return Gfo_invk_.Rv_unhandled; return this; } private static final String Invk__deploy_dir_ = "deploy_dir_" , Invk__pack_html_ = "pack_html_", Invk__pack_file_ = "pack_file_", Invk__pack_file_cutoff_ = "pack_file_cutoff_" + , Invk__pack_fsdb_delete_ = "pack_fsdb_delete_" ; } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java index 63427ad85..42227f192 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java @@ -27,7 +27,7 @@ class Pack_file_mgr { Io_url pack_dir = wiki_dir.GenSubDir_nest("tmp", "pack"); Io_mgr.Instance.DeleteDirDeep(pack_dir); Io_mgr.Instance.CreateDirIfAbsent(pack_dir); String wiki_date = wiki.Props().Modified_latest().XtoStr_fmt("yyyy.MM"); - Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg.Pack_html(), cfg.Pack_file(), cfg.Pack_file_cutoff()); + Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg.Pack_html(), cfg.Pack_file(), cfg.Pack_file_cutoff(), cfg.Pack_fsdb_delete()); // get import_tbl byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo(); @@ -51,9 +51,9 @@ class Pack_file_mgr { // build tasks if (cfg.Pack_html()) - Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html); + Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html, Xobc_import_type.Tid__wiki__ctg); if (cfg.Pack_file()) - Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data); + Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data); // , Xobc_import_type.Tid__fsdb__delete bc_conn.Txn_end(); // deploy @@ -163,7 +163,7 @@ class Pack_file_mgr { } } class Pack_hash_bldr { - public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff) { + public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff, boolean pack_fsdb_delete) { Pack_hash rv = new Pack_hash(); Pack_zip_name_bldr zip_name_bldr = new Pack_zip_name_bldr(pack_dir, wiki.Domain_str(), String_.new_a7(wiki.Domain_itm().Abrv_wm()), wiki_date); Xow_db_mgr db_mgr = wiki.Data__core_mgr(); @@ -200,6 +200,13 @@ class Pack_hash_bldr { } } } + + // bld pack_fsdb_delete + if (pack_fsdb_delete) { + gplx.xowa.bldrs.Xob_db_file fsdb_deletion_db = gplx.xowa.bldrs.Xob_db_file.New__deletion_db(wiki); + if (!Io_mgr.Instance.ExistsFil(fsdb_deletion_db.Url())) throw Err_.new_wo_type("deletion db does not exists: url=" + fsdb_deletion_db.Url().Raw()); + rv.Add(zip_name_bldr, Xobc_import_type.Tid__fsdb__delete, fsdb_deletion_db.Url()); + } return rv; } private static int Get_pack_tid(byte db_file_tid) { @@ -209,6 +216,9 @@ class Pack_hash_bldr { case Xow_db_file_.Tid__search_link: return Xobc_import_type.Tid__wiki__srch; case Xow_db_file_.Tid__html_solo: case Xow_db_file_.Tid__html_data: return Xobc_import_type.Tid__wiki__html; + case Xow_db_file_.Tid__cat: + case Xow_db_file_.Tid__cat_core: + case Xow_db_file_.Tid__cat_link: return Xobc_import_type.Tid__wiki__ctg; case Xow_db_file_.Tid__file_core: return Xobc_import_type.Tid__file__core; case Xow_db_file_.Tid__file_solo: case Xow_db_file_.Tid__file_data: return Xobc_import_type.Tid__file__data; @@ -216,20 +226,3 @@ class Pack_hash_bldr { } } } -class Pack_zip_name_bldr { // en.wikipedia.org-file-ns.000-db.001.xowa -> Xowa_enwiki_2016-05_file_ns.000_db.001.zip - private final Io_url pack_dir; - private final byte[] wiki_domain, zip_name_prefix; - - public Pack_zip_name_bldr(Io_url pack_dir, String wiki_domain_str, String wiki_abrv, String wiki_date) { - this.pack_dir = pack_dir; - this.wiki_domain = Bry_.new_u8(wiki_domain_str); - this.zip_name_prefix = Bry_.new_u8("Xowa_" + wiki_abrv + "_" + String_.Replace(wiki_date, ".", "-")); - } - public Io_url Bld(Io_url orig_url) { - String orig_str = String_.Replace(orig_url.NameAndExt(), ".xowa", ".zip"); - byte[] orig_bry = Bry_.new_u8(orig_str); - orig_bry = Bry_.Replace(orig_bry, Byte_ascii.Dash, Byte_ascii.Underline); - orig_bry = Bry_.Replace(orig_bry, wiki_domain, zip_name_prefix); - return pack_dir.GenSubFil(String_.new_u8(orig_bry)); - } -} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr.java new file mode 100644 index 000000000..577b4b043 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr.java @@ -0,0 +1,56 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*; +public class Pack_zip_name_bldr { // en.wikipedia.org-file-ns.000-db.001.xowa -> Xowa_enwiki_2016-05_file_ns.000_db.001.zip + private final Io_url pack_dir; + private final byte[] wiki_domain, zip_name_prefix; + + public Pack_zip_name_bldr(Io_url pack_dir, String wiki_domain_str, String wiki_abrv, String wiki_date) { + this.pack_dir = pack_dir; + this.wiki_domain = Bry_.new_u8(wiki_domain_str); + this.zip_name_prefix = Bry_.new_u8("Xowa_" + wiki_abrv + "_" + String_.Replace(wiki_date, ".", "-")); + } + public Io_url Bld(Io_url orig_url) { + String orig_str = orig_url.NameOnly() + ".zip"; + byte[] orig_bry = Bry_.new_u8(orig_str); + orig_bry = Bry_.Replace(orig_bry, Byte_ascii.Dash, Byte_ascii.Underline); + orig_bry = Bry_.Replace(orig_bry, wiki_domain, zip_name_prefix); + return pack_dir.GenSubFil(String_.new_u8(orig_bry)); + } + public static Io_url To_wiki_url(Io_url wiki_dir, Io_url zip_dir) { + // get wiki_url based on wiki_dir and xobc_zip_fil; EX: "/wiki/en.wikipedia.org/", "/wiki/tmp/Xowa_enwiki_2016-09_file_core_deletion_2016-09/" -> "/wiki/en.wikipedia.org-file-core-deletion-2016.09.zip" + String name_str = zip_dir.NameOnly() + ".xowa"; + byte[] name_bry = Bry_.new_u8(name_str); + int pos = Bry_find__Find_fwd_idx(name_bry, Byte_ascii.Underline, 2); + name_bry = Bry_.Mid(name_bry, pos, name_bry.length); + name_bry = Bry_.Add(Bry_.new_u8(wiki_dir.NameOnly()), name_bry); + name_bry = Bry_.Replace(name_bry, Byte_ascii.Underline, Byte_ascii.Dash); + return wiki_dir.GenSubFil(String_.new_u8(name_bry)); + } + private static int Bry_find__Find_fwd_idx(byte[] src, byte val, int find_max) { + int src_len = src.length; + int find_cur = 0; + int cur_pos = 0; + while (true) { + int new_pos = Bry_find_.Find_fwd(src, val, cur_pos, src_len); + if (new_pos == -1) throw Err_.New("failed to find value; src={0} val={1}", src, val); + if (find_cur++ == find_max) return new_pos; + cur_pos = new_pos + 1; + } + } +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr__tst.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr__tst.java new file mode 100644 index 000000000..36c9d10ec --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr__tst.java @@ -0,0 +1,30 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*; +import org.junit.*; import gplx.core.tests.*; +public class Pack_zip_name_bldr__tst { + private Pack_zip_name_bldr__fxt fxt = new Pack_zip_name_bldr__fxt(); + @Test public void Basic() { + fxt.Test__to_wiki_url("mem/wiki/en.wikipedia.org/", "mem/wiki/en.wikipedia.org/tmp/Xowa_enwiki_2016-09_file_deletion_2016.09/", "mem/wiki/en.wikipedia.org/en.wikipedia.org-file-deletion-2016.09.xowa"); + } +} +class Pack_zip_name_bldr__fxt { + public void Test__to_wiki_url(String wiki_dir, String zip_fil, String expd) { + Gftest.Eq__str(expd, Pack_zip_name_bldr.To_wiki_url(Io_url_.mem_fil_(wiki_dir), Io_url_.mem_dir_(zip_fil)).Raw(), "wiki_url"); + } +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/splits/files/Split_meta_wkr__fil.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/splits/files/Split_meta_wkr__fil.java index 98bf28c51..8ff34d1fb 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/exports/splits/files/Split_meta_wkr__fil.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/splits/files/Split_meta_wkr__fil.java @@ -47,7 +47,7 @@ class Split_meta_wkr__fil extends Split_meta_wkr_base { ); } @Override protected Object Load_itm(Db_rdr rdr) { - return tbl.Load_by_rdr__full(Fsm_mnt_mgr.Mnt_idx_main, rdr); + return tbl.New_by_rdr(Fsm_mnt_mgr.Mnt_idx_main, rdr); } @Override protected void Save_itm(Split_ctx ctx, Split_rslt_mgr rslt_mgr, Object itm_obj) { Fsd_fil_itm itm = (Fsd_fil_itm)itm_obj; diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__fsdb_db__create_data.java b/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__fsdb_db__create_data.java index ea3646e8c..da0f149f5 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__fsdb_db__create_data.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__fsdb_db__create_data.java @@ -185,7 +185,7 @@ public class Xobldr__fsdb_db__create_data extends Xob_cmd__base implements Xob_c tier_id_val = fsdb.Lnki_tier_id(); page_id_val = fsdb.Lnki_page_id(); lnki_id_val = fsdb.Lnki_id(); - fsdb.Orig_repo_name_(fsdb.Orig_repo_id() == Xof_repo_itm_.Repo_local ? wiki.Domain_bry() : Xow_domain_itm_.Bry__commons); + fsdb.Orig_repo_name_(fsdb.Orig_repo_id() == Xof_repo_tid_.Tid__local ? wiki.Domain_bry() : Xow_domain_itm_.Bry__commons); Download_exec(fsdb); ++exec_count; if (exec_count % progress_interval == 0) Print_progress(fsdb); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__image__create.java b/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__image__create.java index e23849c74..27ae3d37d 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__image__create.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__image__create.java @@ -17,16 +17,20 @@ along with this program. If not, see . */ package gplx.xowa.addons.bldrs.files.cmds; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.dbs.*; import gplx.core.ios.*; import gplx.xowa.files.*; -import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*; +import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sql_dumps.*; import gplx.xowa.addons.bldrs.files.dbs.*; -public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd, Gfo_invk, Sql_file_parser_cmd { +public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd, Gfo_invk, Xosql_dump_cbk { + private Xosql_dump_parser parser; private Db_conn conn = null; private Db_stmt stmt = null; private Xob_image_tbl tbl_image = new Xob_image_tbl(); private byte[] cur_ttl, cur_media_type, cur_minor_mime, cur_timestamp; private int cur_size, cur_width, cur_height, cur_bits, cur_ext_id; private int commit_count = 10000; - public Xobldr__image__create(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);} + public Xobldr__image__create(Xob_bldr bldr, Xowe_wiki wiki) { + this.parser = new Xosql_dump_parser(this, "img_name", "img_size", "img_width", "img_height", "img_bits", "img_media_type", "img_minor_mime", "img_timestamp"); + this.Cmd_ctor(bldr, wiki); + } public Io_url Src_fil() {return src_fil;} public Xobldr__image__create Src_fil_(Io_url v) {src_fil = v; return this;} private Io_url src_fil; - public Sql_file_parser Parser() {return parser;} private Sql_file_parser parser = new Sql_file_parser(); + public Xosql_dump_parser Parser() {return parser;} public void Cmd_init(Xob_bldr bldr) {} public void Cmd_bgn(Xob_bldr bldr) { wiki.Init_assert(); // NOTE: must init wiki for db_mgr_as_sql @@ -35,7 +39,7 @@ public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd, src_fil = Xob_page_wkr_cmd.Find_fil_by(wiki.Fsys_mgr().Root_dir(), "*-image.sql"); if (src_fil == null) throw Err_.new_wo_type(".sql file not found in dir", "dir", wiki.Fsys_mgr().Root_dir()); } - parser.Src_fil_(src_fil).Trg_fil_gen_(dump_url_gen).Fld_cmd_(this).Flds_req_idx_(20, Fld_img_name, Fld_img_size, Fld_img_width, Fld_img_height, Fld_img_bits, Fld_img_media_type, Fld_img_minor_mime, Fld_img_timestamp); + parser.Src_fil_(src_fil); this.conn = Xob_db_file.New__wiki_image(wiki.Fsys_mgr().Root_dir()).Conn(); conn.Txn_bgn("bldr__image"); this.tbl_image = new Xob_image_tbl(); @@ -47,16 +51,16 @@ public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd, tbl_image.Create_index(conn); conn.Txn_end(); } - public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) { + public void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end) { switch (fld_idx) { - case Fld_img_name: cur_ttl = Bry_.Mid(src, fld_bgn, fld_end); break; - case Fld_img_size: cur_size = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break; - case Fld_img_width: cur_width = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break; - case Fld_img_height: cur_height = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break; - case Fld_img_bits: cur_bits = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break; - case Fld_img_media_type: cur_media_type = Bry_.Mid(src, fld_bgn, fld_end); break; - case Fld_img_minor_mime: cur_minor_mime = Bry_.Mid(src, fld_bgn, fld_end); break; - case Fld_img_timestamp: cur_timestamp = Bry_.Mid(src, fld_bgn, fld_end); + case Fld_img_name: cur_ttl = Bry_.Mid(src, val_bgn, val_end); break; + case Fld_img_size: cur_size = Bry_.To_int_or(src, val_bgn, val_end, -1); break; + case Fld_img_width: cur_width = Bry_.To_int_or(src, val_bgn, val_end, -1); break; + case Fld_img_height: cur_height = Bry_.To_int_or(src, val_bgn, val_end, -1); break; + case Fld_img_bits: cur_bits = Bry_.To_int_or(src, val_bgn, val_end, -1); break; + case Fld_img_media_type: cur_media_type = Bry_.Mid(src, val_bgn, val_end); break; + case Fld_img_minor_mime: cur_minor_mime = Bry_.Mid(src, val_bgn, val_end); break; + case Fld_img_timestamp: cur_timestamp = Bry_.Mid(src, val_bgn, val_end); cur_ext_id = Calc_ext_id(show_issues ? app.Usr_dlg() : Gfo_usr_dlg_.Noop, cur_ttl, cur_media_type, cur_minor_mime, cur_width, cur_height); tbl_image.Insert(stmt, cur_ttl, cur_media_type, cur_minor_mime, cur_size, cur_width, cur_height, cur_bits, cur_ext_id, cur_timestamp); ++commit_count; @@ -70,7 +74,8 @@ public class Xobldr__image__create extends Xob_itm_dump_base implements Xob_cmd, public void Cmd_end() {} public void Cmd_term() {} private boolean show_issues = true; - private static final int Fld_img_name = 0, Fld_img_size = 1, Fld_img_width = 2, Fld_img_height = 3, Fld_img_bits = 5, Fld_img_media_type = 6, Fld_img_minor_mime = 8, Fld_img_timestamp = 12; + private static final int Fld_img_name = 0, Fld_img_size = 1, Fld_img_width = 2, Fld_img_height = 3, Fld_img_bits = 4, Fld_img_media_type = 5, Fld_img_minor_mime = 6, Fld_img_timestamp = 7; + // Fld_img_name = 0, Fld_img_size = 1, Fld_img_width = 2, Fld_img_height = 3, Fld_img_bits = 5, Fld_img_media_type = 6, Fld_img_minor_mime = 8, Fld_img_timestamp = 12; @Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { if (ctx.Match(k, Invk_src_fil_)) src_fil = m.ReadIoUrl("v"); else if (ctx.Match(k, Invk_show_issues_)) show_issues = m.ReadYn("v"); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__page_regy__create.java b/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__page_regy__create.java index 320e53f4f..01c90e05f 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__page_regy__create.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__page_regy__create.java @@ -28,14 +28,14 @@ public class Xobldr__page_regy__create extends Xob_cmd__base { commons_wiki.Init_assert(); if (build_commons) { Xob_page_regy_tbl.Reset_table(page_regy_provider); - Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_itm_.Repo_remote, commons_wiki); + Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_tid_.Tid__remote, commons_wiki); Sqlite_engine_.Idx_create(usr_dlg, page_regy_provider, "repo_page", Xob_page_regy_tbl.Idx_main); } else { if (!Bry_.Eq(commons_wiki.Domain_bry(), wiki.Domain_bry())) { // skip local wiki if cur wiki is commons wiki.Init_assert(); Xob_page_regy_tbl.Delete_local(page_regy_provider); - Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_itm_.Repo_local, wiki); + Xob_page_regy_tbl.Create_data(bldr.Usr_dlg(), page_regy_provider, Xof_repo_tid_.Tid__local, wiki); } } } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_orig_regy_tbl.java b/400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_orig_regy_tbl.java index 214ac0ed3..8720d3f0c 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_orig_regy_tbl.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_orig_regy_tbl.java @@ -29,7 +29,7 @@ public class Xob_orig_regy_tbl { Sqlite_engine_.Idx_create(usr_dlg, p, "orig_regy", Idx_ttl_local); Sqlite_engine_.Db_attach(p, "page_db", file_registry_db.Url().Raw()); Io_url repo_0_dir = repo_0_wiki.Fsys_mgr().Root_dir(), repo_1_dir = repo_1_wiki.Fsys_mgr().Root_dir(); - byte repo_0_tid = Xof_repo_itm_.Repo_local, repo_1_tid = Xof_repo_itm_.Repo_remote; + byte repo_0_tid = Xof_repo_tid_.Tid__local, repo_1_tid = Xof_repo_tid_.Tid__remote; boolean local_is_remote = Bry_.Eq(repo_0_wiki.Domain_bry(), repo_1_wiki.Domain_bry()); Xowe_wiki local_wiki = repo_0_wiki; if ( repo_0_is_remote // .gfs manually marked specifes repo_0 as remote @@ -37,8 +37,8 @@ public class Xob_orig_regy_tbl { && local_is_remote // repo_0 = repo_1 ) ) { - repo_0_tid = Xof_repo_itm_.Repo_remote; - repo_1_tid = Xof_repo_itm_.Repo_local; + repo_0_tid = Xof_repo_tid_.Tid__remote; + repo_1_tid = Xof_repo_tid_.Tid__local; local_wiki = repo_1_wiki; } Create_data_for_repo(usr_dlg, p, local_wiki, Byte_.By_int(repo_0_tid), repo_0_dir.GenSubFil(Xob_db_file.Name__wiki_image)); @@ -54,18 +54,18 @@ public class Xob_orig_regy_tbl { } private static void Create_data_for_repo(Gfo_usr_dlg usr_dlg, Db_conn conn, Xowe_wiki local_wiki, byte repo_tid, Io_url join) { usr_dlg.Note_many("", "", "inserting page for xowa.wiki.image: ~{0}", join.OwnerDir().NameOnly()); - boolean wiki_has_cs_file = repo_tid == Xof_repo_itm_.Repo_remote && local_wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all; + boolean wiki_has_cs_file = repo_tid == Xof_repo_tid_.Tid__remote && local_wiki.Ns_mgr().Ns_file().Case_match() == Xow_ns_case_.Tid__all; String lnki_ttl_fld = wiki_has_cs_file ? "Coalesce(o.lnki_commons_ttl, o.lnki_ttl)" : "o.lnki_ttl"; // NOTE: use lnki_commons_ttl if [[File]] is cs PAGE:en.d:water EX:[[image:wikiquote-logo.png|50px|none|alt=]]; DATE:2014-09-05 if (wiki_has_cs_file) Sqlite_engine_.Idx_create(usr_dlg, conn, "orig_regy", Idx_ttl_remote); new Db_attach_mgr(conn, new Db_attach_itm("image_db", join)) - .Exec_sql_w_msg("orig_regy:updating page" , Sql_update_repo_page, repo_tid, lnki_ttl_fld) - .Exec_sql_w_msg("orig_regy:updating redirect" , Sql_update_repo_redirect, repo_tid, lnki_ttl_fld); + .Exec_sql_w_msg("orig_regy:updating page" , Sql_update_repo_page , repo_tid, lnki_ttl_fld) + .Exec_sql_w_msg("orig_regy:updating redirect" , Sql_update_repo_redirect , repo_tid, lnki_ttl_fld); } private static void Create_data_for_cs(Gfo_usr_dlg usr_dlg, Db_conn p, Xowe_wiki local_wiki, Io_url repo_remote_dir) { p.Exec_sql(Xob_orig_regy_tbl.Sql_cs_mark_dupes); // orig_regy: find dupes; see note in SQL p.Exec_sql(Xob_orig_regy_tbl.Sql_cs_update_ttls); // orig_regy: update lnki_ttl with lnki_commons_ttl - Create_data_for_repo(usr_dlg, p, local_wiki, Xof_repo_itm_.Repo_remote, repo_remote_dir.GenSubFil(Xob_db_file.Name__wiki_image)); + Create_data_for_repo(usr_dlg, p, local_wiki, Xof_repo_tid_.Tid__remote, repo_remote_dir.GenSubFil(Xob_db_file.Name__wiki_image)); p.Exec_sql(Xob_lnki_regy_tbl.Sql_cs_mark_changed); // lnki_regy: update lnki_commons_flag p.Exec_sql(Xob_lnki_regy_tbl.Sql_cs_update_ttls); // lnki_regy: update cs } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_page_regy_tbl.java b/400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_page_regy_tbl.java index 7aab08b3a..8c45e875b 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_page_regy_tbl.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/files/dbs/Xob_page_regy_tbl.java @@ -26,7 +26,7 @@ public class Xob_page_regy_tbl { Create_data__insert_redirect(usr_dlg, p, repo_tid, wiki.Fsys_mgr().Root_dir().GenSubFil(Xob_db_file.Name__wiki_redirect)); } public static void Delete_local(Db_conn p) { - p.Exec_sql("DELETE FROM page_regy WHERE repo_id = " + Xof_repo_itm_.Repo_local); + p.Exec_sql("DELETE FROM page_regy WHERE repo_id = " + Xof_repo_tid_.Tid__local); } private static void Create_data__insert_page(Gfo_usr_dlg usr_dlg, Db_conn cur, byte repo_tid, Io_url join) { usr_dlg.Note_many("", "", "inserting page: ~{0}", join.NameOnly()); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_bin_db_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_bin_db_mgr.java index 16404f05e..96f173e66 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_bin_db_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_bin_db_mgr.java @@ -41,7 +41,8 @@ public class Xob_bin_db_mgr { Xob_bin_db_itm itm = schema_is_1 ? Xob_bin_db_itm.new_v1(fil) : Xob_bin_db_itm.new_v2(fil); int ns_id = itm.Ns_id(); Xob_bin_db_itm nth = (Xob_bin_db_itm)nth_hash.Get_by(tier_key.Val_(ns_id)); - if (itm.Pt_id() > nth.Pt_id()) // update max pt_id + if ( nth != null // occurs when existing fsdb_dbb has "file-ns.014-db.001", but 14 no longer specified in fsdb_make; DATE:2016-09-23 + && itm.Pt_id() > nth.Pt_id()) // update max pt_id nth.Set(itm.Id(), itm.Pt_id(), itm.Db_url()); // note that ns_id is same } len = nth_hash.Count(); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_xfer_temp_itm_tst.java b/400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_xfer_temp_itm_tst.java index d18b65fa8..b3d57676b 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_xfer_temp_itm_tst.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/files/utls/Xob_xfer_temp_itm_tst.java @@ -134,7 +134,7 @@ class Xob_xfer_temp_itm_fxt { public Xob_xfer_temp_itm_fxt Init_rdr_image() { GfoFldList flds = GfoFldList_.str_(Flds); nde = GfoNde_.vals_(flds, Object_.Ary - ( Xof_ext_.Id_png, 1, 1, Xof_repo_itm_.Repo_remote + ( Xof_ext_.Id_png, 1, 1, Xof_repo_tid_.Tid__remote , "A.png", Xof_ext_.Id_png, "A.png", Xop_lnki_type.Id_thumb, Xop_file_logger_.Tid__file , 220, 200, 1, 2, 440, 400, 3 , Xop_lnki_tkn.Upright_null, Xof_lnki_time.Null, Xof_lnki_page.Null diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_mgr_db.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_mgr_db.java index 5816445ff..1f83f44a6 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_mgr_db.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_mgr_db.java @@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; -import gplx.dbs.*; +import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.pools.*; import gplx.xowa.addons.bldrs.mass_parses.parses.locks.*; public class Xomp_mgr_db { public Xomp_mgr_db(Io_url url) { @@ -24,18 +24,20 @@ public class Xomp_mgr_db { this.conn = Db_conn_bldr.Instance.Get_or_autocreate(true, url); this.page_tbl = new Xomp_page_tbl(conn); this.wkr_tbl = new Xomp_wkr_tbl(conn); + this.cfg_tbl = new Db_cfg_tbl(conn, "xowa_cfg"); // this.lock_mgr = new Xomp_lock_mgr__db(conn, 5000); this.lock_mgr = new Xomp_lock_mgr__fsys(5000, this.Dir()); } - public Db_conn Conn() {return conn;} private final Db_conn conn; - public Io_url Url() {return url;} private final Io_url url; - public Io_url Dir() {return url.OwnerDir();} - public Xomp_page_tbl Page_tbl() {return page_tbl;} private final Xomp_page_tbl page_tbl; - public Xomp_wkr_tbl Wkr_tbl() {return wkr_tbl;} private final Xomp_wkr_tbl wkr_tbl; - public Xomp_lock_mgr Lock_mgr() {return lock_mgr;} private final Xomp_lock_mgr lock_mgr; + public Db_conn Conn() {return conn;} private final Db_conn conn; + public Io_url Url() {return url;} private final Io_url url; + public Io_url Dir() {return url.OwnerDir();} + public Xomp_page_tbl Tbl__page() {return page_tbl;} private final Xomp_page_tbl page_tbl; + public Xomp_wkr_tbl Tbl__wkr() {return wkr_tbl;} private final Xomp_wkr_tbl wkr_tbl; + public Db_cfg_tbl Tbl__cfg() {return cfg_tbl;} private final Db_cfg_tbl cfg_tbl; + public Xomp_lock_mgr Lock_mgr() {return lock_mgr;} private final Xomp_lock_mgr lock_mgr; public void Remake() { - conn.Meta_tbl_remake_many(page_tbl, wkr_tbl); + conn.Meta_tbl_remake_many(page_tbl, wkr_tbl, cfg_tbl); lock_mgr.Remake(); } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_mgr.java index bdf9c771c..7b260d987 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/inits/Xomp_init_mgr.java @@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.addons.bldrs.mass_parses.inits; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; -import gplx.dbs.*; +import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*; class Xomp_init_mgr { @@ -27,9 +27,12 @@ class Xomp_init_mgr { Xomp_mgr_db mgr_db = Xomp_mgr_db.New__make(wiki); Db_conn mgr_conn = mgr_db.Conn(); - // remake all + // remake all tbls mgr_db.Remake(); + // insert ns into cfg; need for lnki_temp.tier in xomp.make + mgr_db.Tbl__cfg().Insert_str("", gplx.xowa.addons.bldrs.mass_parses.parses.wkrs.Xomp_parse_wkr.Cfg__ns_ids, Int_.Ary_concat("|", cfg.Ns_ids())); + // fill page tbl Db_attach_mgr attach_mgr = new Db_attach_mgr(mgr_conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Conn())); int[] ns_ary = cfg.Ns_ids(); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_rdr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_rdr.java index 61bd591fd..20eb180b4 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_rdr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_rdr.java @@ -23,7 +23,7 @@ class Xomp_html_db_rdr { private final Xomp_mgr_db mgr_db; public Xomp_html_db_rdr(Xowe_wiki wiki) { this.mgr_db = Xomp_mgr_db.New__load(wiki); - this.src_tbls = new Xowd_html_tbl[mgr_db.Wkr_tbl().Select_count()]; + this.src_tbls = new Xowd_html_tbl[mgr_db.Tbl__wkr().Select_count()]; } public void Rows__get(Xowd_html_row rv, int wkr_uid, int page_id) { Xowd_html_tbl src_tbl = src_tbls[wkr_uid]; diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_wtr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_wtr.java index d3496ff79..48ab957e3 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_wtr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_wtr.java @@ -17,11 +17,14 @@ along with this program. If not, see . */ package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.wikis.data.*; +import gplx.xowa.bldrs.cmds.*; class Xomp_html_db_wtr { private final long len_max; private final Xowe_wiki wiki; private final Xow_db_mgr db_mgr; private long len_cur; + private int prv_ns_id = -1; private Xow_db_file html_db; private Xowd_html_tbl html_tbl; + private Xob_ns_file_itm ns_itm; public Xomp_html_db_wtr(Xowe_wiki wiki) { this.wiki = wiki; this.db_mgr = wiki.Data__core_mgr(); this.len_max = wiki.Appe().Api_root().Bldr().Wiki().Import().Html_db_max(); @@ -34,18 +37,19 @@ class Xomp_html_db_wtr { public Xowd_html_tbl Tbls__get_or_new(int ns_id, long html_len) { long len_new = len_cur + html_len; boolean not_inited = html_tbl == null, out_of_space = len_new > len_max; - if (not_inited || out_of_space) { - if (out_of_space) - Commit(); - if ( db_mgr.Props().Layout_html().Tid_is_all_or_few() // is not "lot" - && not_inited // not_inited; set html_db + boolean is_all_or_few = db_mgr.Props().Layout_html().Tid_is_all_or_few(); + boolean ns_changed = ns_id != prv_ns_id; + if (not_inited || out_of_space || ns_changed) { + Commit(); + if ( is_all_or_few // is not "lot" + && not_inited // not_inited; set html_db ) { this.html_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__html_data); if (html_db == null) - this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data); + Make_html_db(is_all_or_few, ns_id); } else - this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data); + Make_html_db(is_all_or_few, ns_id); this.html_tbl = new Xowd_html_tbl(html_db.Conn()); html_tbl.Create_tbl(); @@ -56,10 +60,20 @@ class Xomp_html_db_wtr { len_cur = len_new; return html_tbl; } + private void Make_html_db(boolean is_all_or_few, int ns_id) { + if (prv_ns_id != ns_id + || ns_itm == null) { + prv_ns_id = ns_id; + ns_itm = new Xob_ns_file_itm(Xow_db_file_.Tid__html_data, "ns." + Int_.To_str_pad_bgn_zero(ns_id, 3), Int_.Ary(ns_id)); + } + String file_name = is_all_or_few ? "-html.xowa" : ns_itm.Make_file_name(); + this.html_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__html_data, Int_.To_str(ns_id), ns_itm.Nth_db_idx(), file_name); + } public void Rls() { this.Commit(); } private void Commit() { + if (html_tbl == null) return; html_tbl.Conn().Txn_end(); html_tbl.Conn().Rls_conn(); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_html.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_html.java index 62deac351..905bf55e2 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_html.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_html.java @@ -26,10 +26,10 @@ class Xomp_make_html { Db_conn mgr_conn = mgr_db.Conn(); // update wkr_uid; note that this cannot be done in parse_wkr, b/c multiple-writer-errors for xomp.db|page - int wkrs_len = mgr_db.Wkr_tbl().Select_count(); + int wkrs_len = mgr_db.Tbl__wkr().Select_count(); for (int i = 0; i < wkrs_len; ++i) { Xomp_wkr_db wkr_db = Xomp_wkr_db.New(mgr_db.Dir(), i); - mgr_db.Page_tbl().Update_wkr_uid(i, wkr_db.Conn()); + mgr_db.Tbl__page().Update_wkr_uid(i, wkr_db.Conn()); } // init more diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java index 94370e1eb..184229cc0 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_lnki.java @@ -30,7 +30,7 @@ class Xomp_make_lnki { lnki_temp_tbl.Insert_bgn(); // create ary; add index - int wkr_count = xomp_db.Wkr_tbl().Select_count(); + int wkr_count = xomp_db.Tbl__wkr().Select_count(); Xomp_wkr_db[] db_ary = new Xomp_wkr_db[wkr_count]; for (int i = 0; i < wkr_count; ++i) { Xomp_wkr_db wkr_db = Xomp_wkr_db.New(xomp_db.Dir(), i); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java index 7431acf66..cdb742ea4 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java @@ -51,10 +51,13 @@ public class Xomp_parse_mgr { // assert wkr_tbl int wkr_len = cfg.Num_wkrs(); - int wkr_uid_bgn = mgr_db.Wkr_tbl().Init_wkrs(cfg.Wkr_machine_name(), wkr_len); + int wkr_uid_bgn = mgr_db.Tbl__wkr().Init_wkrs(cfg.Wkr_machine_name(), wkr_len); latch = new Gfo_countdown_latch(wkr_len); Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len]; + // init ns_ord_mgr + Xomp_ns_ord_mgr ns_ord_mgr = new Xomp_ns_ord_mgr(Int_.Ary_parse(mgr_db.Tbl__cfg().Select_str("", Xomp_parse_wkr.Cfg__ns_ids), "|")); + // init parse_wkrs for (int i = 0; i < wkr_len; ++i) { // make wiki @@ -62,7 +65,7 @@ public class Xomp_parse_mgr { wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache); // make wkr - Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, cfg, mgr_db, page_pool, prog_mgr, file_orig_wkr, wkr_wiki, i + wkr_uid_bgn); + Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, cfg, mgr_db, page_pool, prog_mgr, file_orig_wkr, ns_ord_mgr, wkr_wiki, i + wkr_uid_bgn); wkrs[i] = wkr; } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_ns_ord_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_ns_ord_mgr.java new file mode 100644 index 000000000..ae1e27261 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_ns_ord_mgr.java @@ -0,0 +1,29 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; +import gplx.core.primitives.*; import gplx.core.lists.hashs.*; +public class Xomp_ns_ord_mgr { + private final Hash_adp__int hash = new Hash_adp__int(); + public Xomp_ns_ord_mgr(int[] ns_id_ary) { + int len = ns_id_ary.length; + for (int i = 0; i < len; ++i) { + hash.Add(ns_id_ary[i], new Int_obj_val(i)); + } + } + public int Get_ord_by_ns_id(int ns_id) {return ((Int_obj_val)hash.Get_by_or_fail(ns_id)).Val();} +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java index bb7b8547c..b0065bca6 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java @@ -28,6 +28,7 @@ public class Xomp_parse_wkr implements Gfo_invk { private final Xomp_prog_mgr prog_mgr; private final Xomp_page_pool page_pool; private final Xof_orig_wkr file_orig_wkr; + private final Xomp_ns_ord_mgr ns_ord_mgr; // cfg vars private final Xomp_parse_mgr_cfg cfg; @@ -42,10 +43,11 @@ public class Xomp_parse_wkr implements Gfo_invk { private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0; private int done_count; private long done_time; - public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg, Xomp_mgr_db mgr_db, Xomp_page_pool page_pool, Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xowe_wiki wiki, int uid) { + public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xomp_parse_mgr_cfg cfg, Xomp_mgr_db mgr_db, Xomp_page_pool page_pool, Xomp_prog_mgr prog_mgr, Xof_orig_wkr file_orig_wkr, Xomp_ns_ord_mgr ns_ord_mgr, Xowe_wiki wiki, int uid) { // mgr vars this.mgr = mgr; this.mgr_db = mgr_db; this.page_pool = page_pool; this.prog_mgr = prog_mgr; this.file_orig_wkr = file_orig_wkr; + this.ns_ord_mgr = ns_ord_mgr; // cfg vars this.cfg = cfg; @@ -67,7 +69,7 @@ public class Xomp_parse_wkr implements Gfo_invk { wiki.File__orig_mgr().Wkrs__set(file_orig_wkr); wiki.File_mgr().Fsdb_mode().Tid__v2__mp__y_(); - // disable categories b/c categories will be retrieved at run-time + // enable disable categories according to flag wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(cfg.Hdump_catboxs()); // enable lnki_temp @@ -84,14 +86,15 @@ public class Xomp_parse_wkr implements Gfo_invk { wkr_db.Conn().Txn_bgn("xomp"); // set status to running - mgr_db.Wkr_tbl().Update_status(uid, Xomp_wkr_tbl.Status__running); + mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running); // main loop + int prv_ns = -1; while (true) { // get page from page pool Xomp_page_itm ppg = Get_next(); if (ppg == Xomp_page_itm.Null) { - mgr_db.Wkr_tbl().Update_status(uid, Xomp_wkr_tbl.Status__sleeping); + mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__sleeping); break; // no more pages } if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05 @@ -99,8 +102,16 @@ public class Xomp_parse_wkr implements Gfo_invk { try { // init page long done_bgn = gplx.core.envs.System_.Ticks(); - Xoa_ttl ttl = wiki.Ttl_parse(ppg.Ns_id(), ppg.Ttl_bry()); + int cur_ns = ppg.Ns_id(); + Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry()); + // if ns changed and prv_ns is main + if (cur_ns != prv_ns) { + if (prv_ns == gplx.xowa.wikis.nss.Xow_ns_.Tid__main) + wiki.Cache_mgr().Free_mem_all(Bool_.Y); // NOTE: this clears all caches, include imglinks + prv_ns = cur_ns; + } Xoae_page wpg = Xoae_page.New(wiki, ttl); + wpg.Bldr__ns_ord_(ns_ord_mgr.Get_ord_by_ns_id(cur_ns)); // NOTE: must set ns_id for tier_id in lnki_temp; DATE:2016-09-19 wpg.Db().Text().Text_bry_(ppg.Text()); wpg.Db().Page().Id_(ppg.Id()); @@ -153,7 +164,7 @@ public class Xomp_parse_wkr implements Gfo_invk { } private Xomp_page_itm Get_next() { if (list_idx == list_len) { - mgr_db.Wkr_tbl().Update_exec(uid, done_count, done_time); + mgr_db.Tbl__wkr().Update_exec(uid, done_count, done_time); list.Clear(); page_pool.Get_next(mgr_db, cfg.Wkr_machine_name(), list); list_len = list.Len(); @@ -168,4 +179,5 @@ public class Xomp_parse_wkr implements Gfo_invk { return this; } public static final String Invk__exec = "exec"; + public static final String Cfg__ns_ids = "xomp.ns_ids"; } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__exec.java b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__exec.java deleted file mode 100644 index 20bacda51..000000000 --- a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__exec.java +++ /dev/null @@ -1,139 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*; -import gplx.dbs.*; import gplx.dbs.cfgs.*; -import gplx.fsdb.*; import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*; -import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; -import gplx.xowa.addons.bldrs.files.cmds.*; -public class Xobldr__deletion_db__exec extends Xob_cmd__base { - private Io_url deletion_db_url; - public Xobldr__deletion_db__exec(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, null);} - @Override public void Cmd_run() { - if (deletion_db_url == null) throw Err_.new_("bldr", "no file specified"); - Db_conn conn = Db_conn_bldr.Instance.Get_or_noop(deletion_db_url); - if (conn == Db_conn_.Noop) throw Err_.new_("bldr", "file does not exist", "file", deletion_db_url.Raw()); - - Db_cfg_tbl cfg_tbl = gplx.xowa.wikis.data.Xowd_cfg_tbl_.Get_or_null(conn); - if (cfg_tbl == null) throw Err_.new_("bldr", "xowa_cfg tbl does not exist", "file", deletion_db_url.Raw()); - byte[] domain_bry = cfg_tbl.Select_bry_or("", Xobldr__deletion_db__make.Cfg__deletion_db__domain, null); - if (domain_bry == null) throw Err_.new_("bldr", "no domain found in deletion db", "file", deletion_db_url.Raw()); - this.wiki = bldr.App().Wiki_mgr().Get_by_or_make(domain_bry); - wiki.Init_assert(); - Delete_by_url(wiki, deletion_db_url); - conn.Rls_conn(); - Io_mgr.Instance.DeleteFil(deletion_db_url); - } - private void Delete_by_url(Xowe_wiki wiki, Io_url delete_url) { - Xof_fsdb_mgr fsdb_mgr = wiki.File_mgr().Fsdb_mgr(); - Fsm_mnt_itm mnt_itm = fsdb_mgr.Mnt_mgr().Mnts__get_main(); - Db_conn core_conn = mnt_itm.Atr_mgr().Db__core().Conn(); - Db_conn delete_conn = Db_conn_bldr.Instance.Get_or_new(delete_url).Conn(); - core_conn.Env_db_attach("delete_db", delete_conn); - int dbs_len = mnt_itm.Bin_mgr().Dbs__len(); - String dbs_len_str = Int_.To_str(dbs_len - Int_.Base1); - for (int i = 0; i < dbs_len; ++i) { - Fsm_bin_fil bin_db = mnt_itm.Bin_mgr().Dbs__get_at(i); - Delete_by_db(core_conn, bin_db, dbs_len_str); - } - core_conn.Env_db_detach("delete_db"); - } - private void Delete_by_db(Db_conn conn, Fsm_bin_fil bin_db, String dbs_len_str) { - Gfo_usr_dlg usr_dlg = Xoa_app_.Usr_dlg(); - // get rows to delete in db - List_adp list = List_adp_.New(); - int bin_db_id = bin_db.Id(); - String bin_db_id_str = Int_.To_str(bin_db_id); - usr_dlg.Prog_many("", "", "processing files for deletion in database " + bin_db_id_str + " of " + dbs_len_str); - Db_rdr rdr = conn.Exec_rdr(String_.Concat_lines_nl - ( "SELECT ff.fil_id AS item_id" - , ", 1 AS item_is_orig" - , "FROM fsdb_fil ff" - , "JOIN delete_db.delete_regy dr ON ff.fil_id = dr.fil_id AND dr.thm_id = -1" - , "WHERE ff.fil_bin_db_id = " + bin_db_id_str - , "UNION" - , "SELECT ft.thm_id AS item_id" - , ", 0 AS item_is_orig" - , "FROM fsdb_thm ft" - , " JOIN delete_db.delete_regy dr ON ft.thm_owner_id = dr.fil_id AND ft.thm_id = dr.thm_id" - , "WHERE ft.thm_bin_db_id = " + bin_db_id_str - )); - try { - while (rdr.Move_next()) { - int item_id = rdr.Read_int("item_id"); - int item_is_orig = rdr.Read_int("item_is_orig"); - list.Add(new Xob_img_prune_itm(item_id, item_is_orig == 1)); - } - } finally {rdr.Rls();} - - int len = list.Count(); - if (len == 0) return; // no files; exit, else will vacuum below - - conn.Env_db_attach("bin_db", bin_db.Conn()); - conn.Txn_bgn("img_prune__" + bin_db_id_str); - // delete bin - Db_stmt delete_bin_stmt = conn.Stmt_sql("DELETE FROM bin_db.fsdb_bin WHERE bin_owner_id = ?"); - for (int i = 0; i < len; ++i) { - Xob_img_prune_itm itm = (Xob_img_prune_itm)list.Get_at(i); - delete_bin_stmt.Clear().Crt_int("bin_owner_id", itm.Item_id); - delete_bin_stmt.Exec_delete(); - - if (i % 10000 == 0) usr_dlg.Prog_many("", "", "deleting data in database " + bin_db_id_str + " of " + dbs_len_str); - } - delete_bin_stmt.Rls(); - - // delete meta - Db_stmt delete_fil_stmt = conn.Stmt_sql("DELETE FROM fsdb_fil WHERE fil_id = ?"); - Db_stmt delete_thm_stmt = conn.Stmt_sql("DELETE FROM fsdb_thm WHERE thm_id = ?"); - for (int i = 0; i < len; ++i) { - Xob_img_prune_itm itm = (Xob_img_prune_itm)list.Get_at(i); - if (itm.Item_is_orig) { - delete_fil_stmt.Clear().Crt_int("fil_id", itm.Item_id); - delete_fil_stmt.Exec_delete(); - } - else { - delete_thm_stmt.Clear().Crt_int("thm_id", itm.Item_id); - delete_thm_stmt.Exec_delete(); - } - if (i % 10000 == 0) usr_dlg.Prog_many("", "", "deleting meta in database " + bin_db_id_str + " of " + dbs_len_str); - } - delete_fil_stmt.Rls(); - delete_thm_stmt.Rls(); - - conn.Txn_end(); - conn.Env_db_detach("bin_db"); - bin_db.Conn().Env_vacuum(); - } - @Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { - if (ctx.Match(k, Invk__file_)) this.deletion_db_url = Io_url_.new_any_(m.ReadStr("v")); - else return Gfo_invk_.Rv_unhandled; - return this; - } private static final String Invk__file_ = "file_"; - - public static final String BLDR_CMD_KEY = "fsdb.deletion_db.exec"; - @Override public String Cmd_key() {return BLDR_CMD_KEY;} - public static final Xob_cmd Prototype = new Xobldr__deletion_db__exec(null, null); - @Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xobldr__deletion_db__exec(bldr, wiki);} -} -class Xob_img_prune_itm { - public Xob_img_prune_itm(int item_id, boolean item_is_orig) { - this.Item_id = item_id; - this.Item_is_orig = item_is_orig; - } - public final int Item_id; - public final boolean Item_is_orig; -} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xoax_updates_files_addon.java b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_addon.java similarity index 82% rename from 400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xoax_updates_files_addon.java rename to 400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_addon.java index 145e75d39..72a13852c 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xoax_updates_files_addon.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_addon.java @@ -18,12 +18,12 @@ along with this program. If not, see . package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.addons.bldrs.utils_rankings.bldrs.*; -public class Xoax_updates_files_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr { +public class Xodel_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr { public Xob_cmd[] Bldr_cmds() { return new Xob_cmd[] - { Xobldr__deletion_db__make.Prototype - , Xobldr__deletion_db__exec.Prototype - , Xobldr__deletion_db__small_files.Prototype + { Xodel_make_cmd.Prototype + , Xodel_exec_cmd.Prototype + , Xodel_small_cmd.Prototype }; } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_exec_cmd.java b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_exec_cmd.java new file mode 100644 index 000000000..2522ecdbb --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_exec_cmd.java @@ -0,0 +1,36 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*; +import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; +public class Xodel_exec_cmd extends Xob_cmd__base { + private Io_url deletion_db_url; + public Xodel_exec_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);} + @Override public void Cmd_run() { + new Xodel_exec_mgr().Exec_delete(gplx.core.progs.Gfo_prog_ui_.Noop, bldr, deletion_db_url); + } + @Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { + if (ctx.Match(k, Invk__file_)) this.deletion_db_url = Io_url_.new_any_(m.ReadStr("v")); + else return Gfo_invk_.Rv_unhandled; + return this; + } private static final String Invk__file_ = "file_"; + + public static final String BLDR_CMD_KEY = "fsdb.deletion_db.exec"; + @Override public String Cmd_key() {return BLDR_CMD_KEY;} + public static final Xob_cmd Prototype = new Xodel_exec_cmd(null, null); + @Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xodel_exec_cmd(bldr, wiki);} +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_exec_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_exec_mgr.java new file mode 100644 index 000000000..b7a05c807 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_exec_mgr.java @@ -0,0 +1,134 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*; +import gplx.core.progs.*; +import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.xowa.wikis.data.*; +import gplx.xowa.bldrs.*; +import gplx.xowa.files.fsdb.*; import gplx.fsdb.meta.*; +public class Xodel_exec_mgr { + public void Exec_delete(Gfo_prog_ui prog_ui, Xob_bldr bldr, Io_url deletion_db_url) { + // get domain bry from deletion_conn's cfg_tbl + Db_conn deletion_conn = Db_conn_bldr.Instance.Get_or_fail(deletion_db_url); + Db_cfg_tbl cfg_tbl = Xowd_cfg_tbl_.Get_or_fail(deletion_conn); + byte[] domain_bry = cfg_tbl.Select_bry("", Xodel_exec_mgr.Cfg__deletion_db__domain); + + // get wiki; init it; do delete + Xowe_wiki wiki = bldr.App().Wiki_mgr().Get_by_or_make(domain_bry); + wiki.Init_assert(); + Delete_by_url(prog_ui, wiki, deletion_conn, cfg_tbl); + + // cleanup + deletion_conn.Rls_conn(); + Io_mgr.Instance.DeleteFil(deletion_db_url); + } + public void Exec_cleanup(Io_url deletion_db_url) { + Db_conn deletion_conn = Db_conn_bldr.Instance.Get_or_fail(deletion_db_url); + Db_cfg_tbl cfg_tbl = Xowd_cfg_tbl_.Get_or_fail(deletion_conn); + cfg_tbl.Delete_val("", Xodel_exec_mgr.Cfg__deletion_db__db_bgn); + } + private void Delete_by_url(Gfo_prog_ui prog_ui, Xowe_wiki wiki, Db_conn deletion_conn, Db_cfg_tbl cfg_tbl) { + // get fsdb_mgr + Xof_fsdb_mgr fsdb_mgr = wiki.File_mgr().Fsdb_mgr(); + Fsm_mnt_itm mnt_itm = fsdb_mgr.Mnt_mgr().Mnts__get_main(); + Db_conn core_conn = mnt_itm.Atr_mgr().Db__core().Conn(); + try { + core_conn.Env_db_attach("delete_db", deletion_conn); + + // loop dbs + int dbs_len = mnt_itm.Bin_mgr().Dbs__len(); + String dbs_len_str = Int_.To_str(dbs_len - Int_.Base1); + int db_bgn = cfg_tbl.Select_int_or("", Cfg__deletion_db__db_bgn, 0); + for (int i = db_bgn; i < dbs_len; ++i) { + if (prog_ui.Prog_notify_and_chk_if_suspended(i, dbs_len)) return; + Fsm_bin_fil bin_db = mnt_itm.Bin_mgr().Dbs__get_at(i); + Delete_by_db(core_conn, bin_db, dbs_len_str); + cfg_tbl.Upsert_int("", Cfg__deletion_db__db_bgn, i + 1); + } + } finally {core_conn.Env_db_detach("delete_db");} + } + private void Delete_by_db(Db_conn deletion_conn, Fsm_bin_fil bin_db, String dbs_len_str) { + Gfo_usr_dlg usr_dlg = Xoa_app_.Usr_dlg(); + + // get rows to delete in db + List_adp list = List_adp_.New(); + int bin_db_id = bin_db.Id(); + String bin_db_id_str = Int_.To_str(bin_db_id); + usr_dlg.Prog_many("", "", "processing files for deletion in database " + bin_db_id_str + " of " + dbs_len_str); + Db_rdr rdr = deletion_conn.Exec_rdr(String_.Concat_lines_nl + ( "SELECT ff.fil_id AS item_id" + , ", 1 AS item_is_orig" + , "FROM fsdb_fil ff" + , "JOIN delete_db.delete_regy dr ON ff.fil_id = dr.fil_id AND dr.thm_id = -1" + , "WHERE ff.fil_bin_db_id = " + bin_db_id_str + , "UNION" + , "SELECT ft.thm_id AS item_id" + , ", 0 AS item_is_orig" + , "FROM fsdb_thm ft" + , " JOIN delete_db.delete_regy dr ON ft.thm_owner_id = dr.fil_id AND ft.thm_id = dr.thm_id" + , "WHERE ft.thm_bin_db_id = " + bin_db_id_str + )); + try { + while (rdr.Move_next()) { + int item_id = rdr.Read_int("item_id"); + int item_is_orig = rdr.Read_int("item_is_orig"); + list.Add(new Xodel_prune_itm(item_id, item_is_orig == 1)); + } + } finally {rdr.Rls();} + + int len = list.Count(); + if (len == 0) return; // no files; exit, else will vacuum below + + deletion_conn.Env_db_attach("bin_db", bin_db.Conn()); + deletion_conn.Txn_bgn("img_prune__" + bin_db_id_str); + + // delete bin + Db_stmt delete_bin_stmt = deletion_conn.Stmt_sql("DELETE FROM bin_db.fsdb_bin WHERE bin_owner_id = ?"); + for (int i = 0; i < len; ++i) { + Xodel_prune_itm itm = (Xodel_prune_itm)list.Get_at(i); + delete_bin_stmt.Clear().Crt_int("bin_owner_id", itm.Item_id); + delete_bin_stmt.Exec_delete(); + + if (i % 10000 == 0) usr_dlg.Prog_many("", "", "deleting data in database " + bin_db_id_str + " of " + dbs_len_str); + } + delete_bin_stmt.Rls(); + + // delete meta + Db_stmt delete_fil_stmt = deletion_conn.Stmt_sql("DELETE FROM fsdb_fil WHERE fil_id = ?"); + Db_stmt delete_thm_stmt = deletion_conn.Stmt_sql("DELETE FROM fsdb_thm WHERE thm_id = ?"); + for (int i = 0; i < len; ++i) { + Xodel_prune_itm itm = (Xodel_prune_itm)list.Get_at(i); + if (itm.Item_is_orig) { + delete_fil_stmt.Clear().Crt_int("fil_id", itm.Item_id); + delete_fil_stmt.Exec_delete(); + } + else { + delete_thm_stmt.Clear().Crt_int("thm_id", itm.Item_id); + delete_thm_stmt.Exec_delete(); + } + if (i % 10000 == 0) usr_dlg.Prog_many("", "", "deleting meta in database " + bin_db_id_str + " of " + dbs_len_str); + } + delete_fil_stmt.Rls(); + delete_thm_stmt.Rls(); + + // cleanup + deletion_conn.Txn_end(); + deletion_conn.Env_db_detach("bin_db"); + bin_db.Conn().Env_vacuum(); + } + public static final String Cfg__deletion_db__domain = "file.deletion_db.domain", Cfg__deletion_db__db_bgn = "file.deletion_db.db_bgn"; +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_make_cmd.java b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_make_cmd.java new file mode 100644 index 000000000..19ae14b50 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_make_cmd.java @@ -0,0 +1,31 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*; +import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; +public class Xodel_make_cmd extends Xob_cmd__base implements Xob_cmd { + public Xodel_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);} + @Override public void Cmd_run() { + wiki.Init_assert(); + new Xodel_make_mgr().Exec(wiki); + } + + public static final String BLDR_CMD_KEY = "file.deletion_db.make"; + @Override public String Cmd_key() {return BLDR_CMD_KEY;} + public static final Xob_cmd Prototype = new Xodel_make_cmd(null, null); + @Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xodel_make_cmd(bldr, wiki);} +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__make.java b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_make_mgr.java similarity index 70% rename from 400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__make.java rename to 400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_make_mgr.java index 94532e842..4162f062e 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__make.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_make_mgr.java @@ -16,13 +16,10 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*; -import gplx.dbs.*; import gplx.dbs.cfgs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.addons.bldrs.files.dbs.*; -import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; -import gplx.fsdb.meta.*; -public class Xobldr__deletion_db__make extends Xob_cmd__base implements Xob_cmd { - public Xobldr__deletion_db__make(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);} - @Override public void Cmd_run() { - wiki.Init_assert(); +import gplx.dbs.*; +import gplx.xowa.bldrs.*; +class Xodel_make_mgr { + public void Exec(Xow_wiki wiki) { // mark the records deleted Db_conn make_conn = Xob_db_file.New__file_make(wiki.Fsys_mgr().Root_dir()).Conn(); make_conn.Exec_sql_concat_w_msg @@ -39,16 +36,15 @@ public class Xobldr__deletion_db__make extends Xob_cmd__base implements Xob_cmd , "AND xr.file_w = fsdb_regy.fsdb_w" , "AND xr.lnki_time = fsdb_regy.fsdb_time" , "AND xr.lnki_page = fsdb_regy.fsdb_page" - , ")" - ); + , ")"); // create deletion db Xob_db_file deletion_db = Xob_db_file.New__deletion_db(wiki); Db_conn deletion_conn = deletion_db.Conn(); - deletion_db.Tbl__cfg().Upsert_str("", Xobldr__deletion_db__make.Cfg__deletion_db__domain, wiki.Domain_str()); + deletion_db.Tbl__cfg().Upsert_str("", Xodel_exec_mgr.Cfg__deletion_db__domain, wiki.Domain_str()); // copy records over to it - Xob_delete_regy delete_regy_tbl = new Xob_delete_regy(deletion_conn); + Xob_delete_regy_tbl delete_regy_tbl = new Xob_delete_regy_tbl(deletion_conn); deletion_conn.Meta_tbl_remake(delete_regy_tbl.Meta()); deletion_conn.Env_db_attach("make_db", make_conn); deletion_conn.Exec_sql_concat_w_msg @@ -64,19 +60,13 @@ public class Xobldr__deletion_db__make extends Xob_cmd__base implements Xob_cmd deletion_conn.Env_db_detach("make_db"); deletion_conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(delete_regy_tbl.tbl_name, "main", delete_regy_tbl.fld_fil_id, delete_regy_tbl.fld_thm_id)); } - public static final String Cfg__deletion_db__domain = "file.deletion_db.domain"; - - public static final String BLDR_CMD_KEY = "file.deletion_db.make"; - @Override public String Cmd_key() {return BLDR_CMD_KEY;} - public static final Xob_cmd Prototype = new Xobldr__deletion_db__make(null, null); - @Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xobldr__deletion_db__make(bldr, wiki);} } -class Xob_delete_regy { +class Xob_delete_regy_tbl { public final String tbl_name = "delete_regy"; public final Dbmeta_fld_list flds = new Dbmeta_fld_list(); public final String fld_fil_id, fld_thm_id, fld_reason; public final Db_conn conn; - public Xob_delete_regy(Db_conn conn) { + public Xob_delete_regy_tbl(Db_conn conn) { this.conn = conn; this.fld_fil_id = flds.Add_int("fil_id"); this.fld_thm_id = flds.Add_int("thm_id"); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_prune_itm.java b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_prune_itm.java new file mode 100644 index 000000000..c371aec19 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_prune_itm.java @@ -0,0 +1,26 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*; +class Xodel_prune_itm { + public Xodel_prune_itm(int item_id, boolean item_is_orig) { + this.Item_id = item_id; + this.Item_is_orig = item_is_orig; + } + public final int Item_id; + public final boolean Item_is_orig; +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_small_cmd.java b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_small_cmd.java new file mode 100644 index 000000000..a527c7d23 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_small_cmd.java @@ -0,0 +1,33 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*; +import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; +public class Xodel_small_cmd extends Xob_cmd__base { + public Xodel_small_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);} + private final int[] ext_max_ary = Xobldr__fsdb_db__delete_small_files_.New_ext_max_ary(); + @Override public void Cmd_run() { + wiki.Init_assert(); + new Xodel_small_mgr().Exec(wiki, ext_max_ary); + } + @Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return this;} + + public static final String BLDR_CMD_KEY = "file.deletion_db.small_files"; + @Override public String Cmd_key() {return BLDR_CMD_KEY;} + public static final Xob_cmd Prototype = new Xodel_small_cmd(null, null); + @Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xodel_small_cmd(bldr, wiki);} +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__small_files.java b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_small_mgr.java similarity index 77% rename from 400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__small_files.java rename to 400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_small_mgr.java index ac0e38263..c61a57b63 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xobldr__deletion_db__small_files.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/updates/files/Xodel_small_mgr.java @@ -17,12 +17,10 @@ along with this program. If not, see . */ package gplx.xowa.addons.bldrs.updates.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.updates.*; import gplx.dbs.*; -import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; +import gplx.xowa.bldrs.*; import gplx.fsdb.*; import gplx.fsdb.meta.*; import gplx.xowa.files.*; -public class Xobldr__deletion_db__small_files extends Xob_cmd__base { - public Xobldr__deletion_db__small_files(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);} - private final int[] ext_max_ary = Xobldr__fsdb_db__delete_small_files_.New_ext_max_ary(); - @Override public void Cmd_run() { +class Xodel_small_mgr { + public void Exec(Xowe_wiki wiki, int[] ext_max_ary) { wiki.Init_assert(); // get atr_conn Fsdb_db_mgr db_core_mgr = Fsdb_db_mgr_.new_detect(wiki, wiki.Fsys_mgr().Root_dir(), wiki.Fsys_mgr().File_dir()); @@ -54,13 +52,7 @@ public class Xobldr__deletion_db__small_files extends Xob_cmd__base { , "AND t.thm_size <= " + Int_.To_str(max) ); } - @Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {return this;} - - public static final String BLDR_CMD_KEY = "file.deletion_db.small_files"; - @Override public String Cmd_key() {return BLDR_CMD_KEY;} - public static final Xob_cmd Prototype = new Xobldr__deletion_db__small_files(null, null); - @Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xobldr__deletion_db__small_files(bldr, wiki);} -} +} class Xobldr__fsdb_db__delete_small_files_ { public static int[] New_ext_max_ary() { int[] rv = new int[Xof_ext_.Id__max]; diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_cmd.java b/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_cmd.java index a987cfa81..884429094 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_cmd.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_cmd.java @@ -17,24 +17,24 @@ along with this program. If not, see . */ package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*; import gplx.dbs.*; -import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*; -public class Imglnk_bldr_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd { +import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sql_dumps.*; +public class Imglnk_bldr_cmd extends Xob_sql_dump_base implements Xosql_dump_cbk { private Imglnk_bldr_mgr mgr; private int tmp_page_id; private int rows = 0; public Imglnk_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;} @Override public String Sql_file_name() {return "imagelinks";} + @Override protected Xosql_dump_parser New_parser() {return new Xosql_dump_parser(this, "il_from", "il_to");} - @Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) { - parser.Save_csv_n_().Fld_cmd_(this).Flds_req_idx_(3, 0, 1); + @Override public void Cmd_bgn_hook(Xob_bldr bldr, Xosql_dump_parser parser) { mgr = new Imglnk_bldr_mgr(wiki); } - public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) { + public void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end) { switch (fld_idx) { - case Fld__il_from: this.tmp_page_id = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break; + case Fld__il_from: this.tmp_page_id = Bry_.To_int_or(src, val_bgn, val_end, -1); break; case Fld__il_to: - mgr.Tmp_tbl().Insert_by_batch(tmp_page_id, Bry_.Mid(src, fld_bgn, fld_end)); + mgr.Tmp_tbl().Insert_by_batch(tmp_page_id, Bry_.Mid(src, val_bgn, val_end)); if (++rows % 100000 == 0) usr_dlg.Prog_many("", "", "reading row ~{0}", Int_.To_str_fmt(rows, "#,##0")); break; } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_mgr.java index 7f27e4db3..c84dcf6a4 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_mgr.java @@ -41,8 +41,8 @@ class Imglnk_bldr_mgr { Imglnk_reg_tbl reg_tbl = new Imglnk_reg_tbl(conn); conn.Meta_tbl_remake(reg_tbl); reg_tbl.Create_idx__src_ttl(); - reg_tbl.Insert(conn, Xof_repo_itm_.Repo_local , wiki); - reg_tbl.Insert(conn, Xof_repo_itm_.Repo_remote, wiki.Appe().Wiki_mgr().Wiki_commons()); + reg_tbl.Insert(conn, Xof_repo_tid_.Tid__local , wiki); + reg_tbl.Insert(conn, Xof_repo_tid_.Tid__remote, wiki.Appe().Wiki_mgr().Wiki_commons()); reg_tbl.Create_idx__trg_ttl(); // Imglnk_bulk_cmd__img_id.Bulk_exec(conn, Bool_.Y, wiki); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Xof_orig_wkr__img_links_.java b/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Xof_orig_wkr__img_links_.java index 8d8a28a62..b139516d9 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Xof_orig_wkr__img_links_.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Xof_orig_wkr__img_links_.java @@ -23,8 +23,8 @@ public class Xof_orig_wkr__img_links_ { public static void Load_all(Xof_orig_wkr__img_links wkr) { Xowe_wiki wiki = wkr.Wiki(); Db_conn conn = Xob_db_file.New__img_link(wiki).Conn(); - Load_all_by_wiki(wkr, conn, Xof_repo_itm_.Repo_local , wiki); - Load_all_by_wiki(wkr, conn, Xof_repo_itm_.Repo_remote, wiki.Appe().Wiki_mgr().Wiki_commons()); + Load_all_by_wiki(wkr, conn, Xof_repo_tid_.Tid__local , wiki); + Load_all_by_wiki(wkr, conn, Xof_repo_tid_.Tid__remote, wiki.Appe().Wiki_mgr().Wiki_commons()); } public static Xof_orig_itm Load_itm(Xof_orig_wkr__img_links wkr, Db_conn conn, Xowe_wiki wiki, byte[] ttl) { Imglnk_reg_tbl imglnk_reg_tbl = wkr.Tbl__imglnk_reg(); @@ -39,7 +39,7 @@ public class Xof_orig_wkr__img_links_ { else // ttl missing; EX:. */ package gplx.xowa.addons.bldrs.wmdumps.pagelinks.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*; import gplx.xowa.addons.bldrs.wmdumps.pagelinks.*; -import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*; +import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sql_dumps.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.*; import gplx.xowa.addons.bldrs.wmdumps.pagelinks.dbs.*; -public class Pglnk_bldr_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd { +public class Pglnk_bldr_cmd extends Xob_sql_dump_base implements Xosql_dump_cbk { private Db_conn conn; private Pglnk_page_link_temp_tbl temp_tbl; private int tmp_src_id, tmp_trg_ns; private int rows = 0; public Pglnk_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;} @Override public String Sql_file_name() {return Dump_type_key;} public static final String Dump_type_key = "pagelinks"; - @Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) { + @Override protected Xosql_dump_parser New_parser() {return new Xosql_dump_parser(this, "pl_from", "pl_namespace", "pl_title");} + @Override public void Cmd_bgn_hook(Xob_bldr bldr, Xosql_dump_parser parser) { wiki.Init_assert(); - parser.Fld_cmd_(this).Flds_req_idx_(4, 0, 1, 2); Xob_db_file page_link_db = Xob_db_file.New__page_link(wiki); this.conn = page_link_db.Conn(); this.temp_tbl = new Pglnk_page_link_temp_tbl(conn); @@ -49,12 +49,12 @@ public class Pglnk_bldr_cmd extends Xob_sql_dump_base implements Sql_file_parser actl_tbl.Create_idx__trg_src(); conn.Env_vacuum(); } - public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) { + public void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end) { switch (fld_idx) { - case Fld__pl_from: this.tmp_src_id = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break; - case Fld__pl_namespace: this.tmp_trg_ns = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break; + case Fld__pl_from: this.tmp_src_id = Bry_.To_int_or(src, val_bgn, val_end, -1); break; + case Fld__pl_namespace: this.tmp_trg_ns = Bry_.To_int_or(src, val_bgn, val_end, -1); break; case Fld__pl_title: - byte[] tmp_trg_ttl = Bry_.Mid(src, fld_bgn, fld_end); + byte[] tmp_trg_ttl = Bry_.Mid(src, val_bgn, val_end); temp_tbl.Insert(tmp_src_id, tmp_trg_ns, tmp_trg_ttl); if (++rows % 100000 == 0) usr_dlg.Prog_many("", "", "reading row ~{0}", Int_.To_str_fmt(rows, "#,##0")); break; diff --git a/400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt.java b/400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt.java index 07327d411..18838b0d5 100644 --- a/400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt.java +++ b/400_xowa/src/gplx/xowa/addons/htmls/tocs/Xoh_toc_wkr__txt.java @@ -140,7 +140,7 @@ class Xoh_toc_wkr__txt { int rhs_bgn = -1, rhs_end = -1, new_pos = lhs_end; if (lhs_is_pair) { // get rhs unless inline if (tag_id == Gfh_tag_.Id__any) { - Gfo_usr_dlg_.Instance.Warn_many("", "", "unknown tag: page=~{0} tag=~{1}", page_name, lhs_bry); + Gfo_usr_dlg_.Instance.Warn_many("", "", "unknown tag: page=~{0} tag=~{1}", page_name, lhs_bry); Gfh_tag rhs = tag_rdr.Tag__peek_fwd_tail(lhs_bry); if (rhs.Name_id() == Gfh_tag_.Id__eos) return false; rhs_bgn = rhs.Src_bgn(); rhs_end = rhs.Src_end(); diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/Xoa_ctg_mgr.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/Xoa_ctg_mgr.java index 3ecbbf087..f082685fd 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/Xoa_ctg_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/Xoa_ctg_mgr.java @@ -58,7 +58,7 @@ public class Xoa_ctg_mgr implements Gfo_invk { } } public static final byte Version_null = Byte_.Zero, Version_1 = 1, Version_2 = 2; - public static final byte Tid_null = Byte_.Max_value_127, Tid_subc = 0, Tid_file = 1, Tid_page = 2, Tid__max = 3; // SERIALIZED + public static final byte Tid__subc = 0, Tid__file = 1, Tid__page = 2, Tid___max = 3; // SERIALIZED; cat_link.cl_type_id public static final byte Hidden_n = Byte_.Zero, Hidden_y = (byte)1; public static final String Html__cls__str = "CategoryTreeLabel CategoryTreeLabelNs14 CategoryTreeLabelCategory"; public static final byte[] Html__cls__bry = Bry_.new_a7(Html__cls__str); diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_catlink_cmd.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_catlink_cmd.java index f0c06955e..c9cdae77f 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_catlink_cmd.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_catlink_cmd.java @@ -16,33 +16,33 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.addons.wikis.ctgs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; -import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*; -public class Xob_catlink_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd { +import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sql_dumps.*; +public class Xob_catlink_cmd extends Xob_sql_dump_base implements Xosql_dump_cbk { private final Xob_catlink_mgr mgr = new Xob_catlink_mgr(); private int tmp_page_id; private byte[] tmp_ctg_ttl, tmp_sortkey, tmp_timestamp, tmp_sortkey_prefix, tmp_collation, tmp_type; public Xob_catlink_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);} @Override public String Sql_file_name() {return Dump_file_name;} public static final String Dump_file_name = "categorylinks"; + @Override protected Xosql_dump_parser New_parser() {return new Xosql_dump_parser(this, "cl_from", "cl_to", "cl_sortkey", "cl_timestamp", "cl_sortkey_prefix", "cl_collation", "cl_type");} - @Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) { + @Override public void Cmd_bgn_hook(Xob_bldr bldr, Xosql_dump_parser parser) { wiki.Init_assert(); - parser.Save_csv_n_().Fld_cmd_(this).Flds_req_idx_(7, 0, 1, 2, 3, 4, 5, 6); mgr.On_cmd_bgn(wiki); } @Override public void Cmd_end() { mgr.On_cmd_end(); this.Cmd_cleanup_sql(); } - public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) { + public void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end) { switch (fld_idx) { - case Fld__cl_from: this.tmp_page_id = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break; - case Fld__cl_to: this.tmp_ctg_ttl = Bry_.Mid(src, fld_bgn, fld_end); break; - case Fld__cl_sortkey: this.tmp_sortkey = Bry_.Mid(src, fld_bgn, fld_end); break; - case Fld__cl_timestamp: this.tmp_timestamp = Bry_.Mid(src, fld_bgn, fld_end); break; - case Fld__cl_sortkey_prefix: this.tmp_sortkey_prefix = Bry_.Mid(src, fld_bgn, fld_end); break; - case Fld__cl_collation: this.tmp_collation = Bry_.Mid(src, fld_bgn, fld_end); break; - case Fld__cl_type: this.tmp_type = Bry_.Mid(src, fld_bgn, fld_end); + case Fld__cl_from: this.tmp_page_id = Bry_.To_int_or(src, val_bgn, val_end, -1); break; + case Fld__cl_to: this.tmp_ctg_ttl = Bry_.Mid(src, val_bgn, val_end); break; + case Fld__cl_sortkey: this.tmp_sortkey = Bry_.Mid(src, val_bgn, val_end); break; + case Fld__cl_timestamp: this.tmp_timestamp = Bry_.Mid(src, val_bgn, val_end); break; + case Fld__cl_sortkey_prefix: this.tmp_sortkey_prefix = Bry_.Mid(src, val_bgn, val_end); break; + case Fld__cl_collation: this.tmp_collation = Bry_.Mid(src, val_bgn, val_end); break; + case Fld__cl_type: this.tmp_type = Bry_.Mid(src, val_bgn, val_end); mgr.On_cmd_row(tmp_page_id, tmp_ctg_ttl, tmp_sortkey, tmp_timestamp, tmp_sortkey_prefix, tmp_collation, tmp_type); break; } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_catlink_mgr.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_catlink_mgr.java index 17ee24b4b..7a562cbbb 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_catlink_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_catlink_mgr.java @@ -37,7 +37,8 @@ class Xob_catlink_mgr { } public void On_cmd_row(int page_id, byte[] ctg_ttl, byte[] sortkey_orig, byte[] timestamp_bry, byte[] sortkey_prefix, byte[] collation_bry, byte[] type_bry) { // convert strings to numbers - long timestamp = DateAdp_.parse_fmt(String_.new_u8(timestamp_bry), "YYYY-MM-dd HH:mm:ss").Timestamp_unix(); + String timestamp_str = String_.new_u8(timestamp_bry); + long timestamp = String_.Len_eq_0(timestamp_str) ? 0 : DateAdp_.parse_fmt(timestamp_str, "YYYY-MM-dd HH:mm:ss").Timestamp_unix(); byte collation_id = collation_enum.To_tid_or_fail(collation_bry); byte type_id = type_enum.To_tid_or_fail(type_bry); @@ -64,11 +65,17 @@ class Xob_catlink_mgr { public void On_cmd_end() { tmp_link_tbl.Insert_end(); - // create cat_sort + // get cat_core conn tmp_link_tbl.Create_idx__sortkey(); // index should make SELECT DISTINCT faster - Db_conn cat_core_conn = wiki.Data__core_mgr().Props().Layout_text().Tid_is_lot() - ? wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__cat_core).Conn() - : wiki.Data__core_mgr().Db__core().Conn(); + Db_conn cat_core_conn = wiki.Data__core_mgr().Db__core().Conn(); + if (wiki.Data__core_mgr().Props().Layout_text().Tid_is_lot()) { + Xow_db_file cat_core_db = wiki.Data__core_mgr().Dbs__get_by_tid_or_null(Xow_db_file_.Tid__cat_core); + if (cat_core_db == null) + cat_core_db = wiki.Data__core_mgr().Dbs__make_by_tid(Xow_db_file_.Tid__cat_core); + cat_core_conn = cat_core_db.Conn(); + } + + // create tbl Xodb_cat_sort_tbl cat_sort_tbl = new Xodb_cat_sort_tbl(cat_core_conn); cat_core_conn.Meta_tbl_remake(cat_sort_tbl); cat_sort_tbl.Insert_by_select(tmp_conn); diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_pageprop_cmd.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_pageprop_cmd.java index 0a60524b5..5baa6763b 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_pageprop_cmd.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/bldrs/Xob_pageprop_cmd.java @@ -16,9 +16,9 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.addons.wikis.ctgs.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; -import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*; +import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sql_dumps.*; import gplx.dbs.*; import gplx.xowa.addons.wikis.ctgs.dbs.*; -public class Xob_pageprop_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd { +public class Xob_pageprop_cmd extends Xob_sql_dump_base implements Xosql_dump_cbk { private int tmp_id; private boolean tmp_key_is_hiddencat; private int rows; @@ -26,10 +26,10 @@ public class Xob_pageprop_cmd extends Xob_sql_dump_base implements Sql_file_pars public Xob_pageprop_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki);} @Override public String Sql_file_name() {return Dump_file_name;} public static final String Dump_file_name = "page_props"; + @Override protected Xosql_dump_parser New_parser() {return new Xosql_dump_parser(this, "pp_page", "pp_propname", "pp_value");} // NOTE: 4 b/c MW added fld_3:pp_sortkey; DATE:2014-04-28 - @Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) { + @Override public void Cmd_bgn_hook(Xob_bldr bldr, Xosql_dump_parser parser) { wiki.Init_assert(); - parser.Save_csv_n_().Fld_cmd_(this).Flds_req_idx_(4, 0, 1, 2); // NOTE: 4 b/c MW added fld_3:pp_sortkey; DATE:2014-04-28 Xodb_tmp_cat_db tmp_db = new Xodb_tmp_cat_db(wiki); tbl = new Xodb_tmp_cat_hidden_tbl(tmp_db.Conn()); tbl.Insert_bgn(); @@ -38,18 +38,18 @@ public class Xob_pageprop_cmd extends Xob_sql_dump_base implements Sql_file_pars tbl.Insert_end(); this.Cmd_cleanup_sql(); } - public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) { + public void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end) { switch (fld_idx) { - case Fld__id: this.tmp_id = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break; - case Fld__key: this.tmp_key_is_hiddencat = Bry_.Eq(src, fld_bgn, fld_end, Key_hiddencat); break; - case Fld__val: - if (!tmp_key_is_hiddencat) {data.Cancel_row_y_(); return;} - tbl.Insert_cmd_by_batch(tmp_id); + case Fld__pp_page: this.tmp_id = Bry_.To_int_or(src, val_bgn, val_end, -1); break; + case Fld__pp_propname: this.tmp_key_is_hiddencat = Bry_.Eq(src, val_bgn, val_end, Key_hiddencat); break; + case Fld__pp_value: + if (tmp_key_is_hiddencat) + tbl.Insert_cmd_by_batch(tmp_id); if (++rows % 10000 == 0) usr_dlg.Prog_many("", "", "parsing pageprops sql: row=~{0}", Int_.To_str_fmt(rows, "#,##0")); break; } } - private static final byte Fld__id = 0, Fld__key = 1, Fld__val = 2; + private static final byte Fld__pp_page = 0, Fld__pp_propname = 1, Fld__pp_value = 2; public static final String BLDR_CMD_KEY = "wiki.page_props"; @Override public String Cmd_key() {return BLDR_CMD_KEY;} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr.java index cae7c7d0f..645ed2f5a 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr.java @@ -19,34 +19,56 @@ package gplx.xowa.addons.wikis.ctgs.htmls.catpages; import gplx.*; import gplx.x import gplx.xowa.wikis.dbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.langs.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.wikis.nss.*; -import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.fmts.*; +import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.fmts.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.utls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls.*; public class Xoctg_catpage_mgr { + private final Hash_adp_bry cache = Hash_adp_bry.cs(); private final Xoctg_catpage_loader loader = new Xoctg_catpage_loader(); - private final Xoctg_catpage_url tmp_catpage_url = new Xoctg_catpage_url(); private final Xoctg_fmt_grp fmt_subcs = Xoctg_fmt_grp.New__subc(), fmt_pages = Xoctg_fmt_grp.New__page(), fmt_files = Xoctg_fmt_grp.New__file(); - public Xoctg_fmt_grp Fmt(byte tid) { // TEST: + public int Grp_max() {return grp_max;} private int grp_max = Grp_max_dflt; + public Xoctg_fmt_grp Fmt(byte tid) { switch (tid) { - case Xoa_ctg_mgr.Tid_subc: return fmt_subcs; - case Xoa_ctg_mgr.Tid_page: return fmt_pages; - case Xoa_ctg_mgr.Tid_file: return fmt_files; + case Xoa_ctg_mgr.Tid__subc: return fmt_subcs; + case Xoa_ctg_mgr.Tid__page: return fmt_pages; + case Xoa_ctg_mgr.Tid__file: return fmt_files; default: throw Err_.new_unhandled(tid); } } + public void Free_mem_all() {cache.Clear();} + public Xoctg_catpage_ctg Get_or_load_or_null(Xow_wiki wiki, Xoa_ttl cat_ttl) { + // load categories from cat dbs; exit if not found + Xoctg_catpage_ctg ctg = (Xoctg_catpage_ctg)cache.Get_by(cat_ttl.Full_db()); + if (ctg == null) { + if (gplx.core.envs.Env_.Mode_testing()) return null; // needed for dpl test + ctg = loader.Load_by_ttl_or_null(wiki, cat_ttl); + if (ctg == null) return null; // not in cache or db; exit + cache.Add(cat_ttl.Full_db(), ctg); + } + return ctg; + } public void Write_catpage(Bry_bfr bfr, Xow_wiki wiki, Xoa_page page, Xoh_wtr_ctx hctx) { try { // load categories from cat dbs; exit if not found - tmp_catpage_url.Parse(wiki.App().Usr_dlg(), page.Url()); - Xoctg_catpage_ctg dom_ctg = loader.Load_by_ttl_or_null(wiki, page.Ttl()); - if (dom_ctg == null) return; + Xoctg_catpage_ctg ctg = Get_or_load_or_null(wiki, page.Ttl()); + if (ctg == null) return; + + // filter subs; need for large categories like de.w:Category:Mann + Xoctg_catpage_url catpage_url = Xoctg_catpage_url_parser.Parse(page.Url()); + Xoctg_catpage_filter.Filter(grp_max, catpage_url, ctg); // write html Xol_lang_itm lang = page.Lang(); - fmt_subcs.Write_catpage_grp(bfr, wiki, lang, dom_ctg); - fmt_pages.Write_catpage_grp(bfr, wiki, lang, dom_ctg); - fmt_files.Write_catpage_grp(bfr, wiki, lang, dom_ctg); + fmt_subcs.Write_catpage_grp(bfr, wiki, lang, ctg, grp_max); + fmt_pages.Write_catpage_grp(bfr, wiki, lang, ctg, grp_max); + fmt_files.Write_catpage_grp(bfr, wiki, lang, ctg, grp_max); } catch (Exception e) { Xoa_app_.Usr_dlg().Warn_many("", "", "failed to generate category: title=~{0} err=~{1}", page.Url_bry_safe(), Err_.Message_gplx_log(e)); } } + public void Cache__add(byte[] ttl, Xoctg_catpage_ctg ctg) { + cache.Del(ttl); + cache.Add(ttl, ctg); + } + public void Grp_max_(int v) {grp_max = v;} // TEST: + public static int Grp_max_dflt = 200; } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr_tst.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr__basic__tst.java similarity index 75% rename from 400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr_tst.java rename to 400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr__basic__tst.java index 567f4da24..598a5ace8 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr_tst.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr__basic__tst.java @@ -18,16 +18,16 @@ along with this program. If not, see . package gplx.xowa.addons.wikis.ctgs.htmls.catpages; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import org.junit.*; import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.fmts.*; -public class Xoctg_catpage_mgr_tst { - @Before public void init() {fxt.Clear();} private Xoh_ctg_page_fxt fxt = new Xoh_ctg_page_fxt(); +public class Xoctg_catpage_mgr__basic__tst { + @Before public void init() {fxt.Clear();} private Xoctg_catpage_mgr_fxt fxt = new Xoctg_catpage_mgr_fxt(); @Test public void Page_itm() { fxt .Init_itms__pages("A1") - .Test_html_page(Xoa_ctg_mgr.Tid_page, Byte_ascii.Ltr_A, "\n
  • A1
    • "); + .Test__html__page(Xoa_ctg_mgr.Tid__page, Byte_ascii.Ltr_A, "\n
  • A1
    • "); } @Test public void Page_itm_missing() { fxt .Init_itms__pages("A1"); - fxt .Ctg().Grp_by_tid(Xoa_ctg_mgr.Tid_page).Itms()[0].Missing_y_(); - fxt .Test_html_page(Xoa_ctg_mgr.Tid_page, Byte_ascii.Ltr_A, "\n
  • A1 (missing)
    • "); + fxt .Ctg().Grp_by_tid(Xoa_ctg_mgr.Tid__page).Itms__get_at(0).Missing_y_(); + fxt .Test__html__page(Xoa_ctg_mgr.Tid__page, Byte_ascii.Ltr_A, "\n
  • A1 (missing)
    • "); } @Test public void Visited_doesnt_work_for_space() {// PURPOSE: xowa-visited not inserted for pages with space byte[] page_bry = Bry_.new_a7("A 1"); @@ -35,7 +35,7 @@ public class Xoctg_catpage_mgr_tst { Xoa_ttl ttl = Xoa_ttl.Parse(fxt.Wiki(), page_bry); fxt.Wiki().Appe().Usere().History_mgr().Add(url, ttl, page_bry); fxt .Init_itms__pages("A_1").Init_grp__pages(0, 1) - .Test_html_all(Xoa_ctg_mgr.Tid_page, String_.Concat_lines_nl_skip_last + .Test__html__all(Xoa_ctg_mgr.Tid__page, String_.Concat_lines_nl_skip_last ( "" , "
    " , "

    Pages in category \"Ctg_1\"

    " @@ -57,7 +57,7 @@ public class Xoctg_catpage_mgr_tst { } @Test public void Page_all() { fxt .Init_itms__pages("A1").Init_grp__pages(0, 1) - .Test_html_all(Xoa_ctg_mgr.Tid_page, String_.Concat_lines_nl_skip_last + .Test__html__all(Xoa_ctg_mgr.Tid__page, String_.Concat_lines_nl_skip_last ( "" , "
    " , "

    Pages in category \"Ctg_1\"

    " @@ -79,7 +79,7 @@ public class Xoctg_catpage_mgr_tst { } @Test public void File_all() { fxt .Init_itms__files("File:A1.png").Init_grp__files(0, 1) - .Test_html_all(Xoa_ctg_mgr.Tid_file, String_.Concat_lines_nl_skip_last + .Test__html__all(Xoa_ctg_mgr.Tid__file, String_.Concat_lines_nl_skip_last ( "" , "
    " , "

    Media in category \"Ctg_1\"

    " @@ -117,7 +117,7 @@ public class Xoctg_catpage_mgr_tst { } @Test public void Subc_all() { fxt .Init_itms__subcs("Category:Subc_1").Init_grp__files(0, 1) - .Test_html_all(Xoa_ctg_mgr.Tid_subc, String_.Concat_lines_nl_skip_last + .Test__html__all(Xoa_ctg_mgr.Tid__subc, String_.Concat_lines_nl_skip_last ( "" , "
    " , "

    Subcategories

    " @@ -153,7 +153,7 @@ public class Xoctg_catpage_mgr_tst { } @Test public void Page_all_cols() { fxt .Init_itms__pages("A1", "A2", "A3", "B1", "C1").Init_grp__pages(0, 5) - .Test_html_all(Xoa_ctg_mgr.Tid_page, String_.Concat_lines_nl_skip_last + .Test__html__all(Xoa_ctg_mgr.Tid__page, String_.Concat_lines_nl_skip_last ( "" , "
    " , "

    Pages in category \"Ctg_1\"

    " @@ -192,7 +192,7 @@ public class Xoctg_catpage_mgr_tst { } @Test public void Title__escape_quotes() {// PURPOSE: quotes in title should be escaped; DATE:2015-12-28 fxt .Init_itms__pages("A\"1").Init_grp__pages(0, 1) - .Test_html_all(Xoa_ctg_mgr.Tid_page, String_.Concat_lines_nl_skip_last + .Test__html__all(Xoa_ctg_mgr.Tid__page, String_.Concat_lines_nl_skip_last ( "" , "
    " , "

    Pages in category \"Ctg_1\"

    " @@ -212,22 +212,6 @@ public class Xoctg_catpage_mgr_tst { , "
    " )); } - @Test public void Bld_rslts_lnk() { - fxt .Init_itms__pages("A1").Init_grp__pages(1, 1) - .Test_bld_rslts_lnk(Bool_.Y, "Category:Ctg_1", String_.Concat_lines_nl - ( "" - , " (previous 0)" - , " (next 0)" - )); - } - @Test public void Bld_rslts_lnk__encoded() { // escape quotes and spaces; DATE:2016-01-11 - fxt .Init_itms__pages("A\" b").Init_grp__pages(1, 1) - .Test_bld_rslts_lnk(Bool_.Y, "Category:Ctg_1", String_.Concat_lines_nl - ( "" - , " (previous 0)" - , " (next 0)" - )); - } @Test public void Calc_col_len() { fxt.Test__calc_col_len(10, 0, 4); // for 10 items, col 0 has 4 items fxt.Test__calc_col_len(10, 1, 3); // for 10 items, col 1 has 3 items @@ -240,31 +224,32 @@ public class Xoctg_catpage_mgr_tst { fxt.Test__calc_col_len(12, 2, 4); } } -class Xoh_ctg_page_fxt { - public Xoh_ctg_page_fxt Clear() { +class Xoctg_catpage_mgr_fxt { + private int grp_max; + public Xoctg_catpage_mgr_fxt Clear() { if (app == null) { app = Xoa_app_fxt.Make__app__edit(); wiki = Xoa_app_fxt.Make__wiki__edit(app); - ctg_html = wiki.Html_mgr().Ns_ctg(); + ctg_html = wiki.Html_mgr().Catpage_mgr(); } ctg = new Xoctg_catpage_ctg(Bry_.new_a7("Ctg_1")); + grp_max = Xoctg_catpage_mgr.Grp_max_dflt; return this; } private Xoae_app app; private Xoctg_catpage_mgr ctg_html; public void Test__calc_col_len(int grp_len, int col_idx, int expd) {Tfds.Eq(expd, Xoctg_fmt_itm_base.Calc_col_len(grp_len, col_idx, 3));} public Xowe_wiki Wiki() {return wiki;} private Xowe_wiki wiki; public Xoctg_catpage_ctg Ctg() {return ctg;} private Xoctg_catpage_ctg ctg; - public void Test_bld_rslts_lnk(boolean next, String ctg_str, String expd) { - Xoctg_fmt_grp.Grp__max = 0; - byte[] actl = ctg_html.Fmt(Xoa_ctg_mgr.Tid_page).Bld_bwd_fwd(wiki, Xoa_ttl.Parse(wiki, Bry_.new_a7(ctg_str)), ctg.Grp_by_tid(Xoa_ctg_mgr.Tid_page)); + public void Test__navlink(boolean next, String ctg_str, String expd) { + byte[] actl = ctg_html.Fmt(Xoa_ctg_mgr.Tid__page).Bld_bwd_fwd(wiki, Xoa_ttl.Parse(wiki, Bry_.new_a7(ctg_str)), ctg.Grp_by_tid(Xoa_ctg_mgr.Tid__page), grp_max); Tfds.Eq_str_lines(expd, String_.new_u8(actl)); - Xoctg_fmt_grp.Grp__max = 200; } - public Xoh_ctg_page_fxt Init_grp__pages(int bgn, int count) {ctg.Pages().Init(bgn, count); return this;} - public Xoh_ctg_page_fxt Init_grp__files(int bgn, int count) {ctg.Files().Init(bgn, count); return this;} - public Xoh_ctg_page_fxt Init_itms__pages(String... titles) {return Init_itms(Xoa_ctg_mgr.Tid_page, titles);} - public Xoh_ctg_page_fxt Init_itms__files(String... titles) {return Init_itms(Xoa_ctg_mgr.Tid_file, titles);} - public Xoh_ctg_page_fxt Init_itms__subcs(String... titles) {return Init_itms(Xoa_ctg_mgr.Tid_subc, titles);} - private Xoh_ctg_page_fxt Init_itms(byte tid, String[] ttls) { + public Xoctg_catpage_mgr_fxt Init_grp_max(int v) {this.grp_max = v; return this;} + public Xoctg_catpage_mgr_fxt Init_grp__pages(int bgn, int count) {ctg.Pages().Rng_(bgn, count); return this;} + public Xoctg_catpage_mgr_fxt Init_grp__files(int bgn, int count) {ctg.Files().Rng_(bgn, count); return this;} + public Xoctg_catpage_mgr_fxt Init_itms__pages(String... titles) {return Init_itms(Xoa_ctg_mgr.Tid__page, titles);} + public Xoctg_catpage_mgr_fxt Init_itms__files(String... titles) {return Init_itms(Xoa_ctg_mgr.Tid__file, titles);} + public Xoctg_catpage_mgr_fxt Init_itms__subcs(String... titles) {return Init_itms(Xoa_ctg_mgr.Tid__subc, titles);} + private Xoctg_catpage_mgr_fxt Init_itms(byte tid, String[] ttls) { int len = ttls.length; Xoctg_catpage_grp grp = ctg.Grp_by_tid(tid); for (int i = 0; i < len; ++i) { @@ -274,7 +259,7 @@ class Xoh_ctg_page_fxt { grp.Itms__make(); return this; } - public void Test_html_page(byte tid, byte grp_char_0, String expd) { + public void Test__html__page(byte tid, byte grp_char_0, String expd) { Xoctg_fmt_grp list_mgr = ctg_html.Fmt(tid); Xoctg_fmt_itm_base itm_fmt = list_mgr.Itm_fmt(); Xoctg_catpage_grp list = ctg.Grp_by_tid(tid); @@ -285,7 +270,7 @@ class Xoh_ctg_page_fxt { itm_fmt.Bfr_arg__add(bfr); Tfds.Eq_str_lines(expd, bfr.To_str_and_rls()); } - public void Test_html_grp(byte tid, String expd) { + public void Test__html_grp(byte tid, String expd) { Xoctg_fmt_grp list_mgr = ctg_html.Fmt(tid); Xoctg_fmt_ltr fmtr_grp = new Xoctg_fmt_ltr(list_mgr.Itm_fmt()); fmtr_grp.Init_from_grp(wiki, ctg.Grp_by_tid(tid)); @@ -293,9 +278,9 @@ class Xoh_ctg_page_fxt { fmtr_grp.Bfr_arg__add(bfr); Tfds.Eq_str_lines(expd, bfr.To_str_and_rls()); } - public void Test_html_all(byte tid, String expd) { + public void Test__html__all(byte tid, String expd) { Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_b512(); - ctg_html.Fmt(tid).Write_catpage_grp(bfr, wiki, wiki.Lang(), ctg); + ctg_html.Fmt(tid).Write_catpage_grp(bfr, wiki, wiki.Lang(), ctg, grp_max); Tfds.Eq_str_lines(expd, bfr.To_str_and_rls()); } } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr__navlink__tst.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr__navlink__tst.java new file mode 100644 index 000000000..c33f88423 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_mgr__navlink__tst.java @@ -0,0 +1,63 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.wikis.ctgs.htmls.catpages; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; +import org.junit.*; import gplx.xowa.htmls.core.htmls.*; +import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.fmts.*; +public class Xoctg_catpage_mgr__navlink__tst { + @Before public void init() {fxt.Clear();} private Xoctg_catpage_mgr_fxt fxt = new Xoctg_catpage_mgr_fxt(); + @Test public void Navlink__basic() { + fxt .Init_itms__pages("A1", "A2", "A3").Init_grp_max(1).Init_grp__pages(1, 2) + .Test__navlink(Bool_.Y, "Category:Ctg_1", String_.Concat_lines_nl + ( "" + , "(previous 1)" + , "(next 1)" + )); + } + @Test public void Navlink__encoded() { // escape quotes and spaces; DATE:2016-01-11 + fxt .Init_itms__pages("A\" 1", "A\" 2", "A\" 3").Init_grp_max(1).Init_grp__pages(1, 2) + .Test__navlink(Bool_.Y, "Category:Ctg_1", String_.Concat_lines_nl + ( "" + , "(previous 1)" + , "(next 1)" + )); + } + @Test public void Navlink__bos() { + fxt .Init_itms__pages("A1", "A2", "A3").Init_grp_max(1).Init_grp__pages(0, 1) + .Test__navlink(Bool_.Y, "Category:Ctg_1", String_.Concat_lines_nl + ( "" + , "(previous 1)" + , "(next 1)" + )); + } + @Test public void Navlink__eos() { + fxt .Init_itms__pages("A1", "A2", "A3").Init_grp_max(2).Init_grp__pages(2, 3) + .Test__navlink(Bool_.Y, "Category:Ctg_1", String_.Concat_lines_nl + ( "" + , "(previous 2)" + , "(next 2)" + )); + } + @Test public void Navlink__eos__2() { // PURPOSE.fix: do not suppress next page on penultimate page; DATE:2016-09-25 + fxt .Init_itms__pages("A1", "A2", "A3", "A4").Init_grp_max(2).Init_grp__pages(2, 3) + .Test__navlink(Bool_.Y, "Category:Ctg_1", String_.Concat_lines_nl + ( "" + , "(previous 2)" + , "(next 2)" + )); + } +} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_url.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_url.java deleted file mode 100644 index 9dcaaf231..000000000 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_url.java +++ /dev/null @@ -1,83 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.addons.wikis.ctgs.htmls.catpages; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; -import gplx.core.primitives.*; import gplx.core.net.*; import gplx.core.net.qargs.*; -import gplx.xowa.addons.wikis.ctgs.htmls.catpages.fmts.*; -public class Xoctg_catpage_url { - public byte[][] Grp_idxs() {return grp_idxs;} private byte[][] grp_idxs = new byte[3][]; - public byte[] Grp_fwds() {return grp_fwds;} private byte[] grp_fwds = new byte[3]; - private void Clear() { - for (int i = 0; i < 3; i++) { - grp_fwds[i] = Bool_.__byte; - grp_idxs[i] = null; - } - } - public Xoctg_catpage_url Parse(Gfo_usr_dlg usr_dlg, Xoa_url url) { - this.Clear(); - Gfo_qarg_itm[] args = url.Qargs_ary(); - int len = args.length; - for (int i = 0; i < len; i++) { - Gfo_qarg_itm arg = args[i]; - byte[] arg_key = arg.Key_bry(); - Object tid_obj = Arg_keys.Get_by_bry(arg_key); - if (tid_obj == null) {usr_dlg.Warn_many("", "", "unknown arg_key: ~{0}", String_.new_u8(arg_key)); continue;} // ignore invalid args - byte[] arg_val = arg.Val_bry(); - byte tid = ((Byte_obj_val)tid_obj).Val(); - switch (tid) { - case Tid_all_bgn: Set_grp(arg_val, Bool_.Y_byte, Xoa_ctg_mgr.Tid_subc, Xoa_ctg_mgr.Tid_file, Xoa_ctg_mgr.Tid_page); break; // if "from", default all grps to val; DATE:2014-02-05 - case Tid_all_end: Set_grp(arg_val, Bool_.N_byte, Xoa_ctg_mgr.Tid_subc, Xoa_ctg_mgr.Tid_file, Xoa_ctg_mgr.Tid_page); break; - case Tid_subc_bgn: Set_grp(arg_val, Bool_.Y_byte, Xoa_ctg_mgr.Tid_subc); break; - case Tid_subc_end: Set_grp(arg_val, Bool_.N_byte, Xoa_ctg_mgr.Tid_subc); break; - case Tid_file_bgn: Set_grp(arg_val, Bool_.Y_byte, Xoa_ctg_mgr.Tid_file); break; - case Tid_file_end: Set_grp(arg_val, Bool_.N_byte, Xoa_ctg_mgr.Tid_file); break; - case Tid_page_bgn: Set_grp(arg_val, Bool_.Y_byte, Xoa_ctg_mgr.Tid_page); break; - case Tid_page_end: Set_grp(arg_val, Bool_.N_byte, Xoa_ctg_mgr.Tid_page); break; - } - } - return this; - } - private void Set_grp(byte[] val, byte fwd, byte... tids) { - int tids_len = tids.length; - for (int i = 0; i < tids_len; i++) { - byte tid = tids[i]; - grp_fwds[tid] = fwd; - grp_idxs[tid] = val; - } - } - public static final byte Tid_all_bgn = 0, Tid_subc_bgn = 1, Tid_subc_end = 2, Tid_file_bgn = 3, Tid_file_end = 4, Tid_page_bgn = 5, Tid_page_end = 6, Tid_all_end = 8; - public static final byte[] - Url_arg_from = Bry_.new_a7("from") - , Url_arg_until = Bry_.new_a7("until") - , Url_arg_subc_bgn = Bry_.new_a7("subcatfrom") - , Url_arg_subc_end = Bry_.new_a7("subcatuntil") - , Url_arg_page_bgn = Bry_.new_a7("pagefrom") - , Url_arg_page_end = Bry_.new_a7("pageuntil") - , Url_arg_file_bgn = Bry_.new_a7("filefrom") - , Url_arg_file_end = Bry_.new_a7("fileuntil") - ; - public static final Hash_adp_bry Arg_keys = Hash_adp_bry.ci_a7() - .Add_bry_byte(Url_arg_from , Tid_all_bgn) - .Add_bry_byte(Url_arg_until , Tid_all_end) - .Add_bry_byte(Url_arg_subc_bgn , Tid_subc_bgn) - .Add_bry_byte(Url_arg_subc_end , Tid_subc_end) - .Add_bry_byte(Url_arg_file_bgn , Tid_file_bgn) - .Add_bry_byte(Url_arg_file_end , Tid_file_end) - .Add_bry_byte(Url_arg_page_bgn , Tid_page_bgn) - .Add_bry_byte(Url_arg_page_end , Tid_page_end) - ; -} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_url_tst.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_url_tst.java deleted file mode 100644 index faaa55d53..000000000 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_url_tst.java +++ /dev/null @@ -1,72 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.addons.wikis.ctgs.htmls.catpages; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; -import org.junit.*; import gplx.xowa.apps.urls.*; -public class Xoctg_catpage_url_tst { - @Before public void init() {fxt.Clear();} private Xoctg_catpage_url_fxt fxt = new Xoctg_catpage_url_fxt(); - @Test public void Basic() { - fxt.Test_parse("A?subcatfrom=B&filefrom=C&pagefrom=D", fxt.url().Grp_idxs_("B", "C", "D").Grp_fwds_(Bool_.Y_byte, Bool_.Y_byte, Bool_.Y_byte)); - fxt.Test_parse("A?subcatuntil=B&fileuntil=C&pageuntil=D", fxt.url().Grp_idxs_("B", "C", "D").Grp_fwds_(Bool_.N_byte, Bool_.N_byte, Bool_.N_byte)); - fxt.Test_parse("A?from=B", fxt.url().Grp_idxs_("B", "B", "B").Grp_fwds_(Bool_.Y_byte, Bool_.Y_byte, Bool_.Y_byte)); - fxt.Test_parse("A?until=B", fxt.url().Grp_idxs_("B", "B", "B").Grp_fwds_(Bool_.N_byte, Bool_.N_byte, Bool_.N_byte)); - } -} -class Xoctg_catpage_url_fxt { - public Xoctg_catpage_url_chkr url() {return expd.Clear();} private Xoctg_catpage_url_chkr expd; - public void Clear() { - if (parser == null) { - Xoa_app app = Xoa_app_fxt.Make__app__edit(); - parser = app.User().Wikii().Utl__url_parser(); - page_url = Xoa_url.blank(); - ctg_url = new Xoctg_catpage_url(); - expd = new Xoctg_catpage_url_chkr(); - } - } private Xow_url_parser parser; Xoa_url page_url; Xoctg_catpage_url ctg_url; - public void Test_parse(String url_str, Xoctg_catpage_url_chkr expd) { - page_url = parser.Parse(Bry_.new_u8(url_str)); - ctg_url.Parse(Gfo_usr_dlg_.Test(), page_url); - expd.Chk(ctg_url); - expd.Clear(); - } -} -class Xoctg_catpage_url_chkr { - public Xoctg_catpage_url_chkr Grp_idxs_(String subc, String file, String page) { - grp_idxs[Xoa_ctg_mgr.Tid_subc] = Bry_.new_a7(subc); - grp_idxs[Xoa_ctg_mgr.Tid_file] = Bry_.new_a7(file); - grp_idxs[Xoa_ctg_mgr.Tid_page] = Bry_.new_a7(page); - return this; - } byte[][] grp_idxs = new byte[Xoa_ctg_mgr.Tid__max][]; - public Xoctg_catpage_url_chkr Grp_fwds_(byte subc, byte file, byte page) { - grp_fwds[Xoa_ctg_mgr.Tid_subc] = subc; - grp_fwds[Xoa_ctg_mgr.Tid_file] = file; - grp_fwds[Xoa_ctg_mgr.Tid_page] = page; - return this; - } byte[] grp_fwds = new byte[Xoa_ctg_mgr.Tid__max]; - public void Chk(Xoctg_catpage_url actl) { - Tfds.Eq_ary_str(String_.Ary(grp_idxs), String_.Ary(actl.Grp_idxs())); - Tfds.Eq_ary(grp_fwds, actl.Grp_fwds()); - } - public Xoctg_catpage_url_chkr Clear() { - int len = Xoa_ctg_mgr.Tid__max; - for (int i = 0; i < len; i++) { - grp_idxs[i] = null; - grp_fwds[i] = Bool_.__byte; - } - return this; - } -} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_ctg.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_ctg.java index 9c517d680..0dadf10c2 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_ctg.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_ctg.java @@ -19,20 +19,20 @@ package gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms; import gplx.*; import g public class Xoctg_catpage_ctg { public Xoctg_catpage_ctg(byte[] name) {this.name = name;} public byte[] Name() {return name;} private final byte[] name; - public Xoctg_catpage_grp Subcs() {return subcs;} private final Xoctg_catpage_grp subcs = new Xoctg_catpage_grp(Xoa_ctg_mgr.Tid_subc); - public Xoctg_catpage_grp Pages() {return pages;} private final Xoctg_catpage_grp pages = new Xoctg_catpage_grp(Xoa_ctg_mgr.Tid_page); - public Xoctg_catpage_grp Files() {return files;} private final Xoctg_catpage_grp files = new Xoctg_catpage_grp(Xoa_ctg_mgr.Tid_file); - public int Total() {return subcs.Total() + pages.Total() + files.Total();} + public Xoctg_catpage_grp Subcs() {return subcs;} private final Xoctg_catpage_grp subcs = new Xoctg_catpage_grp(Xoa_ctg_mgr.Tid__subc); + public Xoctg_catpage_grp Pages() {return pages;} private final Xoctg_catpage_grp pages = new Xoctg_catpage_grp(Xoa_ctg_mgr.Tid__page); + public Xoctg_catpage_grp Files() {return files;} private final Xoctg_catpage_grp files = new Xoctg_catpage_grp(Xoa_ctg_mgr.Tid__file); + public int Total() {return subcs.Itms__len() + pages.Itms__len() + files.Itms__len();} public Xoctg_catpage_grp Grp_by_tid(byte tid) { switch (tid) { - case Xoa_ctg_mgr.Tid_subc: return subcs; - case Xoa_ctg_mgr.Tid_page: return pages; - case Xoa_ctg_mgr.Tid_file: return files; + case Xoa_ctg_mgr.Tid__subc: return subcs; + case Xoa_ctg_mgr.Tid__page: return pages; + case Xoa_ctg_mgr.Tid__file: return files; default: throw Err_.new_unhandled(tid); } } public void Make_itms() { - for (byte i = 0; i < Xoa_ctg_mgr.Tid__max; ++i) { + for (byte i = 0; i < Xoa_ctg_mgr.Tid___max; ++i) { Xoctg_catpage_grp grp = Grp_by_tid(i); grp.Itms__make(); } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_grp.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_grp.java index 0dd19c1f0..cd4556f18 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_grp.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_grp.java @@ -17,28 +17,27 @@ along with this program. If not, see . */ package gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; public class Xoctg_catpage_grp { + private final List_adp tmp_list = List_adp_.New(); public Xoctg_catpage_grp(byte tid) {this.tid = tid;} - public byte Tid() {return tid;} private byte tid; // subc|page|file - public int Bgn() {return bgn;} private int bgn; - public int End() {return end;} private int end; - public int Len() {return end - bgn;} - public int Total() {return total;} private int total; + public byte Tid() {return tid;} private byte tid; // subc|page|file + public int Bgn() {return bgn;} private int bgn; // idx of 1st item; EX: 200 + public int End() {return end;} private int end; // idx of nth item + 1; EX: 399 + public int Count_by_page() {return end - bgn;} // count of items on page; EX: 200 + public int Itms__len() {return itms_len;} private int itms_len; // count of items in entire category; EX: 456 public Xoctg_catpage_itm[] Itms() {return itms;} private Xoctg_catpage_itm[] itms = Xoctg_catpage_itm.Ary_empty; - public byte[] Itms__nth_sortkey() {return itms__nth_sortkey;} - public Xoctg_catpage_itm Itms__get_at_0th() {return Itms__get_at(0);} - public Xoctg_catpage_itm Itms__get_at_nth() {return Itms__get_at(itms.length - 1);} - private Xoctg_catpage_itm Itms__get_at(int i) { - if (i < 0 || i >= itms.length) throw Err_.new_wo_type("ctg.view: i is out of bounds", "i", i, "len", itms.length, "tid", tid); + public Xoctg_catpage_itm Itms__get_at(int i) { + if (i < 0 || i >= itms.length) throw Err_.new_wo_type("catpage: i is out of bounds", "i", i, "len", itms.length, "tid", tid); Xoctg_catpage_itm rv = itms[i]; if (rv == null) throw Err_.new_wo_type("ctg.view: itm is null", "i", i, "len", itms.length, "tid", tid); return rv; } public void Itms__add(Xoctg_catpage_itm sub) {tmp_list.Add(sub);} public void Itms__make() { - itms = (Xoctg_catpage_itm[])tmp_list.To_ary(Xoctg_catpage_itm.class); - total = end = itms.length; + tmp_list.Sort_by(Xoctg_catpage_itm_sorter__sort_key.Sorter); + itms = (Xoctg_catpage_itm[])tmp_list.To_ary_and_clear(Xoctg_catpage_itm.class); + bgn = 0; + end = itms_len = itms.length; + } + public void Rng_(int bgn, int end) { + this.bgn = bgn; this.end = end; } - public Xoctg_catpage_grp Itms__nth_sortkey_(byte[] v) {itms__nth_sortkey = v; return this;} private byte[] itms__nth_sortkey; - - public void Init(int bgn, int end) {this.bgn = bgn; this.end = end;} // TEST: - public List_adp Itms_list() {return tmp_list;} private final List_adp tmp_list = List_adp_.New(); // TEST } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_itm.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_itm.java index cb8906b7a..8fb9a12cb 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_itm.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_itm.java @@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; -public class Xoctg_catpage_itm implements gplx.CompareAble { +public class Xoctg_catpage_itm { public Xoctg_catpage_itm(int page_id, Xoa_ttl page_ttl, byte[] sort_key) { this.page_id = page_id; this.page_ttl = page_ttl; @@ -28,6 +28,5 @@ public class Xoctg_catpage_itm implements gplx.CompareAble { public boolean Missing() {return missing;} private boolean missing; // not used; remove?; public void Missing_y_() {missing = true;} - public int compareTo(Object obj) {Xoctg_catpage_itm comp = (Xoctg_catpage_itm)obj; return Int_.Compare(page_id, comp.Page_id());} public static final Xoctg_catpage_itm[] Ary_empty = new Xoctg_catpage_itm[0]; } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_catpage_itm_sorter.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_itm_sorter__sort_key.java similarity index 70% rename from 400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_catpage_itm_sorter.java rename to 400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_itm_sorter__sort_key.java index 9ad764e5b..3bdb77ede 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_catpage_itm_sorter.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/doms/Xoctg_catpage_itm_sorter__sort_key.java @@ -15,13 +15,16 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package gplx.xowa.addons.wikis.ctgs.htmls.catpages.fmts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; -import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; -class Xoctg_catpage_itm_sorter implements gplx.core.lists.ComparerAble { +package gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; +public class Xoctg_catpage_itm_sorter__sort_key implements gplx.core.lists.ComparerAble, gplx.core.lists.binary_searches.Binary_comparer { public int compare(Object lhsObj, Object rhsObj) { Xoctg_catpage_itm lhs = (Xoctg_catpage_itm)lhsObj; Xoctg_catpage_itm rhs = (Xoctg_catpage_itm)rhsObj; return Bry_.Compare(lhs.Sort_key(), rhs.Sort_key()); } - public static final Xoctg_catpage_itm_sorter Sorter = new Xoctg_catpage_itm_sorter(); + public int Compare_val_to_obj(Object val, Object obj) { + Xoctg_catpage_itm itm = (Xoctg_catpage_itm)obj; + return Bry_.Compare((byte[])val, itm.Sort_key()); + } + public static final Xoctg_catpage_itm_sorter__sort_key Sorter = new Xoctg_catpage_itm_sorter__sort_key(); } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_grp.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_grp.java index 71c1a4321..44425d2ac 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_grp.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_grp.java @@ -18,7 +18,7 @@ along with this program. If not, see . package gplx.xowa.addons.wikis.ctgs.htmls.catpages.fmts; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.xowa.langs.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.htmls.core.htmls.*; import gplx.langs.htmls.encoders.*; import gplx.xowa.wikis.nss.*; -import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; +import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls.*; public class Xoctg_fmt_grp { // subc|page|file private final byte tid; private final byte[] div_id, url_arg_bgn, url_arg_end; @@ -32,45 +32,47 @@ public class Xoctg_fmt_grp { // subc|page|file this.url_arg_bgn = url_arg_bgn; this.url_arg_end = url_arg_end; this.div_id = div_id; } public Xoctg_fmt_itm_base Itm_fmt() {return itm_fmt;} private final Xoctg_fmt_itm_base itm_fmt; - public void Write_catpage_grp(Bry_bfr bfr, Xow_wiki wiki, Xol_lang_itm lang, Xoctg_catpage_ctg dom_ctg) { // TEST: + public void Write_catpage_grp(Bry_bfr bfr, Xow_wiki wiki, Xol_lang_itm lang, Xoctg_catpage_ctg dom_ctg, int grp_max) { // TEST: Xoctg_catpage_grp dom_grp = dom_ctg.Grp_by_tid(tid); - if (dom_grp.Itms().length == 0) return; // no items in grp; EX: 0 items in File + if (dom_grp.Itms__len() == 0) return; // no items in grp; EX: 0 items in File // get msgs Xow_msg_mgr msg_mgr = wiki.Msg_mgr(); byte[] msg_label_bry = msg_mgr.Val_by_id_args(msg_label_id, dom_ctg.Name()); - byte[] msg_stats_bry = msg_mgr.Val_by_id_args(msg_stats_id, dom_grp.Len(), dom_grp.Total()); + byte[] msg_stats_bry = msg_mgr.Val_by_id_args(msg_stats_id, dom_grp.Count_by_page(), dom_grp.Itms__len()); // get nav html; next / previous 200 Xoa_ttl ctg_ttl = wiki.Ttl_parse(Xow_ns_.Tid__category, dom_ctg.Name()); - byte[] nav_html = this.Bld_bwd_fwd(wiki, ctg_ttl, dom_grp); + byte[] nav_html = this.Bld_bwd_fwd(wiki, ctg_ttl, dom_grp, grp_max); - // sort; init grp; write - Array_.Sort(dom_grp.Itms(), Xoctg_catpage_itm_sorter.Sorter); // NOTE: must assert sorted for v1; PAGE:s.w:Category:Computer_science; DATE:2015-11-22 + // init grp; write itms_fmt.Init_from_grp(wiki, dom_grp); Fmt__ctg.Bld_many(bfr, div_id, msg_label_bry, msg_stats_bry, nav_html, lang.Key_bry(), lang.Dir_ltr_bry(), itms_fmt); } - public byte[] Bld_bwd_fwd(Xow_wiki wiki, Xoa_ttl ttl, Xoctg_catpage_grp view_grp) { // TEST: - if (view_grp.Total() < Grp__max) return Bry_.Empty; + public byte[] Bld_bwd_fwd(Xow_wiki wiki, Xoa_ttl ttl, Xoctg_catpage_grp view_grp, int grp_max) { // TEST: + if (view_grp.Itms__len() < grp_max) return Bry_.Empty; // < 200; never show; Bry_bfr bfr = wiki.Utl__bfr_mkr().Get_k004(); - Html_nav_bry(bfr, wiki, ttl, view_grp, Bool_.N); - Html_nav_bry(bfr, wiki, ttl, view_grp, Bool_.Y); + Html_nav_bry(bfr, wiki, ttl, view_grp, grp_max, Bool_.N); + Html_nav_bry(bfr, wiki, ttl, view_grp, grp_max, Bool_.Y); return bfr.To_bry_and_rls(); } - private void Html_nav_bry(Bry_bfr bfr, Xow_wiki wiki, Xoa_ttl ttl, Xoctg_catpage_grp view_grp, boolean type_is_bgn) { + private void Html_nav_bry(Bry_bfr bfr, Xow_wiki wiki, Xoa_ttl ttl, Xoctg_catpage_grp grp, int grp_max, boolean type_is_next) { Bry_bfr href_bfr = wiki.Utl__bfr_mkr().Get_b512(); // get nav_href; EX:href="/wiki/Category:Ctg_1?pageuntil=A1#mw-pages" wiki.Html__href_wtr().Build_to_bfr(href_bfr, wiki.App(), Xoh_wtr_ctx.Basic, wiki.Domain_bry(), ttl); byte[] arg_idx_lbl = null; byte[] arg_sortkey = null; - if (type_is_bgn) { + if (type_is_next) { arg_idx_lbl = url_arg_bgn; - arg_sortkey = view_grp.Itms__nth_sortkey(); - if (arg_sortkey == null) arg_sortkey = view_grp.Itms__get_at_nth().Sort_key(); + + // get next category after last one on page; needed for "Next 200 (href=Cat_201)" + int nxt_idx = grp.End(); + if (nxt_idx == grp.Itms__len()) --nxt_idx; // if last item, then grp.End() does not exist; just use last one + arg_sortkey = grp.Itms__get_at(nxt_idx).Sort_key(); } else { arg_idx_lbl = url_arg_end; - arg_sortkey = view_grp.Itms__get_at_0th().Sort_key(); + arg_sortkey = grp.Itms__get_at(grp.Bgn()).Sort_key(); // use 1st item as sortkey for "until" args } href_bfr.Add_byte(Byte_ascii.Question).Add(arg_idx_lbl).Add_byte(Byte_ascii.Eq); // filefrom= Gfo_url_encoder_.Http_url.Encode(href_bfr, arg_sortkey); // Abc @@ -78,19 +80,20 @@ public class Xoctg_fmt_grp { // subc|page|file byte[] nav_href = href_bfr.To_bry_and_rls(); // get nav_text - int nav_text_id = type_is_bgn ? Xol_msg_itm_.Id_next_results : Xol_msg_itm_.Id_prev_results; - byte[] nav_text = wiki.Msg_mgr().Val_by_id_args(nav_text_id, Grp__max); // next 200 / previous 200 + int nav_text_id = type_is_next ? Xol_msg_itm_.Id_next_results : Xol_msg_itm_.Id_prev_results; + byte[] nav_text = wiki.Msg_mgr().Val_by_id_args(nav_text_id, grp_max); // next 200 / previous 200 - // fmt - Fmt__nav.Bld_many(bfr, nav_href, ttl.Full_url(), nav_text); + // print text if 1st / zth page; else, print html + if ( ( type_is_next && grp.Bgn() + grp_max > grp.Itms__len()) + || (!type_is_next && grp.Bgn() - grp_max < 0) + ) + Fmt__nav__text.Bld_many(bfr, nav_text); + else + Fmt__nav__href.Bld_many(bfr, nav_href, ttl.Full_url(), nav_text); } - - public static int Grp__max = 200; private static final Bry_fmt - Fmt__nav = Bry_fmt.Auto_nl_skip_last - ( "" - , " (~{nav_text})" - ) + Fmt__nav__href = Bry_fmt.New("\n(~{nav_text})") + , Fmt__nav__text = Bry_fmt.New("\n(~{nav_text})") , Fmt__ctg = Bry_fmt.Auto_nl_skip_last ( "" , "
    " @@ -104,7 +107,7 @@ public class Xoctg_fmt_grp { // subc|page|file , "
    ~{nav_html}" , "
    " ); - public static Xoctg_fmt_grp New__subc() {return new Xoctg_fmt_grp(Xoa_ctg_mgr.Tid_subc, new Xoctg_fmt_itm_subc(), Xol_msg_itm_.Id_ctg_subc_header, Xol_msg_itm_.Id_ctg_subc_count, Xoctg_catpage_url.Url_arg_subc_bgn, Xoctg_catpage_url.Url_arg_subc_end, Bry_.new_a7("mw-subcategories"));} - public static Xoctg_fmt_grp New__page() {return new Xoctg_fmt_grp(Xoa_ctg_mgr.Tid_page, new Xoctg_fmt_itm_page(), Xol_msg_itm_.Id_ctg_page_header, Xol_msg_itm_.Id_ctg_page_count, Xoctg_catpage_url.Url_arg_page_bgn, Xoctg_catpage_url.Url_arg_page_end, Bry_.new_a7("mw-pages"));} - public static Xoctg_fmt_grp New__file() {return new Xoctg_fmt_grp(Xoa_ctg_mgr.Tid_file, new Xoctg_fmt_itm_file(), Xol_msg_itm_.Id_ctg_file_header, Xol_msg_itm_.Id_ctg_file_count, Xoctg_catpage_url.Url_arg_file_bgn, Xoctg_catpage_url.Url_arg_file_end, Bry_.new_a7("mw-category-media"));} + public static Xoctg_fmt_grp New__subc() {return new Xoctg_fmt_grp(Xoa_ctg_mgr.Tid__subc, new Xoctg_fmt_itm_subc(), Xol_msg_itm_.Id_ctg_subc_header, Xol_msg_itm_.Id_ctg_subc_count, Xoctg_catpage_url_parser.Bry__arg_subc_bgn, Xoctg_catpage_url_parser.Bry__arg_subc_end, Bry_.new_a7("mw-subcategories"));} + public static Xoctg_fmt_grp New__page() {return new Xoctg_fmt_grp(Xoa_ctg_mgr.Tid__page, new Xoctg_fmt_itm_page(), Xol_msg_itm_.Id_ctg_page_header, Xol_msg_itm_.Id_ctg_page_count, Xoctg_catpage_url_parser.Bry__arg_page_bgn, Xoctg_catpage_url_parser.Bry__arg_page_end, Bry_.new_a7("mw-pages"));} + public static Xoctg_fmt_grp New__file() {return new Xoctg_fmt_grp(Xoa_ctg_mgr.Tid__file, new Xoctg_fmt_itm_file(), Xol_msg_itm_.Id_ctg_file_header, Xol_msg_itm_.Id_ctg_file_count, Xoctg_catpage_url_parser.Bry__arg_file_bgn, Xoctg_catpage_url_parser.Bry__arg_file_end, Bry_.new_a7("mw-category-media"));} } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_itm_base.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_itm_base.java index 1672f54da..1d3c9b98e 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_itm_base.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_itm_base.java @@ -29,7 +29,7 @@ public abstract class Xoctg_fmt_itm_base implements gplx.core.brys.Bfr_arg { public int Loop_end_idx() {return loop_end_idx;} private int loop_end_idx; public boolean Loop_ends_at_col() {return loop_ends_at_col;} private boolean loop_ends_at_col; public void Col_end_(int col_bgn, int col_idx) { - this.col_end = col_bgn + Calc_col_len(grp.Len(), col_idx, Cols_max); + this.col_end = col_bgn + Calc_col_len(grp.Count_by_page(), col_idx, Cols_max); } public void Init_from_ltr(Xow_wiki wiki, Xoctg_catpage_grp grp) {this.wiki = wiki; this.grp = grp;} public void Set_ltr_and_bgn(byte[] ltr_cur, int loop_bgn) {this.ltr_cur = ltr_cur; this.loop_bgn = loop_bgn;} @@ -37,10 +37,10 @@ public abstract class Xoctg_fmt_itm_base implements gplx.core.brys.Bfr_arg { // init vars Xoh_href_parser href_parser = wiki.App().Html__href_parser(); Xou_history_mgr history_mgr = wiki.App().User().History_mgr(); - int len = grp.Len(); + int grp_end = grp.End(); // loop over itms; - for (int i = loop_bgn; i < len; i++) { + for (int i = loop_bgn; i < grp_end; i++) { // reached end of col; exit if (i == col_end) { loop_end_idx = i; @@ -49,7 +49,7 @@ public abstract class Xoctg_fmt_itm_base implements gplx.core.brys.Bfr_arg { } // get sortkey - Xoctg_catpage_itm itm = grp.Itms()[i]; + Xoctg_catpage_itm itm = grp.Itms__get_at(i); byte[] itm_sortkey = itm.Sort_key(); // reached end of ltr; exit @@ -61,7 +61,7 @@ public abstract class Xoctg_fmt_itm_base implements gplx.core.brys.Bfr_arg { Bld_html(bfr, wiki, history_mgr, href_parser, itm, itm.Page_ttl()); } - loop_end_idx = len; + loop_end_idx = grp_end; loop_ends_at_col = true; } @gplx.Virtual public void Bld_html(Bry_bfr bfr, Xow_wiki wiki, Xou_history_mgr history_mgr, Xoh_href_parser href_parser, Xoctg_catpage_itm itm, Xoa_ttl ttl) { diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_ltr.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_ltr.java index d3ee78af3..370b0da14 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_ltr.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/fmts/Xoctg_fmt_ltr.java @@ -32,16 +32,17 @@ public class Xoctg_fmt_ltr implements gplx.core.brys.Bfr_arg { // "A", "B", "C c itm_fmt.Init_from_ltr(wiki, grp); } public void Bfr_arg__add(Bry_bfr bfr) { - int itms_len = grp.Len(); if (itms_len == 0) return; // no items; exit - int itm_idx = grp.Bgn(); // itm idx; EX: idx=201 in len=500 + int itm_end = grp.End(); + int itms_len = itm_end - itm_idx; if (itms_len == 0) return; // no items; exit + int col_idx = 0; // col idx; EX: 3 cols; idx = 0, 1, 2 boolean start_new_col = true; byte[] ltr_prv = Bry_.Empty; // loop itms until no more itms - while (itm_idx < itms_len) { - Xoctg_catpage_itm itm = grp.Itms()[itm_idx]; + while (itm_idx < itm_end) { + Xoctg_catpage_itm itm = grp.Itms__get_at(itm_idx); // get ltr_head; EX: "C" or "C cont." byte[] itm_sortkey = itm.Sort_key(); diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/urls/Xoctg_catpage_url.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/urls/Xoctg_catpage_url.java new file mode 100644 index 000000000..c75640f95 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/urls/Xoctg_catpage_url.java @@ -0,0 +1,23 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; +public class Xoctg_catpage_url { + public Xoctg_catpage_url(byte[][] keys, boolean[] fwds) {this.keys = keys; this.fwds = fwds;} + public byte[][] Grp_keys() {return keys;} private final byte[][] keys; + public boolean[] Grp_fwds() {return fwds;} private final boolean[] fwds; +} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/urls/Xoctg_catpage_url__tst.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/urls/Xoctg_catpage_url__tst.java new file mode 100644 index 000000000..c1139287b --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/urls/Xoctg_catpage_url__tst.java @@ -0,0 +1,47 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; +import org.junit.*; import gplx.core.tests.*; import gplx.xowa.apps.urls.*; +public class Xoctg_catpage_url__tst { + @Before public void init() {fxt.Clear();} private Xoctg_catpage_url__fxt fxt = new Xoctg_catpage_url__fxt(); + @Test public void Specific() { + fxt.Exec__parse("A?subcatfrom=B&filefrom=C&pagefrom=D" ).Test__keys("B", "C", "D").Test__fwds(Bool_.Y, Bool_.Y, Bool_.Y); + fxt.Exec__parse("A?subcatuntil=B&fileuntil=C&pageuntil=D" ).Test__keys("B", "C", "D").Test__fwds(Bool_.N, Bool_.N, Bool_.N); + } + @Test public void General() { + fxt.Exec__parse("A?from=B" ).Test__keys("B", "B", "B").Test__fwds(Bool_.Y, Bool_.Y, Bool_.Y); + fxt.Exec__parse("A?until=B" ).Test__keys("B", "B", "B").Test__fwds(Bool_.N, Bool_.N, Bool_.N); + } + @Test public void Url_encoded() { + fxt.Exec__parse("A?from=B+C").Test__keys("B C", "B C", "B C").Test__fwds(Bool_.Y, Bool_.Y, Bool_.Y); + } +} +class Xoctg_catpage_url__fxt { + private Xow_url_parser xo_url_parser; private Xoctg_catpage_url ctg_url; + public void Clear() { + Xoa_app app = Xoa_app_fxt.Make__app__edit(); + this.xo_url_parser = app.User().Wikii().Utl__url_parser(); + } + public Xoctg_catpage_url__fxt Exec__parse(String url_str) { + Xoa_url page_url = xo_url_parser.Parse(Bry_.new_u8(url_str)); + this.ctg_url = Xoctg_catpage_url_parser.Parse(page_url); + return this; + } + public Xoctg_catpage_url__fxt Test__keys(String... expd) {Gftest.Eq__ary(Bry_.Ary(expd), ctg_url.Grp_keys(), "keys"); return this;} + public Xoctg_catpage_url__fxt Test__fwds(boolean... expd) {Gftest.Eq__ary(expd, ctg_url.Grp_fwds(), "fwds"); return this;} +} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/urls/Xoctg_catpage_url_parser.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/urls/Xoctg_catpage_url_parser.java new file mode 100644 index 000000000..25e90b494 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/urls/Xoctg_catpage_url_parser.java @@ -0,0 +1,90 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; +import gplx.core.net.*; import gplx.core.net.qargs.*; +import gplx.langs.htmls.encoders.*; +public class Xoctg_catpage_url_parser { + public static Xoctg_catpage_url Parse(Xoa_url url) { + Gfo_qarg_itm[] args = url.Qargs_ary(); + if (args == null) return null; + + // init caturl structs + byte[][] keys = new byte[Xoa_ctg_mgr.Tid___max][]; + boolean[] fwds = new boolean[Xoa_ctg_mgr.Tid___max]; + Bry_bfr tmp_bfr = Bry_bfr_.New(); + + // loop qargs; EX: "?subcatfrom=B&filefrom=C&pagefrom=D" + int len = args.length; + for (int i = 0; i < len; ++i) { + Gfo_qarg_itm arg = args[i]; + + // get tid from arg; EX: "pagefrom" -> Tid__page_bgn + byte[] key = arg.Key_bry(); + byte tid = Key_hash.Get_as_byte_or(key, Byte_ascii.Max_7_bit); + if (tid == Byte_ascii.Max_7_bit) { // if invalid, warn and skip + Gfo_usr_dlg_.Instance.Warn_many("", "", "catpage:unknown url arg: raw~={0} key=~{1}", url.To_bry(true, true), key); + continue; + } + + // set val + byte[] val = arg.Val_bry(); + Gfo_url_encoder_.Http_url.Decode(tmp_bfr, Bool_.N, val, 0, val.length); + val = tmp_bfr.To_bry_and_clear(); + + // set struct + switch (tid) { + case Tid__each_bgn: Set_grp(keys, fwds, val, Bool_.Y, Xoa_ctg_mgr.Tid__subc, Xoa_ctg_mgr.Tid__file, Xoa_ctg_mgr.Tid__page); break; // if "from", default all grps to val; DATE:2014-02-05 + case Tid__each_end: Set_grp(keys, fwds, val, Bool_.N, Xoa_ctg_mgr.Tid__subc, Xoa_ctg_mgr.Tid__file, Xoa_ctg_mgr.Tid__page); break; + case Tid__subc_bgn: Set_grp(keys, fwds, val, Bool_.Y, Xoa_ctg_mgr.Tid__subc); break; + case Tid__subc_end: Set_grp(keys, fwds, val, Bool_.N, Xoa_ctg_mgr.Tid__subc); break; + case Tid__file_bgn: Set_grp(keys, fwds, val, Bool_.Y, Xoa_ctg_mgr.Tid__file); break; + case Tid__file_end: Set_grp(keys, fwds, val, Bool_.N, Xoa_ctg_mgr.Tid__file); break; + case Tid__page_bgn: Set_grp(keys, fwds, val, Bool_.Y, Xoa_ctg_mgr.Tid__page); break; + case Tid__page_end: Set_grp(keys, fwds, val, Bool_.N, Xoa_ctg_mgr.Tid__page); break; + } + } + return new Xoctg_catpage_url(keys, fwds); + } + private static void Set_grp(byte[][] keys, boolean[] fwds, byte[] key, boolean fwd, byte... tids) { + int len = tids.length; + for (int i = 0; i < len; ++i) { + byte tid = tids[i]; + keys[tid] = key; + fwds[tid] = fwd; + } + } + + private static final byte + Tid__each_bgn = 0, Tid__each_end = 1 + , Tid__subc_bgn = 2, Tid__subc_end = 3 + , Tid__file_bgn = 4, Tid__file_end = 5 + , Tid__page_bgn = 6, Tid__page_end = 7 + ; + public static final byte[] + Bry__arg_each_bgn = Bry_.new_a7("from") , Bry__arg_each_end = Bry_.new_a7("until") + , Bry__arg_subc_bgn = Bry_.new_a7("subcatfrom") , Bry__arg_subc_end = Bry_.new_a7("subcatuntil") + , Bry__arg_page_bgn = Bry_.new_a7("pagefrom") , Bry__arg_page_end = Bry_.new_a7("pageuntil") + , Bry__arg_file_bgn = Bry_.new_a7("filefrom") , Bry__arg_file_end = Bry_.new_a7("fileuntil") + ; + private static final Hash_adp_bry Key_hash = Hash_adp_bry.ci_a7() + .Add_bry_byte(Bry__arg_each_bgn , Tid__each_bgn) .Add_bry_byte(Bry__arg_each_end , Tid__each_end) + .Add_bry_byte(Bry__arg_subc_bgn , Tid__subc_bgn) .Add_bry_byte(Bry__arg_subc_end , Tid__subc_end) + .Add_bry_byte(Bry__arg_page_bgn , Tid__page_bgn) .Add_bry_byte(Bry__arg_page_end , Tid__page_end) + .Add_bry_byte(Bry__arg_file_bgn , Tid__file_bgn) .Add_bry_byte(Bry__arg_file_end , Tid__file_end) + ; +} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/utls/Xoctg_catpage_filter.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/utls/Xoctg_catpage_filter.java new file mode 100644 index 000000000..e5f039e1f --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/utls/Xoctg_catpage_filter.java @@ -0,0 +1,62 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.wikis.ctgs.htmls.catpages.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; +import gplx.core.lists.binary_searches.*; +import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls.*; +public class Xoctg_catpage_filter { + public static void Filter(int limit, Xoctg_catpage_url cat_url, Xoctg_catpage_ctg ctg) { + int len = Xoa_ctg_mgr.Tid___max; + for (byte i = 0; i < len; ++i) { + Filter_by_grp(limit, cat_url, ctg.Grp_by_tid(i)); + } + } + private static void Filter_by_grp(int grp_len, Xoctg_catpage_url cat_url, Xoctg_catpage_grp grp) { + byte grp_tid = grp.Tid(); + byte[] grp_key = cat_url.Grp_keys()[grp_tid]; + + // dflt to bos; EX: grp_bgn=0 grp_end=200 + int grp_bgn = 0; + int grp_end = grp_bgn + grp_len; + + // key specified; calc new grp_bgn, grp_end + if (grp_key != null) { + // get idx of key + int key_idx = Binary_search_.Search(grp.Itms(), Xoctg_catpage_itm_sorter__sort_key.Sorter, grp_key); + + // if fwd, set grp_bgn to key_idx, and add grp_len + if (cat_url.Grp_fwds()[grp_tid]) { + grp_bgn = key_idx; + grp_end = grp_bgn + grp_len; + } + // if bwd, set grp_end to key_idx, and subtract grp_len + else { + grp_end = key_idx; + grp_bgn = grp_end - grp_len; + + // assert new grp_bgn is not negative + if (grp_bgn < 0) grp_bgn = 0; + } + } + + // assert new grp_end is not > grp_max; note that this needs to be specified for when grp_key is null and not null + int grp_max = grp.Itms__len(); + if (grp_end > grp_max) + grp_end = grp_max; + grp.Rng_(grp_bgn, grp_end); + } +} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/utls/Xoctg_catpage_filter__tst.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/utls/Xoctg_catpage_filter__tst.java new file mode 100644 index 000000000..8f02083c8 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/utls/Xoctg_catpage_filter__tst.java @@ -0,0 +1,107 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.wikis.ctgs.htmls.catpages.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; +import org.junit.*; import gplx.core.tests.*; +import gplx.xowa.apps.urls.*; +import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls.*; +public class Xoctg_catpage_filter__tst { + private final Xoctg_catpage_filter__fxt fxt = new Xoctg_catpage_filter__fxt(); + private Xoctg_catpage_ctg ctg; + @Before public void init() { + this.ctg = fxt.Make__ctg(25, 25, 25); + } + @Test public void Initial() { + fxt.Exec__filter(5, "A", ctg); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__subc, 0, 5); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__page, 0, 5); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__file, 0, 5); + } + @Test public void Fwd__page__05() { + fxt.Exec__filter(5, "A?pagefrom=05", ctg); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__page, 5, 10); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__subc, 0, 5); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__file, 0, 5); + } + @Test public void Fwd__page__10() { + fxt.Exec__filter(5, "A?pagefrom=10", ctg); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__page, 10, 15); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__subc, 0, 5); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__file, 0, 5); + } + @Test public void Fwd__page__23() { + fxt.Exec__filter(5, "A?pagefrom=23", ctg); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__page, 23, 25); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__subc, 0, 5); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__file, 0, 5); + } + @Test public void Fwd__full__06() { + fxt.Exec__filter(5, "A?from=06", ctg); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__page, 6, 11); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__subc, 6, 11); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__file, 6, 11); + } + @Test public void Bwd__page__20() { + fxt.Exec__filter(5, "A?pageuntil=20", ctg); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__page, 15, 20); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__subc, 0, 5); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__file, 0, 5); + } + @Test public void Bwd__page__2() { + fxt.Exec__filter(5, "A?pageuntil=01", ctg); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__page, 0, 1); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__subc, 0, 5); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__file, 0, 5); + } + @Test public void Bwd__full__11() { + fxt.Exec__filter(5, "A?until=11", ctg); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__page, 6, 11); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__subc, 6, 11); + fxt.Test__cat_grp(ctg, Xoa_ctg_mgr.Tid__file, 6, 11); + } +} +class Xoctg_catpage_filter__fxt { + private Xow_url_parser url_parser; + public Xoctg_catpage_filter__fxt() { + Xoa_app app = Xoa_app_fxt.Make__app__edit(); + this.url_parser = app.User().Wikii().Utl__url_parser(); + } + public Xoctg_catpage_ctg Make__ctg(int subc, int page, int file) { + Xoctg_catpage_ctg ctg = new Xoctg_catpage_ctg(Bry_.new_a7("A")); + Make__ctg_grp(ctg, Xoa_ctg_mgr.Tid__subc, subc); + Make__ctg_grp(ctg, Xoa_ctg_mgr.Tid__page, page); + Make__ctg_grp(ctg, Xoa_ctg_mgr.Tid__file, file); + return ctg; + } + private void Make__ctg_grp(Xoctg_catpage_ctg ctg, byte tid, int count) { + Xoctg_catpage_grp grp = ctg.Grp_by_tid(tid); + for (int i = 0; i < count; ++i) { + Xoctg_catpage_itm itm = new Xoctg_catpage_itm(i * tid, Xoa_ttl.Null, Bry_.new_a7(Int_.To_str_pad_bgn_zero(i, 2))); + grp.Itms__add(itm); + } + grp.Itms__make(); + } + public void Exec__filter(int limit, String cat_url_str, Xoctg_catpage_ctg ctg) { + Xoctg_catpage_url cat_url = Xoctg_catpage_url_parser.Parse(url_parser.Parse(Bry_.new_a7(cat_url_str))); + Xoctg_catpage_filter.Filter(limit, cat_url, ctg); + } + public void Test__cat_grp(Xoctg_catpage_ctg ctg, byte tid, int expd_bgn, int expd_end) { + Xoctg_catpage_grp grp = ctg.Grp_by_tid(tid); + Gftest.Eq__int(expd_bgn, grp.Bgn(), "bgn failed; tid={0}", tid); + Gftest.Eq__int(expd_end, grp.End(), "end failed; tid={0}", tid); + } +} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_loader.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/utls/Xoctg_catpage_loader.java similarity index 76% rename from 400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_loader.java rename to 400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/utls/Xoctg_catpage_loader.java index 65298c63c..aba13c4c7 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/Xoctg_catpage_loader.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/catpages/utls/Xoctg_catpage_loader.java @@ -15,10 +15,11 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package gplx.xowa.addons.wikis.ctgs.htmls.catpages; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; +package gplx.xowa.addons.wikis.ctgs.htmls.catpages.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.addons.wikis.ctgs.htmls.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.addons.wikis.ctgs.dbs.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; -class Xoctg_catpage_loader { +public class Xoctg_catpage_loader { + private static final Object thread_lock = new Object(); public Xoctg_catpage_ctg Load_by_ttl_or_null(Xow_wiki wiki, Xoa_ttl cat_ttl) { // get cat_id for cat_ttl from page_tbl Xow_db_mgr db_mgr = wiki.Data__core_mgr(); @@ -34,7 +35,7 @@ class Xoctg_catpage_loader { int cat_id = page_itm.Id(); Xowd_category_itm cat_core_itm = cat_core_tbl.Select(cat_id); if (cat_core_itm == Xowd_category_itm.Null) { - Gfo_usr_dlg_.Instance.Warn_many("", "", "category does not exist in cat_core table; ttl=~{0}", cat_ttl.Full_db()); + Gfo_usr_dlg_.Instance.Log_many("", "", "category does not exist in cat_core table; ttl=~{0}", cat_ttl.Full_db()); // NOTE: this is not rare as Category pages can be created as deliberately empty, or as redirects; fr.w:Catégorie:Utilisateur_hess-4; DATE:2016-09-12 return null; } @@ -72,21 +73,23 @@ class Xoctg_catpage_loader { // prep sql Db_attach_mgr attach_mgr = new Db_attach_mgr(cat_link_conn, new Db_attach_itm("page_db", page_conn), new Db_attach_itm("cat_core_db", cat_core_conn)); sql = attach_mgr.Resolve_sql(sql); - attach_mgr.Attach(); // run sql and create itms based on cat_link Db_rdr rdr = Db_rdr_.Empty; - try { - rdr = cat_link_conn.Stmt_sql(sql).Exec_select__rls_auto(); - while (rdr.Move_next()) { - Xoa_ttl page_ttl = wiki.Ttl_parse(rdr.Read_int("page_namespace"), rdr.Read_bry_by_str("page_title")); - Xoctg_catpage_itm itm = new Xoctg_catpage_itm(rdr.Read_int("cl_from"), page_ttl, Bry_.new_u8(rdr.Read_str("cl_sortkey"))); - rv.Grp_by_tid(rdr.Read_byte("cl_type_id")).Itms__add(itm); + synchronized (thread_lock) { // LOCK:used by multiple wrks; DATE:2016-09-12 + try { + attach_mgr.Attach(); + rdr = cat_link_conn.Stmt_sql(sql).Exec_select__rls_auto(); + while (rdr.Move_next()) { + Xoa_ttl page_ttl = wiki.Ttl_parse(rdr.Read_int("page_namespace"), rdr.Read_bry_by_str("page_title")); + Xoctg_catpage_itm itm = new Xoctg_catpage_itm(rdr.Read_int("cl_from"), page_ttl, Bry_.new_u8(rdr.Read_str("cl_sortkey"))); + rv.Grp_by_tid(rdr.Read_byte("cl_type_id")).Itms__add(itm); + } + } + finally { + rdr.Rls(); + attach_mgr.Detach(); } - } - finally { - rdr.Rls(); - attach_mgr.Detach(); } } private static String Sql_for_v3(int cat_id) { diff --git a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/pageboxs/Xoctg_pagebox_hash.java b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/pageboxs/Xoctg_pagebox_hash.java index e68dff9dd..bdcdba216 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/pageboxs/Xoctg_pagebox_hash.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/pageboxs/Xoctg_pagebox_hash.java @@ -23,7 +23,7 @@ class Xoctg_pagebox_hash { public int Len() {return hash_by_ttl.Len();} public Xoctg_pagebox_itm Get_at(int i) {return (Xoctg_pagebox_itm)hash_by_ttl.Get_at(i);} public Xoctg_pagebox_itm Get_by_ttl(byte[] full_db) {return (Xoctg_pagebox_itm)hash_by_ttl.Get_by(full_db);} - public Xoctg_pagebox_itm Get_by_id(int page_id) {return (Xoctg_pagebox_itm)hash_by_id.Get_by(page_id);} + public Xoctg_pagebox_itm Get_by_id(int page_id) {return (Xoctg_pagebox_itm)hash_by_id.Get_by_or_fail(page_id);} public Xoctg_pagebox_itm[] To_ary_and_clear() { hash_by_id.Clear(); return (Xoctg_pagebox_itm[])hash_by_ttl.To_ary_and_clear(Xoctg_pagebox_itm.class); diff --git a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader.java b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader.java index b0f4bde14..c9d04704f 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader.java @@ -16,9 +16,9 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.addons.wikis.pages.syncs.core.loaders; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*; -import gplx.core.brys.*; +import gplx.core.brys.*; import gplx.core.btries.*; import gplx.langs.htmls.*; import gplx.xowa.htmls.*; import gplx.langs.htmls.docs.*; import gplx.xowa.htmls.core.wkrs.*; import gplx.xowa.htmls.core.wkrs.imgs.atrs.*; -import gplx.xowa.files.*; import gplx.xowa.files.repos.*; +import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.imgs.*; import gplx.xowa.wikis.domains.*; import gplx.xowa.addons.wikis.pages.syncs.core.parsers.*; public class Xosync_page_loader { @@ -36,6 +36,7 @@ public class Xosync_page_loader { // loop for all int pos = 0; + Btrie_rv trv = new Btrie_rv(); while (true) { // get next "" Gfh_tag img_tag = tag_rdr.Tag__find_fwd_head(pos, src_len, Gfh_tag_.Id__img); @@ -52,22 +53,16 @@ public class Xosync_page_loader { // do simple replace for @src Gfh_atr img_src_atr = img_tag.Atrs__get_by_or_fail(Gfh_atr_.Bry__src); byte[] img_src_val = img_src_atr.Val(); - img_src_val = Bry_.Replace(img_src_val, Xosync_img_src_parser.Bry__xowa_src_bgn, wiki.App().Fsys_mgr().File_dir().To_http_file_bry()); - - // parse other atrs for fsdb - img_src_parser.Parse(err_wkr, hctx, wiki.Domain_bry(), img_src_atr.Val_bgn(), img_src_atr.Val_end()); - if (img_src_parser.File_ttl_bry() != null) { // NOTE: need to skip images that don't follow MW image format ("commons.wikimedia.org/thumb/7/70/A.png"); for example, math images - Xof_fsdb_itm img = hpg.Img_mgr().Make_img(false); - byte[] file_ttl_bry = gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url.Decode(img_src_parser.File_ttl_bry()); - img.Init_by_wm_parse(hctx.Wiki__domain_itm().Abrv_xo(), img_src_parser.Repo_is_commons(), img_src_parser.File_is_orig(), file_ttl_bry, img_src_parser.File_w(), img_src_parser.File_time(), img_src_parser.File_page()); - hctx.File__url_bldr().Init_by_root(img_src_parser.Repo_is_commons() ? hctx.Fsys__file__comm() : hctx.Fsys__file__wiki(), Bool_.N, Byte_ascii.Slash, Bool_.N, Bool_.N, 4); - hctx.File__url_bldr().Init_by_itm(img_src_parser.File_is_orig() ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb, file_ttl_bry, Xof_file_wkr_.Md5(file_ttl_bry), Xof_ext_.new_by_ttl_(file_ttl_bry), img_src_parser.File_w(), img_src_parser.File_time(), img_src_parser.File_page()); - img.Orig_repo_name_(img_src_parser.Repo_is_commons() ? Xow_domain_itm_.Bry__commons : wiki.Domain_bry()); - Io_url html_view_url = hctx.File__url_bldr().Xto_url_by_http(); - img.Init_at_gallery_end(img_tag.Atrs__get_as_int(Gfh_atr_.Bry__width), img_tag.Atrs__get_as_int(Gfh_atr_.Bry__height), html_view_url, html_view_url); - - Xosync_hdoc_parser.Write_img_tag(tmp_bfr, img_tag, img_src_val, img.Html_uid()); + byte path_tid = Xosync_img_src_parser.Src_xo_trie.Match_byte_or(trv, img_src_val, Xosync_img_src_parser.Path__unknown); + switch (path_tid) { + case Xosync_img_src_parser.Path__file: + Add_img(wiki, hpg, img_tag, img_src_atr, img_src_val, path_tid, Xosync_img_src_parser.Bry__xowa_file, wiki.App().Fsys_mgr().File_dir().To_http_file_bry()); + break; + case Xosync_img_src_parser.Path__math: + Add_img(wiki, hpg, img_tag, img_src_atr, img_src_val, path_tid, Xosync_img_src_parser.Bry__xowa_math, wiki.App().Fsys_mgr().File_dir().GenSubDir_nest("math").To_http_file_bry()); + break; } + pos = img_tag.Src_end(); } @@ -76,4 +71,41 @@ public class Xosync_page_loader { hpg.Db().Html().Html_bry_(src); return src; } + private Xof_fsdb_itm Add_img(Xow_wiki wiki, Xoh_page hpg, Gfh_tag img_tag, Gfh_atr img_src_atr, byte[] img_src_val, byte path_tid, byte[] src_find, byte[] src_repl) { + // replace "xowa:/file" with "file:////xowa/file/" + img_src_val = Bry_.Replace(img_src_val, src_find, src_repl); + + // parse src + img_src_parser.Parse(err_wkr, hctx, wiki.Domain_bry(), img_src_atr.Val_bgn(), img_src_atr.Val_end()); + if (img_src_parser.File_ttl_bry() == null) return null; // skip images that don't follow format of "commons.wikimedia.org/thumb/7/70/A.png"; for example, enlarge buttons + + // create img + Xof_fsdb_itm img = hpg.Img_mgr().Make_img(false); + + // use repo_tid to get fsys_root, orig_repo_name + byte repo_tid = img_src_parser.Repo_tid(); + byte[] orig_repo_name = null, fsys_root = null; + switch (repo_tid) { + case Xof_repo_tid_.Tid__remote: fsys_root = hctx.Fsys__file__comm(); orig_repo_name = Xow_domain_itm_.Bry__commons; break; + case Xof_repo_tid_.Tid__local: fsys_root = hctx.Fsys__file__wiki(); orig_repo_name = wiki.Domain_bry(); break; + case Xof_repo_tid_.Tid__math: fsys_root = hctx.Fsys__file__math(); orig_repo_name = Xof_repo_tid_.Bry__math; break; + } + + // set vals + img.Orig_repo_name_(orig_repo_name); + byte[] file_ttl_bry = gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url.Decode(img_src_parser.File_ttl_bry()); + Xof_ext file_ext = Xosync_img_src_parser.Ext_by_ttl(file_ttl_bry, repo_tid); + img.Init_by_wm_parse(hctx.Wiki__domain_itm().Abrv_xo(), img_src_parser.Repo_is_commons(), img_src_parser.File_is_orig(), file_ttl_bry, file_ext, img_src_parser.File_w(), img_src_parser.File_time(), img_src_parser.File_page()); + + // recalc src based on "file:////xowa/file/" + hctx.File__url_bldr().Init_by_repo(repo_tid, fsys_root, Bool_.N, Byte_ascii.Slash, Bool_.N, Bool_.N, 4); + hctx.File__url_bldr().Init_by_itm(img_src_parser.File_is_orig() ? Xof_img_mode_.Tid__orig : Xof_img_mode_.Tid__thumb, file_ttl_bry, Xof_file_wkr_.Md5(file_ttl_bry), Xof_ext_.new_by_ttl_(file_ttl_bry), img_src_parser.File_w(), img_src_parser.File_time(), img_src_parser.File_page()); + Io_url html_view_url = hctx.File__url_bldr().Xto_url_by_http(); + + // if (path_tid == Xosync_img_src_parser.Path__file) + img.Init_at_gallery_end(img_tag.Atrs__get_as_int_or(Gfh_atr_.Bry__width,0), img_tag.Atrs__get_as_int_or(Gfh_atr_.Bry__height, 0), html_view_url, html_view_url); + + Xosync_hdoc_parser.Write_img_tag(tmp_bfr, img_tag, img_src_val, img.Html_uid()); + return img; + } } \ No newline at end of file diff --git a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__fxt.java b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__fxt.java index 6206f45d4..e9a6ff25e 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__fxt.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__fxt.java @@ -38,9 +38,10 @@ public class Xosync_page_loader__fxt { Gftest.Eq__ary__lines(expd, hpg.Db().Html().Html_bry(), "converted html"); return this; } - public Xof_fsdb_itm Make__fsdb(boolean repo_is_commons, boolean file_is_thumb, String file_ttl, int file_w, double file_time, int file_page) { + public Xof_fsdb_itm Make__fsdb(boolean repo_is_commons, boolean file_is_orig, String file_ttl, String orig_ext_str, int file_w, double file_time, int file_page) { Xof_fsdb_itm itm = new Xof_fsdb_itm(); - itm.Init_by_wm_parse(wiki.Domain_itm().Abrv_xo(), repo_is_commons, file_is_thumb, Bry_.new_u8(file_ttl), file_w, file_time, file_page); + Xof_ext orig_ext = Xof_ext_.new_by_ext_(Bry_.new_u8(orig_ext_str)); + itm.Init_by_wm_parse(wiki.Domain_itm().Abrv_xo(), repo_is_commons, file_is_orig, Bry_.new_u8(file_ttl), orig_ext, file_w, file_time, file_page); return itm; } public Xosync_page_loader__fxt Test__fsdb(Xof_fsdb_itm expd) { diff --git a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__tst.java b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__tst.java index 9bbe668d1..1a74a672f 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__tst.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/loaders/Xosync_page_loader__tst.java @@ -20,10 +20,22 @@ import org.junit.*; import gplx.langs.htmls.*; public class Xosync_page_loader__tst { @Before public void init() {fxt.Clear();} private final Xosync_page_loader__fxt fxt = new Xosync_page_loader__fxt(); - @Test public void Basic() { + @Test public void File() { fxt.Exec__parse(Gfh_utl.Replace_apos("ab")) .Test__html(Gfh_utl.Replace_apos("ab")) - .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "Commons-logo.svg", 12, -1, -1)) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "Commons-logo.svg", "svg", 12, -1, -1)) + ; + } + @Test public void Math() { + fxt.Exec__parse(Gfh_utl.Replace_apos("ab")) + .Test__html(Gfh_utl.Replace_apos("ab")) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.Y, "596f8baf206a81478afd4194b44138715dc1a05c", "svg", -1, -1, -1)) + ; + } + @Test public void Ogg() { + fxt.Exec__parse(Gfh_utl.Replace_apos("ab")) + .Test__html(Gfh_utl.Replace_apos("ab")) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A.ogg", "ogv", 320, -1, -1)) ; } } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__err__tst.java b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__err__tst.java index 6c5c87e05..39a564930 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__err__tst.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__err__tst.java @@ -22,7 +22,7 @@ public class Xosync_hdoc_parser__err__tst { @Before public void init() {fxt.Clear();} private final Xosync_hdoc_parser__fxt fxt = new Xosync_hdoc_parser__fxt(); @Test public void Url_does_not_start_with_upload_wikimedia_org() { fxt.Exec__parse(Gfh_utl.Replace_apos("")) - .Test__html(Gfh_utl.Replace_apos("")); + .Test__html(Gfh_utl.Replace_apos("")); } @Test public void Unknown_repo() { fxt.Exec__parse(Gfh_utl.Replace_apos("")) diff --git a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__file__tst.java b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__file__tst.java index a76dfadee..208f747e9 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__file__tst.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__file__tst.java @@ -21,14 +21,14 @@ import gplx.langs.htmls.*; public class Xosync_hdoc_parser__file__tst { @Before public void init() {fxt.Clear();} private final Xosync_hdoc_parser__fxt fxt = new Xosync_hdoc_parser__fxt(); @Test public void Commons__thumb() { - fxt.Exec__parse(Gfh_utl.Replace_apos("")) - .Test__html(Gfh_utl.Replace_apos("")) - .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A.png", 220, -1, -1)); + fxt.Exec__parse(Gfh_utl.Replace_apos("")) + .Test__html(Gfh_utl.Replace_apos("")) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A.png", 320, -1, -1)); } @Test public void Url_encoded() { - fxt.Exec__parse(Gfh_utl.Replace_apos("")) - .Test__html(Gfh_utl.Replace_apos("")) - .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A,B.png", 220, -1, -1)); + fxt.Exec__parse(Gfh_utl.Replace_apos("")) + .Test__html(Gfh_utl.Replace_apos("")) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A,B.png", 320, -1, -1)); } @Test public void Local__orig() { fxt.Exec__parse(Gfh_utl.Replace_apos("")) @@ -36,11 +36,23 @@ public class Xosync_hdoc_parser__file__tst { .Test__fsdb(fxt.Make__fsdb(Bool_.N, Bool_.Y, "A.png", -1, -1, -1)); } @Test public void Svg() { - fxt.Exec__parse(Gfh_utl.Replace_apos("")) - .Test__html(Gfh_utl.Replace_apos("")) - .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "Commons-logo.svg", 12, -1, -1)); - // https://upload.wikimedia.org/wikipedia/commons/thumb/f/fc/Papilio_dardanus_emerging.ogg/320px--Papilio_dardanus_emerging.ogg.jpg - // https://upload.wikimedia.org/wikipedia/commons/thumb/3/30/Clip_from_My_Man_Godfrey.ogg/240px-seek%3D67-Clip_from_My_Man_Godfrey.ogg.jpg - // https://upload.wikimedia.org/wikipedia/commons/thumb/7/7a/PL_Henryk_Sienkiewicz-Pisma_zapomniane_i_niewydane.djvu/page6-250px-PL_Henryk_Sienkiewicz-Pisma_zapomniane_i_niewydane.djvu.jpg + fxt.Exec__parse(Gfh_utl.Replace_apos("")) + .Test__html(Gfh_utl.Replace_apos("")) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A.svg", 12, -1, -1)); + } + @Test public void Ogg() { + fxt.Exec__parse(Gfh_utl.Replace_apos("")) + .Test__html(Gfh_utl.Replace_apos("")) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A.ogg", "ogv", 320, -1, -1)); + } + @Test public void Ogg__time() { + fxt.Exec__parse(Gfh_utl.Replace_apos("")) + .Test__html(Gfh_utl.Replace_apos("")) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A.ogg", "ogv", 320, 1.2, -1)); + } + @Test public void Pdf__page() { + fxt.Exec__parse(Gfh_utl.Replace_apos("")) + .Test__html(Gfh_utl.Replace_apos("")) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "A.djvu", 320, -1, 1)); } } diff --git a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__fxt.java b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__fxt.java index a74d88ac3..9b31b55e6 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__fxt.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__fxt.java @@ -39,9 +39,15 @@ public class Xosync_hdoc_parser__fxt { Gftest.Eq__ary__lines(expd, hpg.Db().Html().Html_bry(), "converted html"); return this; } - public Xof_fsdb_itm Make__fsdb(boolean repo_is_commons, boolean file_is_thumb, String file_ttl, int file_w, double file_time, int file_page) { + public Xof_fsdb_itm Make__fsdb(boolean repo_is_commons, boolean file_is_orig, String file_ttl, int file_w, double file_time, int file_page) { + return Make__fsdb(repo_is_commons, file_is_orig, file_ttl, Xof_ext_.new_by_ttl_(Bry_.new_u8(file_ttl)), file_w, file_time, file_page); + } + public Xof_fsdb_itm Make__fsdb(boolean repo_is_commons, boolean file_is_orig, String file_ttl, String file_ext, int file_w, double file_time, int file_page) { + return Make__fsdb(repo_is_commons, file_is_orig, file_ttl, Xof_ext_.new_by_ext_(Bry_.new_u8(file_ext)), file_w, file_time, file_page); + } + public Xof_fsdb_itm Make__fsdb(boolean repo_is_commons, boolean file_is_orig, String file_ttl, Xof_ext file_ext, int file_w, double file_time, int file_page) { Xof_fsdb_itm itm = new Xof_fsdb_itm(); - itm.Init_by_wm_parse(wiki.Domain_itm().Abrv_xo(), repo_is_commons, file_is_thumb, Bry_.new_u8(file_ttl), file_w, file_time, file_page); + itm.Init_by_wm_parse(wiki.Domain_itm().Abrv_xo(), repo_is_commons, file_is_orig, Bry_.new_u8(file_ttl), file_ext, file_w, file_time, file_page); return itm; } public Xosync_hdoc_parser__fxt Test__fsdb(Xof_fsdb_itm expd) { @@ -54,9 +60,10 @@ public class Xosync_hdoc_parser__fxt { return tmp_bfr.To_str_and_clear(); } private static void To_bfr(Bry_bfr bfr, Xof_fsdb_itm itm) { - bfr.Add_str_a7(itm.Orig_repo_id() == Xof_repo_itm_.Repo_remote ? "remote" : "local").Add_byte_pipe(); + bfr.Add_str_a7(itm.Orig_repo_id() == Xof_repo_tid_.Tid__remote ? "remote" : "local").Add_byte_pipe(); bfr.Add_str_a7(itm.File_is_orig() ? "orig" : "thumb").Add_byte_pipe(); bfr.Add(itm.Orig_ttl()).Add_byte_pipe(); + bfr.Add(itm.Orig_ext().Ext()).Add_byte_pipe(); bfr.Add_int_variable(itm.File_w()).Add_byte_pipe(); bfr.Add_double(itm.Lnki_time()).Add_byte_pipe(); bfr.Add_int_variable(itm.Lnki_page()).Add_byte_pipe(); diff --git a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__misc__tst.java b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__misc__tst.java new file mode 100644 index 000000000..d12c83f95 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__misc__tst.java @@ -0,0 +1,28 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*; +import org.junit.*; +import gplx.langs.htmls.*; +public class Xosync_hdoc_parser__misc__tst { + @Before public void init() {fxt.Clear();} private final Xosync_hdoc_parser__fxt fxt = new Xosync_hdoc_parser__fxt(); + @Test public void Math() { + fxt.Exec__parse(Gfh_utl.Replace_apos("")) + .Test__html(Gfh_utl.Replace_apos("")) + .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.Y, "596f8baf206a81478afd4194b44138715dc1a05c.svg", -1, -1, -1)); + } +} diff --git a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__tst.java b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__tst.java index f50718a13..ef2e82533 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__tst.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_hdoc_parser__tst.java @@ -34,14 +34,6 @@ public class Xosync_hdoc_parser__tst { , "" )); } - @Test public void File() { - fxt.Exec__parse(Gfh_utl.Replace_apos("")) - .Test__html(Gfh_utl.Replace_apos("")) - .Test__fsdb(fxt.Make__fsdb(Bool_.Y, Bool_.N, "Commons-logo.svg", 12, -1, -1)); - // https://upload.wikimedia.org/wikipedia/commons/thumb/f/fc/Papilio_dardanus_emerging.ogg/320px--Papilio_dardanus_emerging.ogg.jpg - // https://upload.wikimedia.org/wikipedia/commons/thumb/3/30/Clip_from_My_Man_Godfrey.ogg/240px-seek%3D67-Clip_from_My_Man_Godfrey.ogg.jpg - // https://upload.wikimedia.org/wikipedia/commons/thumb/7/7a/PL_Henryk_Sienkiewicz-Pisma_zapomniane_i_niewydane.djvu/page6-250px-PL_Henryk_Sienkiewicz-Pisma_zapomniane_i_niewydane.djvu.jpg - } // @Test public void Smoke() { // fxt.Exec__parse(Io_mgr.Instance.LoadFilStr("C:\\xowa\\dev\\wm.updater.src.html")); // Io_mgr.Instance.SaveFilBry("C:\\xowa\\dev\\wm.updater.trg.html", fxt.Hdoc().Converted()); diff --git a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_img_src_parser.java b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_img_src_parser.java index f4bc8f6c3..57e3d144d 100644 --- a/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_img_src_parser.java +++ b/400_xowa/src/gplx/xowa/addons/wikis/pages/syncs/core/parsers/Xosync_img_src_parser.java @@ -16,16 +16,18 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.addons.wikis.pages.syncs.core.parsers; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.wikis.*; import gplx.xowa.addons.wikis.pages.*; import gplx.xowa.addons.wikis.pages.syncs.*; import gplx.xowa.addons.wikis.pages.syncs.core.*; -import gplx.core.brys.*; -import gplx.xowa.files.*; import gplx.xowa.files.repos.*; +import gplx.core.brys.*; import gplx.core.btries.*; +import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.imgs.*; import gplx.langs.htmls.*; import gplx.xowa.htmls.core.wkrs.*; import gplx.xowa.wikis.domains.*; public class Xosync_img_src_parser { private final Bry_rdr rdr = new Bry_rdr().Dflt_dlm_(Byte_ascii.Slash); private final Xof_url_bldr url_bldr = Xof_url_bldr.new_v2(); private final Bry_bfr tmp_bfr = Bry_bfr_.New(); + private final byte[] wiki_abrv_commons; private Xoh_hdoc_ctx hctx; + private byte path_tid; private byte[] img_src_bgn_local, img_src_bgn_remote; private byte[] page_url, repo_local; private byte[] raw; @@ -33,6 +35,7 @@ public class Xosync_img_src_parser { public boolean Repo_is_commons() {return repo_is_commons;} private boolean repo_is_commons; public byte[] File_ttl_bry() {return file_ttl_bry;} private byte[] file_ttl_bry; public boolean File_is_orig() {return file_is_orig;} private boolean file_is_orig; + public Xof_ext File_ext() {return file_ext;} private Xof_ext file_ext; public int File_w() {return file_w;} private int file_w; public double File_time() {return file_time;} private double file_time; public int File_page() {return file_page;} private int file_page; @@ -40,14 +43,16 @@ public class Xosync_img_src_parser { public Xosync_img_src_parser() { rdr.Err_wkr().Fail_throws_err_(false); - img_src_bgn_remote = tmp_bfr.Add(Bry__xowa_src_bgn).Add(Xow_domain_itm_.Bry__commons).Add_byte_slash().To_bry_and_clear(); + img_src_bgn_remote = tmp_bfr.Add(Bry__xowa_file).Add(Xow_domain_itm_.Bry__commons).Add_byte_slash().To_bry_and_clear(); + wiki_abrv_commons = Xow_abrv_xo_.To_bry(Xow_domain_itm_.Bry__commons); } public void Init_by_page(Xoh_hdoc_ctx hctx) { this.hctx = hctx; this.page_url = hctx.Page__url(); + this.path_tid = Path__unknown; this.repo_local = To_wmf_repo_or_null(tmp_bfr, hctx.Wiki__domain_itm()); if (repo_local == null) Gfo_usr_dlg_.Instance.Warn_many("", "", "unsupported wmf repo; domain=~{0}", hctx.Wiki__domain_itm().Domain_bry()); - img_src_bgn_local = tmp_bfr.Add(Bry__xowa_src_bgn).Add(hctx.Wiki__domain_bry()).Add_byte_slash().To_bry_and_clear(); // EX: "xowa:/file/en.wikipedia.org/" + img_src_bgn_local = tmp_bfr.Add(Bry__xowa_file).Add(hctx.Wiki__domain_bry()).Add_byte_slash().To_bry_and_clear(); // EX: "xowa:/file/en.wikipedia.org/" } public boolean Parse(byte[] raw) { // init @@ -60,8 +65,14 @@ public class Xosync_img_src_parser { rdr.Init_by_src(raw); // check "//upload.wikimedia.org/" at bgn - if (!rdr.Is(Bry__upload_wikimedia_org)) return Fail("url does not start with //upload.wikimedia.org"); - + this.path_tid = rdr.Chk_or(path_trie, Path__unknown); + switch (path_tid) { + case Path__file: return Parse_file(raw_len); + case Path__math: return Parse_math(raw_len); + default: return Fail("img src does not start with known sequence"); + } + } + private boolean Parse_file(int raw_len) { // get repo: either "wikipedia/commons/" or "wiki_type/wiki_lang/"; EX:"wiktionary/fr" if (rdr.Is(Bry__repo_remote)) this.repo_is_commons = true; @@ -80,35 +91,74 @@ public class Xosync_img_src_parser { int file_ttl_end = rdr.Find_fwd_lr_or(Byte_ascii.Slash, raw_len); file_ttl_bry = Bry_.Mid(raw, file_ttl_bgn, file_ttl_end); file_ttl_bry = gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url.Decode(file_ttl_bry); // NOTE: @src is always url-encoded; file_ttl_bry is un-encoded (for MD5, database lookups, etc.) + this.file_ext = Xof_ext_.new_by_ttl_(file_ttl_bry); + if (file_ext.Id_is_ogg()) file_ext = Xof_ext_.new_by_id_(Xof_ext_.Id_ogv); - // if thumb, get width, time, page + + // if thumb, get file_w, file_time, file_page if (!file_is_orig) { + // if "page", then file_page exists; EX: // "page1-320px" + if (rdr.Is(Bry__page)) { + int file_page_bgn = rdr.Pos(); + int file_page_end = rdr.Find_fwd_lr(Byte_ascii.Dash); + file_page = Bry_.To_int_or_fail(raw, file_page_bgn, file_page_end); + } + + // get file_w; EX: "320px-" int file_w_bgn = rdr.Pos(); int file_w_end = rdr.Find_fwd_lr(Bry__px); if (file_w_end == -1) return Fail("missing px"); file_w = Bry_.To_int_or(raw, file_w_bgn, file_w_end, -1); if (file_w == -1) return Fail("invalid file_w"); + + // get time via "-seek%3D"; EX: "320px-seek%3D67-" + int seek_end = rdr.Find_fwd_rr(Bry__seek); + if (seek_end != Bry_find_.Not_found) { + int file_time_bgn = rdr.Pos(); + int file_time_end = rdr.Find_fwd_lr(Byte_ascii.Dash); + file_time = Bry_.To_double(raw, file_time_bgn, file_time_end); + } } // make image - Xof_fsdb_itm itm = new Xof_fsdb_itm(); - itm.Init_by_wm_parse(hctx.Wiki__domain_itm().Abrv_xo(), repo_is_commons, file_is_orig, file_ttl_bry, file_w, file_time, file_page); - hctx.Page().Hdump_mgr().Imgs().Add(itm); - + Add_img(hctx.Wiki__domain_itm().Abrv_xo()); return true; } + private boolean Parse_math(int raw_len) { + // set file_ttl_bry to rest of src + ".svg"; EX: "https://wikimedia.org/api/rest_v1/media/math/render/svg/596f8baf206a81478afd4194b44138715dc1a05c" -> "596f8baf206a81478afd4194b44138715dc1a05c.svg" + this.file_ttl_bry = Bry_.Add(Bry_.Mid(raw, rdr.Pos(), raw_len), Byte_ascii.Dot_bry, Xof_ext_.Bry_svg); + this.repo_is_commons = true; + this.file_is_orig = true; + this.file_ext = Xof_ext_.new_by_id_(Xof_ext_.Id_svg); + + Add_img(wiki_abrv_commons); + return true; + } + private void Add_img(byte[] wiki_abrv) { + Xof_fsdb_itm itm = new Xof_fsdb_itm(); + hctx.Page().Hdump_mgr().Imgs().Add(itm); + itm.Init_by_wm_parse(wiki_abrv, repo_is_commons, file_is_orig, file_ttl_bry, file_ext, file_w, file_time, file_page); + } public byte[] To_bry() { - To_bfr(tmp_bfr); + switch (path_tid) { + case Path__file: To_bfr_file(tmp_bfr); break; + case Path__math: To_bfr_math(tmp_bfr); break; + } return tmp_bfr.To_bry_and_clear(); } - public void To_bfr(Bry_bfr bfr) { // EX:'xowa:file/commons.wikimedia.org/thumb/7/0/1/c/A.png/220px.png' - // init repo; either "xowa:file/commons.wikimedia.org" or "xowa:file/en.wikipedia.org" - url_bldr.Init_by_root(repo_is_commons ? img_src_bgn_remote : img_src_bgn_local, Bool_.N, Byte_ascii.Slash, Bool_.N, Bool_.N, 4); + private void To_bfr_file(Bry_bfr bfr) { // EX:'xowa:/file/commons.wikimedia.org/thumb/7/0/1/c/A.png/220px.png' + // init repo; either "xowa:/file/commons.wikimedia.org" or "xowa:/file/en.wikipedia.org" + byte repo_tid = repo_is_commons ? Xof_repo_tid_.Tid__remote : Xof_repo_tid_.Tid__local; + byte[] fsys_root = repo_is_commons ? img_src_bgn_remote : img_src_bgn_local; + url_bldr.Init_by_repo(repo_tid, fsys_root, Bool_.N, Byte_ascii.Slash, Bool_.N, Bool_.N, 4); // set other props and generate url; - url_bldr.Init_by_itm(file_is_orig ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb, gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url.Encode(file_ttl_bry), Xof_file_wkr_.Md5(file_ttl_bry), Xof_ext_.new_by_ttl_(file_ttl_bry), file_w, file_time, file_page); + url_bldr.Init_by_itm(file_is_orig ? Xof_img_mode_.Tid__orig : Xof_img_mode_.Tid__thumb, gplx.langs.htmls.encoders.Gfo_url_encoder_.Http_url.Encode(file_ttl_bry), Xof_file_wkr_.Md5(file_ttl_bry), Xof_ext_.new_by_ttl_(file_ttl_bry), file_w, file_time, file_page); bfr.Add(url_bldr.Xto_bry()); } + private void To_bfr_math(Bry_bfr bfr) { // EX:'xowa:/math/596f8baf206a81478afd4194b44138715dc1a05c + bfr.Add(Bry__xowa_math).Add(file_ttl_bry); + } private void Clear() { this.file_ttl_bry = null; this.repo_is_commons = false; @@ -136,12 +186,23 @@ public class Xosync_img_src_parser { } private static final byte[] - Bry__upload_wikimedia_org = Bry_.new_a7("//upload.wikimedia.org/") - , Bry__repo_remote = Bry_.new_a7("wikipedia/commons/") + Bry__repo_remote = Bry_.new_a7("wikipedia/commons/") , Bry__thumb = Bry_.new_a7("thumb/") , Bry__px = Bry_.new_a7("px") + , Bry__seek = Bry_.new_a7("-seek%3D") + , Bry__page = Bry_.new_a7("page") + ; + public static final byte Path__unknown = 0, Path__file = 1, Path__math = 2; + private final Btrie_slim_mgr path_trie = Btrie_slim_mgr.cs() + .Add_str_byte("//upload.wikimedia.org/", Path__file) + .Add_str_byte("https://wikimedia.org/api/rest_v1/media/math/render/svg/", Path__math) + ; + + public static final byte[] Bry__xowa_file = Bry_.new_a7("xowa:/file/"), Bry__xowa_math = Bry_.new_a7("xowa:/math/"); + public static Btrie_slim_mgr Src_xo_trie = Btrie_slim_mgr.cs() + .Add_bry_byte(Bry__xowa_file, Path__file) + .Add_bry_byte(Bry__xowa_math, Path__math) ; - public static final byte[] Bry__xowa_src_bgn = Bry_.new_a7("xowa:/file/"); private static byte[] To_wmf_repo_or_null(Bry_bfr bfr, Xow_domain_itm domain_itm) { // add type; EX: "fr.wiktionary.org" -> "wiktionary/" @@ -164,4 +225,10 @@ public class Xosync_img_src_parser { bfr.Add(domain_itm.Lang_orig_key()).Add_byte_slash(); return bfr.To_bry_and_clear(); } + public static Xof_ext Ext_by_ttl(byte[] file_ttl_bry, byte repo_tid) { + Xof_ext rv = Xof_ext_.new_by_ttl_(file_ttl_bry); + if (rv.Id_is_ogg()) rv = Xof_ext_.new_by_id_(Xof_ext_.Id_ogv); + if (repo_tid == Xof_repo_tid_.Tid__math) rv = Xof_ext_.new_by_id_(Xof_ext_.Id_svg); + return rv; + } } diff --git a/400_xowa/src/gplx/xowa/apps/boots/Xoa_cmd_arg_mgr.java b/400_xowa/src/gplx/xowa/apps/boots/Xoa_cmd_arg_mgr.java index a47591192..030a42169 100644 --- a/400_xowa/src/gplx/xowa/apps/boots/Xoa_cmd_arg_mgr.java +++ b/400_xowa/src/gplx/xowa/apps/boots/Xoa_cmd_arg_mgr.java @@ -94,6 +94,7 @@ public class Xoa_cmd_arg_mgr { case Op_sys.Tid_lnx: rv = "linux"; break; case Op_sys.Tid_wnt: rv = "windows"; break; case Op_sys.Tid_osx: rv = "macosx"; break; + case Op_sys.Tid_arm: rv = "arm"; break; default: throw Err_.new_unhandled("unknown platform " + Op_sys.Cur()); } if (op_sys.Bitness() == Op_sys.Bitness_64) rv += "_64"; diff --git a/400_xowa/src/gplx/xowa/apps/cfgs/Xoa_cfg_db_txt.java b/400_xowa/src/gplx/xowa/apps/cfgs/Xoa_cfg_db_txt.java index 5f0f4412f..31fc4a37d 100644 --- a/400_xowa/src/gplx/xowa/apps/cfgs/Xoa_cfg_db_txt.java +++ b/400_xowa/src/gplx/xowa/apps/cfgs/Xoa_cfg_db_txt.java @@ -35,8 +35,14 @@ public class Xoa_cfg_db_txt implements Xoa_cfg_db { bfr.ClearAndReset(); } private Bry_bfr bfr = Bry_bfr_.New(); public void Cfg_save_end(Xoa_cfg_mgr cfg_mgr) { - Xoa_app_.Usr_dlg().Log_many("", "", "shutting down app; saving cfg: len=~{0}", bfr.Len()); - Io_mgr.Instance.SaveFilBfr(Cfg_url(cfg_mgr), bfr); + Io_url cfg_url = Cfg_url(cfg_mgr); + if (Io_mgr.Instance.QueryFil(cfg_url).ReadOnly()) { + Xoa_app_.Usr_dlg().Log_many("", "", "shutting down app; skipping cfg b/c file is marked read-only; src=~{0}", cfg_url); + } + else { + Xoa_app_.Usr_dlg().Log_many("", "", "shutting down app; saving cfg: len=~{0}", bfr.Len()); + Io_mgr.Instance.SaveFilBfr(Cfg_url(cfg_mgr), bfr); + } } public void Cfg_save_run(Xoa_cfg_mgr cfg_mgr, Xoa_cfg_grp cfg_grp, Xoa_cfg_itm cfg_itm) { fmtr.Bld_bfr_many(bfr, Xoa_gfs_wtr_.Escape(cfg_grp.Key_bry()), Xoa_gfs_wtr_.Escape(cfg_itm.Key()), Xoa_gfs_wtr_.Escape(cfg_itm.Val())); diff --git a/400_xowa/src/gplx/xowa/apps/gfs/Xoa_gfs_php_mgr.java b/400_xowa/src/gplx/xowa/apps/gfs/Gfs_php_converter.java similarity index 52% rename from 400_xowa/src/gplx/xowa/apps/gfs/Xoa_gfs_php_mgr.java rename to 400_xowa/src/gplx/xowa/apps/gfs/Gfs_php_converter.java index 745cc9292..773c9a0c1 100644 --- a/400_xowa/src/gplx/xowa/apps/gfs/Xoa_gfs_php_mgr.java +++ b/400_xowa/src/gplx/xowa/apps/gfs/Gfs_php_converter.java @@ -18,8 +18,8 @@ along with this program. If not, see . package gplx.xowa.apps.gfs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*; import gplx.core.brys.fmtrs.*; import gplx.langs.phps.*; -public class Xoa_gfs_php_mgr { - public static byte[] Xto_php(Bry_bfr bfr, boolean escape_backslash, byte[] src) { +public class Gfs_php_converter { +public static byte[] Xto_php(Bry_bfr bfr, boolean escape_backslash, byte[] src) { int len = src.length; int pos = 0; boolean dirty = false; @@ -87,26 +87,54 @@ public class Xoa_gfs_php_mgr { } } } - public static byte[] Xto_gfs(Bry_bfr bfr, byte[] raw) { + public static byte[] To_gfs(Bry_bfr bfr, byte[] raw) { int raw_len = raw.length; - for (int i = 0; i < raw_len; i++) { + for (int i = 0; i < raw_len; ++i) { byte b = raw[i]; switch (b) { - case Byte_ascii.Backslash: + case Byte_ascii.Backslash: // unescape; EX: '\"' -> '"' ++i; - if (i < raw_len) - bfr.Add_byte(raw[i]); - else + if (i < raw_len){ + byte escape_byte = raw[i]; + switch (escape_byte) { // REF: http://php.net/manual/en/language.types.String.php + case Byte_ascii.Ltr_t: escape_byte = Byte_ascii.Tab; break; + case Byte_ascii.Ltr_n: escape_byte = Byte_ascii.Nl; break; + case Byte_ascii.Ltr_r: escape_byte = Byte_ascii.Cr; break; + case Byte_ascii.Ltr_v: escape_byte = 11; break; // 11=vertical tab + case Byte_ascii.Ltr_e: escape_byte = 27; break; // 27=escape + case Byte_ascii.Ltr_f: escape_byte = 12; break; // 12=form fed + case Byte_ascii.Backslash: + case Byte_ascii.Quote: + case Byte_ascii.Apos: + case Byte_ascii.Dollar: break; + // FUTURE: + // //\[0-7]{1,3} the sequence of characters matching the regular expression is a character in octal notation, which silently overflows to fit in a byte (e.g. "\400" === "\000") + // \ x[0-9A-Fa-f]{1,2} the sequence of characters matching the regular expression is a character in hexadecimal notation + // \ u{[0-9A-Fa-f]+} the sequence of characters matching the regular expression is a Unicode codepoint, which will be output to the String as that codepoint's UTF-8 representation (added in PHP 7.0.0) + default: // all else seems to be rendered literally; EX:"You do not need to put \ before a /."; PAGE:en.w:MediaWiki:Spam-whitelist; DATE:2016-09-12 + bfr.Add_byte(Byte_ascii.Backslash); + bfr.Add_byte(escape_byte); + continue; + } + bfr.Add_byte(escape_byte); + } + else // if eos, just output "\"; don't fail; EX: "a\" -> "a\" bfr.Add_byte(Byte_ascii.Backslash); break; - case Byte_ascii.Tilde: + case Byte_ascii.Tilde: // double up tilde; EX: '~' -> '~~' bfr.Add_byte_repeat(Bry_fmtr.char_escape, 2); // escape tilde; EX: ~u -> ~~u; DATE:2013-11-11 break; - case Byte_ascii.Dollar: - int end_pos = Php_text_itm_parser.Find_fwd_non_int(raw, i + 1, raw_len); - int int_val = Bry_.To_int_or(raw, i + 1, end_pos, -1); - bfr.Add_byte(Bry_fmtr.char_escape).Add_byte(Bry_fmtr.char_arg_bgn).Add_int_variable(int_val - 1).Add_byte(Bry_fmtr.char_arg_end); - i = end_pos - 1; + case Byte_ascii.Dollar: // convert php args to gfs args; EX: $1 -> ~{0} + int int_bgn = i + 1; + int int_end = Php_text_itm_parser.Find_fwd_non_int(raw, int_bgn, raw_len); + if (int_bgn == int_end ) // no numbers after $; EX: "$ "; "$a" + bfr.Add_byte(b); + else { + int int_val = Bry_.To_int_or(raw, int_bgn, int_end, -1); + if (int_val == -1) throw Err_.new_wo_type(String_.Format("unknown php dollar sequence: raw=~{0}", raw)); + bfr.Add_byte(Bry_fmtr.char_escape).Add_byte(Bry_fmtr.char_arg_bgn).Add_int_variable(int_val - List_adp_.Base1).Add_byte(Bry_fmtr.char_arg_end); // convert "$1" -> "~{0}" + i = int_end - 1; // -1 b/c Find_fwd_non_int positions after non-int + } break; default: bfr.Add_byte(b); diff --git a/400_xowa/src/gplx/xowa/apps/gfs/Gfs_php_converter__to_gfs__tst.java b/400_xowa/src/gplx/xowa/apps/gfs/Gfs_php_converter__to_gfs__tst.java new file mode 100644 index 000000000..7fe6a21db --- /dev/null +++ b/400_xowa/src/gplx/xowa/apps/gfs/Gfs_php_converter__to_gfs__tst.java @@ -0,0 +1,55 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.apps.gfs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*; +import org.junit.*; +public class Gfs_php_converter__to_gfs__tst { + @Before public void init() {fxt.Clear();} private final Gfs_php_converter_fxt fxt = new Gfs_php_converter_fxt(); + @Test public void Escape_sequences() { + fxt.Test__to_gfs("a\\\"b" , "a\"b"); + fxt.Test__to_gfs("a\\'b" , "a'b"); + fxt.Test__to_gfs("a\\$b" , "a$b"); + fxt.Test__to_gfs("a\\\\b" , "a\\b"); + fxt.Test__to_gfs("a\\nb" , "a\nb"); + fxt.Test__to_gfs("a\\tb" , "a\tb"); + fxt.Test__to_gfs("a\\rb" , "a\rb"); + fxt.Test__to_gfs("a\\ b" , "a\\ b"); // "\ " seems to be rendered literally; EX:"You do not need to put \ before a /."; PAGE:en.w:MediaWiki:Spam-whitelist; DATE:2016-09-12 + fxt.Test__to_gfs("a\\\\b\\'c\\\"d\\$e" , "a\\b'c\"d$e"); // backslash.escape + fxt.Test__to_gfs("\\" , "\\"); // backslash.eos; eos, but nothing to escape; render self but dont fail + } + @Test public void Tilde() { + fxt.Test__to_gfs("a~b" , "a~~b"); // tilde.escape + } + @Test public void Arguments() { + fxt.Test__to_gfs("a$1b" , "a~{0}b"); // dollar + fxt.Test__to_gfs("a $ b" , "a $ b"); // noop + } +} +class Gfs_php_converter_fxt { + private final Bry_bfr bfr = Bry_bfr_.New(); + public void Clear() {} + public void Test__to_gfs(String raw, String expd) { + byte[] actl = Gfs_php_converter.To_gfs(bfr, Bry_.new_u8(raw)); + Tfds.Eq(expd, String_.new_u8(actl)); + } + public void Test_Xto_php_escape_y(String raw, String expd) {Test_Xto_php(raw, Bool_.Y, expd);} + public void Test_Xto_php_escape_n(String raw, String expd) {Test_Xto_php(raw, Bool_.N, expd);} + public void Test_Xto_php(String raw, boolean escape_backslash, String expd) { + byte[] actl = Gfs_php_converter.Xto_php(bfr, escape_backslash, Bry_.new_u8(raw)); + Tfds.Eq(expd, String_.new_u8(actl)); + } +} diff --git a/400_xowa/src/gplx/xowa/apps/gfs/Xoa_gfs_php_mgr_tst.java b/400_xowa/src/gplx/xowa/apps/gfs/Gfs_php_converter__to_php__tst.java similarity index 53% rename from 400_xowa/src/gplx/xowa/apps/gfs/Xoa_gfs_php_mgr_tst.java rename to 400_xowa/src/gplx/xowa/apps/gfs/Gfs_php_converter__to_php__tst.java index 05f143004..8a525a597 100644 --- a/400_xowa/src/gplx/xowa/apps/gfs/Xoa_gfs_php_mgr_tst.java +++ b/400_xowa/src/gplx/xowa/apps/gfs/Gfs_php_converter__to_php__tst.java @@ -17,14 +17,8 @@ along with this program. If not, see . */ package gplx.xowa.apps.gfs; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*; import org.junit.*; -public class Xoa_gfs_php_mgr_tst { - @Before public void init() {fxt.Clear();} private Xoa_gfs_php_mgr_fxt fxt = new Xoa_gfs_php_mgr_fxt(); - @Test public void Xto_gfs() { - fxt.Test_Xto_gfs("a\\\\b\\'c\\\"d\\$e" , "a\\b'c\"d$e"); // backslash.escape - fxt.Test_Xto_gfs("\\" , "\\"); // backslash.eos; eos, but nothing to escape; render self but dont fail - fxt.Test_Xto_gfs("a~b" , "a~~b"); // tilde.escape - fxt.Test_Xto_gfs("a$1b" , "a~{0}b"); // dollar - } +public class Gfs_php_converter__to_php__tst { + @Before public void init() {fxt.Clear();} private final Gfs_php_converter_fxt fxt = new Gfs_php_converter_fxt(); @Test public void Xto_php() { fxt.Test_Xto_php_escape_y("a~{0}b" , "a$1b"); // tilde.arg.one fxt.Test_Xto_php_escape_y("a~{0}b~{1}c~{2}d" , "a$1b$2c$3d"); // tilde.arg.many @@ -34,17 +28,3 @@ public class Xoa_gfs_php_mgr_tst { fxt.Test_Xto_php_escape_n("a\\b'c\"d$e" , "a\\b'c\"d$e"); // backslash.escape_n } } -class Xoa_gfs_php_mgr_fxt { - private Bry_bfr bfr = Bry_bfr_.New(); - public void Clear() {} - public void Test_Xto_gfs(String raw, String expd) { - byte[] actl = Xoa_gfs_php_mgr.Xto_gfs(bfr, Bry_.new_u8(raw)); - Tfds.Eq(expd, String_.new_u8(actl)); - } - public void Test_Xto_php_escape_y(String raw, String expd) {Test_Xto_php(raw, Bool_.Y, expd);} - public void Test_Xto_php_escape_n(String raw, String expd) {Test_Xto_php(raw, Bool_.N, expd);} - public void Test_Xto_php(String raw, boolean escape_backslash, String expd) { - byte[] actl = Xoa_gfs_php_mgr.Xto_php(bfr, escape_backslash, Bry_.new_u8(raw)); - Tfds.Eq(expd, String_.new_u8(actl)); - } -} diff --git a/400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_itm__interwikimap.java b/400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_itm__interwikimap.java index ea86a2eca..5441d51b0 100644 --- a/400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_itm__interwikimap.java +++ b/400_xowa/src/gplx/xowa/apps/site_cfgs/Xoa_site_cfg_itm__interwikimap.java @@ -28,7 +28,7 @@ class Xoa_site_cfg_itm__interwikimap extends Xoa_site_cfg_itm__base { if (i != 0) bfr.Add_byte_nl(); byte[] iw_key = nde.Get_bry_or_null("prefix"); if (iw_key == null) throw Err_.new_("site_meta", "invalid interwiki", "key", iw_key); byte[] iw_url = nde.Get_bry_or_null("url"); if (iw_url == null) throw Err_.new_("site_meta", "invalid interwiki", "url", iw_key); - bfr.Add(iw_key).Add_byte_pipe().Add(Xoa_gfs_php_mgr.Xto_gfs(tmp_bfr, iw_url)); + bfr.Add(iw_key).Add_byte_pipe().Add(Gfs_php_converter.To_gfs(tmp_bfr, iw_url)); } @Override public void Exec_csv(Xow_wiki wiki, int loader_tid, byte[] dsv_bry) { if (loader_tid == Xoa_site_cfg_loader_.Tid__fallback) diff --git a/400_xowa/src/gplx/xowa/apps/urls/Xoa_url_encoder.java b/400_xowa/src/gplx/xowa/apps/utls/Xoa_url_encoder.java similarity index 55% rename from 400_xowa/src/gplx/xowa/apps/urls/Xoa_url_encoder.java rename to 400_xowa/src/gplx/xowa/apps/utls/Xoa_url_encoder.java index b0dbb7be5..e00200e7e 100644 --- a/400_xowa/src/gplx/xowa/apps/urls/Xoa_url_encoder.java +++ b/400_xowa/src/gplx/xowa/apps/utls/Xoa_url_encoder.java @@ -15,23 +15,39 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package gplx.xowa.apps.urls; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*; -public class Xoa_url_encoder { +package gplx.xowa.apps.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*; +public class Xoa_url_encoder { // NOTE: redundant with Gfo_url_encoder, but is simpler; DATE:2016-09-15 private final Bry_bfr bfr = Bry_bfr_.New(); public byte[] Encode(byte[] src) { int src_len = src.length; - for (int i = 0; i < src_len; i++) { + boolean dirty = false; + for (int i = 0; i < src_len; ++i) { byte b = src[i]; + byte[] repl = null; switch (b) { - case Byte_ascii.Space: bfr.Add(Bry__underline); break; - case Byte_ascii.Amp: bfr.Add(Bry__amp); break; - case Byte_ascii.Apos: bfr.Add(Bry__apos); break; - case Byte_ascii.Eq: bfr.Add(Bry__eq); break; - case Byte_ascii.Plus: bfr.Add(Bry__plus); break; - default: bfr.Add_byte(b); break; + case Byte_ascii.Space: repl = Bry__underline; break; + case Byte_ascii.Amp: repl = Bry__amp; break; + case Byte_ascii.Apos: repl = Bry__apos; break; + case Byte_ascii.Eq: repl = Bry__eq; break; + case Byte_ascii.Plus: repl = Bry__plus; break; + } + + // not a replacement sequence + if (repl == null) { + // if dirty, add to bfr; else, ignore + if (dirty) + bfr.Add_byte(b); + } + else { + // if clean, add everything before cur_pos to bfr + if (!dirty) { + bfr.Add_mid(src, 0, i); + dirty = true; + } + bfr.Add(repl); } } - return bfr.To_bry_and_clear(); + return dirty ? bfr.To_bry_and_clear() : src; } private static final byte[] Bry__amp = Bry_.new_a7("%26"), Bry__eq = Bry_.new_a7("%3D") , Bry__plus = Bry_.new_a7("%2B"), Bry__apos = Bry_.new_a7("%27") diff --git a/400_xowa/src/gplx/xowa/apps/utls/Xoa_url_encoder__tst.java b/400_xowa/src/gplx/xowa/apps/utls/Xoa_url_encoder__tst.java new file mode 100644 index 000000000..7eed486a2 --- /dev/null +++ b/400_xowa/src/gplx/xowa/apps/utls/Xoa_url_encoder__tst.java @@ -0,0 +1,31 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.apps.utls; import gplx.*; import gplx.xowa.*; import gplx.xowa.apps.*; +import org.junit.*; import gplx.core.tests.*; +public class Xoa_url_encoder__tst { + private final Xoa_url_encoder__fxt fxt = new Xoa_url_encoder__fxt(); + @Test public void Syms__diff() {fxt.Test__encode(" &'=+", "_%26%27%3D%2B");} + @Test public void Syms__same() {fxt.Test__encode("!\"#$%()*,-./:;<>?@[\\]^_`{|}~", "!\"#$%()*,-./:;<>?@[\\]^_`{|}~");} + @Test public void Mixed() {fxt.Test__encode("a &'=+b", "a_%26%27%3D%2Bb");} // PURPOSE: make sure dirty flag is set +} +class Xoa_url_encoder__fxt { + private final Xoa_url_encoder encoder = new Xoa_url_encoder(); + public void Test__encode(String raw, String expd) { + Gftest.Eq__bry(Bry_.new_u8(expd), encoder.Encode(Bry_.new_u8(raw))); + } +} diff --git a/400_xowa/src/gplx/xowa/bldrs/Db_mgr_fxt.java b/400_xowa/src/gplx/xowa/bldrs/Db_mgr_fxt.java index 4beac85f6..da21d9dda 100644 --- a/400_xowa/src/gplx/xowa/bldrs/Db_mgr_fxt.java +++ b/400_xowa/src/gplx/xowa/bldrs/Db_mgr_fxt.java @@ -76,26 +76,6 @@ public class Db_mgr_fxt { } return rv; } - public void Test_category_v1(String ctg_name_str, int... expd) { - Xowe_wiki wiki = bldr_fxt.Wiki(); - byte[] ctg_name_bry = Bry_.new_a7(ctg_name_str); - Xoctg_catpage_ctg ctg = new Xoctg_catpage_ctg(ctg_name_bry); - wiki.Db_mgr_as_sql().Load_mgr().Load_ctg_v1(ctg, ctg_name_bry); - Tfds.Eq_ary(expd, Xto_int_ary(ctg)); - } - int[] Xto_int_ary(Xoctg_catpage_ctg ctg) { - List_adp list = List_adp_.New(); - byte tid_max = Xoa_ctg_mgr.Tid__max; - for (byte tid = 0; tid < tid_max; tid++) { - Xoctg_catpage_grp grp = ctg.Grp_by_tid(tid); if (grp == null) continue; - int len = grp.Itms_list().Count(); - for (int i = 0; i < len; i++) { - Xoctg_catpage_itm itm = (Xoctg_catpage_itm)grp.Itms_list().Get_at(i); - list.Add(itm.Page_id()); - } - } - return (int[])list.To_ary_and_clear(int.class); - } public void Test_file(String url, String expd) { String actl = Io_mgr.Instance.LoadFilStr(url); Tfds.Eq_str_lines(expd, actl); diff --git a/400_xowa/src/gplx/xowa/bldrs/Xob_db_file.java b/400_xowa/src/gplx/xowa/bldrs/Xob_db_file.java index 99fb4a871..c55b00e53 100644 --- a/400_xowa/src/gplx/xowa/bldrs/Xob_db_file.java +++ b/400_xowa/src/gplx/xowa/bldrs/Xob_db_file.java @@ -35,7 +35,7 @@ public class Xob_db_file { public static Xob_db_file New__img_link(Xow_wiki wiki) {return New(wiki.Fsys_mgr().Root_dir(), "xowa.wiki.imglinks.sqlite3");} public static Xob_db_file New__page_file_map(Xow_wiki wiki) {return New(wiki.Fsys_mgr().Root_dir(), wiki.Domain_str() + "-file-page_map.xowa");} public static Xob_db_file New__deletion_db(Xow_wiki wiki) { - String name = String_.Format("{0}-file-core-deletion_db-{1}.xowa", wiki.Domain_str(), Datetime_now.Get().XtoStr_fmt("yyyy.MM")); + String name = String_.Format("{0}-file-deletion-{1}.xowa", wiki.Domain_str(), Datetime_now.Get().XtoStr_fmt("yyyy.MM")); return New(wiki.Fsys_mgr().Root_dir(), name); } public static Xob_db_file New(Io_url dir, String name) { diff --git a/400_xowa/src/gplx/xowa/bldrs/Xob_ns_to_db_mgr.java b/400_xowa/src/gplx/xowa/bldrs/Xob_ns_to_db_mgr.java index 66a04e790..c04a15af8 100644 --- a/400_xowa/src/gplx/xowa/bldrs/Xob_ns_to_db_mgr.java +++ b/400_xowa/src/gplx/xowa/bldrs/Xob_ns_to_db_mgr.java @@ -16,7 +16,8 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.bldrs; import gplx.*; import gplx.xowa.*; -import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.bldrs.cmds.*; +import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.data.tbls.*; +import gplx.xowa.bldrs.cmds.*; public class Xob_ns_to_db_mgr { private final Xob_ns_to_db_wkr wkr; private final Xow_db_mgr db_mgr; private final long db_max; private boolean one_file_conn_init = true; private final Ordered_hash db_list = Ordered_hash_.New(); diff --git a/400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_deploy_zip_cmd.java b/400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_deploy_zip_cmd.java index 36dd1588d..7ea1ad05c 100644 --- a/400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_deploy_zip_cmd.java +++ b/400_xowa/src/gplx/xowa/bldrs/cmds/utils/Xob_deploy_zip_cmd.java @@ -17,7 +17,7 @@ along with this program. If not, see . */ package gplx.xowa.bldrs.cmds.utils; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.core.ios.zips.*; -import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*; +import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sql_dumps.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.tdbs.*; public class Xob_deploy_zip_cmd extends Xob_itm_basic_base implements Xob_cmd { diff --git a/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_dump_cbk.java b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_dump_cbk.java new file mode 100644 index 000000000..dd4bdfe35 --- /dev/null +++ b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_dump_cbk.java @@ -0,0 +1,22 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.bldrs.sql_dumps; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; +import gplx.core.strings.*; +public interface Xosql_dump_cbk { + void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end); +} diff --git a/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_dump_parser.java b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_dump_parser.java new file mode 100644 index 000000000..c085f2a78 --- /dev/null +++ b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_dump_parser.java @@ -0,0 +1,160 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.bldrs.sql_dumps; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; +import gplx.core.flds.*; import gplx.core.ios.*; import gplx.core.ios.streams.*; +public class Xosql_dump_parser { + private Xosql_dump_cbk cbk; + private Io_url src_fil; private int src_rdr_bfr_len = 8 * Io_mgr.Len_mb; + private Xosql_fld_hash cbk_flds; + private Ordered_hash tbl_flds; + public Xosql_dump_parser(Xosql_dump_cbk cbk, String... cbk_keys) { + this.cbk = cbk; + this.cbk_flds = Xosql_fld_hash.New(cbk_keys); + } + public void Src_fil_(Io_url v) {this.src_fil = v;} + public void Parse(Gfo_usr_dlg usr_dlg) { + Io_buffer_rdr rdr = Io_buffer_rdr.Null; + try { + // init bfrs, rdr + Bry_bfr val_bfr = Bry_bfr_.New(); + rdr = Io_buffer_rdr.new_(Io_stream_rdr_.new_by_url_(src_fil), src_rdr_bfr_len); + byte[] bfr = rdr.Bfr(); int bfr_len = rdr.Bfr_len(), fld_idx = 0, cur_pos = 0; + + this.tbl_flds = Identify_flds(cbk_flds, bfr); + + // init fld_rdr + Gfo_fld_rdr fld_rdr = Gfo_fld_rdr.sql_(); + byte[] decode_regy = fld_rdr.Escape_decode(); + + byte mode_prv = Mode__sql_bgn; byte mode = Mode__sql_bgn; + boolean reading_file = true; + while (reading_file) { + if (cur_pos + 256 > bfr_len && rdr.Fil_pos() != rdr.Fil_len()) { // buffer 256 characters; can be 0, but erring on side of simplicity + rdr.Bfr_load_from(cur_pos); + cur_pos = 0; + bfr = rdr.Bfr(); + bfr_len = rdr.Bfr_len(); + } + if (cur_pos == bfr_len) break; + + byte b = bfr[cur_pos]; + switch (mode) { + case Mode__sql_bgn:// skip over header to 1st "VALUES" + cur_pos = Bry_find_.Find_fwd(bfr, Bry_insert_into, cur_pos); + if (cur_pos == Bry_find_.Not_found || cur_pos > bfr_len) {reading_file = false; continue;} + cur_pos = Bry_find_.Find_fwd(bfr, Bry_values, cur_pos); + if (cur_pos == Bry_find_.Not_found || cur_pos > bfr_len) throw Err_.new_wo_type("VALUES not found"); // something went wrong; + mode = Mode__fld; + cur_pos += Bry_values.length; + break; + case Mode__row_bgn: // assert "(" + switch (b) { + case Byte_ascii.Paren_bgn: mode = Mode__fld; break; + default: throw Err_.new_unhandled(mode); + } + ++cur_pos; + break; + case Mode__row_end: // handle 1st char after ")"; + switch (b) { + case Byte_ascii.Nl: break; // ignore \n + case Byte_ascii.Comma: mode = Mode__row_bgn; break; // handle ","; EX: "(1),(2)" + case Byte_ascii.Semic: mode = Mode__sql_bgn; break; // handle ";"; EX: "(1);INSERT INTO" + default: throw Err_.new_unhandled(mode); + } + ++cur_pos; + break; + case Mode__fld: // handle fld chars; EX: "(1,'ab')" + switch (b) { + case Byte_ascii.Space: // ws: skip; EX: "(1 , 2)"; "(1,\n2)" + case Byte_ascii.Nl: + break; + case Byte_ascii.Apos: // apos: switch modes; NOTE: never escape apos by doubling; will fail for empty fields; EX: ", '', ''"; DATE:2013-07-06 + mode = Mode__quote; + break; + case Byte_ascii.Backslash: // backslash: switch modes; + mode_prv = mode; + mode = Mode__escape; + break; + case Byte_ascii.Comma: // comma: end fld + Commit_fld(fld_idx++, val_bfr); + break; + case Byte_ascii.Paren_end: // paren_end: end fld and row + Commit_fld(fld_idx++, val_bfr); + fld_idx = 0; + mode = Mode__row_end; + break; + default: // all other chars; add to val_bfr + val_bfr.Add_byte(b); + break; + } + ++cur_pos; + break; + case Mode__quote: // add to val_bfr until quote encountered; also, handle backslashes; + switch (b) { + case Byte_ascii.Apos: mode = Mode__fld; break; + case Byte_ascii.Backslash: mode_prv = mode; mode = Mode__escape; break; + default: val_bfr.Add_byte(b); break; + } + ++cur_pos; + break; + case Mode__escape: // get escape_val from decode_regy; if unknown, just add original + byte escape_val = decode_regy[b]; + if (escape_val == Byte_ascii.Null) + val_bfr.Add_byte(Byte_ascii.Backslash).Add_byte(b); + else + val_bfr.Add_byte(escape_val); + mode = mode_prv; // switch back to prv_mode + ++cur_pos; + break; + default: throw Err_.new_unhandled(mode); + } + } + } + finally {rdr.Rls();} + } + private void Commit_fld(int fld_idx, Bry_bfr val_bfr) { + Xosql_fld_itm fld = (Xosql_fld_itm)tbl_flds.Get_at(fld_idx); // handle new flds added by MW, but not supported by XO; EX:hiddencat and pp_sortkey; DATE:2014-04-28 + if (fld.Uid() != Int_.Max_value) + cbk.On_fld_done(fld.Uid(), val_bfr.Bfr(), 0, val_bfr.Len()); + val_bfr.Clear(); + } + private static Ordered_hash Identify_flds(Xosql_fld_hash cbk_hash, byte[] raw) { + // parse tbl def + Xosql_tbl_parser tbl_parser = new Xosql_tbl_parser(); + Ordered_hash tbl_flds = tbl_parser.Parse(raw); + + // loop over tbl_flds + int len = tbl_flds.Len(); + for (int i = 0; i < len; ++i) { + Xosql_fld_itm tbl_itm = (Xosql_fld_itm)tbl_flds.Get_at(i); + // get cbk_itm + Xosql_fld_itm cbk_itm = cbk_hash.Get_by_key(tbl_itm.Key()); + if (cbk_itm == null) continue;// throw Err_.New("sql_dump_parser: failed to find fld; src={0} fld={1}", src_fil.Raw(), tbl_itm.Key()); + + // set tbl_def's uid to cbk_itm's uid + tbl_itm.Uid_(cbk_itm.Uid()); + } + + tbl_flds.Sort(); + return tbl_flds; + } + public Xosql_dump_parser Src_rdr_bfr_len_(int v) {src_rdr_bfr_len = v; return this;} // TEST: + + private static final byte[] Bry_insert_into = Bry_.new_a7("INSERT INTO "), Bry_values = Bry_.new_a7(" VALUES ("); + private static final byte Mode__sql_bgn = 0, Mode__row_bgn = 1, Mode__row_end = 2, Mode__fld = 3, Mode__quote = 4, Mode__escape = 5; +} diff --git a/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_dump_parser__tst.java b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_dump_parser__tst.java new file mode 100644 index 000000000..8774d6843 --- /dev/null +++ b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_dump_parser__tst.java @@ -0,0 +1,88 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.bldrs.sql_dumps; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; +import org.junit.*; import gplx.core.ios.*; import gplx.core.tests.*; +public class Xosql_dump_parser__tst { + private final Xosql_dump_parser__fxt fxt = new Xosql_dump_parser__fxt(); + @Test public void One() { + fxt.Init(String_.Ary("c1", "c2"), "c2").Test__parse("INSERT INTO 'tbl_1' VALUES (1,2);", "2|"); + } + @Test public void Many() { + fxt.Init(String_.Ary("c1", "c2"), "c2").Test__parse("INSERT INTO 'tbl_1' VALUES (1,2),(3,4),(5,6);", "2|\n4|\n6|"); + } + @Test public void Quote_basic() { + fxt.Init(String_.Ary("c1", "c2", "c3"), "c2", "c3").Test__parse("INSERT INTO 'tbl_1' VALUES (1,'a','b');", "a|b|"); + } + @Test public void Escape_backslash() { + fxt.Init(String_.Ary("c1", "c2", "c3"), "c2", "c3").Test__parse("INSERT INTO 'tbl_1' VALUES (1,'a\\\\b','c');", "a\\b|c|"); + } + @Test public void Escape_quote() { + fxt.Init(String_.Ary("c1", "c2", "c3"), "c2", "c3").Test__parse("INSERT INTO 'tbl_1' VALUES (1,'a\"b','c');", "a\"b|c|"); + } + @Test public void Fld_paren_end() { + fxt.Init(String_.Ary("c1", "c2", "c3"), "c2", "c3").Test__parse("INSERT INTO 'tbl_1' VALUES (1,'Психостимуляторы_(лекарственные_средства)','c');", "Психостимуляторы_(лекарственные_средства)|c|"); + } + @Test public void Insert_multiple() { + fxt.Init(String_.Ary("c1", "c2"), "c2").Test__parse("INSERT INTO 'tbl_1' VALUES (1,2);INSERT INTO 'tbl_1' VALUES (3,4)", "2|\n4|"); + } +} +class Xosql_dump_parser__fxt { + private Xosql_dump_parser parser; + private Xosql_dump_cbk__test cbk; + private String[] tbl_flds; + public Xosql_dump_parser__fxt Init(String[] tbl_flds, String... cbk_flds) { + this.tbl_flds = tbl_flds; + this.cbk = new Xosql_dump_cbk__test(); + this.parser = new Xosql_dump_parser(cbk, cbk_flds); + return this; + } + public void Test__parse(String raw_str, String expd) { + Io_url src_fil = Io_url_.new_fil_("mem/test.sql"); + Io_mgr.Instance.SaveFilBry(src_fil, Make_dump(tbl_flds, raw_str)); + parser.Src_fil_(src_fil); + parser.Parse(Gfo_usr_dlg_.Test()); + Gftest.Eq__str(expd, cbk.To_bry_and_clear()); + } + private byte[] Make_dump(String[] tbl_flds, String insert) { + Bry_bfr bfr = Bry_bfr_.New(); + bfr.Add_str_a7("CREATE TABLE tbl_0 ("); + for (int i = 0; i < tbl_flds.length; ++i) { + bfr.Add_byte_nl(); + bfr.Add_byte(Byte_ascii.Tick); + bfr.Add_str_a7(tbl_flds[i]); + bfr.Add_byte(Byte_ascii.Tick); + bfr.Add_byte_comma(); + } + bfr.Add_str_a7("\nUNIQUE KEY idx_0 (fld_0));\n"); + bfr.Add_str_u8(insert); + return bfr.To_bry_and_clear(); + } +} +class Xosql_dump_cbk__test implements Xosql_dump_cbk { + private int prv_idx = -1; + private final Bry_bfr bfr = Bry_bfr_.New(); + public void Clear() {prv_idx = -1; bfr.Clear();} + public void On_fld_done(int fld_idx, byte[] src, int val_bgn, int val_end) { + if (fld_idx <= prv_idx) { + if (prv_idx != -1) bfr.Add_byte_nl(); + } + bfr.Add_mid(src, val_bgn, val_end).Add_byte_pipe(); + prv_idx = fld_idx; + } + public byte[] To_bry_and_clear() {return bfr.To_bry_and_clear();} +} diff --git a/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_fld_itm.java b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_fld_itm.java new file mode 100644 index 000000000..7cc46026f --- /dev/null +++ b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_fld_itm.java @@ -0,0 +1,55 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.bldrs.sql_dumps; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; +class Xosql_fld_itm implements gplx.CompareAble { + public Xosql_fld_itm(int uid, byte[] key, int idx) { + this.uid = uid; + this.key = key; + this.idx = idx; + } + public int Uid() {return uid;} private int uid; + public byte[] Key() {return key;} private final byte[] key; + public int Idx() {return idx;} private int idx; + public void Idx_(int v) {this.idx = v;} + public void Uid_(int v) {this.uid = v;} + + public int compareTo(Object obj) { + Xosql_fld_itm comp = (Xosql_fld_itm)obj; + return Int_.Compare(idx, comp.idx); + } +} +class Xosql_fld_hash { + private final Ordered_hash hash = Ordered_hash_.New_bry(); + private int hash_len; + public int Len() {return hash.Len();} + public Xosql_fld_itm Get_by_key(byte[] k) {return (Xosql_fld_itm)hash.Get_by(k);} + public Xosql_fld_itm Get_by_idx_or_null(int i) { + return i > -1 && i < hash_len ? (Xosql_fld_itm)hash.Get_at(i) : null; + } + public void Add(Xosql_fld_itm itm) {hash.Add(itm.Key(), itm); hash_len = hash.Len();} + public void Sort() {hash.Sort();} + public static Xosql_fld_hash New(String[] keys) { // NOTE: keys must be passed in uid order + int len = keys.length; + Xosql_fld_hash rv = new Xosql_fld_hash(); + for (int i = 0; i < len; ++i) { + Xosql_fld_itm itm = new Xosql_fld_itm(i, Bry_.new_u8(keys[i]), -1); + rv.Add(itm); + } + return rv; + } +} diff --git a/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_tbl_parser.java b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_tbl_parser.java new file mode 100644 index 000000000..45ef5b2af --- /dev/null +++ b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_tbl_parser.java @@ -0,0 +1,61 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.bldrs.sql_dumps; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; +import gplx.core.ios.*; +class Xosql_tbl_parser { + public Ordered_hash Parse(byte[] raw) { + Ordered_hash rv = Ordered_hash_.New_bry(); + // get bgn of fields def; assume after "CREATE TABLE" + int bgn = Bry_find_.Find_fwd(raw, Tkn__create_table); if (bgn == Bry_find_.Not_found) throw Err_.new_wo_type("could not find 'CREATE TABLE'"); + bgn = Bry_find_.Find_fwd(raw, Byte_ascii.Nl, bgn); if (bgn == Bry_find_.Not_found) throw Err_.new_wo_type("could not find new line after 'CREATE TABLE'"); + bgn += 1; // position after character + + // get end of fields def; assume before "UNIQUE KEY" or "PRIMARY KEY" + int end = Bry_find_.Find_fwd(raw, Tkn__primary_key); + if (end == Bry_find_.Not_found) { // as of 2016-07, en.w:categorylinks no longer has UNIQUE KEY; try PRIMARY KEY; DATE:2016-07-08 + end = Bry_find_.Find_fwd(raw, Tkn__unique_index); + if (end == Bry_find_.Not_found) throw Err_.new_wo_type("could not find 'UNIQUE KEY' or 'PRIMARY KEY'"); + } + end = Bry_find_.Find_bwd(raw, Byte_ascii.Nl, end); if (bgn == Bry_find_.Not_found) throw Err_.new_wo_type("could not find new line before 'UNIQUE KEY'"); + + // do parse + Parse_flds(rv, Bry_.Mid(raw, bgn, end)); + return rv; + } + private void Parse_flds(Ordered_hash rv, byte[] raw) { + byte[][] lines = Bry_split_.Split(raw, Byte_ascii.Nl); + int lines_len = lines.length; + int fld_idx = 0; + for (int i = 0; i < lines_len; i++) { + byte[] line = lines[i]; + // get fld bgn / end; EX: "`fld_1`" + int bgn = Bry_find_.Find_fwd(line, Byte_ascii.Tick); if (bgn == Bry_find_.Not_found) continue; // skip blank lines + bgn += Int_.Const_position_after_char; + int end = Bry_find_.Find_fwd(line, Byte_ascii.Tick, bgn); if (end == Bry_find_.Not_found) continue; // skip blank lines + + // add fld + byte[] key = Bry_.Mid(line, bgn, end); + rv.Add(key, new Xosql_fld_itm(Int_.Max_value, key, fld_idx++)); + } + } + private static final byte[] + Tkn__create_table = Bry_.new_a7("CREATE TABLE") + , Tkn__unique_index = Bry_.new_a7("UNIQUE KEY") + , Tkn__primary_key = Bry_.new_a7("PRIMARY KEY") + ; +} diff --git a/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_tbl_parser__tst.java b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_tbl_parser__tst.java new file mode 100644 index 000000000..d86db26aa --- /dev/null +++ b/400_xowa/src/gplx/xowa/bldrs/sql_dumps/Xosql_tbl_parser__tst.java @@ -0,0 +1,64 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.bldrs.sql_dumps; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; +import org.junit.*; import gplx.core.tests.*; +public class Xosql_tbl_parser__tst { + private final Xosql_tbl_parser__fxt fxt = new Xosql_tbl_parser__fxt(); + @Test public void Basic() { + fxt.Exec__parse(String_.Concat_lines_nl + ( "ignore" + , "CREATE TABLE tbl_0 (" + , " `fld_2` int," + , " `fld_1` int," + , " `fld_0` int," + , " UNIQUE KEY idx_0 (fld_2)" + , ");" + )); + fxt.Test__count(3); + fxt.Test__get("fld_0", 2); + fxt.Test__get("fld_1", 1); + fxt.Test__get("fld_2", 0); + fxt.Test__get("fld_3", -1); + } + @Test public void Primary_key() { + fxt.Exec__parse(String_.Concat_lines_nl + ( "ignore" + , "CREATE TABLE tbl_0 (" + , " `fld_2` int," + , " `fld_1` int," + , " `fld_0` int," + , " PRIMARY KEY idx_0 (fld_2)" + , ");" + )); + fxt.Test__count(3); + fxt.Test__get("fld_0", 2); + fxt.Test__get("fld_1", 1); + fxt.Test__get("fld_2", 0); + fxt.Test__get("fld_3", -1); + } +} +class Xosql_tbl_parser__fxt { + private final Xosql_tbl_parser parser = new Xosql_tbl_parser(); + private Ordered_hash tbl_flds; + public void Exec__parse(String v) {this.tbl_flds = parser.Parse(Bry_.new_a7(v));} + public void Test__count(int expd) {Gftest.Eq__int(expd, tbl_flds.Len());} + public void Test__get(String key, int expd) { + Xosql_fld_itm actl_itm = (Xosql_fld_itm)tbl_flds.Get_by(Bry_.new_u8(key)); + Gftest.Eq__int(expd, actl_itm == null ? Bry_find_.Not_found : actl_itm.Idx()); + } +} diff --git a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser.java b/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser.java deleted file mode 100644 index ac1bf954a..000000000 --- a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser.java +++ /dev/null @@ -1,173 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; -import gplx.core.flds.*; import gplx.core.ios.*; -public class Sql_file_parser { - private final Gfo_fld_rdr sql_parser = Gfo_fld_rdr.sql_(); private final Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_(); - private Sql_fld_itm[] flds_all; private int flds_all_len; - public boolean Save_csv() {return save_csv;} private boolean save_csv = true; public Sql_file_parser Save_csv_n_() {save_csv = false; return this;} - public Sql_file_parser Fld_cmd_(Sql_file_parser_cmd v) {this.fld_cmd = v; return this;} private Sql_file_parser_cmd fld_cmd; - public Sql_file_parser Flds_req_(byte[]... v) {flds_req = v; return this;} private byte[][] flds_req; - public Sql_file_parser Flds_req_idx_(int flds_all_len, int... idxs) { - new_flds_all(flds_all_len); - int len = idxs.length; - for (int i = 0; i < len; i++) { - int idx = idxs[i]; - Sql_fld_itm itm = new Sql_fld_itm(idx, Bry_.Empty); - flds_all[idx] = itm; - } - return this; - } - private void Identify_flds(byte[] raw) { - Sql_fld_mgr fld_mgr = new Sql_fld_mgr().Parse(raw); - new_flds_all(fld_mgr.Count()); - int len = flds_req.length; - for (int i = 0; i < len; i++) { - byte[] fld = flds_req[i]; - Sql_fld_itm itm = fld_mgr.Get_by_key(fld); if (itm == null) throw Err_.new_wo_type("could not find field", "fld", fld); - flds_all[itm.Idx()] = itm; - } - } - private void new_flds_all(int len) { - this.flds_all_len = len; // NOTE: must set flds_all_len, else Commit_fld will not be correct; - this.flds_all = new Sql_fld_itm[len]; - } - public void Parse(Gfo_usr_dlg usr_dlg) { - Io_buffer_rdr rdr = Io_buffer_rdr.Null; - try { - rdr = Io_buffer_rdr.new_(gplx.core.ios.streams.Io_stream_rdr_.new_by_url_(src_fil), src_len); - Bry_bfr fil_bfr = Bry_bfr_.New(), val_bfr = Bry_bfr_.New(); - byte[] bfr = rdr.Bfr(); int bfr_len = rdr.Bfr_len(), fld_idx = 0, cur_pos = 0; - if (flds_req != null) Identify_flds(bfr); - byte mode = Mode_sql_bgn; byte[] decode_regy = sql_parser.Escape_decode(); - boolean reading_file = true; byte mode_prv = Mode_sql_bgn; - Sql_file_parser_data data = new Sql_file_parser_data(); - while (reading_file) { - if (cur_pos + 256 > bfr_len && rdr.Fil_pos() != rdr.Fil_len()) { // buffer 256 characters; can be 0, but erring on side of simplicity - rdr.Bfr_load_from(cur_pos); - cur_pos = 0; - bfr = rdr.Bfr(); - bfr_len = rdr.Bfr_len(); - } - if (cur_pos == bfr_len) break; - byte b = bfr[cur_pos]; - switch (mode) { - case Mode_sql_bgn: - cur_pos = Bry_find_.Find_fwd(bfr, Bry_insert_into, cur_pos); - if (cur_pos == Bry_find_.Not_found || cur_pos > bfr_len) {reading_file = false; continue;} - cur_pos = Bry_find_.Find_fwd(bfr, Bry_values, cur_pos); - if (cur_pos == Bry_find_.Not_found || cur_pos > bfr_len) throw Err_.new_wo_type("VALUES not found"); // something went wrong; - mode = Mode_fld; - cur_pos += Bry_values.length; - break; - case Mode_row_bgn: - switch (b) { - case Byte_ascii.Paren_bgn: mode = Mode_fld; break; - default: throw Err_.new_unhandled(mode); - } - ++cur_pos; - break; - case Mode_row_end: - switch (b) { - case Byte_ascii.Nl: break; // ignore \n - case Byte_ascii.Comma: mode = Mode_row_bgn; break; - case Byte_ascii.Semic: mode = Mode_sql_bgn; break; - default: throw Err_.new_unhandled(mode); - } - ++cur_pos; - break; - case Mode_fld: - switch (b) { - case Byte_ascii.Apos: mode = Mode_quote; break; // NOTE: never escape apos by doubling; will fail for empty fields; EX: ", '', ''"; DATE:2013-07-06 - case Byte_ascii.Backslash: mode_prv = mode; mode = Mode_escape; break; - default: val_bfr.Add_byte(b); break; - case Byte_ascii.Space: case Byte_ascii.Nl: break; - case Byte_ascii.Comma: - Commit_fld(fld_idx++, val_bfr, fil_bfr, data); - break; - case Byte_ascii.Paren_end: - Commit_fld(fld_idx++, val_bfr, fil_bfr, data); - if (!data.Cancel_row()) - Commit_row(usr_dlg, fil_bfr); - fld_idx = 0; - mode = Mode_row_end; - break; - } - ++cur_pos; - break; - case Mode_quote: - switch (b) { - case Byte_ascii.Apos: mode = Mode_fld; break; - case Byte_ascii.Backslash: mode_prv = mode; mode = Mode_escape; break; - default: val_bfr.Add_byte(b); break; - } - ++cur_pos; - break; - case Mode_escape: - byte escape_val = decode_regy[b]; - if (escape_val == Byte_ascii.Null) {val_bfr.Add_byte(Byte_ascii.Backslash).Add_byte(b);} - else val_bfr.Add_byte(escape_val); - mode = mode_prv; - ++cur_pos; - break; - default: throw Err_.new_unhandled(mode); - } - } - if (save_csv) - Io_mgr.Instance.AppendFilByt(trg_fil_gen.Nxt_url(), fil_bfr.To_bry_and_clear()); - else - fil_bfr.Clear(); - } - finally {rdr.Rls();} - } - private void Commit_row(Gfo_usr_dlg usr_dlg, Bry_bfr fil_bfr) { - fil_bfr.Add_byte(Byte_ascii.Nl); - if (fil_bfr.Len() > trg_len) { - Io_url trg_fil = trg_fil_gen.Nxt_url(); - usr_dlg.Prog_one("", "", "making ~{0}", trg_fil.NameAndExt()); - if (save_csv) - Io_mgr.Instance.AppendFilByt(trg_fil, fil_bfr.To_bry_and_clear()); - else - fil_bfr.Clear(); - } - } - private void Commit_fld(int fld_idx, Bry_bfr val_bfr, Bry_bfr fil_bfr, Sql_file_parser_data data) { - Sql_fld_itm fld = fld_idx < flds_all_len ? flds_all[fld_idx] : null; // handle new flds added by MW, but not supported by XO; EX:hiddencat and pp_sortkey; DATE:2014-04-28 - if (fld != null) { - data.Cancel_row_n_(); - if (fld_cmd == null) { // no custom cmd; assume append; - fld_wtr.Bfr_(fil_bfr); - fld_wtr.Write_bry_escape_fld(val_bfr.Bfr(), 0, val_bfr.Len()); - } - else - fld_cmd.Exec(val_bfr.Bfr(), fld.Key(), fld_idx, 0, val_bfr.Len(), fil_bfr, data); - } - val_bfr.Clear(); - } - - private Io_url_gen trg_fil_gen; private int trg_len = 4 * Io_mgr.Len_mb; - public Io_url Src_fil() {return src_fil;} private Io_url src_fil; - public int Src_len() {return src_len;} private int src_len = 8 * Io_mgr.Len_mb; - public Sql_file_parser Src_fil_(Io_url v) {src_fil = v; return this;} - public Sql_file_parser Src_len_(int v) {src_len = v; return this;} - public Sql_file_parser Trg_fil_gen_(Io_url_gen v) {trg_fil_gen = v; return this;} - public Sql_file_parser Trg_len_(int v) {trg_len = v; return this;} - - private static final byte[] Bry_insert_into = Bry_.new_a7("INSERT INTO "), Bry_values = Bry_.new_a7(" VALUES ("); - private static final byte Mode_sql_bgn = 0, Mode_row_bgn = 1, Mode_row_end = 2, Mode_fld = 3, Mode_quote = 4, Mode_escape = 5; -} diff --git a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_cmd.java b/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_cmd.java deleted file mode 100644 index d565c737a..000000000 --- a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_cmd.java +++ /dev/null @@ -1,43 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; -import gplx.core.strings.*; -public interface Sql_file_parser_cmd { - void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data); -} -class Sql_file_parser_cmd_max_len implements Sql_file_parser_cmd { - public void Log_len_(int v) {log_len = v;} private int log_len = 141; - public void Log_print(Io_url url) { - String_bldr sb = String_bldr_.new_(); - for (int i = 0; i < log.Count(); i++) { - String itm = (String)log.Get_at(i); - sb.Add(String_.Len(itm) + "|" + itm + "\n"); - } - Io_mgr.Instance.SaveFilStr(url, sb.To_str()); - } - public int Max_len() {return max_len;} private int max_len; - public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) { - int fld_len = fld_end - fld_bgn; - if (fld_len > max_len) max_len = fld_len; - if (fld_len > log_len) { - log.Add(String_.new_u8(src, fld_bgn, fld_end)); - } - file_bfr.Add_mid(src, fld_bgn, fld_end).Add_byte(Byte_ascii.Pipe); - } - List_adp log = List_adp_.New(); -} diff --git a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_tst.java b/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_tst.java deleted file mode 100644 index 1819959f9..000000000 --- a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_file_parser_tst.java +++ /dev/null @@ -1,73 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; -import org.junit.*; import gplx.core.ios.*; -public class Sql_file_parser_tst { - Sql_file_parser_fxt fxt = new Sql_file_parser_fxt(); - @Before public void init() {fxt.Clear();} - @Test public void One() { - fxt.Init_flds_req_idx_(2, 1).Test_parse("INSERT INTO 'tbl_1' VALUES (1,2);", "2|\n"); - } - @Test public void Many() { - fxt.Init_flds_req_idx_(2, 1).Test_parse("INSERT INTO 'tbl_1' VALUES (1,2),(3,4),(5,6);", "2|\n4|\n6|\n"); - } - @Test public void Quote_basic() { - fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a','b');", "a|b|\n"); - } - @Test public void Escape_pipe() { - fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a|b','c');", "a~pb|c|\n"); - } - @Test public void Escape_nl() { - fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a\\nb','c');", "a~nb|c|\n"); - } - @Test public void Escape_tab() { - fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a\\tb','c');", "a~tb|c|\n"); - } - @Test public void Escape_backslash() { - fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a\\\\b','c');", "a\\b|c|\n"); - } - @Test public void Escape_quote() { - fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a\"b','c');", "a\"b|c|\n"); - } - @Test public void Fld_paren_end() { - fxt.Init_flds_req_idx_(3, 1, 2).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'Психостимуляторы_(лекарственные_средства)','c');", "Психостимуляторы_(лекарственные_средства)|c|\n"); - } - @Test public void Insert_multiple() { - fxt.Init_flds_req_idx_(2, 1).Test_parse("INSERT INTO 'tbl_1' VALUES (1,2);INSERT INTO 'tbl_1' VALUES (3,4)", "2|\n4|\n"); - } - @Test public void Cmds() { - Sql_file_parser_cmd_max_len cmd = new Sql_file_parser_cmd_max_len(); - fxt.Init_flds_req_idx_(2, 1).Init_cmd_(cmd).Test_parse("INSERT INTO 'tbl_1' VALUES (1,'a'),(3,'abc');", "a|\nabc|\n"); - Tfds.Eq(3, cmd.Max_len()); - } -} -class Sql_file_parser_fxt { - Sql_file_parser parser = new Sql_file_parser().Src_len_(Io_mgr.Len_kb).Trg_len_(Io_mgr.Len_kb); - public Sql_file_parser_fxt Clear() {Io_mgr.Instance.InitEngine_mem(); return this;} - public Sql_file_parser_fxt Init_flds_req_idx_(int flds_all_len, int... idxs) {parser.Flds_req_idx_(flds_all_len, idxs); return this;} - public Sql_file_parser_fxt Init_cmd_(Sql_file_parser_cmd cmd) {parser.Fld_cmd_(cmd); return this;} - public void Test_parse(String raw_str, String expd) { - Io_url src_fil = Io_url_.new_fil_("mem/test.sql"); - Io_mgr.Instance.SaveFilBry(src_fil, Bry_.new_u8(raw_str)); - Io_url trg_fil = Io_url_.new_fil_("mem/test.csv"); - parser.Src_fil_(src_fil).Trg_fil_gen_(Io_url_gen_.fil_(trg_fil)); - parser.Parse(Gfo_usr_dlg_.Test()); - byte[] actl = Io_mgr.Instance.LoadFilBry(trg_fil); - Tfds.Eq(expd, String_.new_u8(actl)); - } -} diff --git a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_fld_mgr.java b/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_fld_mgr.java deleted file mode 100644 index 06e54284c..000000000 --- a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_fld_mgr.java +++ /dev/null @@ -1,66 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; -import gplx.core.ios.*; -class Sql_fld_mgr { - public int Count() {return hash.Count();} - public Sql_fld_itm Get_by_key(String fld) {return Get_by_key(Bry_.new_u8(fld));} - public Sql_fld_itm Get_by_key(byte[] fld) { - return (Sql_fld_itm)hash.Get_by(fld); - } private Ordered_hash hash = Ordered_hash_.New_bry(); - public Sql_fld_mgr Parse(byte[] raw) { - hash.Clear(); - int bgn = Bry_find_.Find_fwd(raw, Tkn_create_table); if (bgn == Bry_find_.Not_found) throw Err_.new_wo_type("could not find 'CREATE TABLE'"); - bgn = Bry_find_.Find_fwd(raw, Byte_ascii.Nl, bgn); if (bgn == Bry_find_.Not_found) throw Err_.new_wo_type("could not find new line after 'CREATE TABLE'"); - bgn += Int_.Const_position_after_char; - int end = Bry_find_.Find_fwd(raw, Tkn_unique_index); - if (end == Bry_find_.Not_found) { // as of 2016-07, en.w:categorylinks no longer has UNIQUE KEY; try PRIMARY KEY; DATE:2016-07-08 - end = Bry_find_.Find_fwd(raw, Tkn_primary_key); - if (end == Bry_find_.Not_found) - throw Err_.new_wo_type("could not find 'UNIQUE KEY' or 'PRIMARY KEY'"); - } - end = Bry_find_.Find_bwd(raw, Byte_ascii.Nl, end); if (bgn == Bry_find_.Not_found) throw Err_.new_wo_type("could not find new line before 'UNIQUE KEY'"); - Parse_lines(Bry_.Mid(raw, bgn, end)); - return this; - } - private void Parse_lines(byte[] raw) { - byte[][] lines = Bry_split_.Split(raw, Byte_ascii.Nl); - int lines_len = lines.length; - int fld_idx = 0; - for (int i = 0; i < lines_len; i++) { - byte[] line = lines[i]; - int bgn = Bry_find_.Find_fwd(line, Byte_ascii.Tick); if (bgn == Bry_find_.Not_found) continue; // skip blank lines - bgn += Int_.Const_position_after_char; - int end = Bry_find_.Find_fwd(line, Byte_ascii.Tick, bgn); if (end == Bry_find_.Not_found) continue; // skip blank lines - byte[] key = Bry_.Mid(line, bgn, end); - Sql_fld_itm fld = new Sql_fld_itm(fld_idx++, key); - hash.Add(fld.Key(), fld); - } - } - private static final byte[] - Tkn_create_table = Bry_.new_a7("CREATE TABLE") - , Tkn_unique_index = Bry_.new_a7("UNIQUE KEY") - , Tkn_primary_key = Bry_.new_a7("PRIMARY KEY") - ; - public static final int Not_found = -1; -} -class Sql_fld_itm { - public Sql_fld_itm(int idx, byte[] key) {this.idx = idx; this.key = key;} - public int Idx() {return idx;} private int idx; - public byte[] Key() {return key;} private byte[] key; -} diff --git a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_fld_mgr_tst.java b/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_fld_mgr_tst.java deleted file mode 100644 index 38ee10cbb..000000000 --- a/400_xowa/src/gplx/xowa/bldrs/sqls/Sql_fld_mgr_tst.java +++ /dev/null @@ -1,63 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.bldrs.sqls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; -import org.junit.*; -public class Sql_fld_mgr_tst { -Sql_fld_mgr_fxt fxt = new Sql_fld_mgr_fxt(); - @Test public void Basic() { - fxt.Exec_parse(String_.Concat_lines_nl - ( "ignore" - , "CREATE TABLE tbl_0 (" - , " `fld_2` int," - , " `fld_1` int," - , " `fld_0` int," - , " UNIQUE KEY idx_0 (fld_2)" - , ");" - )); - fxt.Test_count(3); - fxt.Exec_get("fld_0", 2); - fxt.Exec_get("fld_1", 1); - fxt.Exec_get("fld_2", 0); - fxt.Exec_get("fld_3", -1); - } - @Test public void Primary_key() { - fxt.Exec_parse(String_.Concat_lines_nl - ( "ignore" - , "CREATE TABLE tbl_0 (" - , " `fld_2` int," - , " `fld_1` int," - , " `fld_0` int," - , " PRIMARY KEY idx_0 (fld_2)" - , ");" - )); - fxt.Test_count(3); - fxt.Exec_get("fld_0", 2); - fxt.Exec_get("fld_1", 1); - fxt.Exec_get("fld_2", 0); - fxt.Exec_get("fld_3", -1); - } -} -class Sql_fld_mgr_fxt { - Sql_fld_mgr fld_mgr = new Sql_fld_mgr(); - public void Exec_parse(String v) {fld_mgr.Parse(Bry_.new_a7(v));} - public void Exec_get(String key, int expd) { - Sql_fld_itm actl_itm = fld_mgr.Get_by_key(key); - Tfds.Eq(expd, actl_itm == null ? Sql_fld_mgr.Not_found : actl_itm.Idx()); - } - public void Test_count(int expd) {Tfds.Eq(expd, fld_mgr.Count());} -} diff --git a/400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java b/400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java index 1ed89285d..8d3d5bfd9 100644 --- a/400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java +++ b/400_xowa/src/gplx/xowa/bldrs/wkrs/Xob_sql_dump_base.java @@ -17,13 +17,17 @@ along with this program. If not, see . */ package gplx.xowa.bldrs.wkrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*; import gplx.core.flds.*; import gplx.core.ios.*; import gplx.xowa.wikis.tdbs.*; -import gplx.xowa.bldrs.sqls.*; import gplx.xowa.wikis.tdbs.bldrs.*; +import gplx.xowa.bldrs.sql_dumps.*; import gplx.xowa.wikis.tdbs.bldrs.*; public abstract class Xob_sql_dump_base extends Xob_itm_dump_base implements Xob_cmd, Gfo_invk { - private final Sql_file_parser parser = new Sql_file_parser(); protected boolean fail = false; + private final Xosql_dump_parser parser; protected boolean fail = false; public abstract String Cmd_key(); + public Xob_sql_dump_base() { + this.parser = New_parser(); + } public Io_url Src_fil() {return src_fil;} private Io_url src_fil; public Io_url_gen Make_url_gen() {return make_url_gen;} private Io_url_gen make_url_gen; public abstract String Sql_file_name(); + protected abstract Xosql_dump_parser New_parser(); public void Cmd_init(Xob_bldr bldr) {} public void Cmd_bgn(Xob_bldr bldr) { this.Init_dump(this.Cmd_key()); @@ -38,10 +42,10 @@ public abstract class Xob_sql_dump_base extends Xob_itm_dump_base implements Xob return; } } - parser.Src_fil_(src_fil).Trg_fil_gen_(dump_url_gen); + parser.Src_fil_(src_fil); Cmd_bgn_hook(bldr, parser); } protected Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_(); - public abstract void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser); + public abstract void Cmd_bgn_hook(Xob_bldr bldr, Xosql_dump_parser parser); public void Cmd_run() { if (fail) return; parser.Parse(bldr.Usr_dlg()); diff --git a/400_xowa/src/gplx/xowa/files/Xof_bin_updater.java b/400_xowa/src/gplx/xowa/files/Xof_bin_updater.java index 476756445..cc294e5a1 100644 --- a/400_xowa/src/gplx/xowa/files/Xof_bin_updater.java +++ b/400_xowa/src/gplx/xowa/files/Xof_bin_updater.java @@ -19,18 +19,18 @@ package gplx.xowa.files; import gplx.*; import gplx.xowa.*; import gplx.core.ios.*; import gplx.core.ios.streams.*; import gplx.fsdb.data.*; import gplx.fsdb.meta.*; import gplx.xowa.files.fsdb.*; public class Xof_bin_updater { - private final Fsd_img_itm tmp_img_itm = new Fsd_img_itm(); private final Fsd_thm_itm tmp_thm_itm = Fsd_thm_itm.new_(); private final Fsd_fil_itm tmp_fil_itm = new Fsd_fil_itm(); + private final Fsd_thm_itm tmp_thm_itm = Fsd_thm_itm.new_(); public int Save_bin(Fsm_mnt_itm mnt, Fsm_atr_fil atr_fil, Fsm_bin_fil bin_fil, Xof_fsdb_itm fsdb, Io_stream_rdr rdr, long rdr_len) { int db_uid = -1; int orig_ext_id = fsdb.Orig_ext().Id(); if (fsdb.File_is_orig()) { - if (fsdb.Orig_ext().Id_is_image()) { // does not add .pdf and .djvu b/c latter do not have w,h for fsdb_img - mnt.Insert_img(tmp_img_itm, atr_fil, bin_fil, fsdb.Orig_repo_name(), fsdb.Orig_ttl(), orig_ext_id, fsdb.Orig_w(), fsdb.Orig_h(), rdr_len, rdr); - db_uid = tmp_img_itm.Fil_id(); + if (fsdb.Orig_ext().Id_is_image()) { + Fsd_img_itm img = mnt.Insert_img(atr_fil, bin_fil, fsdb.Orig_repo_name(), fsdb.Orig_ttl(), orig_ext_id, fsdb.Orig_w(), fsdb.Orig_h(), rdr_len, rdr); + db_uid = img.Fil_id(); } - else { - mnt.Insert_fil(tmp_fil_itm, atr_fil, bin_fil, fsdb.Orig_repo_name(), fsdb.Orig_ttl(), orig_ext_id, rdr_len, rdr); - db_uid = tmp_fil_itm.Fil_id(); + else { // adds .pdf and .djvu b/c latter does not have w,h for fsdb_img + Fsd_fil_itm fil = mnt.Insert_fil(atr_fil, bin_fil, fsdb.Orig_repo_name(), fsdb.Orig_ttl(), orig_ext_id, rdr_len, rdr); + db_uid = fil.Fil_id(); } } else { diff --git a/400_xowa/src/gplx/xowa/files/Xof_exec_tid.java b/400_xowa/src/gplx/xowa/files/Xof_exec_tid.java index 37aa1a4ca..33e3dd361 100644 --- a/400_xowa/src/gplx/xowa/files/Xof_exec_tid.java +++ b/400_xowa/src/gplx/xowa/files/Xof_exec_tid.java @@ -17,5 +17,9 @@ along with this program. If not, see . */ package gplx.xowa.files; import gplx.*; import gplx.xowa.*; public class Xof_exec_tid { - public static final int Tid_wiki_page = 1, Tid_wiki_file = 2, Tid_viewer_app = 3; + public static final int + Tid_wiki_page = 1 // regular page; EX: en.w:Earth + , Tid_wiki_file = 2 // "File:" ns; EX: en.w:File:A.png + , Tid_viewer_app = 3 // called by viewer_app + ; } diff --git a/400_xowa/src/gplx/xowa/files/Xof_file_fxt.java b/400_xowa/src/gplx/xowa/files/Xof_file_fxt.java index 3bee5c0f7..ddbb181c6 100644 --- a/400_xowa/src/gplx/xowa/files/Xof_file_fxt.java +++ b/400_xowa/src/gplx/xowa/files/Xof_file_fxt.java @@ -44,6 +44,7 @@ public class Xof_file_fxt { return this; } public void Exec_orig_add(boolean repo_is_commons, String orig_ttl, int orig_ext_id, int orig_w, int orig_h, String orig_redirect) { - wiki.File_mgr().Orig_mgr().Insert(Xof_repo_itm_.Repo_by_bool(repo_is_commons), Bry_.new_u8(orig_ttl), orig_ext_id, orig_w, orig_h, Bry_.new_u8(orig_redirect)); + byte repo_tid = repo_is_commons ? Xof_repo_tid_.Tid__remote : Xof_repo_tid_.Tid__local; + wiki.File_mgr().Orig_mgr().Insert(repo_tid, Bry_.new_u8(orig_ttl), orig_ext_id, orig_w, orig_h, Bry_.new_u8(orig_redirect)); } } diff --git a/400_xowa/src/gplx/xowa/files/Xof_file_wkr.java b/400_xowa/src/gplx/xowa/files/Xof_file_wkr.java index f553735dc..2d5bbb8ad 100644 --- a/400_xowa/src/gplx/xowa/files/Xof_file_wkr.java +++ b/400_xowa/src/gplx/xowa/files/Xof_file_wkr.java @@ -17,7 +17,7 @@ along with this program. If not, see . */ package gplx.xowa.files; import gplx.*; import gplx.xowa.*; import gplx.core.threads.*; import gplx.core.ios.*; import gplx.core.ios.streams.*; -import gplx.fsdb.*; import gplx.fsdb.meta.*; import gplx.fsdb.data.*; import gplx.xowa.files.fsdb.*; +import gplx.fsdb.*; import gplx.fsdb.meta.*; import gplx.fsdb.data.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.files.imgs.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.origs.*; import gplx.xowa.files.bins.*; import gplx.xowa.files.caches.*; import gplx.xowa.guis.cbks.js.*; public class Xof_file_wkr implements Gfo_thread_wkr { private final Xof_orig_mgr orig_mgr; private final Xof_bin_mgr bin_mgr; private final Fsm_mnt_mgr mnt_mgr; private final Xou_cache_mgr cache_mgr; @@ -97,7 +97,7 @@ public class Xof_file_wkr implements Gfo_thread_wkr { try { if (file_rdr != Io_stream_rdr_.Noop) { Xof_repo_itm repo = repo_mgr.Get_trg_by_id_or_null(fsdb.Orig_repo_id(), fsdb.Lnki_ttl(), page.Url_bry_safe()); - Io_url file_url = url_bldr.Init_for_trg_file(Xof_repo_itm_.Mode_by_bool(!fsdb.File_is_orig()), repo, fsdb.Orig_ttl(), fsdb.Orig_ttl_md5(), fsdb.Orig_ext(), fsdb.File_w(), fsdb.Lnki_time(), fsdb.Lnki_page()).Xto_url(); + Io_url file_url = url_bldr.Init_for_trg_file(repo, Xof_img_mode_.By_bool(!fsdb.File_is_orig()), fsdb.Orig_ttl(), fsdb.Orig_ttl_md5(), fsdb.Orig_ext(), fsdb.File_w(), fsdb.Lnki_time(), fsdb.Lnki_page()).Xto_url(); Io_stream_wtr_.Save_rdr(file_url, file_rdr, Io_download_fmt.Null); fsdb.File_size_(file_rdr.Len()); // must update file size for cache fsdb.Init_at_lnki_by_near(fsdb.File_w()); // change lnki to be file_w,-1 diff --git a/400_xowa/src/gplx/xowa/files/Xof_fsdb_itm.java b/400_xowa/src/gplx/xowa/files/Xof_fsdb_itm.java index a8aca07a2..3115a7175 100644 --- a/400_xowa/src/gplx/xowa/files/Xof_fsdb_itm.java +++ b/400_xowa/src/gplx/xowa/files/Xof_fsdb_itm.java @@ -31,7 +31,7 @@ public class Xof_fsdb_itm implements Xof_file_itm { public double Lnki_time() {return lnki_time;} private double lnki_time = Xof_lnki_time.Null; public int Lnki_page() {return lnki_page;} private int lnki_page = Xof_lnki_page.Null; public int User_thumb_w() {return user_thumb_w;} private int user_thumb_w = Xof_img_size.Thumb_width_img; - public byte Orig_repo_id() {return orig_repo_id;} private byte orig_repo_id = Xof_repo_itm_.Repo_null; + public byte Orig_repo_id() {return orig_repo_id;} private byte orig_repo_id = Xof_repo_tid_.Tid__null; public byte[] Orig_repo_name() {return orig_repo_name;} private byte[] orig_repo_name; public byte[] Orig_ttl() {return orig_ttl;} private byte[] orig_ttl; public byte[] Orig_ttl_md5() {return orig_ttl_md5;} private byte[] orig_ttl_md5; @@ -123,7 +123,7 @@ public class Xof_fsdb_itm implements Xof_file_itm { this.file_exists_in_cache = file_exists_in_cache; this.html_w = w; this.html_h = h; this.html_view_url = view_url; } - public void Init_by_wm_parse(byte[] lnki_wiki_abrv, boolean repo_is_commons, boolean file_is_orig, byte[] file_ttl_bry, int file_w, double file_time, int file_page) { + public void Init_by_wm_parse(byte[] lnki_wiki_abrv, boolean repo_is_commons, boolean file_is_orig, byte[] file_ttl_bry, Xof_ext orig_ext, int file_w, double file_time, int file_page) { this.hdump_mode = Hdump_mode__v2; // lnki @@ -134,12 +134,13 @@ public class Xof_fsdb_itm implements Xof_file_itm { this.lnki_wiki_abrv = lnki_wiki_abrv; this.lnki_time = file_time; this.lnki_page = file_page; + this.lnki_exec_tid = Xof_exec_tid.Tid_wiki_page; // orig - this.orig_repo_id = repo_is_commons ? Xof_repo_itm_.Repo_remote : Xof_repo_itm_.Repo_local; + this.orig_repo_id = repo_is_commons ? Xof_repo_tid_.Tid__remote : Xof_repo_tid_.Tid__local; this.file_is_orig = file_is_orig; this.Orig_ttl_(file_ttl_bry); - this.orig_ext = Xof_ext_.new_by_ttl_(file_ttl_bry); + this.orig_ext = orig_ext; // html this.file_w = this.html_w = file_w; diff --git a/400_xowa/src/gplx/xowa/files/Xof_fsdb_itm_fxt.java b/400_xowa/src/gplx/xowa/files/Xof_fsdb_itm_fxt.java index 66ea860d9..94e02c900 100644 --- a/400_xowa/src/gplx/xowa/files/Xof_fsdb_itm_fxt.java +++ b/400_xowa/src/gplx/xowa/files/Xof_fsdb_itm_fxt.java @@ -44,7 +44,7 @@ public class Xof_fsdb_itm_fxt { this.lnki_h = Xof_img_size.Size__neg1; this.lnki_time = Xof_lnki_time.Null; this.lnki_page = Xof_lnki_page.Null; - this.orig_repo_id = Xof_repo_itm_.Repo_null; + this.orig_repo_id = Xof_repo_tid_.Tid__null; this.orig_repo_name = orig_ttl = orig_redirect = null; this.orig_ext = null; } @@ -55,7 +55,7 @@ public class Xof_fsdb_itm_fxt { } public Xof_fsdb_itm_fxt Orig__commons__lnki() { this.orig_repo_name = Xow_domain_itm_.Bry__commons; - this.orig_repo_id = Xof_repo_itm_.Repo_remote; + this.orig_repo_id = Xof_repo_tid_.Tid__remote; this.orig_ttl = lnki_ttl; this.orig_ext = Xof_ext_.new_by_ttl_(orig_ttl); this.orig_w = 880; @@ -64,7 +64,7 @@ public class Xof_fsdb_itm_fxt { } public Xof_fsdb_itm_fxt Orig__enwiki__lnki() { this.orig_repo_name = Xow_domain_itm_.Bry__enwiki; - this.orig_repo_id = Xof_repo_itm_.Repo_local; + this.orig_repo_id = Xof_repo_tid_.Tid__local; this.orig_ttl = lnki_ttl; this.orig_ext = Xof_ext_.new_by_ttl_(orig_ttl); this.orig_w = 880; @@ -77,5 +77,5 @@ public class Xof_fsdb_itm_fxt { rv.Init_at_orig(orig_repo_id, orig_repo_name, orig_ttl, orig_ext, orig_w, orig_h, orig_redirect); return rv; } - private final static byte[] Abrv__en_w = Bry_.new_a7("en.w"); + private final static byte[] Abrv__en_w = Bry_.new_a7("en.w"); } diff --git a/400_xowa/src/gplx/xowa/files/Xof_url_bldr.java b/400_xowa/src/gplx/xowa/files/Xof_url_bldr.java index ee2f86430..cec3418ee 100644 --- a/400_xowa/src/gplx/xowa/files/Xof_url_bldr.java +++ b/400_xowa/src/gplx/xowa/files/Xof_url_bldr.java @@ -18,50 +18,54 @@ along with this program. If not, see . package gplx.xowa.files; import gplx.*; import gplx.xowa.*; import gplx.core.envs.*; import gplx.langs.htmls.encoders.*; -import gplx.xowa.files.repos.*; import gplx.xowa.files.fsdb.*; +import gplx.xowa.files.repos.*; import gplx.xowa.files.fsdb.*; import gplx.xowa.files.imgs.*; public class Xof_url_bldr { private final Bry_bfr tmp_bfr = Bry_bfr_.Reset(400); private final Gfo_url_encoder encoder_src_http = Gfo_url_encoder_.New__http_url().Make(); // NOTE: changed from new_html_href_mw_ to new_url_ on 2012-11-19; issues with A%2Cb becoming A%252Cb private byte[] ttl; private byte[] md5; private Xof_ext ext; private boolean file_is_thumb; private int file_w; private double time = Xof_lnki_time.Null; private int page = Xof_lnki_page.Null; private byte time_dlm = Byte_ascii.At; + private byte repo_tid; private byte[] root; private byte dir_spr; private boolean fsys_is_wnt; private boolean wmf_dir_hive; private boolean wmf_protocol_is_file; private int md5_dir_depth; private byte[] area; public Xof_url_bldr Root_(byte[] v) {root = v; return this;} - public Xof_url_bldr Init_by_root(byte[] root, boolean fsys_is_wnt, byte dir_spr, boolean wmf_dir_hive, boolean wmf_protocol_is_file, int md5_dir_depth) { + public Xof_url_bldr Init_by_repo(byte repo_tid, byte[] root, boolean fsys_is_wnt, byte dir_spr, boolean wmf_dir_hive, boolean wmf_protocol_is_file, int md5_dir_depth) { + this.repo_tid = repo_tid; this.root = root; this.dir_spr = dir_spr; this.wmf_dir_hive = wmf_dir_hive; this.wmf_protocol_is_file = wmf_protocol_is_file; this.md5_dir_depth = md5_dir_depth; this.fsys_is_wnt = fsys_is_wnt; return this; } public Xof_url_bldr Init_by_itm(byte mode, byte[] ttl, byte[] md5, Xof_ext ext, int file_w, double time, int page) { this.ttl = ttl; this.md5 = md5; this.ext = ext; this.file_w = file_w; this.time = time; this.page = page; - if (wmf_protocol_is_file && fsys_is_wnt) this.ttl = Xof_repo_itm_.Ttl_invalid_fsys_chars(tmp_bfr, ttl); // NOTE: changed ttl does not change md5 - this.file_is_thumb = mode == Xof_repo_itm_.Mode_thumb; - this.area = Xof_repo_itm_.Mode_names_key[mode]; + if (wmf_protocol_is_file && fsys_is_wnt) this.ttl = Xof_itm_ttl_.Remove_invalid(tmp_bfr, ttl); // NOTE: changed ttl does not change md5 + this.file_is_thumb = mode == Xof_img_mode_.Tid__thumb; + this.area = Xof_img_mode_.Names_ary[mode]; return this; } - public Xof_url_bldr Init_for_src_file(byte mode, Xof_repo_itm repo, byte[] ttl, byte[] md5, Xof_ext ext, int file_w, double time, int page) { + public Xof_url_bldr Init_for_src_file(Xof_repo_itm repo, byte mode, byte[] ttl, byte[] md5, Xof_ext ext, int file_w, double time, int page) { + this.repo_tid = repo.Tid(); this.wmf_dir_hive = Bool_.Y; this.wmf_protocol_is_file = repo.Tarball(); this.dir_spr = repo.Dir_spr(); this.root = repo.Root_bry(); this.area = repo.Mode_names()[mode]; this.ttl = repo.Gen_name_src(tmp_bfr, ttl); this.md5 = md5; this.ext = ext; - this.file_is_thumb = mode == Xof_repo_itm_.Mode_thumb; this.file_w = file_w; this.time = time; this.page = page; + this.file_is_thumb = mode == Xof_img_mode_.Tid__thumb; this.file_w = file_w; this.time = time; this.page = page; return this; } - public Xof_url_bldr Init_for_trg_file(byte mode, Xof_repo_itm repo, byte[] ttl, byte[] md5, Xof_ext ext, int file_w, double time, int page) { - return Init(Bool_.N, Bool_.N, repo.Dir_spr(), repo.Root_bry() + public Xof_url_bldr Init_for_trg_file(Xof_repo_itm repo, byte mode, byte[] ttl, byte[] md5, Xof_ext ext, int file_w, double time, int page) { + return Init(repo.Tid(), Bool_.N, Bool_.N, repo.Dir_spr(), repo.Root_bry() , repo.Mode_names()[mode], repo.Dir_depth(), repo.Gen_name_trg(tmp_bfr, ttl, md5, ext), md5, ext, mode, file_w, time, page); } - public Xof_url_bldr Init_for_trg_html(byte mode, Xof_repo_itm repo, byte[] ttl, byte[] md5, Xof_ext ext, int file_w, double time, int page) { - return Init(Bool_.N, Bool_.N, Byte_ascii.Slash, repo.Root_http() + public Xof_url_bldr Init_for_trg_html(Xof_repo_itm repo, byte mode, byte[] ttl, byte[] md5, Xof_ext ext, int file_w, double time, int page) { + return Init(repo.Tid(), Bool_.N, Bool_.N, Byte_ascii.Slash, repo.Root_http() , repo.Mode_names()[mode], repo.Dir_depth(), repo.Gen_name_trg(tmp_bfr, ttl, md5, ext), md5, ext, mode, file_w, time, page); } - private Xof_url_bldr Init(boolean wmf_dir_hive, boolean wmf_protocol_is_file, byte dir_spr + private Xof_url_bldr Init(byte repo_tid, boolean wmf_dir_hive, boolean wmf_protocol_is_file, byte dir_spr , byte[] root, byte[] area, int md5_dir_depth , byte[] ttl, byte[] md5, Xof_ext ext , byte file_mode, int file_w, double time, int page ) { + this.repo_tid = repo_tid; this.wmf_dir_hive = wmf_dir_hive; this.wmf_protocol_is_file = wmf_protocol_is_file; this.dir_spr = dir_spr; this.root = root; this.area = area; this.md5_dir_depth = md5_dir_depth; this.ttl = ttl; this.md5 = md5; this.ext = ext; - this.file_is_thumb = file_mode == Xof_repo_itm_.Mode_thumb; this.file_w = file_w; this.time = time; this.page = page; + this.file_is_thumb = file_mode == Xof_img_mode_.Tid__thumb; this.file_w = file_w; this.time = time; this.page = page; return this; } public byte[] Xto_bry() {Bld(); byte[] rv = tmp_bfr.To_bry_and_clear(); Clear(); return rv;} @@ -69,27 +73,38 @@ public class Xof_url_bldr { public Io_url Xto_url() {Bld(); Io_url rv = Io_url_.new_fil_(tmp_bfr.To_str()); Clear(); return rv;} public Io_url Xto_url_by_http() {Bld(); Io_url rv = Io_url_.New__http_or_fail(tmp_bfr.To_str()); Clear(); return rv;} public Io_url To_url_trg(Xof_repo_itm repo_itm, Xof_fsdb_itm itm, boolean orig) { - byte mode = orig ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb; - return this.Init_for_trg_file(mode, repo_itm, itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext(), itm.File_w(), itm.Lnki_time(), itm.Lnki_page()).Xto_url(); + byte mode = orig ? Xof_img_mode_.Tid__orig : Xof_img_mode_.Tid__thumb; + return this.Init_for_trg_file(repo_itm, mode, itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext(), itm.File_w(), itm.Lnki_time(), itm.Lnki_page()).Xto_url(); } public Io_url To_url_trg(Xof_repo_itm repo_itm, Xof_file_itm itm, boolean orig) { - byte mode = orig ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb; - return this.Init_for_trg_file(mode, repo_itm, itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext(), itm.File_w(), itm.Lnki_time(), itm.Lnki_page()).Xto_url(); + byte mode = orig ? Xof_img_mode_.Tid__orig : Xof_img_mode_.Tid__thumb; + return this.Init_for_trg_file(repo_itm, mode, itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext(), itm.File_w(), itm.Lnki_time(), itm.Lnki_page()).Xto_url(); } public Io_url To_url_trg(Xof_repo_itm repo_itm, gplx.xowa.files.caches.Xou_cache_itm itm, boolean orig) { - byte mode = orig ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb; - return this.Init_for_trg_file(mode, repo_itm, itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext_itm(), itm.File_w(), itm.Lnki_time(), itm.Lnki_page()).Xto_url(); + byte mode = orig ? Xof_img_mode_.Tid__orig : Xof_img_mode_.Tid__thumb; + return this.Init_for_trg_file(repo_itm, mode, itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext_itm(), itm.File_w(), itm.Lnki_time(), itm.Lnki_page()).Xto_url(); } + private static final byte[] Bry__http = Bry_.new_a7("http"); private void Bld() { - Add_core(); - if (file_is_thumb) { - if (wmf_dir_hive) Add_thumb_wmf(); - else Add_thumb_xowa(); + if (repo_tid == Xof_repo_tid_.Tid__math) { + tmp_bfr.Add(root); // add root; EX: "C:\xowa\file\"; assume trailing dir_spr + boolean root_is_http = Bry_.Has_at_bgn(root, Bry__http); + if (root_is_http) + tmp_bfr.Add_mid(ttl, 0, ttl.length - 4); // -4 to remove ".svg". note that XO stores ".svg", but WM doesn't; EX: "596f8baf206a81478afd4194b44138715dc1a05c.svg" + else + tmp_bfr.Add(ttl); + } + else { + Add_core(); + if (file_is_thumb) { + if (wmf_dir_hive) Add_thumb_wmf(); + else Add_thumb_xowa(); + } } } private Xof_url_bldr Add_core() { tmp_bfr.Add(root); // add root; EX: "C:\xowa\file\"; assume trailing dir_spr - if (area != null && !(wmf_dir_hive && !file_is_thumb)) // !(wmf_dir_hive && !thumb) means never add if wmf_dir_hive and orig + if (area != null && !(wmf_dir_hive && !file_is_thumb)) // !(wmf_dir_hive && !thumb) means never add if wmf_dir_hive and orig tmp_bfr.Add(area).Add_byte(dir_spr); // add area; EX: "thumb\" byte b0 = md5[0]; if (wmf_dir_hive) { @@ -101,9 +116,9 @@ public class Xof_url_bldr { tmp_bfr.Add_byte(md5[i]).Add_byte(dir_spr); // add md5_0123 EX: "0/1/2/3" } if (wmf_dir_hive) { - if (wmf_protocol_is_file) // sitting on local file system (as opposed to http) + if (wmf_protocol_is_file) // sitting on local file system (as opposed to http) tmp_bfr.Add(ttl); // NOTE: file_names are not url-encoded; this includes symbols (') and foreign characters (ö) - else // wmf_http + else // wmf_http tmp_bfr.Add(encoder_src_http.Encode(ttl)); // NOTE: file_names must be url-encoded; JAVA will default to native charset which on Windows will be 1252; foreign character urls will fail due to conversion mismatch (1252 on windows; UTF-8 on WMF); PAGE:en.w:Möbius strip } else @@ -156,7 +171,7 @@ public class Xof_url_bldr { break; } int ttl_len = ttl.length; - if (ttl_len > 160) { // long file name + if (ttl_len > 160) { // long file name tmp_bfr.Add(Bry_thumnbail_w_dot); tmp_bfr.Add(ext.Ext()); } @@ -169,7 +184,7 @@ public class Xof_url_bldr { tmp_bfr.Add_byte(Byte_ascii.Dot).Add(Xof_ext_.Bry_png); // add .png; EX: "A.svg" -> "A.svg.png" NOTE: MediaWiki always adds as lowercase break; case Xof_ext_.Id_pdf: - case Xof_ext_.Id_tif: // add .jpg EX: "A.tif" -> "A.tif.jpg" NOTE: MediaWiki always adds as lowercase + case Xof_ext_.Id_tif: // add .jpg EX: "A.tif" -> "A.tif.jpg" NOTE: MediaWiki always adds as lowercase case Xof_ext_.Id_tiff: case Xof_ext_.Id_ogg: case Xof_ext_.Id_ogv: @@ -195,6 +210,7 @@ public class Xof_url_bldr { time = Xof_lnki_time.Null; ext = null; tmp_bfr.Clear(); + repo_tid = Xof_repo_tid_.Tid__null; return this; } public static final byte[] diff --git a/400_xowa/src/gplx/xowa/files/Xof_url_bldr__tst.java b/400_xowa/src/gplx/xowa/files/Xof_url_bldr__tst.java new file mode 100644 index 000000000..8a24ebfc0 --- /dev/null +++ b/400_xowa/src/gplx/xowa/files/Xof_url_bldr__tst.java @@ -0,0 +1,67 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.files; import gplx.*; import gplx.xowa.*; +import org.junit.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.imgs.*; +public class Xof_url_bldr__tst { + private Xof_url_bldr__fxt fxt = new Xof_url_bldr__fxt(); + @Before public void init() {fxt.Clear().Dir_spr_http_();} + @Test public void Ogv() {fxt.Root_("http://test/").Md5_("d0").Ttl_("A.ogv").W_(220).Expd_src_("http://test/thumb/d/d0/A.ogv/220px--A.ogv.jpg").Test();} + @Test public void Ogv__seek() {fxt.Root_("http://test/").Md5_("d0").Ttl_("A.ogv").W_(220).Expd_src_("http://test/thumb/d/d0/A.ogv/220px-seek%3D5-A.ogv.jpg").Seek_(5).Test();} + @Test public void Ogv__no_w() {fxt.Root_("http://test/").Md5_("d0").Ttl_("A.ogv").W_( -1).Expd_src_("http://test/thumb/d/d0/A.ogv/-1px--A.ogv.jpg").Test();} // TODO_OLD: use orig_w, not -1 + @Test public void Xcf() {fxt.Root_("http://test/").Md5_("44").Ttl_("A.xcf").W_(220).Expd_src_("http://test/thumb/4/44/A.xcf/220px-A.xcf.png").Test();} + @Test public void Bmp() {fxt.Root_("http://test/").Md5_("70").Ttl_("A.bmp").W_(220).Expd_src_("http://test/thumb/7/70/A.bmp/220px-A.bmp.png").Test();} + @Test public void Pdf() {fxt.Root_("http://test/").Md5_("ef").Ttl_("A.pdf").W_(220).Expd_src_("http://test/thumb/e/ef/A.pdf/page1-220px-A.pdf.jpg").Test();} + @Test public void Pdf__page_2() {fxt.Root_("http://test/").Md5_("ef").Ttl_("A.pdf").W_(220).Expd_src_("http://test/thumb/e/ef/A.pdf/page2-220px-A.pdf.jpg").Page_(2).Test();} + @Test public void Long() { + String filename = String_.Repeat("A", 200) + ".png"; + fxt.Root_("http://test/").Md5_("14").Ttl_(filename).W_(220) + .Expd_src_("http://test/thumb/1/14/" + filename + "/220px-thumbnail.png") + .Test() + ; + } + @Test public void Math__http() {fxt.Repo_tid_(Xof_repo_tid_.Tid__math).Fsys_is_wnt_(Bool_.N).Root_("http://test/").Ttl_("random_md5.svg").Expd_src_("http://test/random_md5").Test();} // NOTE: strip ".svg" if online + @Test public void Math__file() {fxt.Repo_tid_(Xof_repo_tid_.Tid__math).Fsys_is_wnt_(Bool_.Y).Root_("file://xowa/").Ttl_("random_md5.svg").Expd_src_("file://xowa/random_md5.svg").Test();} // NOTE: keep ".svg" if online +} +class Xof_url_bldr__fxt { + private final Xof_url_bldr url_bldr = new Xof_url_bldr(); + public Xof_url_bldr__fxt Clear() { + dir_spr = Byte_.Zero; ext = null; root = md5 = ttl = expd_src = null; + seek = Xof_lnki_time.Null; + page = Xof_lnki_page.Null; + w = Xof_img_size.Null; + return this; + } + public Xof_url_bldr__fxt Dir_spr_http_() {return Dir_spr_(Byte_ascii.Slash);} + public Xof_url_bldr__fxt Dir_spr_fsys_wnt_() {return Dir_spr_(Byte_ascii.Backslash);} + public Xof_url_bldr__fxt Dir_spr_(byte v) {dir_spr = v; return this;} private byte dir_spr; + public Xof_url_bldr__fxt Root_(String v) {root = v; return this;} private String root; + public Xof_url_bldr__fxt Md5_(String v) {md5 = v; return this;} private String md5; + public Xof_url_bldr__fxt Ttl_(String v) {ttl = v; ext = Xof_ext_.new_by_ttl_(Bry_.new_u8(v)); return this;} private String ttl; Xof_ext ext; + public Xof_url_bldr__fxt W_(int v) {this.w = v; return this;} private int w; + public Xof_url_bldr__fxt Page_(int v) {page = v; return this;} private int page = Xof_lnki_page.Null; + public Xof_url_bldr__fxt Seek_(int v) {seek = v; return this;} private double seek = Xof_lnki_time.Null; + public Xof_url_bldr__fxt Repo_tid_(byte v) {repo_tid = v; return this;} private byte repo_tid = Xof_repo_tid_.Tid__null; + public Xof_url_bldr__fxt Fsys_is_wnt_(boolean v) {fsys_is_wnt = v; return this;} private boolean fsys_is_wnt; + public Xof_url_bldr__fxt Expd_src_(String v) {expd_src = v; return this;} private String expd_src; + public Xof_url_bldr__fxt Test() { + url_bldr.Init_by_repo(repo_tid, Bry_.new_u8(root), fsys_is_wnt, dir_spr, Bool_.Y, Bool_.N, 2); + url_bldr.Init_by_itm (Xof_img_mode_.Tid__thumb, Bry_.new_u8(ttl), Bry_.new_u8_safe(md5), ext, w, seek, page); + Tfds.Eq(expd_src, url_bldr.Xto_str()); + return this; + } +} diff --git a/400_xowa/src/gplx/xowa/files/Xof_url_bldr_tst.java b/400_xowa/src/gplx/xowa/files/Xof_url_bldr_tst.java deleted file mode 100644 index 21c9f3400..000000000 --- a/400_xowa/src/gplx/xowa/files/Xof_url_bldr_tst.java +++ /dev/null @@ -1,63 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.files; import gplx.*; import gplx.xowa.*; -import org.junit.*; import gplx.xowa.files.repos.*; -public class Xof_url_bldr_tst { - private Xof_url_bldr_fxt fxt = new Xof_url_bldr_fxt(); - @Before public void init() {fxt.Clear();} - @Test public void Ogv() {fxt.Dir_spr_http_().Root_("http://test/").Md5_("d0").Ttl_("A.ogv").W_(220).Expd_src_("http://test/thumb/d/d0/A.ogv/220px--A.ogv.jpg").Test();} - @Test public void Ogv__seek() {fxt.Dir_spr_http_().Root_("http://test/").Md5_("d0").Ttl_("A.ogv").W_(220).Expd_src_("http://test/thumb/d/d0/A.ogv/220px-seek%3D5-A.ogv.jpg").Seek_(5).Test();} - @Test public void Ogv__no_w() {fxt.Dir_spr_http_().Root_("http://test/").Md5_("d0").Ttl_("A.ogv").W_( -1).Expd_src_("http://test/thumb/d/d0/A.ogv/-1px--A.ogv.jpg").Test();} // TODO_OLD: use orig_w, not -1 - @Test public void Xcf() {fxt.Dir_spr_http_().Root_("http://test/").Md5_("44").Ttl_("A.xcf").W_(220).Expd_src_("http://test/thumb/4/44/A.xcf/220px-A.xcf.png").Test();} - @Test public void Bmp() {fxt.Dir_spr_http_().Root_("http://test/").Md5_("70").Ttl_("A.bmp").W_(220).Expd_src_("http://test/thumb/7/70/A.bmp/220px-A.bmp.png").Test();} - @Test public void Pdf() {fxt.Dir_spr_http_().Root_("http://test/").Md5_("ef").Ttl_("A.pdf").W_(220).Expd_src_("http://test/thumb/e/ef/A.pdf/page1-220px-A.pdf.jpg").Test();} - @Test public void Pdf__page_2() {fxt.Dir_spr_http_().Root_("http://test/").Md5_("ef").Ttl_("A.pdf").W_(220).Expd_src_("http://test/thumb/e/ef/A.pdf/page2-220px-A.pdf.jpg").Page_(2).Test();} - @Test public void Long() { - String filename = String_.Repeat("A", 200) + ".png"; - fxt.Dir_spr_http_().Root_("http://test/").Md5_("14").Ttl_(filename).W_(220) - .Expd_src_("http://test/thumb/1/14/" + filename + "/220px-thumbnail.png") - .Test() - ; - } -} -class Xof_url_bldr_fxt { - private final Xof_url_bldr url_bldr = new Xof_url_bldr(); - public Xof_url_bldr_fxt Clear() { - dir_spr = Byte_.Zero; ext = null; root = md5 = ttl = expd_src = null; - seek = Xof_lnki_time.Null; - page = Xof_lnki_page.Null; - w = Xof_img_size.Null; - return this; - } - public Xof_url_bldr_fxt Dir_spr_http_() {return Dir_spr_(Byte_ascii.Slash);} - public Xof_url_bldr_fxt Dir_spr_fsys_wnt_() {return Dir_spr_(Byte_ascii.Backslash);} - public Xof_url_bldr_fxt Dir_spr_(byte v) {dir_spr = v; return this;} private byte dir_spr; - public Xof_url_bldr_fxt Root_(String v) {root = v; return this;} private String root; - public Xof_url_bldr_fxt Md5_(String v) {md5 = v; return this;} private String md5; - public Xof_url_bldr_fxt Ttl_(String v) {ttl = v; ext = Xof_ext_.new_by_ttl_(Bry_.new_u8(v)); return this;} private String ttl; Xof_ext ext; - public Xof_url_bldr_fxt W_(int v) {this.w = v; return this;} private int w; - public Xof_url_bldr_fxt Page_(int v) {page = v; return this;} private int page = Xof_lnki_page.Null; - public Xof_url_bldr_fxt Seek_(int v) {seek = v; return this;} private double seek = Xof_lnki_time.Null; - public Xof_url_bldr_fxt Expd_src_(String v) {expd_src = v; return this;} private String expd_src; - public Xof_url_bldr_fxt Test() { - url_bldr.Init_by_root(Bry_.new_u8(root), Bool_.N, dir_spr, Bool_.Y, Bool_.N, 2); - url_bldr.Init_by_itm (Xof_repo_itm_.Mode_thumb, Bry_.new_u8(ttl), Bry_.new_u8(md5), ext, w, seek, page); - Tfds.Eq(expd_src, url_bldr.Xto_str()); - return this; - } -} diff --git a/400_xowa/src/gplx/xowa/files/Xof_xfer_itm.java b/400_xowa/src/gplx/xowa/files/Xof_xfer_itm.java index 3e0196825..92bdba95f 100644 --- a/400_xowa/src/gplx/xowa/files/Xof_xfer_itm.java +++ b/400_xowa/src/gplx/xowa/files/Xof_xfer_itm.java @@ -17,7 +17,7 @@ along with this program. If not, see . */ package gplx.xowa.files; import gplx.*; import gplx.xowa.*; import gplx.core.primitives.*; -import gplx.xowa.guis.cbks.js.*; import gplx.xowa.files.repos.*; +import gplx.xowa.guis.cbks.js.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.imgs.*; import gplx.xowa.wikis.tdbs.metas.*; import gplx.xowa.parsers.utils.*; public class Xof_xfer_itm implements Xof_file_itm { @@ -181,8 +181,8 @@ public class Xof_xfer_itm implements Xof_file_itm { file_is_orig = img_size.File_is_orig(); } } - private Io_url Trg_view_url(byte mode_id, int width) {return tmp_url_bldr.Init_for_trg_file(mode_id, trg_repo_itm, lnki_ttl, orig_ttl_md5, orig_ext, width, lnki_time, lnki_page).Xto_url();} - public Io_url Trg_orig_url(byte mode_id, int width) {return tmp_url_bldr.Init_for_trg_file(mode_id, trg_repo_itm, lnki_ttl, orig_ttl_md5, orig_ext, width, lnki_time, lnki_page).Xto_url();} + private Io_url Trg_view_url(byte mode_id, int width) {return tmp_url_bldr.Init_for_trg_file(trg_repo_itm, mode_id, lnki_ttl, orig_ttl_md5, orig_ext, width, lnki_time, lnki_page).Xto_url();} + public Io_url Trg_orig_url(byte mode_id, int width) {return tmp_url_bldr.Init_for_trg_file(trg_repo_itm, mode_id, lnki_ttl, orig_ttl_md5, orig_ext, width, lnki_time, lnki_page).Xto_url();} public boolean Calc_by_meta() {return Calc_by_meta(false);} public boolean Calc_by_meta(boolean caller_is_file_page) { file_exists = false; @@ -198,19 +198,19 @@ public class Xof_xfer_itm implements Xof_file_itm { html_w = Xof_img_size.Thumb_width_ogv; if (!file_is_orig) { // file is thumb if (orig_ext.Id_is_video()) { // video is a special case; src is thumb_w but html_w / html_h is based on calc - html_orig_url = Trg_view_url(Xof_repo_itm_.Mode_orig, Xof_img_size.Size__neg1); + html_orig_url = Trg_view_url(Xof_img_mode_.Tid__orig, Xof_img_size.Size__neg1); if (meta_itm.Thumbs_indicates_oga() && orig_ext.Id_is_ogv()) {orig_ext = Xof_ext_.new_by_ext_(Xof_ext_.Bry_oga); return true;} // if audio, do not thumb; NOTE: must happen after html_orig_bry, b/c html must still be generated to auto-download files; NOTE: must change ext to oga b/c ogg may trigger video code elsewhere Xof_meta_thumb thumb = meta_itm.Thumbs_get_vid(Xof_lnki_time.X_int(lnki_time)); if (thumb != null) { Xof_xfer_itm_.Calc_xfer_size(calc_size, lnki_type, thumb_w_img, thumb.Width(), thumb.Height(), html_w, html_h, !file_is_orig, lnki_upright); html_w = calc_size.Val_0(); html_h = calc_size.Val_1(); - html_view_url = Trg_view_url(Xof_repo_itm_.Mode_thumb, thumb.Width()); // NOTE: must pass thumb.Width() not html_w b/c only one thumb generated for a video file + html_view_url = Trg_view_url(Xof_img_mode_.Tid__thumb, thumb.Width()); // NOTE: must pass thumb.Width() not html_w b/c only one thumb generated for a video file file_exists = true; return true; } } else { // regular thumb - html_orig_url = Trg_view_url(Xof_repo_itm_.Mode_orig, Xof_img_size.Size__neg1); + html_orig_url = Trg_view_url(Xof_img_mode_.Tid__orig, Xof_img_size.Size__neg1); if (orig_ext.Id_is_audio()) return true; // if audio, do not thumb; even if user requests thumb; Xof_meta_thumb[] thumbs = meta_itm.Thumbs(); int thumbs_len = thumbs.length; Xof_meta_thumb thumb = null; if (lnki_h > 0 && orig_w < 1 && thumbs_len > 0) { // if height is specified and no orig, then iterate over thumbs to find similar height; NOTE: this is a fallback case; orig_w/h is optimal; EX: c:Jacques-Louis David and @@ -230,7 +230,7 @@ public class Xof_xfer_itm implements Xof_file_itm { Xof_xfer_itm_.Calc_xfer_size(calc_size, lnki_type, thumb_w_img, meta_itm.Orig_w(), meta_itm.Orig_h(), html_w, html_h, !file_is_orig, lnki_upright, limit_size); // calc html_h and html_w; can differ from lnki_w, lnki_h; note that -1 width is handled by thumb_w_img html_w = calc_size.Val_0(); if (html_h != -1) html_h = calc_size.Val_1(); // NOTE: if -1 (no height specified) do not set height; EX:Tokage_2011-07-15.jpg; DATE:2013-06-03 - html_view_url = Trg_view_url(Xof_repo_itm_.Mode_thumb, this.File_w()); + html_view_url = Trg_view_url(Xof_img_mode_.Tid__thumb, this.File_w()); thumb = meta_itm.Thumbs_get_img(html_w, 0); if (thumb == null) { // exact thumb not found if (html_w == meta_itm.Orig_w() // html_w matches orig_w; occurs when thumb,upright requested, but upright size is larger than orig; PAGE:en.w:St. Petersburg @@ -238,7 +238,7 @@ public class Xof_xfer_itm implements Xof_file_itm { && meta_itm.Orig_exists() == Xof_meta_itm.Exists_y ) { html_h = meta_itm.Orig_h(); - html_view_url = Trg_view_url(Xof_repo_itm_.Mode_orig, -1); + html_view_url = Trg_view_url(Xof_img_mode_.Tid__orig, -1); file_exists = true; return true; } @@ -255,7 +255,7 @@ public class Xof_xfer_itm implements Xof_file_itm { } } else { // file is orig - byte mode_id = orig_ext.Id_is_svg() ? Xof_repo_itm_.Mode_thumb : Xof_repo_itm_.Mode_orig; // svgs will always return thumb; EX:[[A.svg]] -> A.svg.png + byte mode_id = orig_ext.Id_is_svg() ? Xof_img_mode_.Tid__thumb : Xof_img_mode_.Tid__orig; // svgs will always return thumb; EX:[[A.svg]] -> A.svg.png html_view_url = html_orig_url = Trg_view_url(mode_id, this.File_w()); if (meta_itm.Thumbs_indicates_oga() && orig_ext.Id_is_ogv()) {orig_ext = Xof_ext_.new_by_ext_(Xof_ext_.Bry_oga); return true;} // if audio, do not thumb; NOTE: must happen after html_orig_bry, b/c html must still be generated to auto-download files; NOTE: must change ext to oga b/c ogg may trigger video code elsewhere if (orig_ext.Id_is_audio()) return true; // if audio, return true; SEE:NOTE_2 @@ -263,7 +263,7 @@ public class Xof_xfer_itm implements Xof_file_itm { Xof_meta_thumb thumb = meta_itm.Thumbs_get_vid(Xof_lnki_time.X_int(lnki_time)); // get thumb at lnki_time; NOTE: in most cases this will just be the 1st thumb; note that orig video files don't have an official thumb if (thumb != null) { html_w = thumb.Width(); html_h = thumb.Height(); // NOTE: take thumb_size; do not rescale to html_w, html_h b/c html_w will default to 220; native width of thumbnail should be used; DATE:2013-04-11 - html_view_url = Trg_view_url(Xof_repo_itm_.Mode_thumb, thumb.Width()); // NOTE: must pass thumb.Width() not html_w b/c only one thumb generated for a video file + html_view_url = Trg_view_url(Xof_img_mode_.Tid__thumb, thumb.Width()); // NOTE: must pass thumb.Width() not html_w b/c only one thumb generated for a video file file_exists = true; return true; } @@ -283,7 +283,7 @@ public class Xof_xfer_itm implements Xof_file_itm { private boolean Calc_by_meta_found(byte lnki_type, int model_w, int model_h) { Xof_xfer_itm_.Calc_xfer_size(calc_size, lnki_type, thumb_w_img, model_w, model_h, html_w, html_h, !file_is_orig, lnki_upright, false); // recalc html_w, html_h; note that false passed b/c truncation is not needed html_w = calc_size.Val_0(); html_h = calc_size.Val_1(); - html_view_url = Trg_view_url(Xof_repo_itm_.Mode_thumb, model_w); // note that thumb.Width is used (the actual file width), not html_w + html_view_url = Trg_view_url(Xof_img_mode_.Tid__thumb, model_w); // note that thumb.Width is used (the actual file width), not html_w file_exists = true; return true; } diff --git a/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__fsys_base.java b/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__fsys_base.java index e2e194695..c1298865c 100644 --- a/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__fsys_base.java +++ b/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__fsys_base.java @@ -17,21 +17,21 @@ along with this program. If not, see . */ package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; import gplx.core.ios.*; import gplx.core.ios.streams.*; import gplx.core.envs.*; -import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*; +import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.imgs.*; public abstract class Xof_bin_wkr__fsys_base implements Xof_bin_wkr, Gfo_invk { public Xof_bin_wkr__fsys_base() {} public abstract byte Tid(); public abstract String Key(); public boolean Resize_allowed() {return resize_allowed;} public void Resize_allowed_(boolean v) {resize_allowed = v;} private boolean resize_allowed = false; public Io_stream_rdr Get_as_rdr(Xof_fsdb_itm itm, boolean is_thumb, int w) { - Io_url src_url = this.Get_src_url(Xof_repo_itm_.Mode_by_bool(is_thumb), String_.new_u8(itm.Orig_repo_name()), itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext(), w, itm.Lnki_time(), itm.Lnki_page()); + Io_url src_url = this.Get_src_url(Xof_img_mode_.By_bool(is_thumb), String_.new_u8(itm.Orig_repo_name()), itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext(), w, itm.Lnki_time(), itm.Lnki_page()); return (src_url == Io_url_.Empty) ? gplx.core.ios.streams.Io_stream_rdr_.Noop : gplx.core.ios.streams.Io_stream_rdr_.file_(src_url); } public boolean Get_to_fsys(Xof_fsdb_itm itm, boolean is_thumb, int w, Io_url bin_url) { return Get_to_fsys(itm.Orig_repo_name(), itm.Orig_ttl(), itm.Orig_ttl_md5(), itm.Orig_ext(), is_thumb, w, itm.Lnki_time(), itm.Lnki_page(), bin_url); } private boolean Get_to_fsys(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) { - Io_url src_url = this.Get_src_url(Xof_repo_itm_.Mode_by_bool(lnki_is_thumb), String_.new_u8(orig_repo), orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page); + Io_url src_url = this.Get_src_url(Xof_img_mode_.By_bool(lnki_is_thumb), String_.new_u8(orig_repo), orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page); if (src_url == Io_url_.Empty) return false; byte[] bin = Io_mgr.Instance.LoadFilBry(src_url); return bin != Io_mgr.LoadFilBry_fail; @@ -56,13 +56,15 @@ class Xof_bin_wkr__fsys_wmf extends Xof_bin_wkr__fsys_wmf_base { @Override public byte Tid() {return Xof_bin_wkr_.Tid_fsys_wmf;} @Override public String Key() {return Xof_bin_wkr_.Key_fsys_wmf;} @Override public void Init_by_root() { - this.Url_bldr().Init_by_root(Bry_.Empty, Op_sys.Cur().Tid_is_wnt(), Op_sys.Cur().Fsys_dir_spr_byte(), Bool_.Y, Bool_.Y, Xof_repo_itm_.Dir_depth_wmf); + this.Url_bldr().Init_by_repo(Xof_repo_tid_.Tid__null, Bry_.Empty, Op_sys.Cur().Tid_is_wnt(), Op_sys.Cur().Fsys_dir_spr_byte(), Bool_.Y, Bool_.Y, Md5_dir_depth__wmf); } + private static final int Md5_dir_depth__wmf = 2; } class Xof_bin_wkr__fsys_xowa extends Xof_bin_wkr__fsys_wmf_base { @Override public byte Tid() {return Xof_bin_wkr_.Tid_fsys_xowa;} @Override public String Key() {return Xof_bin_wkr_.Key_fsys_xowa;} @Override public void Init_by_root() { - this.Url_bldr().Init_by_root(Bry_.Empty, Op_sys.Cur().Tid_is_wnt(), Op_sys.Cur().Fsys_dir_spr_byte(), Bool_.N, Bool_.N, Xof_repo_itm_.Dir_depth_xowa); + this.Url_bldr().Init_by_repo(Xof_repo_tid_.Tid__null, Bry_.Empty, Op_sys.Cur().Tid_is_wnt(), Op_sys.Cur().Fsys_dir_spr_byte(), Bool_.N, Bool_.N, Md5_dir_depth__xowa); } + private static final int Md5_dir_depth__xowa = 4; } diff --git a/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf.java b/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf.java index 534565f44..3f982adca 100644 --- a/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf.java +++ b/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf.java @@ -18,7 +18,7 @@ along with this program. If not, see . package gplx.xowa.files.bins; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; import gplx.core.ios.*; import gplx.core.ios.streams.*; import gplx.core.threads.*; import gplx.xowa.apps.*; -import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*; +import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.imgs.*; import gplx.xowa.wikis.domains.*; public class Xof_bin_wkr__http_wmf implements Xof_bin_wkr { private final Xow_repo_mgr repo_mgr; private final IoEngine_xrg_downloadFil download_wkr; @@ -37,12 +37,12 @@ public class Xof_bin_wkr__http_wmf implements Xof_bin_wkr { boolean rv = rdr.Exists(); // NOTE: use Exists which detects for response_code 200, not content length > 0; DATE:2015-05-20 if (!rv) { Handle_error(); - if (!rv && fsdb.Orig_repo_id() == Xof_repo_itm_.Repo_local) { // image is not found in local; check commons; occurs with bldr which relies on inaccurate data in image dumps; PAGE:en.w:Apollo_13; DATE:2015-08-05 + if (!rv && fsdb.Orig_repo_id() == Xof_repo_tid_.Tid__local) { // image is not found in local; check commons; occurs with bldr which relies on inaccurate data in image dumps; PAGE:en.w:Apollo_13; DATE:2015-08-05 src = Make_src(Xow_domain_itm_.Bry__commons, fsdb.Orig_ttl(), fsdb.Orig_ttl_md5(), fsdb.Orig_ext(), is_thumb, w, fsdb.Lnki_time(), fsdb.Lnki_page(), Io_url_.Empty); rdr = download_mgr.Download_as_rdr(src); rv = rdr.Exists(); if (rv) - fsdb.Change_repo(Xof_repo_itm_.Repo_remote, Xow_domain_itm_.Bry__commons); // set commons.wikimedia.org; DATE:2015-08-05 + fsdb.Change_repo(Xof_repo_tid_.Tid__remote, Xow_domain_itm_.Bry__commons); // set commons.wikimedia.org; DATE:2015-08-05 else Handle_error(); } @@ -66,15 +66,15 @@ public class Xof_bin_wkr__http_wmf implements Xof_bin_wkr { Thread_adp_.Sleep(fail_timeout); // as per WMF policy, pause 1 second for every cache miss; http://lists.wikimedia.org/pipermail/wikitech-l/2013-September/071948.html } private void Init_download(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) { - byte mode = lnki_is_thumb ? Xof_repo_itm_.Mode_thumb : Xof_repo_itm_.Mode_orig; + byte mode = lnki_is_thumb ? Xof_img_mode_.Tid__thumb : Xof_img_mode_.Tid__orig; Xof_repo_pair repo_itm = repo_mgr.Repos_get_by_wiki(orig_repo); - String src = url_bldr.Init_for_src_file(mode, repo_itm.Src(), orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page).Xto_str(); + String src = url_bldr.Init_for_src_file(repo_itm.Src(), mode, orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page).Xto_str(); download_wkr.Init(src, file_url); } private String Make_src(byte[] orig_repo, byte[] orig_ttl, byte[] orig_md5, Xof_ext orig_ext, boolean lnki_is_thumb, int file_w, double lnki_time, int lnki_page, Io_url file_url) { - byte mode = lnki_is_thumb ? Xof_repo_itm_.Mode_thumb : Xof_repo_itm_.Mode_orig; + byte mode = lnki_is_thumb ? Xof_img_mode_.Tid__thumb : Xof_img_mode_.Tid__orig; Xof_repo_pair repo_itm = repo_mgr.Repos_get_by_wiki(orig_repo); - return url_bldr.Init_for_src_file(mode, repo_itm.Src(), orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page).Xto_str(); + return url_bldr.Init_for_src_file(repo_itm.Src(), mode, orig_ttl, orig_md5, orig_ext, file_w, lnki_time, lnki_page).Xto_str(); } public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { if (ctx.Match(k, Invk_fail_timeout_)) fail_timeout = m.ReadInt("v"); diff --git a/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf__tst.java b/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf__tst.java index 1ff6238dc..6d5b216fd 100644 --- a/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf__tst.java +++ b/400_xowa/src/gplx/xowa/files/bins/Xof_bin_wkr__http_wmf__tst.java @@ -31,7 +31,7 @@ public class Xof_bin_wkr__http_wmf__tst { fxt.Exec__Get_as_rdr(fsdb_itm, Bool_.Y, 220); // look in enwiki fxt.Test__Get_as_rdr__rdr("test_data"); // test that enwiki tries commons again Tfds.Eq_str("commons.wikimedia.org", fsdb_itm.Orig_repo_name(), "repo_name"); // test that it's now commons - Tfds.Eq_byte(Xof_repo_itm_.Repo_remote, fsdb_itm.Orig_repo_id(), "repo_tid"); // test that it's now commons + Tfds.Eq_byte(Xof_repo_tid_.Tid__remote, fsdb_itm.Orig_repo_id(), "repo_tid"); // test that it's now commons } @Test public void Long_filename_becomes_thumbnail() { String filename = String_.Repeat("A", 200) + ".png"; diff --git a/400_xowa/src/gplx/xowa/files/caches/Xofc_fil_mgr.java b/400_xowa/src/gplx/xowa/files/caches/Xofc_fil_mgr.java index 9ac5794c7..7e092e892 100644 --- a/400_xowa/src/gplx/xowa/files/caches/Xofc_fil_mgr.java +++ b/400_xowa/src/gplx/xowa/files/caches/Xofc_fil_mgr.java @@ -17,7 +17,8 @@ along with this program. If not, see . */ package gplx.xowa.files.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; import gplx.core.primitives.*; -import gplx.dbs.*; import gplx.fsdb.*; import gplx.xowa.wikis.*; import gplx.xowa.files.repos.*; +import gplx.dbs.*; import gplx.fsdb.*; import gplx.xowa.wikis.*; +import gplx.xowa.files.repos.*; import gplx.xowa.files.imgs.*; class Xofc_fil_mgr { private Xof_cache_mgr cache_mgr; private final Xofc_fil_tbl tbl = new Xofc_fil_tbl(); private final Ordered_hash hash = Ordered_hash_.New_bry(); private final Bry_bfr key_bldr = Bry_bfr_.Reset(255); @@ -118,7 +119,7 @@ class Xofc_fil_mgr { finally {tbl.Conn().Txn_end();} } private void Fsys_delete(Xof_url_bldr url_bldr, Xoae_wiki_mgr wiki_mgr, Xoa_repo_mgr repo_mgr, Xofc_dir_mgr dir_mgr, Xofc_fil_itm itm) { - byte mode_id = itm.Is_orig() ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb; + byte mode_id = itm.Is_orig() ? Xof_img_mode_.Tid__orig : Xof_img_mode_.Tid__thumb; byte[] wiki_domain = dir_mgr.Get_by_id(itm.Dir_id()).Name(); Xowe_wiki wiki = wiki_mgr.Get_by_or_make(wiki_domain); wiki.Init_assert(); @@ -126,7 +127,7 @@ class Xofc_fil_mgr { byte[] ttl = itm.Name(); byte[] md5 = Xof_file_wkr_.Md5(ttl); int itm_ext_id = itm.Ext().Id(); - Io_url fil_url = url_bldr.Init_for_trg_file(mode_id, trg_repo, ttl, md5, itm.Ext(), itm.W() + Io_url fil_url = url_bldr.Init_for_trg_file(trg_repo, mode_id, ttl, md5, itm.Ext(), itm.W() , Xof_lnki_time.Convert_to_xowa_thumbtime (itm_ext_id, itm.Time()) , Xof_lnki_time.Convert_to_xowa_page (itm_ext_id, itm.Time()) ).Xto_url(); diff --git a/400_xowa/src/gplx/xowa/files/caches/Xou_cache_mgr.java b/400_xowa/src/gplx/xowa/files/caches/Xou_cache_mgr.java index 41c7da870..5406ed044 100644 --- a/400_xowa/src/gplx/xowa/files/caches/Xou_cache_mgr.java +++ b/400_xowa/src/gplx/xowa/files/caches/Xou_cache_mgr.java @@ -17,7 +17,7 @@ along with this program. If not, see . */ package gplx.xowa.files.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; import gplx.core.primitives.*; import gplx.dbs.*; import gplx.dbs.cfgs.*; -import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*; +import gplx.xowa.files.fsdb.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.imgs.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.domains.*; import gplx.xowa.users.data.*; public class Xou_cache_mgr { private final Xoa_wiki_mgr wiki_mgr; private final Xou_cache_tbl cache_tbl; private final Db_cfg_tbl cfg_tbl; private final Bry_bfr tmp_bfr = Bry_bfr_.Reset(512); @@ -169,8 +169,8 @@ public class Xou_cache_mgr { byte[] orig_md5 = cache.Orig_ttl_md5(); Xof_ext orig_ext = cache.Orig_ext_itm(); orig_ttl = trg_repo.Gen_name_trg(tmp_bfr, orig_ttl, orig_md5, orig_ext); - byte mode_id = cache.File_is_orig() ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb; - return url_bldr.Init_for_trg_file(mode_id, trg_repo, orig_ttl, orig_md5, orig_ext, cache.File_w() + byte mode_id = cache.File_is_orig() ? Xof_img_mode_.Tid__orig : Xof_img_mode_.Tid__thumb; + return url_bldr.Init_for_trg_file(trg_repo, mode_id, orig_ttl, orig_md5, orig_ext, cache.File_w() , Xof_lnki_time.Convert_to_xowa_thumbtime (orig_ext.Id(), cache.File_time()) , Xof_lnki_time.Convert_to_xowa_page (orig_ext.Id(), cache.File_page()) ).Xto_url(); diff --git a/400_xowa/src/gplx/xowa/files/caches/Xou_cache_mgr_tst.java b/400_xowa/src/gplx/xowa/files/caches/Xou_cache_mgr_tst.java index 99f6ff0df..20f69c742 100644 --- a/400_xowa/src/gplx/xowa/files/caches/Xou_cache_mgr_tst.java +++ b/400_xowa/src/gplx/xowa/files/caches/Xou_cache_mgr_tst.java @@ -78,7 +78,7 @@ class Xou_cache_mgr_fxt { byte[] wiki_domain = Bry_.new_a7(wiki); byte[] file_ttl = Bry_.new_a7(file); rv.Init_at_lnki(Xof_exec_tid.Tid_wiki_page, wiki_domain, file_ttl, Byte_.Zero, 1, 1, 1 ,1, 1, 1); - rv.Init_at_orig(Xof_repo_itm_.Repo_local, wiki_domain, file_ttl, Xof_ext_.new_by_id_(Xof_ext_.Id_png), 120, 120, Bry_.Empty); + rv.Init_at_orig(Xof_repo_tid_.Tid__local, wiki_domain, file_ttl, Xof_ext_.new_by_id_(Xof_ext_.Id_png), 120, 120, Bry_.Empty); rv.File_size_(size); return rv; } diff --git a/400_xowa/src/gplx/xowa/files/fsdb/tsts/Xof_file_fxt.java b/400_xowa/src/gplx/xowa/files/fsdb/tsts/Xof_file_fxt.java index 166870a31..74eaf8810 100644 --- a/400_xowa/src/gplx/xowa/files/fsdb/tsts/Xof_file_fxt.java +++ b/400_xowa/src/gplx/xowa/files/fsdb/tsts/Xof_file_fxt.java @@ -24,7 +24,7 @@ import gplx.xowa.wikis.nss.*; import gplx.xowa.parsers.lnkis.*; class Xof_file_fxt { private Xoae_app app; private Xof_fsdb_mgr__sql fsdb_mgr; private Xowe_wiki wiki; private Xof_orig_mgr orig_mgr; - private final Fsd_thm_itm tmp_thm = Fsd_thm_itm.new_(); private final Fsd_img_itm tmp_img = new Fsd_img_itm(); + private final Fsd_thm_itm tmp_thm = Fsd_thm_itm.new_(); public void Rls() {} public void Reset() { Io_mgr.Instance.InitEngine_mem(); // NOTE: files are downloaded to mem_engine, regardless of Db being mem or sqlite; always reset @@ -61,7 +61,7 @@ class Xof_file_fxt { if (arg.Is_thumb()) mnt_itm.Insert_thm(tmp_thm, atr_fil, bin_fil, arg.Wiki(), arg.Ttl(), arg.Ext(), arg.W(), arg.H(), arg.Time(), arg.Page(), arg.Bin().length, gplx.core.ios.streams.Io_stream_rdr_.mem_(arg.Bin())); else - mnt_itm.Insert_img(tmp_img, atr_fil, bin_fil, arg.Wiki(), arg.Ttl(), arg.Ext(), arg.W(), arg.H(), arg.Bin().length, gplx.core.ios.streams.Io_stream_rdr_.mem_(arg.Bin())); + mnt_itm.Insert_img(atr_fil, bin_fil, arg.Wiki(), arg.Ttl(), arg.Ext(), arg.W(), arg.H(), arg.Bin().length, gplx.core.ios.streams.Io_stream_rdr_.mem_(arg.Bin())); } public void Exec_get(Xof_exec_arg arg) { byte[] ttl_bry = arg.Ttl(); @@ -119,7 +119,7 @@ class Xof_orig_arg { public static Xof_orig_arg new_comm_redirect(String src, String trg) {return new_(Bool_.Y, src, 440, 400, trg);} private static Xof_orig_arg new_(boolean repo_is_commons, String page, int w, int h) {return new_(repo_is_commons, page, w, h, null);} public static Xof_orig_arg new_(boolean repo_is_commons, String page, int w, int h, String redirect_str) { - byte repo = repo_is_commons ? Xof_repo_itm_.Repo_remote : Xof_repo_itm_.Repo_local; + byte repo = repo_is_commons ? Xof_repo_tid_.Tid__remote : Xof_repo_tid_.Tid__local; byte[] redirect = redirect_str == null ? Bry_.Empty : Bry_.new_u8(redirect_str); return new Xof_orig_arg(repo, Bry_.new_u8(page), w, h, redirect); } diff --git a/400_xowa/src/gplx/xowa/files/imgs/Xof_img_mode_.java b/400_xowa/src/gplx/xowa/files/imgs/Xof_img_mode_.java new file mode 100644 index 000000000..d3baa5b1e --- /dev/null +++ b/400_xowa/src/gplx/xowa/files/imgs/Xof_img_mode_.java @@ -0,0 +1,26 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.files.imgs; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; +public class Xof_img_mode_ { + public static final byte + Tid__orig = 0 + , Tid__thumb = 1 + ; + public static byte By_bool(boolean is_thumb) {return is_thumb ? Tid__thumb : Tid__orig;} + public static final byte[][] Names_ary = new byte[][] {Bry_.new_a7("orig"), Bry_.new_a7("thumb")}; +} diff --git a/400_xowa/src/gplx/xowa/files/repos/Xof_itm_ttl_.java b/400_xowa/src/gplx/xowa/files/repos/Xof_itm_ttl_.java new file mode 100644 index 000000000..cf2bf5489 --- /dev/null +++ b/400_xowa/src/gplx/xowa/files/repos/Xof_itm_ttl_.java @@ -0,0 +1,42 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; +public class Xof_itm_ttl_ { + public static byte[] Remove_invalid(Bry_bfr bfr, byte[] ttl) { + int ttl_len = ttl.length; + for (int i = 0; i < ttl_len; i++) { + byte b = ttl[i]; + if (gplx.core.envs.Op_sys_.Wnt_invalid_char(b)) b = Byte_ascii.Underline; + bfr.Add_byte(b); + } + return bfr.To_bry_and_clear(); + } + public static byte[] Shorten(Bry_bfr bfr, byte[] ttl, int ttl_max, byte[] md5, byte[] ext_bry) { + byte[] rv = ttl; + int exceed_len = rv.length - ttl_max; + if (exceed_len > 0) { + bfr.Add_mid(rv, 0, ttl_max - 33); // add truncated title; 33=_.length + md5.length + bfr.Add_byte(Byte_ascii.Underline); // add underline; EX: "_" + bfr.Add(md5); // add md5; EX: "abcdefghijklmnopqrstuvwxyz0123456" + bfr.Add_byte(Byte_ascii.Dot); // add dot; EX: "." + bfr.Add(ext_bry); // add ext; EX: ".png" + rv = bfr.To_bry_and_clear(); + } + return rv; + } +} diff --git a/400_xowa/src/gplx/xowa/files/repos/Xof_repo_itm.java b/400_xowa/src/gplx/xowa/files/repos/Xof_repo_itm.java index 4f8831897..6831079a4 100644 --- a/400_xowa/src/gplx/xowa/files/repos/Xof_repo_itm.java +++ b/400_xowa/src/gplx/xowa/files/repos/Xof_repo_itm.java @@ -16,8 +16,8 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; -import gplx.core.btries.*; import gplx.core.consoles.*; -import gplx.xowa.apps.fsys.*; import gplx.xowa.files.exts.*; +import gplx.core.consoles.*; +import gplx.xowa.apps.fsys.*; import gplx.xowa.files.exts.*; import gplx.xowa.files.imgs.*; import gplx.xowa.wikis.domains.*; public class Xof_repo_itm implements Gfo_invk { private final Xoa_fsys_mgr app_fsys; private final Xof_rule_mgr ext_rule_mgr; @@ -25,23 +25,33 @@ public class Xof_repo_itm implements Gfo_invk { this.key = key; this.app_fsys = app_fsys; this.ext_rule_mgr = ext_rule_mgr; Wiki_domain_(wiki_domain); } - public byte[] Key() {return key;} private final byte[] key; - public byte[] Wiki_domain() {return wiki_domain;} private byte[] wiki_domain; - public byte[] Wiki_abrv_xo() {return wiki_abrv_xo;} private byte[] wiki_abrv_xo; - public byte[] Root_bry() {return root_bry;} private byte[] root_bry; - public byte[] Root_http() {return root_http;} private byte[] root_http = Bry_.Empty; - public String Root_str() {return root_str;} private String root_str; - public byte Dir_spr() {return dir_spr;} private byte dir_spr; - public boolean Fsys_is_wnt() {return fsys_is_wnt;} public Xof_repo_itm Fsys_is_wnt_(boolean v) {fsys_is_wnt = v; return this;} private boolean fsys_is_wnt; - public boolean Shorten_ttl() {return shorten_ttl;} public Xof_repo_itm Shorten_ttl_(boolean v) {shorten_ttl = v; return this;} private boolean shorten_ttl = true; - public boolean Wmf_fsys() {return wmf_fsys;} public Xof_repo_itm Wmf_fsys_(boolean v) {wmf_fsys = v; return this;} private boolean wmf_fsys; - public boolean Wmf_api() {return wmf_api;} public Xof_repo_itm Wmf_api_(boolean v) {wmf_api = v; return this;} private boolean wmf_api; - public boolean Tarball() {return tarball;} public Xof_repo_itm Tarball_(boolean v) {tarball = v; return this;} private boolean tarball; - public byte[][] Mode_names() {return mode_names;} private byte[][] mode_names = new byte[][] {Xof_repo_itm_.Mode_names_key[0], Xof_repo_itm_.Mode_names_key[1]}; - public int Dir_depth() {return dir_depth;} public Xof_repo_itm Dir_depth_(int v) {dir_depth = v; return this;} private int dir_depth = 4; - public Xof_rule_grp Ext_rules() {return ext_rules;} private Xof_rule_grp ext_rules; + public byte[] Key() {return key;} private final byte[] key; // EX: "src_http_commons" + public byte Tid() {return tid;} private byte tid; // EX: Xof_repo_tid_.Tid__remote + public byte[] Wiki_domain() {return wiki_domain;} private byte[] wiki_domain; // EX: "commons.wikimedia.org" + public byte[] Wiki_abrv_xo() {return wiki_abrv_xo;} private byte[] wiki_abrv_xo; // EX: "c" + public byte Dir_spr() {return dir_spr;} private byte dir_spr; // EX: "/" + public int Dir_depth() {return dir_depth;} private int dir_depth = 4; // EX: "/1/2/3/4" vs "/1/2" + public byte[] Root_bry() {return root_bry;} private byte[] root_bry; // EX: + public byte[] Root_http() {return root_http;} private byte[] root_http = Bry_.Empty; // EX: + public String Root_str() {return root_str;} private String root_str; + public boolean Fsys_is_wnt() {return fsys_is_wnt;} private boolean fsys_is_wnt; + public boolean Shorten_ttl() {return shorten_ttl;} private boolean shorten_ttl = true; + public boolean Wmf_fsys() {return wmf_fsys;} private boolean wmf_fsys; + public boolean Wmf_api() {return wmf_api;} private boolean wmf_api; + public boolean Tarball() {return tarball;} private boolean tarball; + public byte[][] Mode_names() {return mode_names;} private byte[][] mode_names = new byte[][] {Xof_img_mode_.Names_ary[0], Xof_img_mode_.Names_ary[1]}; + public Xof_rule_grp Ext_rules() {return ext_rules;} private Xof_rule_grp ext_rules; + public boolean Primary() {return primary;} private boolean primary; + + public Xof_repo_itm Fsys_is_wnt_(boolean v) {fsys_is_wnt = v; return this;} + public Xof_repo_itm Shorten_ttl_(boolean v) {shorten_ttl = v; return this;} + public Xof_repo_itm Wmf_fsys_(boolean v) {wmf_fsys = v; return this;} + public Xof_repo_itm Wmf_api_(boolean v) {wmf_api = v; return this;} + public Xof_repo_itm Tarball_(boolean v) {tarball = v; return this;} public Xof_repo_itm Ext_rules_(byte[] ext_rules_key) {ext_rules = ext_rule_mgr.Get_or_new(ext_rules_key); return this;} - public boolean Primary() {return primary;} public Xof_repo_itm Primary_(boolean v) {primary = v; return this;} private boolean primary; + public Xof_repo_itm Dir_depth_(int v) {dir_depth = v; return this;} + public Xof_repo_itm Primary_(boolean v) {primary = v; return this;} + public void Wiki_domain_(byte[] v) { this.wiki_domain = v; Xow_domain_itm domain_itm = Xow_domain_itm_.parse(v); @@ -70,11 +80,11 @@ public class Xof_repo_itm implements Gfo_invk { } public byte[] Gen_name_src(Bry_bfr tmp_bfr, byte[] name) { if (!fsys_is_wnt || wmf_fsys) return name; - return Xof_repo_itm_.Ttl_invalid_fsys_chars(tmp_bfr, name); + return Xof_itm_ttl_.Remove_invalid(tmp_bfr, name); } public byte[] Gen_name_trg(Bry_bfr tmp_bfr, byte[] bry, byte[] md5, Xof_ext ext) { byte[] rv = Gen_name_src(tmp_bfr, bry); - return shorten_ttl ? Xof_repo_itm_.Ttl_shorten_ttl(tmp_bfr, rv, Ttl__max_len, md5, ext.Ext()) : rv; + return shorten_ttl ? Xof_itm_ttl_.Shorten(tmp_bfr, rv, Ttl__max_len, md5, ext.Ext()) : rv; } public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { if (ctx.Match(k, Invk_owner)) throw Err_.new_unimplemented_w_msg("deprecated repo_itm.owner"); @@ -83,9 +93,10 @@ public class Xof_repo_itm implements Gfo_invk { else if (ctx.Match(k, Invk_ext_rules_)) Ext_rules_(m.ReadBry("v")); else if (ctx.Match(k, Invk_wmf_api_)) wmf_api = m.ReadYn("v"); else if (ctx.Match(k, Invk_tarball_)) tarball = m.ReadYn("v"); + else if (ctx.Match(k, Invk_tid_)) tid = Xof_repo_tid_.By_str(m.ReadStr("v")); else return Gfo_invk_.Rv_unhandled; return this; } - private static final String Invk_owner = "owner", Invk_fsys_ = "fsys_", Invk_ext_rules_ = "ext_rules_", Invk_primary_ = "primary_", Invk_wmf_api_ = "wmf_api_", Invk_tarball_ = "tarball_"; + private static final String Invk_owner = "owner", Invk_fsys_ = "fsys_", Invk_ext_rules_ = "ext_rules_", Invk_primary_ = "primary_", Invk_wmf_api_ = "wmf_api_", Invk_tarball_ = "tarball_", Invk_tid_ = "tid_"; public static final int Ttl__max_len = 180; } diff --git a/400_xowa/src/gplx/xowa/files/repos/Xof_repo_itm_.java b/400_xowa/src/gplx/xowa/files/repos/Xof_repo_itm_.java deleted file mode 100644 index b29717428..000000000 --- a/400_xowa/src/gplx/xowa/files/repos/Xof_repo_itm_.java +++ /dev/null @@ -1,56 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; -import gplx.core.btries.*; import gplx.core.envs.*; -public class Xof_repo_itm_ { - public static final int Dir_depth_null = -1, Dir_depth_wmf = 2, Dir_depth_xowa = 4; - public static final byte Mode_orig = 0, Mode_thumb = 1, Mode_nil = Byte_.Max_value_127; - public static byte Mode_by_bool(boolean is_thumb) {return is_thumb ? Mode_thumb : Mode_orig;} - public static final byte[][] Mode_names_key = new byte[][] {Bry_.new_a7("orig"), Bry_.new_a7("thumb")}; - public static final byte Repo_remote = 0, Repo_local = 1, Repo_unknown = 126, Repo_null = Byte_.Max_value_127; - public static byte Repo_by_bool(boolean is_commons) {return is_commons ? Repo_remote : Repo_local;} - public static boolean Repo_is_known(byte repo) { - switch (repo) { - case Repo_remote: - case Repo_local: return true; - default: return false; - } - } - public static byte[] Ttl_invalid_fsys_chars(Bry_bfr bfr, byte[] ttl) { - int ttl_len = ttl.length; - for (int i = 0; i < ttl_len; i++) { - byte b = ttl[i]; - if (Op_sys_.Wnt_invalid_char(b)) b = Byte_ascii.Underline; - bfr.Add_byte(b); - } - return bfr.To_bry_and_clear(); - } - public static byte[] Ttl_shorten_ttl(Bry_bfr bfr, byte[] ttl, int ttl_max, byte[] md5, byte[] ext_bry) { - byte[] rv = ttl; - int exceed_len = rv.length - ttl_max; - if (exceed_len > 0) { - bfr.Add_mid(rv, 0, ttl_max - 33); // add truncated title; 33=_.length + md5.length - bfr.Add_byte(Byte_ascii.Underline); // add underline; EX: "_" - bfr.Add(md5); // add md5; EX: "abcdefghijklmnopqrstuvwxyz0123456" - bfr.Add_byte(Byte_ascii.Dot); // add dot; EX: "." - bfr.Add(ext_bry); // add ext; EX: ".png" - rv = bfr.To_bry_and_clear(); - } - return rv; - } -} diff --git a/400_xowa/src/gplx/xowa/files/repos/Xof_repo_pair.java b/400_xowa/src/gplx/xowa/files/repos/Xof_repo_pair.java index 4d938b2d3..3bf2a549b 100644 --- a/400_xowa/src/gplx/xowa/files/repos/Xof_repo_pair.java +++ b/400_xowa/src/gplx/xowa/files/repos/Xof_repo_pair.java @@ -17,15 +17,16 @@ along with this program. If not, see . */ package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; public class Xof_repo_pair implements Gfo_invk { - public Xof_repo_pair(byte repo_idx, byte[] wiki_domain, Xof_repo_itm src, Xof_repo_itm trg) { - this.repo_idx = repo_idx; this.wiki_domain = wiki_domain; this.src = src; this.trg = trg; + public Xof_repo_pair(byte id, byte[] wiki_domain, Xof_repo_itm src, Xof_repo_itm trg) { + this.id = id; this.wiki_domain = wiki_domain; this.src = src; this.trg = trg; } - public byte Repo_idx() {return repo_idx;} private byte repo_idx; - public byte[] Wiki_domain() {return wiki_domain;} private final byte[] wiki_domain; - public Xof_repo_itm Src() {return src;} private final Xof_repo_itm src; - public Xof_repo_itm Trg() {return trg;} private final Xof_repo_itm trg; + public byte Id() {return id;} private byte id; + public byte[] Wiki_domain() {return wiki_domain;} private final byte[] wiki_domain; + public Xof_repo_itm Src() {return src;} private final Xof_repo_itm src; + public Xof_repo_itm Trg() {return trg;} private final Xof_repo_itm trg; + public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { - if (ctx.Match(k, Invk_repo_id_)) repo_idx = m.ReadByte("v"); + if (ctx.Match(k, Invk_repo_id_)) id = m.ReadByte("v"); else return Gfo_invk_.Rv_unhandled; return this; } private static final String Invk_repo_id_ = "repo_id_"; diff --git a/400_xowa/src/gplx/xowa/files/repos/Xof_repo_tid_.java b/400_xowa/src/gplx/xowa/files/repos/Xof_repo_tid_.java new file mode 100644 index 000000000..966d12dcd --- /dev/null +++ b/400_xowa/src/gplx/xowa/files/repos/Xof_repo_tid_.java @@ -0,0 +1,36 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.files.repos; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; +public class Xof_repo_tid_ { + public static final byte + Tid__remote = 0 // EX: "https://commons.wikimedia.org/wiki/File:A.png" + , Tid__local = 1 // EX: "https://en.wikipedia.org/wiki/File:A.png" + , Tid__xtns = 2 // EX: "https://en.wikipedia.org/w/extensions/ImageMap/desc-20.png?15600" + , Tid__math = 3 // EX: "https://wikimedia.org/api/rest_v1/media/math/render/svg/596f8baf206a81478afd4194b44138715dc1a05c" + , Tid__null = Byte_.Max_value_127 + ; + public static byte By_str(String v) { + if (String_.Eq(v, "self")) return Tid__local; + else if (String_.Eq(v, "comm")) return Tid__remote; + else if (String_.Eq(v, "math")) return Tid__math; + else throw Err_.new_unhandled_default(v); + } + public static final byte[] + Bry__math = Bry_.new_a7("wikimedia.org/math") // EX: https://wikimedia.org/api/rest_v1/media/math/render/svg/596f8baf206a81478afd4194b44138715dc1a05c + ; +} diff --git a/400_xowa/src/gplx/xowa/files/repos/Xowe_repo_mgr.java b/400_xowa/src/gplx/xowa/files/repos/Xowe_repo_mgr.java index 3803c42a1..ddc7d1870 100644 --- a/400_xowa/src/gplx/xowa/files/repos/Xowe_repo_mgr.java +++ b/400_xowa/src/gplx/xowa/files/repos/Xowe_repo_mgr.java @@ -55,7 +55,7 @@ public class Xowe_repo_mgr implements Xow_repo_mgr, Gfo_invk { int len = repos.Count(); for (int i = 0; i < len; i++) { Xof_repo_pair pair = (Xof_repo_pair)repos.Get_at(i); - if (pair.Repo_idx() == id) return pair; + if (pair.Id() == id) return pair; } return null; } diff --git a/400_xowa/src/gplx/xowa/files/xfers/Xof_xfer_mgr.java b/400_xowa/src/gplx/xowa/files/xfers/Xof_xfer_mgr.java index db667a1cb..555382c60 100644 --- a/400_xowa/src/gplx/xowa/files/xfers/Xof_xfer_mgr.java +++ b/400_xowa/src/gplx/xowa/files/xfers/Xof_xfer_mgr.java @@ -17,7 +17,7 @@ along with this program. If not, see . */ package gplx.xowa.files.xfers; import gplx.*; import gplx.xowa.*; import gplx.xowa.files.*; import gplx.core.primitives.*; import gplx.gfui.*; -import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.exts.*; import gplx.xowa.files.downloads.*; +import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.exts.*; import gplx.xowa.files.downloads.*; import gplx.xowa.files.imgs.*; import gplx.xowa.bldrs.wms.*; import gplx.xowa.apps.wms.apis.*; import gplx.xowa.apps.wms.apis.origs.*; import gplx.xowa.wikis.tdbs.metas.*; public class Xof_xfer_mgr { @@ -74,8 +74,8 @@ public class Xof_xfer_mgr { } // BLOCK: orig; get orig for convert; note that Img_download will not download file again if src exists - src_str = this.Src_url(src_repo, Xof_repo_itm_.Mode_orig, Xof_img_size.Size__neg1); - trg_url = this.Trg_url(trg_repo, Xof_repo_itm_.Mode_orig, Xof_img_size.Size__neg1); + src_str = this.Src_url(src_repo, Xof_img_mode_.Tid__orig, Xof_img_size.Size__neg1); + trg_url = this.Trg_url(trg_repo, Xof_img_mode_.Tid__orig, Xof_img_size.Size__neg1); if (!Img_download(src_str, trg_url, false)) return false; trg_url = rslt.Trg(); @@ -84,7 +84,7 @@ public class Xof_xfer_mgr { rslt.Clear(); // clear error from failed thumb Io_url src_url = trg_url; if (orig_ext.Id_is_djvu()) { // NOTE: this block converts djvu -> tiff b/c vanilla imageMagick cannot do djvu -> jpeg - trg_url = this.Trg_url(trg_repo, Xof_repo_itm_.Mode_thumb, lnki_w).GenNewExt(".tiff"); // NOTE: manually change orig_ext to tiff; note that djvu has view type of jpeg + trg_url = this.Trg_url(trg_repo, Xof_img_mode_.Tid__thumb, lnki_w).GenNewExt(".tiff"); // NOTE: manually change orig_ext to tiff; note that djvu has view type of jpeg wiki.Appe().File_mgr().Img_mgr().Wkr_convert_djvu_to_tiff().Exec(src_url, trg_url); if (!Cmd_query_size(trg_url)) return false; // meta_itm.Update_orig_size(file_w, file_h); // NOTE that thumb size is always orig size @@ -93,7 +93,7 @@ public class Xof_xfer_mgr { boolean limit = !orig_ext.Id_is_svg(); // do not limit if svg Xof_xfer_itm_.Calc_xfer_size(calc_size, xfer_itm.Lnki_type(), wiki.Html_mgr().Img_thumb_width(), file_w, file_h, lnki_w, lnki_h, lnki_thumbable, xfer_itm.Lnki_upright(), limit); // NOTE: always recalc w/h; needed for (a) when width < 1 and (b) when w/h are wrong; xfer=160,160, lnki=65,50, actl should be 50,50; PAGE:en.w:[[Image:Gnome-mime-audio-openclipart.svg|65x50px|center|link=|alt=]] lnki_w = calc_size.Val_0(); lnki_h = calc_size.Val_1(); - trg_url = this.Trg_url(trg_repo, Xof_repo_itm_.Mode_thumb, lnki_w); + trg_url = this.Trg_url(trg_repo, Xof_img_mode_.Tid__thumb, lnki_w); if (!Img_convert(src_url, trg_url)) return false; // convert failed; exit if (orig_ext.Id_is_djvu()) Io_mgr.Instance.DeleteFil(src_url); // convert passed; if djvu, delete intermediary .tiff file; } @@ -110,7 +110,7 @@ public class Xof_xfer_mgr { xfer_itm.Orig_repo_id_(Xof_meta_itm.Repo_same); // set repo to "same" else { // wmf returned other wiki (which is 99% likely to be commons) Xof_repo_pair trg_repo_pair = wiki.File_mgr().Repo_mgr().Repos_get_by_wiki(rslts.Orig_wiki()); // need to do this b/c commons is not always first; see wikinews; DATE:2013-12-04 - int trg_repo_idx = trg_repo_pair == null ? 0 : (int)trg_repo_pair.Repo_idx(); // 0=commons + int trg_repo_idx = trg_repo_pair == null ? 0 : (int)trg_repo_pair.Id(); // 0=commons xfer_itm.Orig_repo_id_(trg_repo_idx); } if (!Bry_.Eq(rslts.Orig_page(), orig_ttl)) { @@ -139,7 +139,7 @@ public class Xof_xfer_mgr { if (!wmf_api_found) return false; // not found in wmf_api; exit now } else if (src_repo.Tarball()) { - String src_str = this.Src_url(src_repo, Xof_repo_itm_.Mode_orig, Xof_img_size.Size__neg1); + String src_str = this.Src_url(src_repo, Xof_img_mode_.Tid__orig, Xof_img_size.Size__neg1); meta_itm.Orig_exists_(Xof_meta_itm.Exists_unknown); // mark exists unknown; note need to assertively mark unknown b/c it may have been marked n in previous pass through multiple repos; DATE:20121227 meta_itm.Vrtl_repo_(Xof_meta_itm.Repo_unknown); // mark repo unknown; if (!Cmd_query_size(Io_url_.new_fil_(src_str))) { @@ -163,8 +163,8 @@ public class Xof_xfer_mgr { lnki_w = calc_size.Val_0(); if (lnki_h != -1) lnki_h = calc_size.Val_1(); // NOTE: if -1 (no height specified) do not set height; EX:Tokage_2011-07-15.jpg; DATE:2013-06-03 - src_str = src_repo.Tarball() ? this.Src_url(src_repo, Xof_repo_itm_.Mode_orig, Xof_img_size.Size_null) : this.Src_url(src_repo, Xof_repo_itm_.Mode_thumb, lnki_w); - trg_url = this.Trg_url(trg_repo, Xof_repo_itm_.Mode_thumb, lnki_w); + src_str = src_repo.Tarball() ? this.Src_url(src_repo, Xof_img_mode_.Tid__orig, Xof_img_size.Size_null) : this.Src_url(src_repo, Xof_img_mode_.Tid__thumb, lnki_w); + trg_url = this.Trg_url(trg_repo, Xof_img_mode_.Tid__thumb, lnki_w); if (Make_img_exec(src_str, trg_url)) { // download passed trg_url = rslt.Trg(); if (lnki_w > 0 && lnki_h > 0) { // lnki specified width and height; check against xfer; needed when w/h are wrong; lnki=65,50 but xfer=160,160; actl should be 50,50; PAGE:en.w:[[Image:Gnome-mime-audio-openclipart.svg|65x50px|center|link=|alt=]]; SEE:NOTE_1 @@ -191,23 +191,23 @@ public class Xof_xfer_mgr { Xof_xfer_itm_.Calc_xfer_size(calc_size, xfer_itm.Lnki_type(), wiki.Html_mgr().Img_thumb_width(), meta_itm.Orig_w(), meta_itm.Orig_h(), lnki_w, lnki_h, lnki_thumbable, lnki_upright, limit); lnki_w = calc_size.Val_0(); lnki_h = calc_size.Val_1(); - src_str = src_repo.Tarball() ? this.Src_url(src_repo, Xof_repo_itm_.Mode_orig, Xof_img_size.Size_null) : this.Src_url(src_repo, Xof_repo_itm_.Mode_thumb, lnki_w); - trg_url = this.Trg_url(trg_repo, Xof_repo_itm_.Mode_thumb, lnki_w); + src_str = src_repo.Tarball() ? this.Src_url(src_repo, Xof_img_mode_.Tid__orig, Xof_img_size.Size_null) : this.Src_url(src_repo, Xof_img_mode_.Tid__thumb, lnki_w); + trg_url = this.Trg_url(trg_repo, Xof_img_mode_.Tid__thumb, lnki_w); return Make_img_exec(src_str, trg_url); } else { // no orig dimensions; do download if (lnki_w == Xof_img_size.Null) lnki_w = wiki.Html_mgr().Img_thumb_width(); // set lnki_w to default thumb_width (220) - src_str = src_repo.Tarball() ? this.Src_url(src_repo, Xof_repo_itm_.Mode_orig, Xof_img_size.Size_null) : this.Src_url(src_repo, Xof_repo_itm_.Mode_thumb, lnki_w); - trg_url = this.Trg_url(trg_repo, Xof_repo_itm_.Mode_thumb, lnki_w); + src_str = src_repo.Tarball() ? this.Src_url(src_repo, Xof_img_mode_.Tid__orig, Xof_img_size.Size_null) : this.Src_url(src_repo, Xof_img_mode_.Tid__thumb, lnki_w); + trg_url = this.Trg_url(trg_repo, Xof_img_mode_.Tid__thumb, lnki_w); if (Make_img_exec(src_str, trg_url)) { // download if (src_repo.Tarball()) return true; // convert worked; no need to download again; int old_lnki_w = lnki_w; lnki_w = (file_w * lnki_h) / file_h; // calculate correct width for specified height; lnki_w = Xof_xfer_itm_.Calc_w(file_w, file_h, lnki_h); if (lnki_w == old_lnki_w) return true; // download at 220 actually worked; this will probably occur very infrequently, but if so, exit - src_str = this.Src_url(src_repo, Xof_repo_itm_.Mode_thumb, lnki_w); - trg_url = this.Trg_url(trg_repo, Xof_repo_itm_.Mode_thumb, lnki_w); + src_str = this.Src_url(src_repo, Xof_img_mode_.Tid__thumb, lnki_w); + trg_url = this.Trg_url(trg_repo, Xof_img_mode_.Tid__thumb, lnki_w); if (Make_img_exec(src_str, trg_url)) { // download again trg_url = rslt.Trg(); Xof_xfer_itm_.Calc_xfer_size(calc_size, xfer_itm.Lnki_type(), wiki.Html_mgr().Img_thumb_width(), file_w, file_h, lnki_w, lnki_h, lnki_thumbable, lnki_upright);// calculate again using width and height @@ -235,8 +235,8 @@ public class Xof_xfer_mgr { boolean thumb_pass = false; Make_other(); // NOTE: must go before thumb b/c rslt.Pass() is modified by both if (src_repo_is_wmf) { // src is wmf >>> copy down thumb; NOTE: thumb not available in tar - String src_str = this.Src_url(src_repo, Xof_repo_itm_.Mode_thumb, lnki_w); - Io_url trg_url = this.Trg_url(trg_repo, Xof_repo_itm_.Mode_thumb, lnki_w); + String src_str = this.Src_url(src_repo, Xof_img_mode_.Tid__thumb, lnki_w); + Io_url trg_url = this.Trg_url(trg_repo, Xof_img_mode_.Tid__thumb, lnki_w); thumb_pass = Cmd_download(src_str, trg_url, false); // NOTE: ogg audios may sometimes have thumb, but 0 size; thumb_pass will be true, but will fail on thumb_rename; PAGE:en.w:Beethoven; [[File:Ludwig van Beethoven - Symphonie 5 c-moll - 1. Allegro con brio.ogg]] if (thumb_pass) { thumb_pass = Img_rename_by_size(trg_url); // NOTE: lnki cites view_w which will rarely match file_w; PAGE:en.w:Earth;Northwest coast of United States to Central South America at Night.ogv|250px; which is atually 640 @@ -257,8 +257,8 @@ public class Xof_xfer_mgr { } boolean Make_other() { if (!Orig_max_download() && !force_orig) return false; - String src_str = this.Src_url(src_repo, Xof_repo_itm_.Mode_orig, Xof_img_size.Size__neg1); - Io_url trg_url = this.Trg_url(trg_repo, Xof_repo_itm_.Mode_orig, Xof_img_size.Size__neg1); + String src_str = this.Src_url(src_repo, Xof_img_mode_.Tid__orig, Xof_img_size.Size__neg1); + Io_url trg_url = this.Trg_url(trg_repo, Xof_img_mode_.Tid__orig, Xof_img_size.Size__neg1); return Cmd_download(src_str, trg_url, true); } boolean Orig_max_download() { @@ -353,8 +353,8 @@ public class Xof_xfer_mgr { file_w = file_size.Width(); file_h = file_size.Height(); return true; } - String Src_url(Xof_repo_itm repo, byte mode, int lnki_w) {return url_bldr.Init_for_src_file(mode, repo, orig_ttl, orig_ttl_md5, orig_ext, lnki_w, lnki_thumbtime, lnki_page).Xto_str();} - Io_url Trg_url(Xof_repo_itm repo, byte mode, int lnki_w) {return url_bldr.Init_for_trg_file(mode, repo, orig_ttl, orig_ttl_md5, orig_ext, lnki_w, lnki_thumbtime, lnki_page).Xto_url();} + String Src_url(Xof_repo_itm repo, byte mode, int lnki_w) {return url_bldr.Init_for_src_file(repo, mode, orig_ttl, orig_ttl_md5, orig_ext, lnki_w, lnki_thumbtime, lnki_page).Xto_str();} + Io_url Trg_url(Xof_repo_itm repo, byte mode, int lnki_w) {return url_bldr.Init_for_trg_file(repo, mode, orig_ttl, orig_ttl_md5, orig_ext, lnki_w, lnki_thumbtime, lnki_page).Xto_url();} private Xof_url_bldr url_bldr = new Xof_url_bldr(); } /* diff --git a/400_xowa/src/gplx/xowa/guis/views/Xog_win_itm.java b/400_xowa/src/gplx/xowa/guis/views/Xog_win_itm.java index 3eea085bf..aa76b23b0 100644 --- a/400_xowa/src/gplx/xowa/guis/views/Xog_win_itm.java +++ b/400_xowa/src/gplx/xowa/guis/views/Xog_win_itm.java @@ -62,7 +62,6 @@ public class Xog_win_itm implements Gfo_invk, Gfo_evt_itm { public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { if (ctx.Match(k, Invk_link_click)) Win__link_click(); else if (ctx.Match(k, Invk_link_print)) Xog_win_itm__prog_href_mgr.Print(this); - else if (ctx.Match(k, Gfui_html.Evt_link_hover)) Xog_win_itm__prog_href_mgr.Hover(app, this.Active_tab().Wiki(), this.Active_page(), Xoh_href_gui_utl.Standardize_xowa_link(m.ReadStr("v"))); else if (ctx.Match(k, Gfui_html.Evt_location_changed)) Win__link_clicked(m.ReadStr("v")); else if (ctx.Match(k, Gfui_html.Evt_location_changing)) Page__navigate_by_href(tab_mgr.Active_tab(), Xoh_href_gui_utl.Standardize_xowa_link(m.ReadStr("v"))); else if (ctx.Match(k, Gfui_html.Evt_win_resized)) Refresh_win_size(); @@ -92,6 +91,10 @@ public class Xog_win_itm implements Gfo_invk, Gfo_evt_itm { else if (ctx.Match(k, Invk_page)) return this.Active_page(); else if (ctx.Match(k, Invk_wiki)) return this.Active_tab().Wiki(); else if (ctx.Match(k, Invk_exit)) App__exit(); + else if (ctx.Match(k, Gfui_html.Evt_link_hover)) { + if (this.Active_tab() != null) // NOTE: this.Active_tab() should not be null, but is null when running on raspberry pi; DATE:2016-09-23 + Xog_win_itm__prog_href_mgr.Hover(app, this.Active_tab().Wiki(), this.Active_page(), Xoh_href_gui_utl.Standardize_xowa_link(m.ReadStr("v"))); + } else return Gfo_invk_.Rv_unhandled; return this; } diff --git a/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java b/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java index d5fec0a01..9af1c6a0a 100644 --- a/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java +++ b/400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java @@ -106,7 +106,7 @@ public class Xoh_page_wtr_wkr { , portal_mgr.Div_ns_bry(wiki.Utl__bfr_mkr(), page_ttl, wiki.Ns_mgr()) , portal_mgr.Div_view_bry(wiki.Utl__bfr_mkr(), html_gen_tid, page.Html_data().Xtn_search_text()) , portal_mgr.Div_logo_bry(), portal_mgr.Div_home_bry(), new Xopg_xtn_skin_fmtr_arg(page, Xopg_xtn_skin_itm_tid.Tid_sidebar) - , portal_mgr.Div_sync_bry(tmp_bfr, wiki, page) + , portal_mgr.Div_sync_bry(tmp_bfr, app.Api_root().Addon().Bldr().Sync().Manual_enabled(), wiki, page) , portal_mgr.Div_wikis_bry(wiki.Utl__bfr_mkr()) , portal_mgr.Sidebar_mgr().Html_bry() , mgr.Edit_rename_div_bry(page_ttl), page.Html_data().Edit_preview_w_dbg(), js_edit_toolbar_bry @@ -132,9 +132,10 @@ public class Xoh_page_wtr_wkr { Write_body_edit(bfr, data_raw, page_ns_id, page_tid); else { switch (page_tid) { + case Xow_page_tid.Tid_msg: case Xow_page_tid.Tid_js: case Xow_page_tid.Tid_css: - case Xow_page_tid.Tid_lua: Write_body_pre (bfr, app, wiki, data_raw, tmp_bfr); page_tid_uses_pre = true; break; + case Xow_page_tid.Tid_lua: Write_body_pre (bfr, app, wiki, hctx, data_raw, tmp_bfr); page_tid_uses_pre = true; break; case Xow_page_tid.Tid_json: app.Wiki_mgr().Wdata_mgr().Write_json_as_html(bfr, page_ttl.Full_db(), data_raw); break; case Xow_page_tid.Tid_wikitext: Write_body_wikitext (bfr, app, wiki, data_raw, ctx, hctx, page, page_tid, page_ns_id); break; } @@ -154,7 +155,7 @@ public class Xoh_page_wtr_wkr { } // dump and exit if MediaWiki message; if (ns_id == Xow_ns_.Tid__mediawiki) { // if MediaWiki and wikitext, must be a message; convert args back to php; DATE:2014-06-13 - bfr.Add(Xoa_gfs_php_mgr.Xto_php(tmp_bfr, Bool_.N, data_raw)); + bfr.Add(Gfs_php_converter.Xto_php(tmp_bfr, Bool_.N, data_raw)); return; } // if [[File]], add boilerplate header; note that html is XOWA-generated so does not need to be tidied @@ -175,7 +176,7 @@ public class Xoh_page_wtr_wkr { } // if [[Category]], render rest of html (Subcategories; Pages; Files); note that a category may have other html which requires wikitext processing - if (ns_id == Xow_ns_.Tid__category) wiki.Html_mgr().Ns_ctg().Write_catpage(tidy_bfr, wiki, page, hctx); + if (ns_id == Xow_ns_.Tid__category) wiki.Html_mgr().Catpage_mgr().Write_catpage(tidy_bfr, wiki, page, hctx); // tidy html wiki.Html_mgr().Tidy_mgr().Exec_tidy(tidy_bfr, !hctx.Mode_is_hdump(), page.Url_bry_safe()); @@ -199,16 +200,19 @@ public class Xoh_page_wtr_wkr { Xol_vnt_mgr vnt_mgr = wiki.Lang().Vnt_mgr(); if (vnt_mgr.Enabled()) bfr.Add(vnt_mgr.Convert_lang().Parse_page(vnt_mgr.Cur_itm(), page.Db().Page().Id(), bfr.To_bry_and_clear())); } - private void Write_body_pre(Bry_bfr bfr, Xoae_app app, Xowe_wiki wiki, byte[] data_raw, Bry_bfr tmp_bfr) { + private void Write_body_pre(Bry_bfr bfr, Xoae_app app, Xowe_wiki wiki, Xoh_wtr_ctx hctx, byte[] data_raw, Bry_bfr tmp_bfr) { Xoh_html_wtr_escaper.Escape(app.Parser_amp_mgr(), tmp_bfr, data_raw, 0, data_raw.length, false, false); - app.Html_mgr().Page_mgr().Content_code_fmtr().Bld_bfr_many(bfr, tmp_bfr); + if (hctx.Mode_is_hdump()) + bfr.Add(data_raw); + else + app.Html_mgr().Page_mgr().Content_code_fmtr().Bld_bfr_many(bfr, tmp_bfr); tmp_bfr.Clear(); } private void Write_body_edit(Bry_bfr bfr, byte[] data_raw, int ns_id, byte page_tid) { if ( ns_id == Xow_ns_.Tid__mediawiki // if MediaWiki and wikitext, must be a message; convert args back to php; DATE:2014-06-13 && page_tid == Xow_page_tid.Tid_wikitext ) - data_raw = Xoa_gfs_php_mgr.Xto_php(tmp_bfr, Bool_.N, data_raw); + data_raw = Gfs_php_converter.Xto_php(tmp_bfr, Bool_.N, data_raw); int data_raw_len = data_raw.length; if (mgr.Html_capable()) Xoh_html_wtr_escaper.Escape(page.Wikie().Appe().Parser_amp_mgr(), bfr, data_raw, 0, data_raw_len, false, false); // NOTE: must escape; assume that browser will automatically escape (<) (which Mozilla does) diff --git a/400_xowa/src/gplx/xowa/htmls/Xow_html_mgr.java b/400_xowa/src/gplx/xowa/htmls/Xow_html_mgr.java index 7fd4e7391..76cfc093d 100644 --- a/400_xowa/src/gplx/xowa/htmls/Xow_html_mgr.java +++ b/400_xowa/src/gplx/xowa/htmls/Xow_html_mgr.java @@ -53,24 +53,19 @@ public class Xow_html_mgr implements Gfo_invk { public Xop_xatr_whitelist_mgr Whitelist_mgr() {return whitelist_mgr;} private final Xop_xatr_whitelist_mgr whitelist_mgr = new Xop_xatr_whitelist_mgr().Ini(); public Xow_portal_mgr Portal_mgr() {return portal_mgr;} private Xow_portal_mgr portal_mgr; public Xow_module_mgr Head_mgr() {return module_mgr;} private Xow_module_mgr module_mgr; + public Xoctg_catpage_mgr Catpage_mgr() {return catpage_mgr;} private final Xoctg_catpage_mgr catpage_mgr = new Xoctg_catpage_mgr(); public boolean Importing_ctgs() {return importing_ctgs;} public void Importing_ctgs_(boolean v) {importing_ctgs = v;} private boolean importing_ctgs; public int Img_thumb_width() {return img_thumb_width;} private int img_thumb_width = 220; public byte[] Img_xowa_protocol() {return img_xowa_protocol;} private byte[] img_xowa_protocol; public boolean Img_suppress_missing_src() {return img_suppress_missing_src;} public Xow_html_mgr Img_suppress_missing_src_(boolean v) {img_suppress_missing_src = v; return this;} private boolean img_suppress_missing_src = true; - public Xoctg_catpage_mgr Ns_ctg() {return ns_ctg;} private final Xoctg_catpage_mgr ns_ctg = new Xoctg_catpage_mgr(); public Xoh_imgs_mgr Imgs_mgr() {return imgs_mgr;} private Xoh_imgs_mgr imgs_mgr; public void Copy_cfg(Xow_html_mgr html_mgr) {imgs_mgr.Copy_cfg(html_mgr.Imgs_mgr());} public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { if (ctx.Match(k, Invk_article)) return page_wtr_mgr; else if (ctx.Match(k, Invk_portal)) return portal_mgr; else if (ctx.Match(k, Invk_imgs)) return imgs_mgr; - else if (ctx.Match(k, Invk_ns_ctg)) return ns_ctg; else if (ctx.Match(k, Invk_modules)) return module_mgr; else return Gfo_invk_.Rv_unhandled; } - public static final String - Invk_article = "article" - , Invk_portal = "portal", Invk_imgs = "imgs", Invk_ns_ctg = "ns_ctg" - , Invk_modules = "modules" - ; + public static final String Invk_article = "article", Invk_portal = "portal", Invk_imgs = "imgs", Invk_modules = "modules"; } diff --git a/400_xowa/src/gplx/xowa/htmls/core/Xow_hdump_mgr__save.java b/400_xowa/src/gplx/xowa/htmls/core/Xow_hdump_mgr__save.java index f25f01653..4fc3ef1b1 100644 --- a/400_xowa/src/gplx/xowa/htmls/core/Xow_hdump_mgr__save.java +++ b/400_xowa/src/gplx/xowa/htmls/core/Xow_hdump_mgr__save.java @@ -34,11 +34,12 @@ public class Xow_hdump_mgr__save { Bld_hdump(page); tmp_hpg.Ctor_by_hdiff(tmp_bfr, page, page.Wikie().Msg_mgr().Val_by_id(gplx.xowa.langs.msgs.Xol_msg_itm_.Id_toc)); Xow_db_file html_db = Get_html_db(wiki, page, html_db_is_new.Val_n_()); - return Save(tmp_hpg, html_db.Tbl__html(), html_db_is_new.Val()); + return Save(tmp_hpg, html_db.Tbl__html(), html_db_is_new.Val(), true); } } - public int Save(Xoh_page hpg, Xowd_html_tbl html_tbl, boolean insert) { - byte[] db_body = Write(tmp_bfr, wiki, hpg, hzip_mgr, zip_mgr, dflt_zip_tid, dflt_hzip_tid, hpg.Db().Html().Html_bry()); + public int Save(Xoh_page hpg, Xowd_html_tbl html_tbl, boolean insert, boolean use_hzip_dflt) { + int hzip_tid = use_hzip_dflt ? dflt_hzip_tid : Xoh_hzip_dict_.Hzip__none; + byte[] db_body = Write(tmp_bfr, wiki, hpg, hzip_mgr, zip_mgr, dflt_zip_tid, hzip_tid, hpg.Db().Html().Html_bry()); if (insert) html_tbl.Insert(hpg, dflt_zip_tid, dflt_hzip_tid, db_body); else html_tbl.Update(hpg, dflt_zip_tid, dflt_hzip_tid, db_body); return db_body.length; diff --git a/400_xowa/src/gplx/xowa/htmls/core/bldrs/Xob_hdump_bldr.java b/400_xowa/src/gplx/xowa/htmls/core/bldrs/Xob_hdump_bldr.java index 4f634a3b8..19d3e775f 100644 --- a/400_xowa/src/gplx/xowa/htmls/core/bldrs/Xob_hdump_bldr.java +++ b/400_xowa/src/gplx/xowa/htmls/core/bldrs/Xob_hdump_bldr.java @@ -19,7 +19,7 @@ package gplx.xowa.htmls.core.bldrs; import gplx.*; import gplx.xowa.*; import gp import gplx.core.brys.*; import gplx.dbs.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.cmds.*; import gplx.xowa.apps.apis.xowa.bldrs.imports.*; import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.htmls.core.hzips.*; import gplx.xowa.htmls.core.dbs.*; -import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.data.*; +import gplx.xowa.wikis.*; import gplx.xowa.wikis.pages.*; import gplx.xowa.wikis.data.*; import gplx.xowa.parsers.*; public class Xob_hdump_bldr implements Gfo_invk { private boolean enabled, hzip_enabled, hzip_diff, hzip_b256; private byte zip_tid = Byte_.Max_value_127; @@ -52,17 +52,25 @@ public class Xob_hdump_bldr implements Gfo_invk { wpg.File_queue().Clear(); // need to reset uid to 0, else xowa_file_# will resume from last // write to html - wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_page_.Tid_read).Write_hdump(tmp_bfr, ctx, Xoh_wtr_ctx.Hdump, wpg); - byte[] orig_bry = tmp_bfr.To_bry_and_clear(); - wpg.Db().Html().Html_bry_(orig_bry); + Xoa_ttl ttl = wpg.Ttl(); + boolean is_wikitext = Xow_page_tid.Identify(wpg.Wiki().Domain_tid(), ttl.Ns().Id(), ttl.Page_db()) == Xow_page_tid.Tid_wikitext; + byte[] orig_bry = Bry_.Empty; + if (is_wikitext) { + wiki.Html_mgr().Page_wtr_mgr().Wkr(Xopg_page_.Tid_read).Write_hdump(tmp_bfr, ctx, Xoh_wtr_ctx.Hdump, wpg); + orig_bry = tmp_bfr.To_bry_and_clear(); + wpg.Db().Html().Html_bry_(orig_bry); + } + else { // not wikitext; EX: pages in MediaWiki: ns; DATE:2016-09-12 + wpg.Db().Html().Html_bry_(wpg.Db().Text().Text_bry()); + } // save to db Xowd_html_tbl html_tbl = html_tbl_retriever.Get_html_tbl(wpg.Ttl().Ns(), prv_row_len); // get html_tbl - this.prv_row_len = hdump_mgr.Save_mgr().Save(tmp_hpg.Ctor_by_hdiff(tmp_bfr, wpg, toc_label), html_tbl, true); // save to db + this.prv_row_len = hdump_mgr.Save_mgr().Save(tmp_hpg.Ctor_by_hdiff(tmp_bfr, wpg, toc_label), html_tbl, true, is_wikitext); // save to db stat_tbl.Insert(tmp_hpg, stat_itm, wpg.Root().Root_src().length, tmp_hpg.Db().Html().Html_bry().length, prv_row_len); // save stats // run hzip diff if enabled - if (hzip_diff) { + if (hzip_diff && is_wikitext) { byte[] expd_bry = op_sys_is_wnt ? Bry_.Replace(tmp_bfr, orig_bry, Byte_ascii.Cr_lf_bry, Byte_ascii.Nl_bry) : orig_bry; // tidy adds crlf if wnt byte[] actl_bry = hdump_mgr.Load_mgr().Decode_as_bry(tmp_bfr, tmp_hpg, hdump_mgr.Save_mgr().Src_as_hzip(), Bool_.Y); byte[][] diff = Bry_diff_.Diff_1st_line(expd_bry, actl_bry); diff --git a/400_xowa/src/gplx/xowa/htmls/core/wkrs/Xoh_hdoc_ctx.java b/400_xowa/src/gplx/xowa/htmls/core/wkrs/Xoh_hdoc_ctx.java index 486d87106..adc64f2af 100644 --- a/400_xowa/src/gplx/xowa/htmls/core/wkrs/Xoh_hdoc_ctx.java +++ b/400_xowa/src/gplx/xowa/htmls/core/wkrs/Xoh_hdoc_ctx.java @@ -26,6 +26,7 @@ public class Xoh_hdoc_ctx { public byte[] Fsys__root() {return fsys__root;} private byte[] fsys__root; public byte[] Fsys__file__comm() {return fsys__file__comm;} private byte[] fsys__file__comm; public byte[] Fsys__file__wiki() {return fsys__file__wiki;} private byte[] fsys__file__wiki; + public byte[] Fsys__file__math() {return fsys__file__math;} private byte[] fsys__file__math; public Xof_repo_itm Fsys__repo(boolean local) {return local ? repo_local : repo_remote;} private Xof_repo_itm repo_remote, repo_local; public boolean Fsys__is_wnt() {return fsys__is_wnt;} private boolean fsys__is_wnt; public Xoa_app App() {return app;} private Xoa_app app; @@ -53,6 +54,7 @@ public class Xoh_hdoc_ctx { this.fsys__root = fsys_mgr.Root_dir().To_http_file_bry(); this.fsys__file = fsys_mgr.File_dir().To_http_file_bry(); this.fsys__file__comm = Bry_.Add(fsys__file, Xow_domain_itm_.Bry__commons, Byte_ascii.Slash_bry); + this.fsys__file__math = Bry_.Add(fsys__file, Bry_.new_a7("math"), Byte_ascii.Slash_bry); this.fsys__res = gplx.core.envs.Op_sys.Cur().Tid_is_drd() ? Fsys__res__drd : fsys__root; this.fsys__is_wnt = gplx.core.envs.Op_sys.Cur().Tid_is_wnt(); Xou_cache_mgr usr_cache_mgr = app.User().User_db_mgr().Cache_mgr(); diff --git a/400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/Xoh_img_hzip.java b/400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/Xoh_img_hzip.java index d869c4ad2..ceffe3719 100644 --- a/400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/Xoh_img_hzip.java +++ b/400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/Xoh_img_hzip.java @@ -21,7 +21,7 @@ import gplx.langs.htmls.*; import gplx.langs.htmls.docs.*; import gplx.langs.htm import gplx.xowa.htmls.hrefs.*; import gplx.xowa.htmls.core.hzips.*; import gplx.xowa.htmls.core.wkrs.bfr_args.*; import gplx.xowa.htmls.core.wkrs.imgs.atrs.*; import gplx.xowa.htmls.core.wkrs.lnkis.*; import gplx.xowa.htmls.core.wkrs.lnkis.anchs.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.ttls.*; import gplx.xowa.xtns.pagebanners.*; -import gplx.xowa.files.*; import gplx.xowa.files.repos.*; +import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.imgs.*; public class Xoh_img_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm { private final Bry_bfr tmp_bfr = Bry_bfr_.New_w_size(32); private final Xoh_img_xoimg_hzip xoimg = new Xoh_img_xoimg_hzip(); @@ -272,13 +272,16 @@ public class Xoh_img_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm { // set url_bldr Xof_url_bldr url_bldr = hctx.File__url_bldr(); - url_bldr.Init_by_root(file__repo_is_local ? hctx.Fsys__file__wiki() : hctx.Fsys__file__comm(), hctx.Fsys__is_wnt(), Byte_ascii.Slash, false, false, Md5_depth); + byte repo_tid = file__repo_is_local ? Xof_repo_tid_.Tid__local : Xof_repo_tid_.Tid__remote; + byte[] fsys_root = file__repo_is_local ? hctx.Fsys__file__wiki() : hctx.Fsys__file__comm(); + + url_bldr.Init_by_repo(repo_tid, fsys_root, hctx.Fsys__is_wnt(), Byte_ascii.Slash, false, false, Md5_depth); // NOTE: set md5 / ext now b/c src_page will be changed for gui mode byte[] md5 = Xof_file_wkr_.Md5(src_page); Xof_ext ext = Xof_ext_.new_by_ttl_(src_page); if (!hpg.Wiki().File__fsdb_mode().Tid__bld()) // if gui, do not url-encode; else "View HTML" will not work, since fsys uses non-url-decoded form; note needs to be url-encoded else hdiff will return different values src_page = Xoa_ttl.Replace_spaces(Gfo_url_encoder_.Href.Decode(src_page)); - url_bldr.Init_for_trg_html(file__is_orig ? Xof_repo_itm_.Mode_orig : Xof_repo_itm_.Mode_thumb, hctx.Fsys__repo(file__repo_is_local), src_page, md5, ext, file_w, file_time, file_page); + url_bldr.Init_for_trg_html(hctx.Fsys__repo(file__repo_is_local), file__is_orig ? Xof_img_mode_.Tid__orig : Xof_img_mode_.Tid__thumb, src_page, md5, ext, file_w, file_time, file_page); // set data vars for wtr data.Init_by_decode(anch_rel_is_nofollow, anch_title_bgn, anch_title_end, img__wo_anch, img__is_vid, img_w, img_h, img_alt_bgn, img_alt_end, img_imap_idx); diff --git a/400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/atrs/Xoh_img_src_data.java b/400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/atrs/Xoh_img_src_data.java index afa88edd3..bd417b425 100644 --- a/400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/atrs/Xoh_img_src_data.java +++ b/400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/atrs/Xoh_img_src_data.java @@ -17,7 +17,7 @@ along with this program. If not, see . */ package gplx.xowa.htmls.core.wkrs.imgs.atrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.*; import gplx.xowa.htmls.core.wkrs.*; import gplx.xowa.htmls.core.wkrs.imgs.*; import gplx.core.brys.*; import gplx.core.btries.*; -import gplx.xowa.files.*; +import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.langs.htmls.*; import gplx.langs.htmls.docs.*; import gplx.xowa.wikis.domains.*; public class Xoh_img_src_data implements Bfr_arg_clearable, Xoh_itm_parser { @@ -29,6 +29,7 @@ public class Xoh_img_src_data implements Bfr_arg_clearable, Xoh_itm_parser { public int Repo_bgn() {return repo_bgn;} private int repo_bgn; public int Repo_end() {return repo_end;} private int repo_end; public boolean Repo_is_commons() {return repo_is_commons;} private boolean repo_is_commons; + public byte Repo_tid() {return repo_tid;} private byte repo_tid; public int File_ttl_bgn() {return file_ttl_bgn;} private int file_ttl_bgn; public int File_ttl_end() {return file_ttl_end;} private int file_ttl_end; public boolean File_ttl_exists() {return file_ttl_end > file_ttl_bgn;} @@ -44,6 +45,7 @@ public class Xoh_img_src_data implements Bfr_arg_clearable, Xoh_itm_parser { src_bgn = src_end = repo_bgn = repo_end = file_ttl_bgn = file_ttl_end = file_w = file_page = -1; file_time = -1; repo_is_commons = file_is_orig = false; + repo_tid = Xof_repo_tid_.Tid__null; file_ttl_bry = null; } public boolean Parse(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, byte[] domain_bry, Gfh_tag tag) { @@ -59,14 +61,23 @@ public class Xoh_img_src_data implements Bfr_arg_clearable, Xoh_itm_parser { // get repo_bgn; note that some may be hiero / enlarge / magnify and should exit rdr.Init_by_wkr(err_wkr, "img.src.xowa", src_bgn, src_end).Fail_throws_err_(Bool_.N); - repo_bgn = rdr.Find_fwd_rr(Bry__file); // skip past /file/; EX: "file:///J:/xowa/file/commons.wikimedia.org/" - if (repo_bgn == -1) return false; + repo_bgn = rdr.Find_fwd_rr(Bry__file); + if (repo_bgn == -1) { + repo_bgn = rdr.Find_fwd_rr(Bry__math); + if (repo_bgn == Bry_find_.Not_found) return false; + this.file_ttl_bry = Bry_.Mid(rdr.Src(), repo_bgn, src_end); + this.repo_is_commons = true; + this.repo_tid = Xof_repo_tid_.Tid__math; + this.file_is_orig = true; + return true; + } rdr.Fail_throws_err_(Bool_.Y); // get repo repo_end = rdr.Find_fwd_lr(); repo_is_commons = Bry_.Match(rdr.Src(), repo_bgn, repo_end, Xow_domain_itm_.Bry__commons); + repo_tid = repo_is_commons ? Xof_repo_tid_.Tid__remote : Xof_repo_tid_.Tid__local; if (!repo_is_commons) { if (!Bry_.Match(rdr.Src(), repo_bgn, repo_end, domain_bry)) rdr.Err_wkr().Fail("repo must be commons or self", "repo", Bry_.Mid(rdr.Src(), repo_bgn, repo_end)); } @@ -120,7 +131,7 @@ public class Xoh_img_src_data implements Bfr_arg_clearable, Xoh_itm_parser { } return rdr.Move_to(pos); } - private static final byte[] Bry__file = Bry_.new_a7("/file/"), Bry__orig = Bry_.new_a7("orig/"), Bry__thumb = Bry_.new_a7("thumb/"); + private static final byte[] Bry__file = Bry_.new_a7("/file/"), Bry__math = Bry_.new_a7("/math/"), Bry__orig = Bry_.new_a7("orig/"), Bry__thumb = Bry_.new_a7("thumb/"); private static final byte Tid__orig = 1, Tid__thumb = 2; private static final Btrie_slim_mgr trie = Btrie_slim_mgr.cs().Add_bry_byte(Bry__orig, Tid__orig).Add_bry_byte(Bry__thumb, Tid__thumb); } diff --git a/400_xowa/src/gplx/xowa/htmls/portal/Xow_portal_mgr.java b/400_xowa/src/gplx/xowa/htmls/portal/Xow_portal_mgr.java index 03faa5447..99d86e746 100644 --- a/400_xowa/src/gplx/xowa/htmls/portal/Xow_portal_mgr.java +++ b/400_xowa/src/gplx/xowa/htmls/portal/Xow_portal_mgr.java @@ -106,15 +106,14 @@ public class Xow_portal_mgr implements Gfo_invk { } public static final byte[] Cls_selected_y = Bry_.new_a7("selected"), Cls_new = Bry_.new_a7("new"), Cls_display_none = Bry_.new_a7("xowa_display_none"); public byte[] Div_logo_bry() {return div_logo_bry;} private byte[] div_logo_bry = Bry_.Empty; public byte[] Div_home_bry() {return api_skin != null && api_skin.Sidebar_home_enabled() ? div_home_bry : Bry_.Empty;} private byte[] div_home_bry = Bry_.Empty; - public byte[] Div_sync_bry(Bry_bfr tmp_bfr, Xow_wiki wiki, Xoa_page page) { + public byte[] Div_sync_bry(Bry_bfr tmp_bfr, boolean manual_enabled, Xow_wiki wiki, Xoa_page page) { // only show update_html if wmf; DATE:2016-08-31 - switch (wiki.Domain_itm().Domain_type().Src()) { - case Xow_domain_tid.Src__wmf: - div_sync_fmtr.Bld_bfr_many(tmp_bfr, page.Ttl().Full_url()); - return tmp_bfr.To_bry_and_clear(); - default: - return Bry_.Empty; + if ( wiki.Domain_itm().Domain_type().Src() == Xow_domain_tid.Src__wmf + && manual_enabled) { + div_sync_fmtr.Bld_bfr_many(tmp_bfr, page.Ttl().Full_url()); + return tmp_bfr.To_bry_and_clear(); } + return Bry_.Empty; } public byte[] Div_wikis_bry(Bry_bfr_mkr bfr_mkr) { if (toggle_itm == null) // TEST:lazy-new b/c Init_by_wiki diff --git a/400_xowa/src/gplx/xowa/langs/msgs/Xol_msg_itm_.java b/400_xowa/src/gplx/xowa/langs/msgs/Xol_msg_itm_.java index 05c48d9c2..91e4939c6 100644 --- a/400_xowa/src/gplx/xowa/langs/msgs/Xol_msg_itm_.java +++ b/400_xowa/src/gplx/xowa/langs/msgs/Xol_msg_itm_.java @@ -514,7 +514,7 @@ case Xol_msg_itm_.Id_xowa_wikidata_links_special: return new_(Xol_msg_itm_.Id_xo } public static byte[] eval_(Bry_bfr bfr, Xol_msg_itm tmp_msg_itm, byte[] val, Object... args) { synchronized (tmp_fmtr) { // LOCK:static-objs; DATE:2016-07-07 - val = gplx.xowa.apps.gfs.Xoa_gfs_php_mgr.Xto_gfs(bfr, val); + val = gplx.xowa.apps.gfs.Gfs_php_converter.To_gfs(bfr, val); update_val_(tmp_msg_itm, val); return tmp_fmtr.Bld_bry_many(bfr, args); } diff --git a/400_xowa/src/gplx/xowa/langs/msgs/Xol_msg_mgr_.java b/400_xowa/src/gplx/xowa/langs/msgs/Xol_msg_mgr_.java index 128be8543..073ed4b42 100644 --- a/400_xowa/src/gplx/xowa/langs/msgs/Xol_msg_mgr_.java +++ b/400_xowa/src/gplx/xowa/langs/msgs/Xol_msg_mgr_.java @@ -22,10 +22,6 @@ import gplx.xowa.apps.gfs.*; import gplx.xowa.htmls.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.pages.dbs.*; public class Xol_msg_mgr_ { -// public static String Get_msg_val_gui_or_null(Xol_lang_itm lang, byte[] pre, byte[] key, byte[] suf) { -// String rv = Get_msg_val_gui_or_null(lang, pre, key, suf); -// return rv == null ? "<" + String_.new_u8(Bry_.Add(pre, key, suf)) + ">" : rv; -// } public static String Get_msg_val_gui_or_empty(Xoa_lang_mgr lang_mgr, Xol_lang_itm lang, byte[] pre, byte[] key, byte[] suf) { // get from lang, else get from en; does not use get_msg_val to skip db lookups; should only be used for gui; DATE:2014-05-28 String rv = Get_msg_val_gui_or_null(lang_mgr, lang, pre, key, suf); return rv == null ? "" : rv; @@ -102,45 +98,12 @@ public class Xol_msg_mgr_ { } } else { // page found; dump entire contents - msg_val = Xoa_gfs_php_mgr.Xto_gfs(tmp_bfr, msg_db); // note that MediaWiki msg's use php arg format ($1); xowa.gfs msgs are already converted + msg_val = Gfs_php_converter.To_gfs(tmp_bfr, msg_db); // note that MediaWiki msg's use php arg format ($1); xowa.gfs msgs are already converted msg_in_wiki.Defined_in_(Xol_msg_itm.Defined_in__wiki); } -// Xoa_page msg_page = Get_msg_itm_from_db(wiki, lang, msg_key, msg_key_sub_root); -// byte[] msg_val = Bry_.Empty; -// if (msg_page.Db().Page().Exists_n()) { // [[MediaWiki:key/fallback]] still not found; search "lang.gfs"; -// Xol_msg_itm msg_in_lang = Get_msg_itm_from_gfs(wiki, lang, msg_key_sub_root); -// if (msg_in_lang == null) { -// msg_val = tmp_bfr.Add_byte(Byte_ascii.Lt).Add(msg_key).Add_byte(Byte_ascii.Gt).To_bry_and_clear(); // set val to -// msg_in_wiki.Defined_in_(Xol_msg_itm.Defined_in__none); -// } -// else { -// msg_val = msg_in_lang.Val(); -// msg_in_wiki.Defined_in_(Xol_msg_itm.Defined_in__lang); -// } -// } -// else { // page found; dump entire contents -// msg_val = Xoa_gfs_php_mgr.Xto_gfs(tmp_bfr, msg_page.Db().Text().Text_bry()); // note that MediaWiki msg's use php arg format ($1); xowa.gfs msgs are already converted -// msg_in_wiki.Defined_in_(Xol_msg_itm.Defined_in__wiki); -// } Xol_msg_itm_.update_val_(msg_in_wiki, msg_val); return msg_in_wiki; } -// private static Xoa_page Get_msg_itm_from_db(Xowe_wiki wiki, Xol_lang_itm lang, byte[] msg_key, byte[] msg_key_sub_root) { -// byte[] ns_bry = wiki.Ns_mgr().Ns_mediawiki().Name_db_w_colon(); -// Xoa_ttl ttl = wiki.Ttl_parse(Bry_.Add(ns_bry, msg_key)); // ttl="MediaWiki:msg_key"; note that there may be "/lang"; EX:pl.d:Wikislownik:Bar/Archiwum_6 and newarticletext/pl -// Xoae_page rv = ttl == null ? Xoae_page.Empty : wiki.Data_mgr().Load_page_by_ttl_for_msg(ttl); -// if (rv.Db().Page().Exists_n()) { // [[MediaWiki:key]] not found; search for [[MediaWiki:key/fallback]] -// byte[][] fallback_ary = lang.Fallback_bry_ary(); -// int fallback_ary_len = fallback_ary.length; -// for (int i = 0; i < fallback_ary_len; i++) { -// byte[] fallback = fallback_ary[i]; -// ttl = wiki.Ttl_parse(Bry_.Add(ns_bry, msg_key_sub_root, Slash_bry, fallback)); // ttl="MediaWiki:msg_key/fallback" -// rv = ttl == null ? Xoae_page.Empty : wiki.Data_mgr().Load_page_by_ttl_for_msg(ttl); -// if (rv.Db().Page().Exists()) break; -// } -// } -// return rv; -// } private static byte[] Get_msg_from_db_or_null(Xow_wiki wiki, Xol_lang_itm lang, byte[] msg_key, byte[] msg_key_sub_root) { byte[] ns_bry = wiki.Ns_mgr().Ns_mediawiki().Name_db_w_colon(); Xoa_ttl ttl = wiki.Ttl_parse(Bry_.Add(ns_bry, msg_key)); // ttl="MediaWiki:msg_key"; note that there may be "/lang"; EX:pl.d:Wikislownik:Bar/Archiwum_6 and newarticletext/pl @@ -162,14 +125,20 @@ public class Xol_msg_mgr_ { } private static byte[] Load_msg_from_db_or_null(Xow_wiki wiki, Xoa_ttl ttl) { Xoa_page pg = null; - if (wiki.Type_is_edit()) { + if (wiki.Type_is_edit()) // NOTE: this check only works when loading pages directly (EX:en.wikipedia.org/wiki/MediaWiki:Sidebar); however, due to way msgs load, wiki is always edit, even if html_dump; DATE:2016-09-17 pg = ((Xowe_wiki)wiki).Data_mgr().Load_page_by_ttl_for_msg(ttl); - } - else { + + // HACK: handle htmp_dump wikis when loading messages such as sidebar; DATE:2016-09-17 + if ( !wiki.Type_is_edit() // app is drd; DATE:2016-09-23 + || ( pg.Db().Page().Exists() // page exists + && Bry_.Len_eq_0(pg.Db().Text().Text_bry()) // but text is empty -> check html_dump + ) + ) { Xoh_page hpg = new Xoh_page(); pg = hpg; hpg.Ctor_by_hview(wiki, Xoa_url.New(wiki, ttl), ttl, -1); - ((Xowv_wiki)wiki).Pages_get(hpg, gplx.core.net.Gfo_url.Empty, ttl); // NOTE: url is only for "Special:" pages + wiki.Html__hdump_mgr().Load_mgr().Load(hpg, ttl); + pg.Db().Text().Text_bry_(pg.Db().Html().Html_bry()); } return pg.Db().Page().Exists() ? pg.Db().Text().Text_bry() : null; } diff --git a/400_xowa/src/gplx/xowa/specials/xowa/diags/Xows_cmd__file_check.java b/400_xowa/src/gplx/xowa/specials/xowa/diags/Xows_cmd__file_check.java index ce805d867..445a482da 100644 --- a/400_xowa/src/gplx/xowa/specials/xowa/diags/Xows_cmd__file_check.java +++ b/400_xowa/src/gplx/xowa/specials/xowa/diags/Xows_cmd__file_check.java @@ -94,7 +94,7 @@ class Xows_cmd__file_check { Object[] row = rows[i]; int owner_id = Int_.cast(row[owner_id_ordinal]); byte[] bin_data = (byte[])row[bin_data_ordinal]; - file_bry = gplx.xowa.files.repos.Xof_repo_itm_.Ttl_invalid_fsys_chars(tmp_bfr, file_bry); + file_bry = gplx.xowa.files.repos.Xof_itm_ttl_.Remove_invalid(tmp_bfr, file_bry); Io_url bin_url = tmp_dir.GenSubFil(String_.new_u8(file_bry)); bin_url = tmp_dir.GenSubFil(bin_url.NameOnly() + "_" + Int_.To_str(owner_id) + bin_url.Ext()); Write_kv(bfr, "fsdb.bin.export", bin_url.Raw()); diff --git a/400_xowa/src/gplx/xowa/wikis/Xow_page_tid.java b/400_xowa/src/gplx/xowa/wikis/Xow_page_tid.java index 4ec16d440..a83ddcd5a 100644 --- a/400_xowa/src/gplx/xowa/wikis/Xow_page_tid.java +++ b/400_xowa/src/gplx/xowa/wikis/Xow_page_tid.java @@ -22,6 +22,8 @@ public class Xow_page_tid { public static byte Identify(int wiki_tid, int ns_id, byte[] ttl) { switch (ns_id) { case Xow_ns_.Tid__mediawiki: + byte rv = Identify_by_ttl(ttl); + return rv == Tid_wikitext ? Tid_msg : rv; // if mediawiki ns, but not css / js, return msg, not wikitext; DATE:2016-09-12 case Xow_ns_.Tid__user: return Identify_by_ttl(ttl); case Xow_ns_.Tid__module: @@ -38,5 +40,5 @@ public class Xow_page_tid { else return Tid_wikitext; } private static final byte[] Ext_js = Bry_.new_a7(".js"), Ext_css = Bry_.new_a7(".css"), Ext_doc= Bry_.new_a7("/doc"); - public static final byte Tid_wikitext = 1, Tid_json = 2, Tid_js = 3, Tid_css = 4, Tid_lua = 5; + public static final byte Tid_wikitext = 1, Tid_json = 2, Tid_js = 3, Tid_css = 4, Tid_lua = 5, Tid_msg = 6; } diff --git a/400_xowa/src/gplx/xowa/wikis/Xowv_repo_mgr.java b/400_xowa/src/gplx/xowa/wikis/Xowv_repo_mgr.java index cca000765..c645a3e3f 100644 --- a/400_xowa/src/gplx/xowa/wikis/Xowv_repo_mgr.java +++ b/400_xowa/src/gplx/xowa/wikis/Xowv_repo_mgr.java @@ -34,7 +34,7 @@ public class Xowv_repo_mgr implements Xow_repo_mgr { int len = repos.Count(); for (int i = 0; i < len; i++) { Xof_repo_pair pair = (Xof_repo_pair)repos.Get_at(i); - if (pair.Repo_idx() == id) return pair; + if (pair.Id() == id) return pair; } return null; } diff --git a/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java b/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java index 885955612..033d9c3ee 100644 --- a/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java +++ b/400_xowa/src/gplx/xowa/wikis/caches/Xow_cache_mgr.java @@ -58,6 +58,7 @@ public class Xow_cache_mgr { ifexist_cache.Clear(); wiki.Appe().Wiki_mgr().Wdata_mgr().Clear(); // moved from ctx.Clear(); DATE:2016-07-21 } + wiki.Html_mgr().Catpage_mgr().Free_mem_all(); tmpl_result_cache.Clear(); defn_cache.Free_mem_all(); misc_cache.Clear(); diff --git a/400_xowa/src/gplx/xowa/wikis/data/Xowd_cfg_tbl_.java b/400_xowa/src/gplx/xowa/wikis/data/Xowd_cfg_tbl_.java index 3c16b7915..4e960f739 100644 --- a/400_xowa/src/gplx/xowa/wikis/data/Xowd_cfg_tbl_.java +++ b/400_xowa/src/gplx/xowa/wikis/data/Xowd_cfg_tbl_.java @@ -21,7 +21,12 @@ public class Xowd_cfg_tbl_ { public static final String Tbl_name = "xowa_cfg"; public static Db_cfg_tbl New(gplx.dbs.Db_conn conn) {return New(conn, Tbl_name);} public static Db_cfg_tbl New(gplx.dbs.Db_conn conn, String tbl_name) {return new Db_cfg_tbl(conn, tbl_name);} - public static Db_cfg_tbl Get_or_null(gplx.dbs.Db_conn conn) { + public static Db_cfg_tbl Get_or_null(Db_conn conn) { return conn.Meta_tbl_exists(Tbl_name) ? new Db_cfg_tbl(conn, Tbl_name) : null; } + public static Db_cfg_tbl Get_or_fail(Db_conn conn) { + Db_cfg_tbl rv = Get_or_null(conn); + if (rv == null) throw Err_.New("xowa_cfg tbl does not exist: file={0}", conn.Conn_info().Raw()); + return rv; + } } diff --git a/400_xowa/src/gplx/xowa/wikis/data/tbls/Xowd_category_itm.java b/400_xowa/src/gplx/xowa/wikis/data/tbls/Xowd_category_itm.java index 06a053fb7..2f490d7be 100644 --- a/400_xowa/src/gplx/xowa/wikis/data/tbls/Xowd_category_itm.java +++ b/400_xowa/src/gplx/xowa/wikis/data/tbls/Xowd_category_itm.java @@ -28,9 +28,9 @@ public class Xowd_category_itm { public int Count_pages() {return count_pages;} private int count_pages; public int Count_by_tid(byte tid) { switch (tid) { - case Xoa_ctg_mgr.Tid_subc: return count_subcs; - case Xoa_ctg_mgr.Tid_page: return count_pages; - case Xoa_ctg_mgr.Tid_file: return count_files; + case Xoa_ctg_mgr.Tid__subc: return count_subcs; + case Xoa_ctg_mgr.Tid__page: return count_pages; + case Xoa_ctg_mgr.Tid__file: return count_files; default: throw Err_.new_unhandled(tid); } } diff --git a/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr.java b/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr.java index 6568c3d49..10796e196 100644 --- a/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr.java +++ b/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr.java @@ -19,7 +19,7 @@ package gplx.xowa.wikis.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa import gplx.core.primitives.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.addons.wikis.ctgs.bldrs.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.wikis.data.tbls.*; -import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; +import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls.*; public interface Xodb_load_mgr { void Load_init (Xowe_wiki wiki); void Load_page (Xowd_page_itm rv, Xow_ns ns); @@ -27,10 +27,7 @@ public interface Xodb_load_mgr { void Load_by_ids (Cancelable cancelable, List_adp rv, int bgn, int end); boolean Load_by_ttl (Xowd_page_itm rv, Xow_ns ns, byte[] ttl); void Load_by_ttls (Cancelable cancelable, Ordered_hash rv, boolean fill_idx_fields_only, int bgn, int end); - int Load_ctg_count (byte[] ttl); boolean Load_ctg_v1 (Xoctg_catpage_ctg rv, byte[] ttl); - void Load_ctg_v2a (Xoctg_catpage_ctg rv, Xoctg_catpage_url url_ctg, byte[] ttl_bry, int limit, boolean app_is_cmd); - Xowd_page_itm[] Load_ctg_list (byte[][] ctg_ttls); void Load_ttls_for_all_pages (Cancelable cancelable, List_adp rslt_list, Xowd_page_itm rslt_nxt, Xowd_page_itm rslt_prv, Int_obj_ref rslt_count, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item); void Load_ttls_for_search_suggest(Cancelable cancelable, List_adp rslt_list, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item); byte[] Find_random_ttl (Xow_ns ns); diff --git a/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_sql.java b/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_sql.java index 33cfb66ce..793c081f4 100644 --- a/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_sql.java +++ b/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_sql.java @@ -21,7 +21,7 @@ import gplx.xowa.apps.gfs.*; import gplx.xowa.addons.wikis.ctgs.bldrs.*; import import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.metas.*; import gplx.xowa.wikis.data.*; import gplx.xowa.addons.wikis.searchs.*; -import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; +import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls.*; public class Xodb_load_mgr_sql implements Xodb_load_mgr { private Xodb_mgr_sql db_mgr; Xow_db_mgr fsys_mgr; public Xodb_load_mgr_sql(Xow_wiki wiki, Xodb_mgr_sql db_mgr, Xow_db_mgr fsys_mgr) {this.db_mgr = db_mgr; this.fsys_mgr = fsys_mgr;} @@ -63,88 +63,17 @@ public class Xodb_load_mgr_sql implements Xodb_load_mgr { Xowd_category_itm ctg = fsys_mgr.Db__cat_core().Tbl__cat_core().Select(cat_page_id); if (ctg == Xowd_category_itm.Null) return false; return Ctg_select_v1(db_mgr.Wiki(), db_mgr.Core_data_mgr(), rv, ctg.File_idx(), ctg); } - public void Load_ctg_v2a(Xoctg_catpage_ctg rv, Xoctg_catpage_url ctg_url, byte[] ctg_ttl, int load_max, boolean app_is_cmd) { - int cat_page_id = db_mgr.Core_data_mgr().Tbl__page().Select_id(Xow_ns_.Tid__category, ctg_ttl); if (cat_page_id == Xowd_page_itm.Id_null) return; - Xowd_category_itm ctg = fsys_mgr.Db__cat_core().Tbl__cat_core().Select(cat_page_id); if (ctg == Xowd_category_itm.Null) return; - List_adp list = List_adp_.New(); - Load_ctg_v2a_db_retrieve(rv, ctg_url, cat_page_id, load_max, ctg.File_idx(), list, app_is_cmd); - Load_ctg_v2a_ui_sift(rv, ctg, list); - } - private void Load_ctg_v2a_db_retrieve(Xoctg_catpage_ctg rv, Xoctg_catpage_url ctg_url, int cat_page_id, int load_max, int cat_link_db_idx, List_adp list, boolean app_is_cmd) { - int len = Xoa_ctg_mgr.Tid__max; - for (byte i = Xoa_ctg_mgr.Tid_subc; i < len; i++) { - boolean arg_is_from = ctg_url.Grp_fwds()[i] == Bool_.N_byte; - byte[] arg_sortkey = ctg_url.Grp_idxs()[i]; - Xowd_cat_link_tbl cat_link_tbl = db_mgr.Core_data_mgr().Dbs__get_by_id_or_fail(cat_link_db_idx).Tbl__cat_link(); - int found = cat_link_tbl.Select_by_type(list, cat_page_id, i, arg_sortkey, arg_is_from, load_max); - if (found > 0 && found == load_max + 1) { - Xowd_page_itm last_page = (Xowd_page_itm)List_adp_.Pop(list); - Xoctg_page_xtn last_ctg = (Xoctg_page_xtn)last_page.Xtn(); - rv.Grp_by_tid(i).Itms__nth_sortkey_(last_ctg.Sortkey()); - } - } - db_mgr.Core_data_mgr().Tbl__page().Select_in__id(Cancelable_.Never, !app_is_cmd, list); - } - private void Load_ctg_v2a_ui_sift(Xoctg_catpage_ctg rv, Xowd_category_itm ctg, List_adp list) { - int len = list.Count(); - Xowe_wiki wiki = this.db_mgr.Wiki(); - byte prv_tid = Byte_.Max_value_127; - Xoctg_catpage_grp view_grp = null; - for (int i = 0; i < len; i++) { - Xowd_page_itm db_page = (Xowd_page_itm)list.Get_at(i); - if (db_page.Ns_id() == Int_.Min_value) continue; // HACK: page not found; ignore - Xoctg_page_xtn db_ctg = (Xoctg_page_xtn)db_page.Xtn(); - byte cur_tid = db_ctg.Tid(); - if (prv_tid != cur_tid) { - view_grp = rv.Grp_by_tid(cur_tid); - prv_tid = cur_tid; - } - Xoa_ttl ttl = Xoa_ttl.Parse(wiki, db_page.Ns_id(), db_page.Ttl_page_db()); - Xoctg_catpage_itm view_itm = new Xoctg_catpage_itm(db_page.Id(), ttl, db_ctg.Sortkey()); - view_grp.Itms__add(view_itm); - } - rv.Make_itms(); - } public void Load_ttls_for_all_pages(Cancelable cancelable, List_adp rslt_list, Xowd_page_itm rslt_nxt, Xowd_page_itm rslt_prv, Int_obj_ref rslt_count, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item) { db_mgr.Core_data_mgr().Tbl__page().Select_for_special_all_pages(cancelable, rslt_list, rslt_nxt, rslt_prv, rslt_count, ns, key, max_results, min_page_len, browse_len, include_redirects, fetch_prv_item); } public void Load_ttls_for_search_suggest(Cancelable cancelable, List_adp rslt_list, Xow_ns ns, byte[] key, int max_results, int min_page_len, int browse_len, boolean include_redirects, boolean fetch_prv_item) { db_mgr.Core_data_mgr().Tbl__page().Select_for_search_suggest(cancelable, rslt_list, ns, key, max_results, min_page_len, browse_len, include_redirects, fetch_prv_item); } - public int Load_ctg_count(byte[] ttl) { - if (db_mgr.Core_data_mgr().Db__cat_core() == null) return 0; - int page_id = db_mgr.Core_data_mgr().Tbl__page().Select_id(Xow_ns_.Tid__category, ttl); - if (page_id == Xowd_page_itm.Id_null) return 0; // title not found; return 0; - return db_mgr.Core_data_mgr().Db__cat_core().Tbl__cat_core().Select(page_id).Count_all(); - } public byte[] Load_qid(byte[] wiki_alias, byte[] ns_num, byte[] ttl) {return db_mgr.Core_data_mgr().Db__wbase().Tbl__wbase_qid().Select_qid(wiki_alias, ns_num, ttl);} public int Load_pid(byte[] lang_key, byte[] pid_name) {return db_mgr.Core_data_mgr().Db__wbase().Tbl__wbase_pid().Select_pid(lang_key, pid_name);} public byte[] Find_random_ttl(Xow_ns ns) {return db_mgr.Core_data_mgr().Tbl__page().Select_random(ns);} public Xodb_page_rdr Get_page_rdr(Xowe_wiki wiki) {return new Xodb_page_rdr__sql(wiki);} public void Clear() {} - public Xowd_page_itm[] Load_ctg_list(byte[][] ctg_ttls) { - if (db_mgr.Core_data_mgr().Db__cat_core() == null) return Xowd_page_itm.Ary_empty; - int len = ctg_ttls.length; - Ordered_hash hash = Ordered_hash_.New_bry(); - for (int i = 0; i < len; i++) { - Xowd_page_itm page = new Xowd_page_itm(); - byte[] ttl = Xoa_ttl.Replace_spaces(ctg_ttls[i]); // NOTE: ctg_ttls has spaces since v1 rendered it literally; - page.Ttl_page_db_(ttl); - if (!hash.Has(ttl)) - hash.Add(ttl, page); - } - len = hash.Count(); // must update len (!hash.Has() may have skipped titles) - db_mgr.Core_data_mgr().Tbl__page().Select_in__ttl(Cancelable_.Never, hash, Xow_ns_.Tid__category, 0, len); - Ordered_hash hash2 = Ordered_hash_.New(); - for (int i = 0; i < len; i++) { - Xowd_page_itm page = (Xowd_page_itm)hash.Get_at(i); - if (!hash2.Has(page.Id_val())) - hash2.Add(page.Id_val(), page); - } - len = hash2.Count(); // must update len (!hash2.Has() may have skipped titles) - db_mgr.Core_data_mgr().Db__cat_core().Tbl__cat_core().Select_by_cat_id_in(Cancelable_.Never, hash2, 0, len); - return (Xowd_page_itm[])hash.To_ary(Xowd_page_itm.class); - } private static boolean Ctg_select_v1(Xowe_wiki wiki, Xow_db_mgr core_data_mgr, Xoctg_catpage_ctg view_ctg, int link_db_id, Xowd_category_itm ctg) { List_adp link_list = List_adp_.New(); core_data_mgr.Dbs__get_by_id_or_fail(link_db_id).Tbl__cat_link().Select_in(link_list, ctg.Id()); @@ -167,9 +96,9 @@ public class Xodb_load_mgr_sql implements Xodb_load_mgr { } public static byte Load_ctg_v1_tid(int ns_id) { switch (ns_id) { - case Xow_ns_.Tid__category: return Xoa_ctg_mgr.Tid_subc; - case Xow_ns_.Tid__file: return Xoa_ctg_mgr.Tid_file; - default: return Xoa_ctg_mgr.Tid_page; + case Xow_ns_.Tid__category: return Xoa_ctg_mgr.Tid__subc; + case Xow_ns_.Tid__file: return Xoa_ctg_mgr.Tid__file; + default: return Xoa_ctg_mgr.Tid__page; } } } diff --git a/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_sql_tst.java b/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_sql_tst.java deleted file mode 100644 index ac36c0443..000000000 --- a/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_sql_tst.java +++ /dev/null @@ -1,174 +0,0 @@ -/* -XOWA: the XOWA Offline Wiki Application -Copyright (C) 2012 gnosygnu@gmail.com - -This program is free software: you can redistribute it and/or modify -it under the terms of the GNU Affero General Public License as -published by the Free Software Foundation, either version 3 of the -License, or (at your option) any later version. - -This program is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU Affero General Public License for more details. - -You should have received a copy of the GNU Affero General Public License -along with this program. If not, see . -*/ -package gplx.xowa.wikis.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; -import org.junit.*; import gplx.core.primitives.*; import gplx.xowa.bldrs.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.dbs.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.nss.*; -import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.dbs.*; -public class Xodb_load_mgr_sql_tst { - @Before public void init() {if (Xoa_test_.Db_skip()) return; fxt.Clear();} private Xodb_load_mgr_sql_fxt fxt = new Xodb_load_mgr_sql_fxt(); - @After public void term() {if (Xoa_test_.Db_skip()) return; fxt.Rls();} - @Test public void Load_ctg_ttls() { - if (Xoa_test_.Db_skip()) return; - Xowd_page_itm[] ctgs = fxt.pages_ - ( fxt.ctg_(1, "Ctg_1", Bool_.Y, 10, 11, 12) - , fxt.ctg_(2, "Ctg_2", Bool_.N, 20, 21, 22) - , fxt.ctg_(3, "Ctg_3", Bool_.Y, 30, 31, 32) - ); - fxt.Init_save_ctgs(ctgs); - fxt.Test_load_ctg_list(ctgs); - } -} -class Xoctg_catpage_url_mok extends Xoctg_catpage_url { public Xoctg_catpage_url_mok Page_bgn_(String v) {return Grp(Xoa_ctg_mgr.Tid_page, Bool_.Y, v);} - public Xoctg_catpage_url_mok Page_end_(String v) {return Grp(Xoa_ctg_mgr.Tid_page, Bool_.N, v);} - Xoctg_catpage_url_mok Grp(byte tid, boolean v, String bmk) { - this.Grp_fwds()[tid] = v ? Bool_.Y_byte : Bool_.N_byte; - this.Grp_idxs()[tid] = Bry_.new_a7(bmk); - return this; - } -} -class Xodb_load_mgr_sql_fxt { - Db_mgr_fxt fxt; Int_obj_ref next_id = Int_obj_ref.New(1); Xoae_app app; Xowe_wiki wiki; - public void Clear() { - if (fxt == null) { - fxt = new Db_mgr_fxt(); - fxt.Ctor_fsys(); - fxt.Init_db_sqlite(); - wiki = fxt.Wiki(); - app = wiki.Appe(); - } - } - public void Rls() {fxt.Rls();} - public Xowd_page_itm[] pages_(Xowd_page_itm... ary) {return ary;} - public Xowd_page_itm ctg_(int id, String ttl, boolean hidden, int count_subcs, int count_files, int count_pages) { - Xowd_page_itm rv = new Xowd_page_itm().Ns_id_(Xow_ns_.Tid__category).Id_(id).Ttl_page_db_(Bry_.new_a7(ttl)); - Xowd_category_itm ctg = Xowd_category_itm.load_(id, 0, hidden, count_subcs, count_files, count_pages); - rv.Xtn_(ctg); - return rv; - } - public void Init_save_ctgs(Xowd_page_itm[] ary) { - int len = ary.length; - Xodb_mgr_sql db_mgr = wiki.Db_mgr_as_sql(); - Xowd_cat_core_tbl cat_core_tbl = Xodb_cat_db_.Get_cat_core_or_fail(db_mgr.Core_data_mgr()); - cat_core_tbl.Create_tbl(); - DateAdp modified = Datetime_now.Get(); - Xowd_page_tbl tbl_page = wiki.Db_mgr_as_sql().Core_data_mgr().Tbl__page(); - tbl_page.Insert_bgn(); - cat_core_tbl.Insert_bgn(); - for (int i = 0; i < len; i++) { - Xowd_page_itm page = ary[i]; - tbl_page.Insert_cmd_by_batch(page.Id(), page.Ns_id(), page.Ttl_page_db(), false, modified, 10, page.Id(), 0, 0); - Xowd_category_itm ctg_itm = (Xowd_category_itm)page.Xtn(); - cat_core_tbl.Insert_cmd_by_batch(ctg_itm.Id(), ctg_itm.Count_pages(), ctg_itm.Count_subcs(), ctg_itm.Count_files(), Bool_.To_byte(ctg_itm.Hidden()), 0); - } - cat_core_tbl.Insert_end(); - tbl_page.Insert_end(); - } - public void Test_load_ctg_list(Xowd_page_itm[] ary) { - int len = ary.length; - byte[][] ttls = new byte[len][]; - for (int i = 0; i < len; i++) { - ttls[i] = ary[i].Ttl_page_db(); - } - Xowd_page_itm[] actl = wiki.Db_mgr_as_sql().Load_mgr().Load_ctg_list(ttls); - Tfds.Eq_str_lines(Xto_str(ary), Xto_str(actl)); - } - private static String Xto_str(Xowd_page_itm[] ary) { - Bry_bfr bfr = Bry_bfr_.New(); - int len = ary.length; - for (int i = 0; i < len; i++) { - Xowd_page_itm page = ary[i]; - Xowd_category_itm ctg_itm = (Xowd_category_itm)page.Xtn(); - bfr.Add_int_variable(page.Id()).Add_byte_pipe(); - bfr.Add(page.Ttl_page_db()).Add_byte_pipe(); - bfr.Add_byte(Bool_.To_byte(ctg_itm.Hidden())).Add_byte_nl(); - } - return bfr.To_str_and_clear(); - } - public Xoctg_catpage_url_mok ctg_url_() {return new Xoctg_catpage_url_mok();} - public Xodb_load_mgr_sql_fxt Init_limit_(int v) {limit = v; return this;} private int limit = 3; - public void Test_select(Xoctg_catpage_url ctg_url, Xoctg_mok_ctg expd) { - Xoctg_catpage_ctg view_ctg = new Xoctg_catpage_ctg(expd.Ttl()); - wiki.Db_mgr_as_sql().Load_mgr().Load_ctg_v2a(view_ctg, ctg_url, expd.Ttl(), limit, false); - for (byte i = 0; i < Xoa_ctg_mgr.Tid__max; i++) { - Xoctg_catpage_grp view_grp = view_ctg.Grp_by_tid(i); - Xoctg_mok_grp mok_grp = expd.Grps_get_or_new(i); - Tfds.Eq_ary_str(Xto_str(mok_grp), Xto_str(view_grp)); - Tfds.Eq(String_.new_a7(mok_grp.Last_plus_one_sortkey()), String_.new_a7(view_grp.Itms__nth_sortkey())); - } - } - String[] Xto_str(Xoctg_catpage_grp grp) { - Xoctg_catpage_itm[] ary = grp.Itms(); - int len = ary.length; - String[] rv = new String[len]; - for (int i = 0; i< len; i++) { - Xoctg_catpage_itm itm = ary[i]; - rv[i] = itm.Page_ttl().Page_db_as_str(); - } - return rv; - } - String[] Xto_str(Xoctg_mok_grp grp) { - List_adp list = grp.Itms(); - int len = list.Count(); - String[] rv = new String[len]; - for (int i = 0; i< len; i++) { - Xowd_page_itm itm = (Xowd_page_itm)list.Get_at(i); - rv[i] = String_.new_a7(itm.Ttl_page_db()); - } - return rv; - } - public Xoctg_mok_ctg ctg_() { - Xoctg_mok_ctg rv = new Xoctg_mok_ctg(next_id); - return rv; - } -} -class Xoctg_mok_grp { - public byte Tid() {return tid;} public Xoctg_mok_grp Tid_(byte v) {this.tid = v; return this;} private byte tid; - public byte[] Last_plus_one_sortkey() {return last_plus_one_sortkey;} public Xoctg_mok_grp Last_plus_one_sortkey_(byte[] v) {this.last_plus_one_sortkey = v; return this;} private byte[] last_plus_one_sortkey; - public List_adp Itms() {return itms;} List_adp itms = List_adp_.New(); -} -class Xoctg_mok_ctg { - public Xoctg_mok_ctg(Int_obj_ref next_id) {this.next_id = next_id;} Int_obj_ref next_id; - public byte[] Ttl() {return ttl;} - public Xoctg_mok_ctg Ttl_(String v) {return Ttl_(Bry_.new_a7(v));} - public Xoctg_mok_ctg Ttl_(byte[] v) {this.ttl = v; return this;} private byte[] ttl; - public Xoctg_mok_grp[] Grps() {return grps;} private Xoctg_mok_grp[] grps = new Xoctg_mok_grp[3]; - public Xoctg_mok_grp Grps_get_or_new(byte tid) { - Xoctg_mok_grp rv = grps[tid]; - if (rv == null) { - rv = new Xoctg_mok_grp().Tid_(tid); - grps[tid] = rv; - } - return rv; - } - public Xoctg_mok_ctg Grp_pages_(int count) {return Grp_pages_(0, count, null);} - public Xoctg_mok_ctg Grp_pages_(int bgn, int end, String last_itm_plus_one_sortkey) { - Xoctg_mok_grp grp = Grps_get_or_new(Xoa_ctg_mgr.Tid_page); - byte[] ttl_prefix = Bry_.new_a7("Page_"); - int ns_id = Xow_ns_.Tid__main; - byte ctg_tid = Xoa_ctg_mgr.Tid_page; - for (int i = bgn; i < end; i++) { - byte[] ttl = Bry_.Add(ttl_prefix, Bry_.new_a7(Int_.To_str_pad_bgn_zero(i, 3))); - Xoctg_page_xtn db_ctg = new Xoctg_page_xtn(ctg_tid, ttl); - Xowd_page_itm page = new Xowd_page_itm(); - int page_id = next_id.Val_add_post(); - page.Id_(page_id).Ns_id_(ns_id).Ttl_page_db_(ttl).Xtn_(db_ctg); - grp.Itms().Add(page); - } - grp.Last_plus_one_sortkey_(Bry_.new_a7(last_itm_plus_one_sortkey)); - return this; - } -} diff --git a/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_txt.java b/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_txt.java index ddab1db4d..0d831f59c 100644 --- a/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_txt.java +++ b/400_xowa/src/gplx/xowa/wikis/dbs/Xodb_load_mgr_txt.java @@ -24,7 +24,7 @@ import gplx.xowa.wikis.tdbs.*; import gplx.xowa.wikis.tdbs.hives.*; import gplx. import gplx.xowa.wikis.pages.*; import gplx.xowa.addons.wikis.searchs.specials.*; import gplx.xowa.guis.views.*; -import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; +import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.urls.*; public class Xodb_load_mgr_txt implements Xodb_load_mgr { private final Xob_xdat_file tmp_xdat_file = new Xob_xdat_file(); private final Xob_xdat_itm tmp_xdat_itm = new Xob_xdat_itm(); private final Xowd_page_itm tmp_page = new Xowd_page_itm(); @@ -439,7 +439,6 @@ public class Xodb_load_mgr_txt implements Xodb_load_mgr { Xob_xdat_itm pid_itm = Load_xdat_itm_by_dir(pids_root.GenSubDir(String_.new_u8(lang_key)), pid_name); if (pid_itm == null) return gplx.xowa.xtns.wbases.Wdata_wiki_mgr.Pid_null; return Bry_.To_int_or(pid_itm.Src(), pid_itm.Itm_bgn() + pid_name.length + 1 + 1, pid_itm.Itm_end(), gplx.xowa.xtns.wbases.Wdata_wiki_mgr.Pid_null); // extract pid; note that all itms have format of "ttl|pid"; +1=skip pipe; +1 skip p } Io_url pids_root; - public int Load_ctg_count(byte[] ttl) {throw Err_.new_deprecated("XOWA no longer supports categories for text databases");} Xob_xdat_itm Load_xdat_itm_by_dir(Io_url dir, byte[] ttl) { Xoa_hive_mgr hive_mgr = wiki.Appe().Hive_mgr(); int fil_idx = hive_mgr.Find_fil(dir, ttl); if (fil_idx == Xowd_regy_mgr.Regy_null) return null; // sub_dir not found; EX: commonswiki if qid; fr if pid; @@ -447,8 +446,6 @@ public class Xodb_load_mgr_txt implements Xodb_load_mgr { rdr.Find(tmp_xdat_itm, ttl, 0, Byte_ascii.Pipe, true); return tmp_xdat_itm.Found_exact() ? tmp_xdat_itm : null; } - public void Load_ctg_v2a(Xoctg_catpage_ctg rv, Xoctg_catpage_url url_ctg, byte[] ttl_bry, int limit, boolean app_is_cmd) {throw Err_.new_deprecated("XOWA no longer supports categories for text databases");} - public Xowd_page_itm[] Load_ctg_list(byte[][] ttls) {throw Err_.new_deprecated("XOWA no longer supports categories for text databases");} public Xodb_page_rdr Get_page_rdr(Xowe_wiki wiki) {return new Xodb_page_rdr__tdb(wiki);} static final String GRP_KEY = "xowa.wiki.db.load"; } diff --git a/400_xowa/src/gplx/xowa/wikis/nss/Xow_ns.java b/400_xowa/src/gplx/xowa/wikis/nss/Xow_ns.java index 7b65d513e..68d492072 100644 --- a/400_xowa/src/gplx/xowa/wikis/nss/Xow_ns.java +++ b/400_xowa/src/gplx/xowa/wikis/nss/Xow_ns.java @@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.wikis.nss; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; -import gplx.xowa.bldrs.cmds.*; import gplx.xowa.apps.urls.*; +import gplx.xowa.bldrs.cmds.*; import gplx.xowa.apps.utls.*; public class Xow_ns implements Gfo_invk { public Xow_ns(int id, byte case_match, byte[] name, boolean is_alias) { this.id = id; this.case_match = case_match; this.is_alias = is_alias; diff --git a/400_xowa/src/gplx/xowa/wikis/tdbs/hives/Xob_hive_mgr.java b/400_xowa/src/gplx/xowa/wikis/tdbs/hives/Xob_hive_mgr.java index c7b460b77..746cb2aa1 100644 --- a/400_xowa/src/gplx/xowa/wikis/tdbs/hives/Xob_hive_mgr.java +++ b/400_xowa/src/gplx/xowa/wikis/tdbs/hives/Xob_hive_mgr.java @@ -18,7 +18,7 @@ along with this program. If not, see . package gplx.xowa.wikis.tdbs.hives; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; import gplx.xowa.wikis.tdbs.*; import gplx.core.ios.zips.*; import gplx.xowa.wikis.data.tbls.*; -import gplx.xowa.bldrs.sqls.*; +import gplx.xowa.bldrs.sql_dumps.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.specials.*; import gplx.xowa.specials.allPages.*; import gplx.xowa.wikis.tdbs.xdats.*; diff --git a/400_xowa/src/gplx/xowa/wikis/xwikis/bldrs/Xow_xwiki_itm_bldr.java b/400_xowa/src/gplx/xowa/wikis/xwikis/bldrs/Xow_xwiki_itm_bldr.java index 28a05c613..89227a931 100644 --- a/400_xowa/src/gplx/xowa/wikis/xwikis/bldrs/Xow_xwiki_itm_bldr.java +++ b/400_xowa/src/gplx/xowa/wikis/xwikis/bldrs/Xow_xwiki_itm_bldr.java @@ -23,7 +23,7 @@ import gplx.xowa.wikis.domains.*; public class Xow_xwiki_itm_bldr { private final Bry_bfr tmp_bfr = Bry_bfr_.New(); private final Gfo_url_parser url_parser = new Gfo_url_parser(); private final Gfo_url url = new Gfo_url(); - public Xow_xwiki_itm Bld_mw(Xow_domain_itm cur_domain, byte[] key, byte[] mw_url, byte[] domain_name) {return Bld_xo(cur_domain, key, Xoa_gfs_php_mgr.Xto_gfs(tmp_bfr, mw_url), domain_name);} // EX: "//commons.wikimedia.org/wiki/Category:$1" -> "//commons.wikimedia.org/wiki/Category:~{0}" + public Xow_xwiki_itm Bld_mw(Xow_domain_itm cur_domain, byte[] key, byte[] mw_url, byte[] domain_name) {return Bld_xo(cur_domain, key, Gfs_php_converter.To_gfs(tmp_bfr, mw_url), domain_name);} // EX: "//commons.wikimedia.org/wiki/Category:$1" -> "//commons.wikimedia.org/wiki/Category:~{0}" public Xow_xwiki_itm Bld_xo(Xow_domain_itm cur_domain, byte[] key, byte[] xo_url, byte[] domain_name) { byte[] domain_bry = Xow_xwiki_mgr.Get_domain_from_url(url_parser, url, xo_url); Xow_domain_itm domain_itm = Xow_domain_itm_.parse(domain_bry); diff --git a/400_xowa/src/gplx/xowa/xtns/dynamicPageList/Dpl_xnde.java b/400_xowa/src/gplx/xowa/xtns/dynamicPageList/Dpl_xnde.java index 2cf7e9e1b..4b7d3a4b4 100644 --- a/400_xowa/src/gplx/xowa/xtns/dynamicPageList/Dpl_xnde.java +++ b/400_xowa/src/gplx/xowa/xtns/dynamicPageList/Dpl_xnde.java @@ -89,12 +89,12 @@ class Dpl_page_finder { List_adp del_list = List_adp_.New(); int ns_filter = itm.Ns_filter(); Ordered_hash exclude_pages = Ordered_hash_.New(); - Find_excludes(exclude_pages, load_mgr, tmp_page, tmp_id, itm.Ctg_excludes()); + Find_excludes(exclude_pages, wiki, load_mgr, tmp_page, tmp_id, itm.Ctg_excludes()); for (int i = 0; i < includes_len; i++) { // loop over includes byte[] include = (byte[])includes.Get_at(i); cur_regy.Clear(); del_list.Clear(); - Find_pages_in_ctg(cur_regy, load_mgr, tmp_page, tmp_id, include); + Find_pages_in_ctg(cur_regy, wiki, load_mgr, tmp_page, tmp_id, include); Del_old_pages_not_in_cur(i, tmp_id, old_regy, cur_regy, del_list); Add_cur_pages_also_in_old(i, tmp_id, old_regy, cur_regy, new_regy, exclude_pages, ns_filter); old_regy = new_regy; @@ -108,30 +108,35 @@ class Dpl_page_finder { wiki.Db_mgr().Load_mgr().Load_by_ids(Cancelable_.Never, rv, 0, pages_len); rv.Sort_by(Xowd_page_itm_sorter.IdAsc); } - private static void Find_excludes(Ordered_hash exclude_pages, Xodb_load_mgr load_mgr, Xowd_page_itm tmp_page, Int_obj_ref tmp_id, List_adp exclude_ctgs) { + private static void Find_excludes(Ordered_hash exclude_pages, Xowe_wiki wiki, Xodb_load_mgr load_mgr, Xowd_page_itm tmp_page, Int_obj_ref tmp_id, List_adp exclude_ctgs) { if (exclude_ctgs == null) return; int exclude_ctgs_len = exclude_ctgs.Count(); for (int i = 0; i < exclude_ctgs_len; i++) { byte[] exclude_ctg = (byte[])exclude_ctgs.Get_at(i); - Find_pages_in_ctg(exclude_pages, load_mgr, tmp_page, tmp_id, exclude_ctg); + Find_pages_in_ctg(exclude_pages, wiki, load_mgr, tmp_page, tmp_id, exclude_ctg); } } - private static void Find_pages_in_ctg(Ordered_hash list, Xodb_load_mgr load_mgr, Xowd_page_itm tmp_page, Int_obj_ref tmp_id, byte[] ctg_ttl) { - Xoctg_catpage_ctg ctg = new Xoctg_catpage_ctg(ctg_ttl); - load_mgr.Load_ctg_v1(ctg, ctg_ttl); + private static void Find_pages_in_ctg(Ordered_hash rv, Xowe_wiki wiki, Xodb_load_mgr load_mgr, Xowd_page_itm tmp_page, Int_obj_ref tmp_id, byte[] ctg_ttl) { + Xoctg_catpage_ctg ctg = wiki.Html_mgr().Catpage_mgr().Get_or_load_or_null(wiki, wiki.Ttl_parse(gplx.xowa.wikis.nss.Xow_ns_.Tid__category, ctg_ttl)); + if (ctg == null) return; - for (byte ctg_tid = 0; ctg_tid < Xoa_ctg_mgr.Tid__max; ctg_tid++) { - Xoctg_catpage_grp ctg_mgr = ctg.Grp_by_tid(ctg_tid); if (ctg_mgr == null) continue; - int itms_len = ctg_mgr.Total(); - for (int i = 0; i < itms_len; i++) { - Xoctg_catpage_itm ctg_itm = ctg_mgr.Itms()[i]; - int ctg_itm_id = ctg_itm.Page_id(); - if (list.Has(tmp_id.Val_(ctg_itm_id))) continue; - list.Add(Int_obj_ref.New(ctg_itm_id), ctg_itm); -// if (ctg_tid == Xoa_ctg_mgr.Tid_subc) { // recurse subcategories -// load_mgr.Load_by_id(tmp_page, ctg_itm_id); -// Find_pages_in_ctg(list, load_mgr, tmp_page, tmp_id, tmp_page.Ttl_wo_ns()); -// } + // loop grps to get grp + for (byte ctg_tid = 0; ctg_tid < Xoa_ctg_mgr.Tid___max; ++ctg_tid) { + Xoctg_catpage_grp ctg_grp = ctg.Grp_by_tid(ctg_tid); + int itms_len = ctg_grp.Itms__len(); + + // loop itms in grp and add to hash + for (int i = 0; i < itms_len; ++i) { + Xoctg_catpage_itm ctg_itm = ctg_grp.Itms__get_at(i); + int itm_page_id = ctg_itm.Page_id(); + if (rv.Has(tmp_id.Val_(itm_page_id))) continue; + rv.Add(Int_obj_ref.New(itm_page_id), ctg_itm); + + // DELETE: recurse subcategories; PAGE:en.b:XML DATE:2016-09-18 + // if (ctg_tid == Xoa_ctg_mgr.Tid__subc) { + // load_mgr.Load_by_id(tmp_page, itm_page_id); + // Find_pages_in_ctg(rv, wiki, load_mgr, tmp_page, tmp_id, tmp_page.Ttl_page_db()); + // } } } } diff --git a/400_xowa/src/gplx/xowa/xtns/dynamicPageList/Dpl_xnde_tst.java b/400_xowa/src/gplx/xowa/xtns/dynamicPageList/Dpl_xnde_tst.java index cb093570f..ca0e0085e 100644 --- a/400_xowa/src/gplx/xowa/xtns/dynamicPageList/Dpl_xnde_tst.java +++ b/400_xowa/src/gplx/xowa/xtns/dynamicPageList/Dpl_xnde_tst.java @@ -17,220 +17,261 @@ along with this program. If not, see . */ package gplx.xowa.xtns.dynamicPageList; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import org.junit.*; import gplx.core.strings.*; import gplx.xowa.apps.cfgs.*; import gplx.xowa.wikis.nss.*; import gplx.langs.htmls.*; +import gplx.xowa.addons.wikis.ctgs.htmls.catpages.*; import gplx.xowa.addons.wikis.ctgs.htmls.catpages.doms.*; public class Dpl_xnde_tst { private final Dpl_xnde_fxt fxt = new Dpl_xnde_fxt(); @Before public void init() {fxt.Clear();} @Test public void Ctg() { - fxt.Ctg_create("Ctg_0", "B", "A"); - fxt.Ul_pages("category=Ctg_0", fxt.Ul(Itm_html_null, "B", "A")); + fxt.Init__create_ctg("Ctg_0", "B", "A"); + fxt.Exec__parse("category=Ctg_0"); + fxt.Test__html(fxt.Make__html__itms__null("B", "A")); } @Test public void Ctg_multiple() { - fxt.Ctg_create_pages("Ctg_0", Dpl_page_mok.new_(101, "A"), Dpl_page_mok.new_(102, "B")); - fxt.Ctg_create_pages("Ctg_1", Dpl_page_mok.new_(101, "A")); - fxt.Ul_pages(String_.Concat_lines_nl + fxt.Init__create_ctg_pages("Ctg_0", Dpl_page_mok.new_(101, "A"), Dpl_page_mok.new_(102, "B")); + fxt.Init__create_ctg_pages("Ctg_1", Dpl_page_mok.new_(101, "A")); + fxt.Exec__parse ( "" , "category=Ctg_0" , "category=Ctg_1" , "" - ), fxt.Ul(Itm_html_null, "A")); + ); + fxt.Test__html(fxt.Make__html__itms__null("A")); } @Test public void Ctg_multiple_none() { // PURPOSE: page must be in both categories - fxt.Ctg_create("Ctg_0", "A"); - fxt.Ctg_create("Ctg_1", "B"); - fxt.Ul_pages(String_.Concat_lines_nl + fxt.Init__create_ctg("Ctg_0", "A"); + fxt.Init__create_ctg("Ctg_1", "B"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "category=Ctg_1" , "" - ), "No pages meet these criteria."); + ); + fxt.Test__html("No pages meet these criteria."); } @Test public void Notcategory() { - fxt.Ctg_create_pages("Ctg_0", Dpl_page_mok.new_(101, "A"), Dpl_page_mok.new_(102, "B")); - fxt.Ctg_create_pages("Ctg_1", Dpl_page_mok.new_(101, "A")); - fxt.Ul_pages(String_.Concat_lines_nl + fxt.Init__create_ctg_pages("Ctg_0", Dpl_page_mok.new_(101, "A"), Dpl_page_mok.new_(102, "B")); + fxt.Init__create_ctg_pages("Ctg_1", Dpl_page_mok.new_(101, "A")); + fxt.Exec__parse ( "" , "category=Ctg_0" , "notcategory=Ctg_1" , "" - ), fxt.Ul(Itm_html_null, "B")); + ); + fxt.Test__html(fxt.Make__html__itms__null("B")); } @Test public void Ctg_ascending() { - fxt.Ctg_create("Ctg_0", "B", "A"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "B", "A"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "order=ascending" - , ""), fxt.Ul(Itm_html_null, "A", "B")); + , "" + ); + fxt.Test__html(fxt.Make__html__itms__null("A", "B")); } @Test public void Ctg_ws() { - fxt.Ctg_create("Ctg_0", "B", "A"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "B", "A"); + fxt.Exec__parse ( "" , " category = Ctg_0 " , " order = ascending " - , ""), fxt.Ul(Itm_html_null, "A", "B")); + , "" + ); + fxt.Test__html(fxt.Make__html__itms__null("A", "B")); } @Test public void Ctg_descending() { - fxt.Ctg_create("Ctg_0", "A", "B"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "A", "B"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "order=descending" - , ""), fxt.Ul(Itm_html_null, "B", "A")); + , "" + ); + fxt.Test__html(fxt.Make__html__itms__null("B", "A")); } @Test public void Nofollow() { - fxt.Ctg_create("Ctg_0", "A", "B"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "A", "B"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "nofollow=true" - , ""), fxt.Ul(" rel=\"nofollow\"", "A", "B")); + , "" + ); + fxt.Test__html(fxt.Make__html(" rel=\"nofollow\"", "A", "B")); } @Test public void Invalid_key() { - fxt.Ctg_create("Ctg_0", "A", "B"); - fxt.Warns("dynamic_page_list:unknown_key: page=Test page key=invalid_key"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "A", "B"); + fxt.Init__warns("dynamic_page_list:unknown_key: page=Test page key=invalid_key"); + fxt.Exec__parse ( "" , "invalid_key=invalid_val" , "category=Ctg_0" - , ""), fxt.Ul(Itm_html_null, "A", "B")); + , "" + ); + fxt.Test__html(fxt.Make__html__itms__null("A", "B")); } @Test public void No_results() { - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Exec__parse ( "" , "category=Ctg_0" - , ""), "No pages meet these criteria."); + , "" + ); + fxt.Test__html("No pages meet these criteria."); } @Test public void Suppress_errors() { - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Exec__parse ( "" , "category=Ctg_0" , "suppresserrors=true" - , ""), ""); + , "" + ); + fxt.Test__html(""); } @Test public void Count() { - fxt.Ctg_create("Ctg_0", "A", "B", "C"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "A", "B", "C"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "count=2" - , ""), fxt.Ul(Itm_html_null, "A", "B")); + , "" + ); + fxt.Test__html(fxt.Make__html__itms__null("A", "B")); } @Test public void Offset() { - fxt.Ctg_create("Ctg_0", "A", "B", "C"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "A", "B", "C"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "offset=2" , "count=2" - , ""), fxt.Ul(Itm_html_null, "C")); + , "" + ); + fxt.Test__html(fxt.Make__html__itms__null("C")); } @Test public void Ns() { - fxt.Ctg_create("Ctg_0", "Talk:A B", "B"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "Talk:A B", "B"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "namespace=Talk" - , ""), Gfh_utl.Replace_apos_concat_lines + , "" + ); + fxt.Test__html(Gfh_utl.Replace_apos_concat_lines ( "" )); } @Test public void Show_ns() { - fxt.Ctg_create("Ctg_0", "Talk:A"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "Talk:A"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "shownamespace=true" - , ""), Gfh_utl.Replace_apos_concat_lines + , "" + ); + fxt.Test__html(Gfh_utl.Replace_apos_concat_lines ( "" )); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + + fxt.Exec__parse ( "" , "category=Ctg_0" , "shownamespace=false" - , ""), Gfh_utl.Replace_apos_concat_lines + , "" + ); + fxt.Test__html(Gfh_utl.Replace_apos_concat_lines ( "
      " , "
    • A
      • " , "
    " )); } @Test public void Comment() { // PURPOSE: comment should be ignored; en.n:Portal:Federally_Administered_Tribal_Areas; DATE:2014-01-18 - fxt.Ctg_create("Ctg_0", "B", "A"); - fxt.Ul_pages(String_.Concat_lines_nl + fxt.Init__create_ctg("Ctg_0", "B", "A"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "" , "" - ), fxt.Ul(Itm_html_null, "B", "A")); + ); + fxt.Test__html(fxt.Make__html__itms__null("B", "A")); } @Test public void Error_skip_line() { // PURPOSE: error should skip rest of line; was failing with array out of bounds; en.n:Portal:Austria/Wikipedia; DATE:2014-01-18 - fxt.Warns("dynamic_page_list:unknown_key: page=Test page key=Ctg_0 order"); // ignore warning message - fxt.Ul_pages(" category=Ctg_0 order=descending", "No pages meet these criteria."); + fxt.Init__warns("dynamic_page_list:unknown_key: page=Test page key=Ctg_0 order"); // ignore warning message + fxt.Exec__parse(" category=Ctg_0 order=descending"); + fxt.Test__html("No pages meet these criteria."); } @Test public void Atr_has_template() { // PURPOSE: attribute also has template; DATE:2014-01-31 - fxt.Ctg_create("Test_page", "B", "A"); - fxt.Ul_pages(String_.Concat_lines_nl + fxt.Init__create_ctg("Test_page", "B", "A"); + fxt.Exec__parse ( "" , "category={{PAGENAME}}" , "" - ), fxt.Ul(Itm_html_null, "B", "A")); + ); + fxt.Test__html(fxt.Make__html__itms__null("B", "A")); } @Test public void Err_page_ns_doesnt_exist() {// PURPOSE: check that is not enabled if wiki does not have Page / Index ns; PAGE:fr.w:Wikipedia:Le_Bistro/novembre_2006 DATE:2014-11-28 - fxt.Wiki().Ns_mgr().Clear().Init(); // call .Clear() to remove ns for Page / Index + // reset categories to (a) remove ns for Page / Index; (b) add back Category (else null error) + fxt.Wiki().Ns_mgr().Clear(); + fxt.Wiki().Ns_mgr().Add_new(14, "Category").Init(); + fxt.Wiki().Cfg_parser().Xtns().Itm_pages().Reset(); // must reset to clear cached valid ns_page from previous tests fxt.Fxt().Test_parse_page_wiki_str("category=a", "No pages meet these criteria."); fxt.Wiki().Cfg_parser().Xtns().Itm_pages().Reset(); // must reset to clear cached invalid ns_page for next tests + + // reset categories for rest of tests fxt.Wiki().Ns_mgr().Add_new(0, "").Init(); // call .Clear() to remove ns for Page / Index } @Test public void Ordermethod__invalid() { // PURPOSE: do not fail if ordermethod is invalid; PAGE:sr.d:Викиречник:Википројекат_1001_арапска_реч/Списак_уноса; DATE:2015-10-16 - fxt.Ctg_create("Ctg_0", "A", "B", "C"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "A", "B", "C"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "ordermethod=ascending" // should not throw error , "ordermethod=sortkey" - , "") - , fxt.Ul(Itm_html_null, "A", "B", "C")); + , "" + ); + fxt.Test__html(fxt.Make__html__itms__null("A", "B", "C")); } @Test public void Encode_spaces() {// PURPOSE:encode spaces in href; PAGE:en.q:Wikiquote:Speedy_deletions DATE:2016-01-19 - fxt.Ctg_create("Ctg_0", "A B"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "A B"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "nofollow=true" - , ""), Gfh_utl.Replace_apos_concat_lines + , "" + ); + fxt.Test__html(Gfh_utl.Replace_apos_concat_lines ( "
      " , "
    • A B
      • " // "/wiki/A_B" not "/wiki/A B" , "
    " )); } @Test public void Encode_quotes() {// PURPOSE:encode quotes; PAGE:en.b:Wikibooks:Alphabetical_classification/All_Books; DATE:2016-01-21 - fxt.Ctg_create("Ctg_0", "A\"B"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__create_ctg("Ctg_0", "A\"B"); + fxt.Exec__parse ( "" , "category=Ctg_0" , "nofollow=true" - , ""), Gfh_utl.Replace_apos_concat_lines + , "" + ); + fxt.Test__html(Gfh_utl.Replace_apos_concat_lines ( "
      " , "
    • A"B
      • " // "/wiki/A_B" not "/wiki/A B" , "
    " )); } @Test public void Err__bad_key_causes_out_of_bound() { // PURPOSE: bad key causes out of bounds error; PAGE:de.n:Portal:Brandenburg DATE:2016-04-21 - fxt.Warns("dynamic_page_list:unknown_key: page=Test page key=category", "dynamic_page_list:unknown_key: page=Test page key=category"); - fxt.Ul_pages(String_.Concat_lines_nl_skip_last + fxt.Init__warns("dynamic_page_list:unknown_key: page=Test page key=category", "dynamic_page_list:unknown_key: page=Test page key=category"); + fxt.Exec__parse ( "" , "category=Aa=b c=d" , "category=B" - ) - , "No pages meet these criteria." ); + fxt.Test__html("No pages meet these criteria."); } - private static final String Itm_html_null = null; } class Dpl_page_mok { public Dpl_page_mok(int id, String ttl) {this.id = id; this.ttl = ttl;} @@ -240,14 +281,17 @@ class Dpl_page_mok { } class Dpl_xnde_fxt { private final Xop_fxt fxt = new Xop_fxt(); + private final Bry_bfr bfr = Bry_bfr_.New(); + private String exec__raw; private int next_id; public void Clear() { next_id = 100; fxt.Reset(); - warns = String_.Ary_empty; + this.exec__raw = String_.Empty; + this.expd_warns = String_.Ary_empty; fxt.App().Usr_dlg().Gui_wkr().Clear(); fxt.Wiki().Hive_mgr().Clear(); - fxt.Wiki().Db_mgr().Save_mgr().Clear(); // NOTE: must clear to reset next_id to 0; Ctg_create assumes clean slate from 0 + fxt.Wiki().Db_mgr().Save_mgr().Clear(); // NOTE: must clear to reset next_id to 0; Init__create_ctg assumes clean slate from 0 fxt.Wiki().Xtn_mgr().Xtn_proofread().Enabled_y_(); fxt.Wiki().Db_mgr().Load_mgr().Clear(); // must clear; otherwise fails b/c files get deleted, but wiki.data_mgr caches the Xowd_regy_mgr (the .reg file) in memory; fxt.Wiki().Ns_mgr().Add_new(Xowc_xtn_pages.Ns_page_id_default, "Page").Add_new(Xowc_xtn_pages.Ns_index_id_default, "Index").Init(); @@ -255,46 +299,50 @@ class Dpl_xnde_fxt { } public Xowe_wiki Wiki() {return fxt.Wiki();} public Xop_fxt Fxt() {return fxt;} - public void Warns(String... v) {warns = v;} private String[] warns; - public void Page_create(String page) {fxt.Init_page_create(page);} - public void Ctg_create(String ctg, String... ttls) { + public void Init__warns(String... v) {expd_warns = v;} private String[] expd_warns; + public void Init__create_ctg(String ctg, String... ttls) { int len = ttls.length; Dpl_page_mok[] ary = new Dpl_page_mok[len]; for (int i = 0; i < len; i++) ary[i] = Dpl_page_mok.new_(++next_id, ttls[i]); - Ctg_create_pages(ctg, ary); + Init__create_ctg_pages(ctg, ary); } - public void Ctg_create_pages(String ctg, Dpl_page_mok... pages) { + public void Init__create_ctg_pages(String ctg, Dpl_page_mok... pages) { + Xoctg_catpage_ctg captage_ctg = new Xoctg_catpage_ctg(Bry_.new_u8(ctg)); int pages_len = pages.length; - int[] page_ids = new int[pages_len]; for (int i = 0; i < pages_len; i++) { Dpl_page_mok page = pages[i]; int id = page.Id(); - String ttl = page.Ttl(); - Xoa_ttl page_ttl = Xoa_ttl.Parse(fxt.Wiki(), Bry_.new_u8(ttl)); - Xoae_page page_obj = fxt.Wiki().Data_mgr().Load_page_by_ttl(page_ttl); + String ttl_str = page.Ttl(); + Xoa_ttl ttl = fxt.Wiki().Ttl_parse(Bry_.new_u8(ttl_str)); + Xoae_page page_obj = fxt.Wiki().Data_mgr().Load_page_by_ttl(ttl); if (page_obj.Db().Page().Exists_n()) { - fxt.Init_page_create(ttl); - fxt.Init_id_create (id, 0, 0, false, 5, Xow_ns_.Tid__main, ttl); + fxt.Init_page_create(ttl_str, ttl_str); + fxt.Init_id_create (id, 0, 0, false, 5, Xow_ns_.Tid__main, ttl_str); } - page_ids[i] = id; + Xoctg_catpage_itm captage_itm = new Xoctg_catpage_itm(page.Id(), ttl, ttl.Page_db()); + captage_ctg.Pages().Itms__add(captage_itm); } - fxt.Init_ctg_create(ctg, page_ids); + captage_ctg.Make_itms(); + Xoctg_catpage_mgr catpage_mgr = fxt.Wiki().Html_mgr().Catpage_mgr(); + catpage_mgr.Cache__add(Bry_.new_u8("Category:" + ctg), captage_ctg); } - public String Ul(String itm_html, String... pages) { - bfr.Add("
      ").Add_char_nl(); + public String Make__html__itms__null(String... pages) {return this.Make__html(null, pages);} + public String Make__html(String itm_html, String... pages) { + bfr.Add_str_a7("").Add_char_nl(); + bfr.Add_str_a7("
    ").Add_byte_nl(); return bfr.To_str_and_clear(); } - public void Ul_pages(String raw, String expd) { - fxt.Test_parse_page_wiki_str(raw, expd); - fxt.tst_Warn(warns); - } String_bldr bfr = String_bldr_.new_(); + public void Exec__parse(String... raw_lines) {this.exec__raw = String_.Concat_lines_nl_skip_last(raw_lines);} + public void Test__html(String expd) { + fxt.Test_parse_page_wiki_str(exec__raw, expd); + fxt.tst_Warn(expd_warns); + } } diff --git a/400_xowa/src/gplx/xowa/xtns/flaggedRevs/scribunto/Flagged_revs_lib.java b/400_xowa/src/gplx/xowa/xtns/flaggedRevs/scribunto/Flagged_revs_lib.java index ce6973cac..bcbc0c6ee 100644 --- a/400_xowa/src/gplx/xowa/xtns/flaggedRevs/scribunto/Flagged_revs_lib.java +++ b/400_xowa/src/gplx/xowa/xtns/flaggedRevs/scribunto/Flagged_revs_lib.java @@ -18,6 +18,7 @@ along with this program. If not, see . package gplx.xowa.xtns.flaggedRevs.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.flaggedRevs.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.libs.*; import gplx.xowa.xtns.pfuncs.exprs.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Flagged_revs_lib implements Scrib_lib { private Scrib_core core; public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; diff --git a/400_xowa/src/gplx/xowa/xtns/pfuncs/scribunto/Pfunc_scrib_lib.java b/400_xowa/src/gplx/xowa/xtns/pfuncs/scribunto/Pfunc_scrib_lib.java index cdc8b8f92..ad3a5e0d2 100644 --- a/400_xowa/src/gplx/xowa/xtns/pfuncs/scribunto/Pfunc_scrib_lib.java +++ b/400_xowa/src/gplx/xowa/xtns/pfuncs/scribunto/Pfunc_scrib_lib.java @@ -18,6 +18,7 @@ along with this program. If not, see . package gplx.xowa.xtns.pfuncs.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.pfuncs.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.libs.*; import gplx.xowa.xtns.pfuncs.exprs.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Pfunc_scrib_lib implements Scrib_lib { private Scrib_core core; public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; diff --git a/400_xowa/src/gplx/xowa/xtns/pfuncs/ttls/Pfunc_filepath.java b/400_xowa/src/gplx/xowa/xtns/pfuncs/ttls/Pfunc_filepath.java index 3c7f5bea2..bf975b8ab 100644 --- a/400_xowa/src/gplx/xowa/xtns/pfuncs/ttls/Pfunc_filepath.java +++ b/400_xowa/src/gplx/xowa/xtns/pfuncs/ttls/Pfunc_filepath.java @@ -19,7 +19,7 @@ package gplx.xowa.xtns.pfuncs.ttls; import gplx.*; import gplx.xowa.*; import gp import gplx.core.envs.*; import gplx.core.caches.*; import gplx.xowa.langs.*; import gplx.xowa.langs.kwds.*; import gplx.xowa.wikis.nss.*; -import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.origs.*; +import gplx.xowa.files.*; import gplx.xowa.files.repos.*; import gplx.xowa.files.origs.*; import gplx.xowa.files.imgs.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*; public class Pfunc_filepath extends Pf_func_base { @Override public boolean Func_require_colon_arg() {return true;} @@ -39,7 +39,7 @@ public class Pfunc_filepath extends Pf_func_base { Xof_repo_itm trg_repo = wiki.File_mgr().Repo_mgr().Repos_get_at(tmp_rslt.Repo_idx()).Trg(); Xof_xfer_itm xfer_itm = ctx.Tmp_mgr().Xfer_itm(); xfer_itm.Orig_ttl_and_redirect_(ttl_bry, Bry_.Empty); // redirect is empty b/c Get_page does all redirect lookups - byte[] url = wiki.Parser_mgr().Url_bldr().Init_for_trg_html(Xof_repo_itm_.Mode_orig, trg_repo, ttl_bry, xfer_itm.Orig_ttl_md5(), xfer_itm.Orig_ext(), Xof_img_size.Size__neg1, Xof_lnki_time.Null, Xof_lnki_page.Null).Xto_bry(); + byte[] url = wiki.Parser_mgr().Url_bldr().Init_for_trg_html(trg_repo, Xof_img_mode_.Tid__orig, ttl_bry, xfer_itm.Orig_ttl_md5(), xfer_itm.Orig_ext(), Xof_img_size.Size__neg1, Xof_lnki_time.Null, Xof_lnki_page.Null).Xto_bry(); bfr.Add(url); } public static Pfunc_filepath_itm Load_page(Xowe_wiki wiki, Xoa_ttl ttl) { diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_core.java b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_core.java index 35cc63806..13f0e90bc 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_core.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_core.java @@ -20,6 +20,7 @@ import gplx.core.envs.*; import gplx.xowa.langs.*; import gplx.xowa.xtns.scribunto.libs.*; import gplx.xowa.xtns.scribunto.engines.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_core { private Hash_adp_bry mods = Hash_adp_bry.cs(); private int expensive_function_count; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func_fxt.java b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func_fxt.java index 9d3480b9f..043743939 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func_fxt.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func_fxt.java @@ -16,8 +16,8 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.xtns.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; -import gplx.xowa.xtns.scribunto.libs.*; import gplx.xowa.xtns.scribunto.engines.process.*; -import gplx.xowa.parsers.tmpls.*; +import gplx.xowa.xtns.scribunto.libs.*; import gplx.xowa.xtns.scribunto.engines.process.*; import gplx.xowa.xtns.scribunto.procs.*; +import gplx.xowa.parsers.tmpls.*; public class Scrib_invoke_func_fxt { private Xop_fxt fxt; Bry_bfr tmp_bfr = Bry_bfr_.Reset(255); Scrib_core core; Process_server_mock server; Scrib_lua_rsp_bldr rsp_bldr = new Scrib_lua_rsp_bldr(); public Xop_fxt Parser_fxt() {return fxt;} diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_lib.java b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_lib.java index 14caa8f00..ea443f668 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_lib.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_lib.java @@ -16,6 +16,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.xtns.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; +import gplx.xowa.xtns.scribunto.procs.*; public interface Scrib_lib { Scrib_proc_mgr Procs(); Scrib_lib Init(); diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/engines/luaj/Luaj_engine.java b/400_xowa/src/gplx/xowa/xtns/scribunto/engines/luaj/Luaj_engine.java index 834e76383..8e999f0af 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/engines/luaj/Luaj_engine.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/engines/luaj/Luaj_engine.java @@ -16,6 +16,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.xtns.scribunto.engines.luaj; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.engines.*; +import gplx.xowa.xtns.scribunto.procs.*; import org.luaj.vm2.*; import org.luaj.vm2.lib.*; import org.luaj.vm2.lib.jse.*; import gplx.xowa.xtns.scribunto.engines.process.*; public class Luaj_engine implements Scrib_engine { diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/engines/mocks/Mock_scrib_fxt.java b/400_xowa/src/gplx/xowa/xtns/scribunto/engines/mocks/Mock_scrib_fxt.java index a154d19e9..c9ab71f57 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/engines/mocks/Mock_scrib_fxt.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/engines/mocks/Mock_scrib_fxt.java @@ -17,6 +17,7 @@ along with this program. If not, see . */ package gplx.xowa.xtns.scribunto.engines.mocks; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.engines.*; import gplx.xowa.parsers.tmpls.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Mock_scrib_fxt { private final Mock_engine engine = new Mock_engine(); private final Mock_server server = new Mock_server(); diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/engines/process/Process_engine.java b/400_xowa/src/gplx/xowa/xtns/scribunto/engines/process/Process_engine.java index d4b943624..6ee450196 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/engines/process/Process_engine.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/engines/process/Process_engine.java @@ -17,7 +17,7 @@ along with this program. If not, see . */ package gplx.xowa.xtns.scribunto.engines.process; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.engines.*; import gplx.core.encoders.*; -import gplx.xowa.xtns.scribunto.libs.*; +import gplx.xowa.xtns.scribunto.libs.*; import gplx.xowa.xtns.scribunto.procs.*; public class Process_engine implements Scrib_engine { private Scrib_core core; private Xoae_app app; private Scrib_xtn_mgr scrib_opts; private Process_recv_msg rsp = new Process_recv_msg(); private Process_send_wtr msg_encoder; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_html.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_html.java index 2e0091786..9ccc8411f 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_html.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_html.java @@ -16,6 +16,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_html implements Scrib_lib { public Scrib_lib_html(Scrib_core core) {} public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_language.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_language.java index adf89fa52..552ed3575 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_language.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_language.java @@ -18,6 +18,7 @@ along with this program. If not, see . package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.langs.*; import gplx.xowa.xtns.pfuncs.times.*; import gplx.xowa.langs.numbers.*; import gplx.xowa.xtns.pfuncs.numbers.*; import gplx.xowa.langs.durations.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_language implements Scrib_lib { public Scrib_lib_language(Scrib_core core) {this.core = core;} private Scrib_core core; public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_message.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_message.java index 5e2c90c29..0e3037de7 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_message.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_message.java @@ -20,6 +20,7 @@ import gplx.core.primitives.*; import gplx.langs.htmls.*; import gplx.xowa.apps.gfs.*; import gplx.xowa.langs.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.parsers.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_message implements Scrib_lib { public Scrib_lib_message(Scrib_core core) {this.core = core;} private Scrib_core core; public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; @@ -105,7 +106,7 @@ class Scrib_lib_message_data { if (raw_msg_key != null) { Xol_msg_itm raw_msg_itm = new Xol_msg_itm(-1, Bry_.Empty); Bry_bfr tmp_bfr = Bry_bfr_.New(); // wiki.Utl__bfr_mkr().Get_b512(); - byte[] raw_msg_val = Xoa_gfs_php_mgr.Xto_gfs(tmp_bfr, raw_msg_key); + byte[] raw_msg_val = Gfs_php_converter.To_gfs(tmp_bfr, raw_msg_key); Xol_msg_itm_.update_val_(raw_msg_itm, raw_msg_val); byte[] raw_msg_rv = wiki.Msg_mgr().Val_by_itm(tmp_bfr, raw_msg_itm, args); tmp_bfr.Mkr_rls(); diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw.java index d49cdeb81..8c733a89e 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_mw.java @@ -19,6 +19,7 @@ package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import import gplx.core.primitives.*; import gplx.core.envs.*; import gplx.core.errs.*; import gplx.xowa.langs.*; import gplx.xowa.langs.funcs.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.tmpls.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_mw implements Scrib_lib { private Scrib_core core; private Scrib_fsys_mgr fsys_mgr; public Scrib_lib_mw(Scrib_core core) {this.core = core; this.fsys_mgr = core.Fsys_mgr();} diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_site.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_site.java index 8d0900b81..8d2155796 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_site.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_site.java @@ -19,6 +19,7 @@ package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import import gplx.xowa.langs.*; import gplx.xowa.langs.msgs.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.wikis.metas.*; import gplx.xowa.wikis.data.site_stats.*; import gplx.xowa.wikis.xwikis.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_site implements Scrib_lib { public Scrib_lib_site(Scrib_core core) {this.core = core;} private final Scrib_core core; public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_text.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_text.java index 0a9c824dd..a7d4ffa41 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_text.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_text.java @@ -18,6 +18,7 @@ along with this program. If not, see . package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; import gplx.core.bits.*; import gplx.xowa.langs.msgs.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_text implements Scrib_lib { private final Scrib_lib_text__json_util json_util = new Scrib_lib_text__json_util(); private final Scrib_lib_text__reindex_data reindex_data = new Scrib_lib_text__reindex_data(); diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title.java index aa6303306..97ca56055 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title.java @@ -21,6 +21,7 @@ import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.caches.*; import gplx.xowa.xtns.pfuncs.ttls.*; import gplx.xowa.wikis.xwikis.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.files.commons.*; import gplx.xowa.files.origs.*; import gplx.xowa.apps.wms.apis.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_title implements Scrib_lib { public Scrib_lib_title(Scrib_core core) {this.core = core;} private Scrib_core core; public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title_tst.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title_tst.java index 858c2eca6..0bb28f4ac 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title_tst.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_title_tst.java @@ -114,7 +114,7 @@ public class Scrib_lib_title_tst { } private static void Wiki_orig_tbl__insert(Xowe_wiki wiki, String ttl_str, int w, int h) { byte[] ttl_bry = Bry_.new_u8(ttl_str); - wiki.File__orig_mgr().Insert(gplx.xowa.files.repos.Xof_repo_itm_.Repo_remote, ttl_bry, gplx.xowa.files.Xof_ext_.new_by_ttl_(ttl_bry).Id(), w, h, Bry_.Empty); + wiki.File__orig_mgr().Insert(gplx.xowa.files.repos.Xof_repo_tid_.Tid__remote, ttl_bry, gplx.xowa.files.Xof_ext_.new_by_ttl_(ttl_bry).Id(), w, h, Bry_.Empty); } private static String ttl_fast(int ns_id, String ns_str, String ttl) {return ttl_fast(ns_id, ns_str, ttl, "", "", ttl);} private static String ttl_fast(int ns_id, String ns_str, String ttl, String anchor) {return ttl_fast(ns_id, ns_str, ttl, anchor, "", ttl);} diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_uri.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_uri.java index 43599cbf2..5fef919c5 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_uri.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_uri.java @@ -19,6 +19,7 @@ package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import import gplx.xowa.xtns.pfuncs.ttls.*; import gplx.xowa.wikis.nss.*; import gplx.xowa.parsers.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_uri implements Scrib_lib { public Scrib_lib_uri(Scrib_core core) {this.core = core;} private Scrib_core core; public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_ustring.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_ustring.java index 2b994ec10..e678abc5e 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_ustring.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_ustring.java @@ -18,6 +18,7 @@ along with this program. If not, see . package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; import gplx.langs.regxs.*; import gplx.core.intls.*; import gplx.xowa.parsers.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_ustring implements Scrib_lib { private final String_surrogate_utl surrogate_utl = new String_surrogate_utl(); public Scrib_lib_ustring(Scrib_core core) {this.core = core;} private Scrib_core core; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_wikibase.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_wikibase.java index d0bba723e..ccdae5c29 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_wikibase.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_wikibase.java @@ -18,6 +18,7 @@ along with this program. If not, see . package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; import gplx.langs.jsons.*; import gplx.xowa.xtns.wbases.*; import gplx.xowa.xtns.wbases.parsers.*; import gplx.xowa.wikis.domains.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_wikibase implements Scrib_lib { public Scrib_lib_wikibase(Scrib_core core) {this.core = core;} private Scrib_core core; public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_wikibase_entity.java b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_wikibase_entity.java index 642ca1b50..b305661a9 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_wikibase_entity.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/libs/Scrib_lib_wikibase_entity.java @@ -19,6 +19,7 @@ package gplx.xowa.xtns.scribunto.libs; import gplx.*; import gplx.xowa.*; import import gplx.xowa.xtns.wbases.*; import gplx.langs.jsons.*; import gplx.xowa.xtns.wbases.core.*; import gplx.xowa.xtns.wbases.claims.*; +import gplx.xowa.xtns.scribunto.procs.*; public class Scrib_lib_wikibase_entity implements Scrib_lib { public Scrib_lib_wikibase_entity(Scrib_core core) {this.core = core;} private Scrib_core core; public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc.java b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc.java similarity index 89% rename from 400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc.java rename to 400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc.java index eb4b8a2a8..0a5b0b0af 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc.java @@ -15,7 +15,7 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package gplx.xowa.xtns.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; +package gplx.xowa.xtns.scribunto.procs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.libs.*; public class Scrib_proc { private Scrib_lib lib; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc_args.java b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_args.java similarity index 68% rename from 400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc_args.java rename to 400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_args.java index 1dd066f8a..16941dfeb 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc_args.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_args.java @@ -15,52 +15,56 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package gplx.xowa.xtns.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; +package gplx.xowa.xtns.scribunto.procs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; public class Scrib_proc_args { private Keyval[] ary; private int ary_len; - public Scrib_proc_args(Keyval[] v) {Init(v);} - public int Len() {return ary_len;} + public Scrib_proc_args(Keyval[] v) { + // if 0 members, standardize to Keyval_.Ary_empty; could possibly find all callers of new Scrib_arg() and make sure that 0 == empty, but this is easier + int v_len = v.length; + if (v_len == 0) { + ary = Keyval_.Ary_empty; + ary_len = 0; + return; + } + + // find ary_len + int v_max = -1; + for (int i = 0; i < v_len; ++i) { + Keyval kv = v[i]; + int idx = Int_.cast(kv.Key_as_obj()); + if (v_max < idx) v_max = idx; + } + this.ary_len = v_max; + + // keys are in sequential order; EX: [1:a,2:b,3:c] + if (v_max == v_len) + this.ary = v; + // keys are not in sequential order, or there are gaps; EX: [1:a,3:c] + else { + ary = new Keyval[ary_len]; + for (int i = 0; i < v_len; i++) { + Keyval kv = v[i]; + int idx = Int_.cast(kv.Key_as_obj()); + ary[idx - List_adp_.Base1] = kv; + } + } + } + public int Len() {return ary_len;} public Keyval[] Ary() {return ary;} - public boolean Pull_bool(int i) {return Bool_.cast(Get_or_fail(i));} + public Object Pull_obj(int i) {return Get_or_fail(i);} + public boolean Pull_bool(int i) {return Bool_.cast(Get_or_fail(i));} public String Pull_str(int i) {return String_.cast(Get_or_fail(i));} public byte[] Pull_bry(int i) {return Bry_.new_u8(String_.cast(Get_or_fail(i)));} - public Keyval[] Pull_kv_ary_safe(int idx) { // NOTE: must check for null array items; EX:[2:v2] -> [1:null,2:v2]; PAGE:en.d:Category:Nouns_by_language DATE:2016-04-29 - Keyval[] rv = (Keyval[])Get_or_fail(idx); - List_adp list = null; - int rv_len = rv.length; - for (int i = 0; i < rv_len; ++i) { - Keyval kv = rv[i]; - if ( kv.Key_tid() == Type_adp_.Tid__int // luaj will be int - || kv.Key_tid() == Type_adp_.Tid__obj) { // lua will be obj; note that luaj will also have other non-key objects - Object key_obj = kv.Key_as_obj(); - if (key_obj.getClass() == Int_.Cls_ref_type) { // key is int; cast it - int key_int = Int_.cast(kv.Key_as_obj()); - if (key_int != i + 1) { // key_int should match i; if not, then gaps exist; EX:[2:v2] should be 2nd element, not 1st - if (list == null) { - list = List_adp_.New(); - rv = null; - } - for (int j = 0; j < key_int - 1; ++j) // add everything up to key_int as null; EX: [2:v2] -> [1:null] - list.Add(Keyval_.int_(j + List_adp_.Base1, null)); - } - } - } - if (rv == null) - list.Add(kv); - } - if (rv == null) - rv = (Keyval[])list.To_ary(Keyval.class); - return rv; - } - public int Pull_int(int i) {Object rv = Get_or_fail(i); - try {return Int_.coerce_(rv);} // coerce to handle "1" and 1; will still fail if "abc" is passed + public int Pull_int(int i) { + try {return Int_.coerce_(Get_or_fail(i));} // coerce to handle "1" and 1; will still fail if "abc" is passed catch (Exception e) { Err_.Noop(e); throw Err_.new_wo_type("bad argument; int expected", "idx", i, "len", ary_len); } } public long Pull_long(int i) {return (long)Pull_double(i);} - public double Pull_double(int i) {Object rv = Get_or_fail(i); + public double Pull_double(int i) { + Object rv = Get_or_fail(i); try {return Int_.coerce_(rv);} // coerce to handle "1" and 1; will still fail if "abc" is passed catch (Exception e) { Err_.Noop(e); @@ -71,18 +75,52 @@ public class Scrib_proc_args { } } } - public Object Pull_obj(int i) {return Get_or_fail(i);} + public Keyval[] Pull_kv_ary_safe(int idx) { // NOTE: must check for null array items; EX:[1:a,3:c] -> [1:a,2:null,3:c]; PAGE:en.d:Category:Nouns_by_language DATE:2016-04-29 + Keyval[] rv = (Keyval[])Get_or_fail(idx); + int rv_len = rv.length; + + List_adp list = null; // list for out-of-order; lazy-instantiation + int prv_key = -1; // mark previous itm_key; needed to reconstruct keys for gaps + // loop itms and handle out-of-order; note that in most cases, itms should be in-order and proc will just return rv; DATE:2016-09-12 + for (int i = 0; i < rv_len; ++i) { + Keyval kv = rv[i]; + // get integer-key; needed to handle gaps + if ( kv.Key_tid() == Type_adp_.Tid__int // luaj will be int + || kv.Key_tid() == Type_adp_.Tid__obj) { // lua will be obj; note that luaj will also have other non-key objects + Object key_obj = kv.Key_as_obj(); + if (key_obj.getClass() == Int_.Cls_ref_type) { // key is int; cast it + int expd_key = i + List_adp_.Base1; // EX: i=1; expd_key=2 + int actl_key = Int_.cast(kv.Key_as_obj()); // EX: i=1; actl_key=3 + if (actl_key != expd_key) { // mismatch; gaps exist; EX:[1:a,3:c] + // 1st mismatch; create list, and add everything before itm in array to list; + if (list == null) { + list = List_adp_.New(); + for (int j = 0; j < i; ++j) + list.Add(rv[j]); // NOTE: everything before i is in sequence, so just add it directly + prv_key = i + 1; // EX: i=1; prv_key=2; + } + for (int j = prv_key; j < actl_key; ++j) // add everything up to actl_key as null; EX: [1:a,3:c] -> [1:a,2:null] + list.Add(Keyval_.int_(j, null)); + prv_key = actl_key + 1; // EX: i=3; prv_key=4 + } + } + } + if (list != null) + list.Add(kv); + } + return (list == null) + ? rv + : (Keyval[])list.To_ary(Keyval.class); + } public String Cast_str_or(int i, String or) {Object rv = Get_or_null(i); return rv == null ? or : String_.cast (rv);} public String Cast_str_or_null(int i) {Object rv = Get_or_null(i); return rv == null ? null : String_.cast (rv);} public byte[] Cast_bry_or_null(int i) {Object rv = Get_or_null(i); return rv == null ? null : Bry_.new_u8(String_.cast (rv));} // NOTE: cast is deliberate; Scrib call checkType whi public byte[] Cast_bry_or_empty(int i) {Object rv = Get_or_null(i); return rv == null ? Bry_.Empty : Bry_.new_u8(String_.cast (rv));} public byte[] Cast_bry_or(int i, byte[] or) {Object rv = Get_or_null(i); return rv == null ? or : Bry_.new_u8(String_.cast (rv));} public Object Cast_obj_or_null(int i) {return Get_or_null(i);} - public boolean Cast_bool_or_y(int i) {Object rv = Get_or_null(i); return rv == null ? Bool_.Y : Bool_.cast(rv);} - public boolean Cast_bool_or_n(int i) {Object rv = Get_or_null(i); return rv == null ? Bool_.N : Bool_.cast(rv);} + public boolean Cast_bool_or_y(int i) {Object rv = Get_or_null(i); return rv == null ? Bool_.Y : Bool_.cast(rv);} + public boolean Cast_bool_or_n(int i) {Object rv = Get_or_null(i); return rv == null ? Bool_.N : Bool_.cast(rv);} public int Cast_int_or(int i, int or) {Object rv = Get_or_null(i); return rv == null ? or : Int_.coerce_(rv);} // coerce to handle "1" and 1; - public String Xstr_str_or_null(int i) {Object rv = Get_or_null(i); return rv == null ? null : Object_.Xto_str_loose_or(rv, null);} // NOTE: Modules can throw exceptions in which return value is nothing; do not fail; return ""; EX: -logy; DATE:2013-10-14 - public byte[] Xstr_bry_or_null(int i) {Object rv = Get_or_null(i); return rv == null ? null : Bry_.new_u8(Object_.Xto_str_loose_or(rv, null));} public Keyval[] Cast_kv_ary_or_null(int i) {Object rv = Get_or_null(i); return rv == null ? null : (Keyval[])rv;} public byte[][] Cast_params_as_bry_ary_or_rest_of_ary(int params_idx) { // PAGE:ru.w:Ленин,_Владимир_Ильич; DATE:2014-07-01 MW:LanguageLibrary.php|ConvertPlural: if (is_array($args[0])) $args = $args[0]; $forms = array_values(array_map('strval', $args)); if (params_idx < 0 || params_idx >= ary_len) return Bry_.Ary_empty; @@ -107,6 +145,8 @@ public class Scrib_proc_args { return rv; } } + public String Xstr_str_or_null(int i) {Object rv = Get_or_null(i); return rv == null ? null : Object_.Xto_str_loose_or(rv, null);} // NOTE: Modules can throw exceptions in which return value is nothing; do not fail; return ""; EX: -logy; DATE:2013-10-14 + public byte[] Xstr_bry_or_null(int i) {Object rv = Get_or_null(i); return rv == null ? null : Bry_.new_u8(Object_.Xto_str_loose_or(rv, null));} public byte[] Extract_qry_args(Xowe_wiki wiki, int idx) { Object qry_args_obj = Cast_obj_or_null(idx); if (qry_args_obj == null) return Bry_.Empty; @@ -131,32 +171,6 @@ public class Scrib_proc_args { return Bry_.Empty; } } - private void Init(Keyval[] v) { - int v_len = v.length; - if (v_len == 0) { - ary = Keyval_.Ary_empty; - ary_len = 0; - return; - } - int v_max = -1; - for (int i = 0; i < v_len; i++) { - Keyval kv = v[i]; - int idx = Int_.cast(kv.Key_as_obj()); - if (v_max < idx) v_max = idx; - } - this.ary_len = v_max; - if (v_max == v_len) { // keys are in sequential order; EX: [1:a,2:b,3:c] - this.ary = v; - } - else { // keys are not in sequential order, or there are gaps; EX: [1:a,3:c] - ary = new Keyval[ary_len]; - for (int i = 0; i < v_len; i++) { - Keyval kv = v[i]; - int idx = Int_.cast(kv.Key_as_obj()); - ary[idx - List_adp_.Base1] = kv; - } - } - } private Object Get_or_null(int i) { if (i < 0 || i >= ary_len) return null; Keyval kv = ary[i]; diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_args__tst.java b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_args__tst.java new file mode 100644 index 000000000..0c225f66e --- /dev/null +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_args__tst.java @@ -0,0 +1,45 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.xtns.scribunto.procs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; +import org.junit.*; import gplx.core.tests.*; +public class Scrib_proc_args__tst { + private final Scrib_proc_args__fxt fxt = new Scrib_proc_args__fxt(); + @Test public void Pull_kv_ary__basic() { // PURPOSE.assert: + fxt.Init(Keyval_.int_(1, Keyval_.Ary(Keyval_.int_(1, "a"), Keyval_.int_(2, "b"), Keyval_.int_(3, "c")))); + fxt.Test__pull_kv_ary(0, Keyval_.int_(1, "a"), Keyval_.int_(2, "b"), Keyval_.int_(3, "c")); + } + @Test public void Pull_kv_ary__gaps() { // PURPOSE.fix: gaps cause null-ref; PAGE:en.w:Shalkar_District DATE:2016-09-12 + fxt.Init(Keyval_.int_(1, Keyval_.Ary(Keyval_.int_(1, "a"), Keyval_.int_(3, "c"), Keyval_.int_(5, "e")))); + fxt.Test__pull_kv_ary(0, Keyval_.int_(1, "a"), Keyval_.int_(2, null), Keyval_.int_(3, "c"), Keyval_.int_(4, null), Keyval_.int_(5, "e")); + } + @Test public void Pull_kv_ary__gaps__many() { // PURPOSE.assert: + fxt.Init(Keyval_.int_(1, Keyval_.Ary(Keyval_.int_(1, "a"), Keyval_.int_(4, "d"), Keyval_.int_(5, "e"), Keyval_.int_(8, "h")))); + fxt.Test__pull_kv_ary(0 + , Keyval_.int_(1, "a"), Keyval_.int_(2, null), Keyval_.int_(3, null), Keyval_.int_(4, "d"), Keyval_.int_(5, "e") + , Keyval_.int_(6, null), Keyval_.int_(7, null), Keyval_.int_(8, "h") + ); + } +} +class Scrib_proc_args__fxt { + private Scrib_proc_args args; + public void Init(Keyval... ary) {this.args = new Scrib_proc_args(ary);} + public void Test__pull_kv_ary(int idx, Keyval... expd) { + Keyval[] actl = args.Pull_kv_ary_safe(idx); + Gftest.Eq__str(Keyval_.Ary_to_str(expd), Keyval_.Ary_to_str(actl)); + } +} diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc_mgr.java b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_mgr.java similarity index 92% rename from 400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc_mgr.java rename to 400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_mgr.java index 4d8cc8532..584e4c921 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc_mgr.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_mgr.java @@ -15,7 +15,7 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package gplx.xowa.xtns.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; +package gplx.xowa.xtns.scribunto.procs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; public class Scrib_proc_mgr { private final Ordered_hash hash = Ordered_hash_.New(); public void Clear() {hash.Clear();} diff --git a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc_rslt.java b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_rslt.java similarity index 92% rename from 400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc_rslt.java rename to 400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_rslt.java index 81105bf3e..ff7be68b4 100644 --- a/400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_proc_rslt.java +++ b/400_xowa/src/gplx/xowa/xtns/scribunto/procs/Scrib_proc_rslt.java @@ -15,7 +15,7 @@ GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ -package gplx.xowa.xtns.scribunto; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; +package gplx.xowa.xtns.scribunto.procs; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; import gplx.xowa.xtns.scribunto.*; public class Scrib_proc_rslt { private Keyval[] ary; public Keyval[] Ary() {return ary;} diff --git a/400_xowa/src/gplx/xowa/xtns/titleBlacklists/Blacklist_scrib_lib.java b/400_xowa/src/gplx/xowa/xtns/titleBlacklists/Blacklist_scrib_lib.java index 667cf142c..5bc77b398 100644 --- a/400_xowa/src/gplx/xowa/xtns/titleBlacklists/Blacklist_scrib_lib.java +++ b/400_xowa/src/gplx/xowa/xtns/titleBlacklists/Blacklist_scrib_lib.java @@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License along with this program. If not, see . */ package gplx.xowa.xtns.titleBlacklists; import gplx.*; import gplx.xowa.*; import gplx.xowa.xtns.*; -import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.libs.*; +import gplx.xowa.xtns.scribunto.*; import gplx.xowa.xtns.scribunto.libs.*; import gplx.xowa.xtns.scribunto.procs.*; public class Blacklist_scrib_lib implements Scrib_lib { public Scrib_lua_mod Mod() {return mod;} private Scrib_lua_mod mod; public Scrib_lib Init() {procs.Init_by_lib(this, Proc_names); return this;} diff --git a/400_xowa/src/gplx/xowa/xtns/wbases/Wdata_wiki_mgr.java b/400_xowa/src/gplx/xowa/xtns/wbases/Wdata_wiki_mgr.java index 7a482a85f..fbdd1a85d 100644 --- a/400_xowa/src/gplx/xowa/xtns/wbases/Wdata_wiki_mgr.java +++ b/400_xowa/src/gplx/xowa/xtns/wbases/Wdata_wiki_mgr.java @@ -27,6 +27,7 @@ public class Wdata_wiki_mgr implements Gfo_evt_itm, Gfo_invk { private final Xoae_app app; private final Wdata_prop_val_visitor prop_val_visitor; private final Wdata_doc_parser wdoc_parser_v1 = new Wdata_doc_parser_v1(), wdoc_parser_v2 = new Wdata_doc_parser_v2(); + private final Object thread_lock =new Object(); private Wdata_hwtr_mgr hwtr_mgr; private final Bry_bfr tmp_bfr = Bry_bfr_.New_w_size(32); public Wdata_wiki_mgr(Xoae_app app) { @@ -50,7 +51,16 @@ public class Wdata_wiki_mgr implements Gfo_evt_itm, Gfo_invk { Doc_mgr.Enabled_(v); } public byte[] Domain() {return domain;} public void Domain_(byte[] v) {domain = v;} private byte[] domain = Bry_.new_a7("www.wikidata.org"); - public Xowe_wiki Wdata_wiki() {if (wdata_wiki == null) wdata_wiki = app.Wiki_mgr().Get_by_or_make(domain).Init_assert(); return wdata_wiki;} private Xowe_wiki wdata_wiki; + public Xowe_wiki Wdata_wiki() { + if (wdata_wiki == null) { + synchronized (thread_lock) { // LOCK:must synchronized b/c multiple threads may init wdata_mgr at same time; + Xowe_wiki tmp_wdata_wiki = app.Wiki_mgr().Get_by_or_make(domain).Init_assert(); + if (wdata_wiki == null) // synchronized is not around "if (wdata_wiki == null)", so multiple threads may try to set; only set if null; DATE:2016-09-12 + wdata_wiki = tmp_wdata_wiki; + } + } + return wdata_wiki; + } private Xowe_wiki wdata_wiki; public Json_parser Jdoc_parser() {return jdoc_parser;} private Json_parser jdoc_parser = new Json_parser(); public void Init_by_app() {} public Wdata_doc_parser Wdoc_parser(Json_doc jdoc) { @@ -61,7 +71,7 @@ public class Wdata_wiki_mgr implements Gfo_evt_itm, Gfo_invk { } public Xop_log_property_wkr Property_wkr() {return property_wkr;} private Xop_log_property_wkr property_wkr; public void Clear() { - synchronized (wdoc_parser_v2) { // LOCK:app-level + synchronized (thread_lock) { // LOCK:app-level Qid_mgr.Clear(); Pid_mgr.Clear(); Doc_mgr.Clear(); @@ -72,14 +82,14 @@ public class Wdata_wiki_mgr implements Gfo_evt_itm, Gfo_invk { Wdata_doc wdoc = Doc_mgr.Get_by_bry_or_null(qid); if (wdoc == null) return or; Wbase_claim_grp claim_grp = wdoc.Claim_list_get(pid); if (claim_grp == null || claim_grp.Len() == 0) return or; Wbase_claim_base claim_itm = claim_grp.Get_at(0); - synchronized (tmp_bfr) { // LOCK:must synchronized b/c prop_val_visitor has member bfr which can get overwritten; DATE:2016-07-06 + synchronized (thread_lock) { // LOCK:must synchronized b/c prop_val_visitor has member bfr which can get overwritten; DATE:2016-07-06 prop_val_visitor.Init(tmp_bfr, hwtr_mgr.Msgs(), domain.Lang_orig_key()); claim_itm.Welcome(prop_val_visitor); return tmp_bfr.To_bry_and_clear(); } } public void Resolve_to_bfr(Bry_bfr bfr, Wbase_claim_grp prop_grp, byte[] lang_key) { - synchronized (this) { // LOCK:must synchronized b/c prop_val_visitor has member bfr which can get overwritten; DATE:2016-07-06 + synchronized (thread_lock) { // LOCK:must synchronized b/c prop_val_visitor has member bfr which can get overwritten; DATE:2016-07-06 Hwtr_mgr_assert(); int len = prop_grp.Len(); Wbase_claim_base selected = null;