From 989ccde83ad8b11e7a9640a8885bb158839b6391 Mon Sep 17 00:00:00 2001 From: gnosygnu Date: Fri, 19 Jul 2019 07:14:24 -0400 Subject: [PATCH] Sqlite: Add option for read-only detection [#509] --- 100_core/src/gplx/Io_mgr.java | 6 +- 100_core/src/gplx/Ordered_hash_base.java | 10 +- .../src/gplx/core}/caches/Lru_cache.java | 54 ++++-- .../src/gplx/core/caches/Lru_cache_root.java | 50 +++++ .../src/gplx/core}/caches/Lru_cache_tst.java | 18 +- 100_core/src/gplx/core/ios/IoEngine.java | 108 +---------- 100_core/src/gplx/core/ios/IoEngineUtl.java | 175 ++++++++++++++++++ 100_core/src/gplx/core/ios/IoEngine_base.java | 4 +- .../src/gplx/core/ios/IoEngine_memory.java | 5 +- .../src/gplx/core/ios/IoEngine_system.java | 11 +- 100_core/src/gplx/core/ios/IoItmDir.java | 3 +- .../gplx/core/ios/atrs/Io_itm_atr_req.java | 35 ++++ .../gplx/core/ios/atrs/Io_itm_atr_wkr.java | 63 +++++++ .../core/ios/atrs/Io_itm_atr_wkr__acl.java | 93 ++++++++++ .../ios/atrs/Io_itm_atr_wkr__acl__tst.java | 80 ++++++++ .../core/ios/atrs/Io_itm_atr_wkr__psx.java | 52 ++++++ .../gplx/core/ios/streams/IoStream_mock.java | 2 +- .../dbs/engines/sqlite/Sqlite_engine.java | 12 +- .../dbs/engines/sqlite/Sqlite_engine_.java | 1 + .../gplx/core/caches/Gfo_cache_mgr_base.java | 8 +- .../gplx/core/caches/Gfo_cache_mgr_bry.java | 10 +- 400_xowa/src/gplx/xowa/Xoae_app.java | 13 ++ .../apps/cfgs/mgrs/types/Xocfg_type_mgr.java | 1 + .../parses/mgrs/Xomp_parse_mgr.java | 3 +- .../parses/mgrs/Xomp_parse_mgr_cfg.java | 4 +- .../parses/utls/Xomp_tmpl_cache_bldr.java | 4 +- .../xowa/wikis/caches/Xow_page_cache.java | 12 +- res/bin/any/xowa/addon/app/cfg/xo.cfg.sqlite3 | Bin 278528 -> 278528 bytes 28 files changed, 669 insertions(+), 168 deletions(-) rename {400_xowa/src/gplx/core/lists => 100_core/src/gplx/core}/caches/Lru_cache.java (64%) create mode 100644 100_core/src/gplx/core/caches/Lru_cache_root.java rename {400_xowa/src/gplx/core/lists => 100_core/src/gplx/core}/caches/Lru_cache_tst.java (83%) create mode 100644 100_core/src/gplx/core/ios/IoEngineUtl.java create mode 100644 100_core/src/gplx/core/ios/atrs/Io_itm_atr_req.java create mode 100644 100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr.java create mode 100644 100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__acl.java create mode 100644 100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__acl__tst.java create mode 100644 100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__psx.java diff --git a/100_core/src/gplx/Io_mgr.java b/100_core/src/gplx/Io_mgr.java index ceea1c679..0be0c5808 100644 --- a/100_core/src/gplx/Io_mgr.java +++ b/100_core/src/gplx/Io_mgr.java @@ -14,7 +14,8 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ package gplx; -import gplx.core.primitives.*; import gplx.core.ios.*; /*IoItmFil, IoItmDir..*/ import gplx.core.ios.streams.*; import gplx.core.ios.loaders.*; +import gplx.core.primitives.*; +import gplx.core.ios.*; /*IoItmFil, IoItmDir..*/ import gplx.core.ios.streams.*; import gplx.core.ios.loaders.*; import gplx.core.ios.atrs.*; public class Io_mgr implements Gfo_evt_mgr_owner { // exists primarily to gather all cmds under gplx namespace; otherwise need to use gplx.core.ios whenever copying/deleting file public Io_mgr() {evt_mgr = new Gfo_evt_mgr(this);} public Gfo_evt_mgr Evt_mgr() {return evt_mgr;} private final Gfo_evt_mgr evt_mgr; @@ -57,6 +58,7 @@ public class Io_mgr implements Gfo_evt_mgr_owner { // exists primarily to gather } public Io_url[] QueryDir_fils(Io_url dir) {return QueryDir_args(dir).ExecAsUrlAry();} public IoEngine_xrg_queryDir QueryDir_args(Io_url dir) {return IoEngine_xrg_queryDir.new_(dir);} + public Io_itm_atr_req Query_itm_atrs(Io_url url, Io_itm_atr_req req) {return IoEnginePool.Instance.Get_by(url.Info().EngineKey()).Query_itm_atrs(url, req);} public void DeleteDirSubs(Io_url url) {IoEngine_xrg_deleteDir.new_(url).Exec();} public IoEngine_xrg_deleteDir DeleteDir_cmd(Io_url url) {return IoEngine_xrg_deleteDir.new_(url);} public void DeleteDirDeep(Io_url url) {IoEngine_xrg_deleteDir.new_(url).Recur_().Exec();} @@ -149,11 +151,13 @@ public class Io_mgr implements Gfo_evt_mgr_owner { // exists primarily to gather } public boolean DownloadFil(String src, Io_url trg) {return IoEngine_xrg_downloadFil.new_(src, trg).Exec();} public IoEngine_xrg_downloadFil DownloadFil_args(String src, Io_url trg) {return IoEngine_xrg_downloadFil.new_(src, trg);} + public boolean Query_read_only(Io_url url, int read_only_type) {return IoEngineUtl.Query_read_only(IoEnginePool.Instance.Get_by(url.Info().EngineKey()), url, read_only_type);} public static final Io_mgr Instance = new Io_mgr(); public static final int Len_kb = 1024, Len_mb = 1048576, Len_gb = 1073741824, Len_gb_2 = 2147483647; public static final long Len_mb_long = Len_mb; public static final long Len_null = -1; public static final String Evt__fil_created = "fil_created"; + public static final int Read_only__basic__file = 1, Read_only__basic__file_and_dirs = 2, Read_only__perms__file = 3; } class Io_mgr_ { public static int Delete_dir_empty(Io_url url) { diff --git a/100_core/src/gplx/Ordered_hash_base.java b/100_core/src/gplx/Ordered_hash_base.java index e22b0e82c..8ce6c9956 100644 --- a/100_core/src/gplx/Ordered_hash_base.java +++ b/100_core/src/gplx/Ordered_hash_base.java @@ -21,14 +21,14 @@ public class Ordered_hash_base extends Hash_adp_base implements Ordered_hash, Gf @Override protected void Add_base(Object key, Object val) { super.Add_base(key, val); ordered.Add(val); - AssertCounts(); + AssertCounts("Add_base", key); } @Override public void Del(Object key) { if (!this.Has_base(key)) return; Object val = this.Fetch_base(key); this.Del_base(key); ordered.Del(val); - AssertCounts(); + AssertCounts("Del", key); } protected Object Get_at_base(int index) {return ordered.Get_at(index);} protected int IndexOf_base(Object obj) {return ordered.Idx_of(obj);} @@ -50,7 +50,7 @@ public class Ordered_hash_base extends Hash_adp_base implements Ordered_hash, Gf if (locked) Lock_fail(); super.Add_base(key, val); ordered.Add_at(i, val); - AssertCounts(); + AssertCounts("Add_at", key); } public Ordered_hash Add_many_str(String... ary) { int ary_len = ary.length; @@ -61,8 +61,8 @@ public class Ordered_hash_base extends Hash_adp_base implements Ordered_hash, Gf } return this; } - void AssertCounts() { - if (super.Count() != ordered.Count()) throw Err_.new_wo_type("counts do not match", "hash", super.Count(), "list", ordered.Count()); + private void AssertCounts(String proc, Object key) { + if (super.Count() != ordered.Count()) throw Err_.new_wo_type("counts do not match; same key is either added twice, or delete failed", "proc", proc, "key", Object_.Xto_str_strict_or_null_mark(key), "hash", super.Count(), "list", ordered.Count()); } public void Resize_bounds(int i) {if (locked) Lock_fail(); ordered.Resize_bounds(i);} public void Lock() {locked = true;} private boolean locked = false; diff --git a/400_xowa/src/gplx/core/lists/caches/Lru_cache.java b/100_core/src/gplx/core/caches/Lru_cache.java similarity index 64% rename from 400_xowa/src/gplx/core/lists/caches/Lru_cache.java rename to 100_core/src/gplx/core/caches/Lru_cache.java index c5532ab93..fa040a9f0 100644 --- a/400_xowa/src/gplx/core/lists/caches/Lru_cache.java +++ b/100_core/src/gplx/core/caches/Lru_cache.java @@ -13,19 +13,24 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*; +package gplx.core.caches; import gplx.*; import gplx.core.*; public class Lru_cache { private final Hash_adp map = Hash_adp_.New(); - private long max; - private long cur = 0; - private Lru_node head; - private Lru_node tail; - private long evicts; - public void Max_(long max) { + private Lru_node head, tail; + private long cur, min, max, evicts; + public Lru_cache(boolean auto_reg, String key, long min, long max) { + this.key = key; + this.min = min; this.max = max; + if (auto_reg) Lru_cache_root.Instance.Add(this); } + public String Key() {return key;} private final String key; public long Evicts() {return evicts;} public long Cur() {return cur;} + public void Min_max_(long min, long max) { + this.min = min; + this.max = max; + } public Object Get_or_null(Object key) { Lru_node nde = (Lru_node)map.Get_by(key); if (nde == null) { @@ -46,10 +51,7 @@ public class Lru_cache { Add_to_tail(nde); } else { - while (cur + size > max) { - Del_node_from_this(head); - evicts++; - } + this.Clear_min(size); nde = new Lru_node(key, val, size); Add_to_tail(nde); @@ -63,12 +65,19 @@ public class Lru_cache { Del_node_from_this(nde); } } - public void Clear() { + public void Clear_all() { map.Clear(); head = null; tail = null; cur = 0; } + public void Clear_min(long size) { + long threshold = min >= 0 ? min : max; + while (cur + size > threshold) { + Del_node_from_this(head); + evicts++; + } + } private void Del_node_from_this(Lru_node nde) { map.Del(nde.Key()); cur -= nde.Size(); @@ -96,6 +105,21 @@ public class Lru_cache { if (head == null) head = tail; } + public void To_str(Bry_bfr bfr, boolean grps_only_or_both) { + bfr.Add_str_a7("g"); + bfr.Add_byte_pipe().Add_str_u8(key); + bfr.Add_byte_pipe().Add_long_variable(cur); + bfr.Add_byte_pipe().Add_long_variable(min); + bfr.Add_byte_pipe().Add_long_variable(max); + bfr.Add_byte_nl(); + if (grps_only_or_both) { + Lru_node nde = head; + while (nde != null) { + nde.To_str(bfr); + nde = nde.Nxt(); + } + } + } } class Lru_node { private final Object key; @@ -114,4 +138,10 @@ class Lru_node { public long Size() {return size;} public Lru_node Prv() {return prv;} public void Prv_(Lru_node v) {this.prv = v;} public Lru_node Nxt() {return nxt;} public void Nxt_(Lru_node v) {this.nxt = v;} + public void To_str(Bry_bfr bfr) { + bfr.Add_str_a7("i"); + bfr.Add_byte_pipe().Add_str_u8(Object_.Xto_str_strict_or_null_mark(key)); + bfr.Add_byte_pipe().Add_long_variable(size); + bfr.Add_byte_nl(); + } } diff --git a/100_core/src/gplx/core/caches/Lru_cache_root.java b/100_core/src/gplx/core/caches/Lru_cache_root.java new file mode 100644 index 000000000..6ce4545d5 --- /dev/null +++ b/100_core/src/gplx/core/caches/Lru_cache_root.java @@ -0,0 +1,50 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012-2017 gnosygnu@gmail.com + +XOWA is licensed under the terms of the General Public License (GPL) Version 3, +or alternatively under the terms of the Apache License Version 2.0. + +You may use XOWA according to either of these licenses as is most appropriate +for your project on a case-by-case basis. + +The terms of each license can be found in the source code repository: + +GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt +Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt +*/ +package gplx.core.caches; import gplx.*; import gplx.core.*; +public class Lru_cache_root { + private final Ordered_hash hash = Ordered_hash_.New(); + public Lru_cache Get_by_key(String key) {return (Lru_cache)hash.Get_by(key);} + public void Add(Lru_cache grp) { + hash.Add(grp.Key(), grp); + } + public void Del(String key) { + hash.Del(key); + } + public void Clear_caches_all() { + int len = hash.Len(); + for (int i = 0; i < len; i++) { + Lru_cache grp = (Lru_cache)hash.Get_at(i); + grp.Clear_all(); + } + } + public void Clear_caches_min() { + int len = hash.Len(); + for (int i = 0; i < len; i++) { + Lru_cache grp = (Lru_cache)hash.Get_at(i); + grp.Clear_min(0); + } + } + public String Print_contents(boolean grps_only_or_both) { + Bry_bfr bfr = Bry_bfr_.New(); + int len = hash.Len(); + for (int i = 0; i < len; i++) { + Lru_cache grp = (Lru_cache)hash.Get_at(i); + grp.To_str(bfr, grps_only_or_both); + } + return bfr.To_str_and_clear(); + } + public static final Lru_cache_root Instance = new Lru_cache_root(); +} diff --git a/400_xowa/src/gplx/core/lists/caches/Lru_cache_tst.java b/100_core/src/gplx/core/caches/Lru_cache_tst.java similarity index 83% rename from 400_xowa/src/gplx/core/lists/caches/Lru_cache_tst.java rename to 100_core/src/gplx/core/caches/Lru_cache_tst.java index 832bc0bd8..923d9cac1 100644 --- a/400_xowa/src/gplx/core/lists/caches/Lru_cache_tst.java +++ b/100_core/src/gplx/core/caches/Lru_cache_tst.java @@ -13,25 +13,21 @@ The terms of each license can be found in the source code repository: GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ -package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*; +package gplx.core.caches; import gplx.*; import gplx.core.*; import org.junit.*; import gplx.core.tests.*; -import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.nss.*; public class Lru_cache_tst { private final Lru_cache_fxt fxt = new Lru_cache_fxt(); @Test public void Get_one() { - fxt.Init__max(10); fxt.Exec__set("a", 5); fxt.Test__get_y("a"); } @Test public void Pop_one() { - fxt.Init__max(10); fxt.Exec__set("a", 10); fxt.Exec__set("b", 10); fxt.Test__get_n("a"); fxt.Test__get_y("b"); } @Test public void Add_many() { - fxt.Init__max(10); fxt.Exec__set("a", 4); fxt.Exec__set("b", 3); fxt.Exec__set("c", 2); @@ -39,7 +35,6 @@ public class Lru_cache_tst { fxt.Test__get_y("a", "b", "c", "d"); } @Test public void Pop_many() { - fxt.Init__max(10); fxt.Exec__set("a", 4); fxt.Exec__set("b", 3); fxt.Exec__set("c", 2); @@ -49,14 +44,12 @@ public class Lru_cache_tst { fxt.Test__get_n("a", "b"); } @Test public void Set_repeatedly() { - fxt.Init__max(10); fxt.Exec__set("a", "a1", 10); fxt.Exec__set("a", "a2", 10); fxt.Exec__set("a", "a3", 10); fxt.Test__get_val("a", "a3"); } @Test public void Set_bumps_priority() { - fxt.Init__max(10); fxt.Exec__set("a", 2); fxt.Exec__set("b", 3); fxt.Exec__set("c", 2); @@ -66,7 +59,6 @@ public class Lru_cache_tst { fxt.Test__get_n("b", "c"); } @Test public void Del() { - fxt.Init__max(10); fxt.Exec__set("a", 2); fxt.Exec__set("b", 2); fxt.Exec__del("b"); @@ -74,7 +66,6 @@ public class Lru_cache_tst { fxt.Test__get_n("b"); } @Test public void Clear() { - fxt.Init__max(10); fxt.Exec__set("a", 2); fxt.Exec__set("b", 2); fxt.Exec__clear(); @@ -82,10 +73,7 @@ public class Lru_cache_tst { } } class Lru_cache_fxt { - private final Lru_cache cache = new Lru_cache(); - public void Init__max(long max) { - cache.Max_(max); - } + private final Lru_cache cache = new Lru_cache(Bool_.N, "test", -1, 10); public void Exec__set(String key, long size) { cache.Set(key, key, size); } @@ -96,7 +84,7 @@ class Lru_cache_fxt { cache.Del(key); } public void Exec__clear() { - cache.Clear(); + cache.Clear_all(); } public void Test__get_y(String... keys) { for (String key : keys) diff --git a/100_core/src/gplx/core/ios/IoEngine.java b/100_core/src/gplx/core/ios/IoEngine.java index a960d263b..e9363527a 100644 --- a/100_core/src/gplx/core/ios/IoEngine.java +++ b/100_core/src/gplx/core/ios/IoEngine.java @@ -15,7 +15,7 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ package gplx.core.ios; import gplx.*; import gplx.core.*; import gplx.core.consoles.*; import gplx.core.criterias.*; -import gplx.core.ios.streams.*; +import gplx.core.ios.streams.*; import gplx.core.ios.atrs.*; public interface IoEngine { String Key(); boolean ExistsFil_api(Io_url url); @@ -32,6 +32,7 @@ public interface IoEngine { void XferFil(IoEngine_xrg_xferFil args); void RecycleFil(IoEngine_xrg_recycleFil xrg); boolean Truncate_fil(Io_url url, long size); + Io_itm_atr_req Query_itm_atrs(Io_url url, Io_itm_atr_req req); boolean ExistsDir(Io_url url); void CreateDir(Io_url url); // creates all folder levels (EX: C:\a\b\c\ will create C:\a\ and C:\a\b\). will not fail if called on already existing folders. @@ -47,108 +48,3 @@ public interface IoEngine { boolean DownloadFil(IoEngine_xrg_downloadFil xrg); Io_stream_rdr DownloadFil_as_rdr(IoEngine_xrg_downloadFil xrg); } -class IoEngineUtl { - public int BufferLength() {return bufferLength;} public void BufferLength_set(int v) {bufferLength = v;} int bufferLength = 4096; // 0x1000 - public void DeleteRecycleGplx(IoEngine engine, IoEngine_xrg_recycleFil xrg) { - Io_url recycleUrl = xrg.RecycleUrl(); - if (recycleUrl.Type_fil()) - engine.MoveFil(IoEngine_xrg_xferFil.move_(xrg.Url(), recycleUrl).Overwrite_(false).ReadOnlyFails_(true)); - else - engine.MoveDirDeep(IoEngine_xrg_xferDir.move_(xrg.Url(), recycleUrl).Overwrite_(false).ReadOnlyFails_(true)); - } - public void DeleteDirDeep(IoEngine engine, Io_url dirUrl, IoEngine_xrg_deleteDir args) { - Console_adp usrDlg = args.UsrDlg(); - IoItmDir dir = engine.QueryDir(dirUrl); if (!dir.Exists()) return; - for (Object subDirObj : dir.SubDirs()) { - IoItmDir subDir = (IoItmDir)subDirObj; - if (!args.SubDirScanCrt().Matches(subDir)) continue; - if (args.Recur()) DeleteDirDeep(engine, subDir.Url(), args); - } - for (Object subFilObj : dir.SubFils()) { - IoItmFil subFil = (IoItmFil)subFilObj; - if (!args.MatchCrt().Matches(subFil)) continue; - Io_url subFilUrl = subFil.Url(); - try {engine.DeleteFil_api(IoEngine_xrg_deleteFil.new_(subFilUrl).ReadOnlyFails_(args.ReadOnlyFails()));} - catch (Exception exc) {usrDlg.Write_fmt_w_nl(Err_.Message_lang(exc));} - } - // all subs deleted; now delete dir - if (!args.MatchCrt().Matches(dir)) return; - try {engine.DeleteDir(dir.Url());} - catch (Exception exc) {usrDlg.Write_fmt_w_nl(Err_.Message_lang(exc));} - } - public void XferDir(IoEngine srcEngine, Io_url src, IoEngine trgEngine, Io_url trg, IoEngine_xrg_xferDir args) { - trgEngine.CreateDir(trg); - IoItmDir srcDir = QueryDirDeep(srcEngine, IoEngine_xrg_queryDir.new_(src).Recur_(false)); - for (Object subSrcObj : srcDir.SubDirs()) { - IoItmDir subSrc = (IoItmDir)subSrcObj; - if (!args.SubDirScanCrt().Matches(subSrc)) continue; - if (!args.MatchCrt().Matches(subSrc)) continue; - Io_url subTrg = trg.GenSubDir_nest(subSrc.Url().NameOnly()); //EX: C:\abc\def\ -> C:\123\ + def\ - if (args.Recur()) XferDir(srcEngine, subSrc.Url(), trgEngine, subTrg, args); - } - IoItmList srcFils = IoItmList.list_(src.Info().CaseSensitive()); - for (Object srcFilObj : srcDir.SubFils()) { - IoItmFil srcFil = (IoItmFil)srcFilObj; - if (args.MatchCrt().Matches(srcFil)) srcFils.Add(srcFil); - } - for (Object srcFilObj : srcFils) { - IoItmFil srcFil = (IoItmFil)srcFilObj; - Io_url srcFilPath = srcFil.Url(); - Io_url trgFilPath = trg.GenSubFil(srcFilPath.NameAndExt()); //EX: C:\abc\fil.txt -> C:\123\ + fil.txt - IoEngine_xrg_xferFil xferArgs = args.Type_move() ? IoEngine_xrg_xferFil.move_(srcFilPath, trgFilPath).Overwrite_(args.Overwrite()) : IoEngine_xrg_xferFil.copy_(srcFilPath, trgFilPath).Overwrite_(args.Overwrite()); - XferFil(srcEngine, xferArgs); - } - if (args.Type_move()) srcEngine.DeleteDirDeep(IoEngine_xrg_deleteDir.new_(src).Recur_(args.Recur()).ReadOnlyFails_(args.ReadOnlyFails()));// this.DeleteDirDeep(srcEngine, src, IoEngine_xrg_deleteItm.new_(src).Recur_(args.Recur()).ReadOnlyIgnored_(args.ReadOnlyIgnored())); - } - public void XferFil(IoEngine srcEngine, IoEngine_xrg_xferFil args) { - Io_url src = args.Src(), trg = args.Trg(); - if (String_.Eq(srcEngine.Key(), trg.Info().EngineKey())) { - if (args.Type_move()) - srcEngine.MoveFil(args); - else - srcEngine.CopyFil(args); - } - else { - TransferStream(src, trg); - if (args.Type_move()) srcEngine.DeleteFil_api(IoEngine_xrg_deleteFil.new_(src)); - } - } - public IoItmDir QueryDirDeep(IoEngine engine, IoEngine_xrg_queryDir args) { - IoItmDir rv = IoItmDir_.top_(args.Url()); - rv.Exists_set(QueryDirDeepCore(rv, args.Url(), engine, args.Recur(), args.SubDirScanCrt(), args.DirCrt(), args.FilCrt(), args.UsrDlg(), args.DirInclude())); - return rv; - } - static boolean QueryDirDeepCore(IoItmDir ownerDir, Io_url url, IoEngine engine, boolean recur, Criteria subDirScanCrt, Criteria dirCrt, Criteria filCrt, Console_adp usrDlg, boolean dirInclude) { - if (usrDlg.Canceled_chk()) return false; - if (usrDlg.Enabled()) usrDlg.Write_tmp(String_.Concat("scan: ", url.Raw())); - IoItmDir scanDir = engine.QueryDir(url); - for (Object subDirObj : scanDir.SubDirs()) { - IoItmDir subDir = (IoItmDir)subDirObj; - if (!subDirScanCrt.Matches(subDir)) continue; - if (dirCrt.Matches(subDir)) { - ownerDir.SubDirs().Add(subDir); // NOTE: always add subDir; do not use dirCrt here, else its subFils will be added to non-existent subDir - } - if (recur) - QueryDirDeepCore(subDir, subDir.Url(), engine, recur, subDirScanCrt, dirCrt, filCrt, usrDlg, dirInclude); - } - for (Object subFilObj : scanDir.SubFils()) { - IoItmFil subFil = (IoItmFil)subFilObj; - if (filCrt.Matches(subFil)) ownerDir.SubFils().Add(subFil); - } - return scanDir.Exists(); - } - void TransferStream(Io_url src, Io_url trg) { - IoStream srcStream = null; - IoStream trgStream = null; - try { - srcStream = IoEnginePool.Instance.Get_by(src.Info().EngineKey()).OpenStreamRead(src); - trgStream = IoEngine_xrg_openWrite.new_(trg).Exec(); - srcStream.Transfer(trgStream, bufferLength); - } - finally { - if (srcStream != null) srcStream.Rls(); - if (trgStream != null) trgStream.Rls(); - } - } - public static IoEngineUtl new_() {return new IoEngineUtl();} IoEngineUtl() {} -} diff --git a/100_core/src/gplx/core/ios/IoEngineUtl.java b/100_core/src/gplx/core/ios/IoEngineUtl.java new file mode 100644 index 000000000..44fdc140a --- /dev/null +++ b/100_core/src/gplx/core/ios/IoEngineUtl.java @@ -0,0 +1,175 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012-2017 gnosygnu@gmail.com + +XOWA is licensed under the terms of the General Public License (GPL) Version 3, +or alternatively under the terms of the Apache License Version 2.0. + +You may use XOWA according to either of these licenses as is most appropriate +for your project on a case-by-case basis. + +The terms of each license can be found in the source code repository: + +GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt +Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt +*/ +package gplx.core.ios; import gplx.*; import gplx.core.*; +import gplx.core.ios.streams.*; import gplx.core.ios.atrs.*; +import gplx.core.consoles.*; import gplx.core.criterias.*; import gplx.core.caches.*; +import gplx.core.envs.*; +public class IoEngineUtl { + public int BufferLength() {return bufferLength;} public void BufferLength_set(int v) {bufferLength = v;} int bufferLength = 4096; // 0x1000 + public void DeleteRecycleGplx(IoEngine engine, IoEngine_xrg_recycleFil xrg) { + Io_url recycleUrl = xrg.RecycleUrl(); + if (recycleUrl.Type_fil()) + engine.MoveFil(IoEngine_xrg_xferFil.move_(xrg.Url(), recycleUrl).Overwrite_(false).ReadOnlyFails_(true)); + else + engine.MoveDirDeep(IoEngine_xrg_xferDir.move_(xrg.Url(), recycleUrl).Overwrite_(false).ReadOnlyFails_(true)); + } + public void DeleteDirDeep(IoEngine engine, Io_url dirUrl, IoEngine_xrg_deleteDir args) { + Console_adp usrDlg = args.UsrDlg(); + IoItmDir dir = engine.QueryDir(dirUrl); if (!dir.Exists()) return; + for (Object subDirObj : dir.SubDirs()) { + IoItmDir subDir = (IoItmDir)subDirObj; + if (!args.SubDirScanCrt().Matches(subDir)) continue; + if (args.Recur()) DeleteDirDeep(engine, subDir.Url(), args); + } + for (Object subFilObj : dir.SubFils()) { + IoItmFil subFil = (IoItmFil)subFilObj; + if (!args.MatchCrt().Matches(subFil)) continue; + Io_url subFilUrl = subFil.Url(); + try {engine.DeleteFil_api(IoEngine_xrg_deleteFil.new_(subFilUrl).ReadOnlyFails_(args.ReadOnlyFails()));} + catch (Exception exc) {usrDlg.Write_fmt_w_nl(Err_.Message_lang(exc));} + } + // all subs deleted; now delete dir + if (!args.MatchCrt().Matches(dir)) return; + try {engine.DeleteDir(dir.Url());} + catch (Exception exc) {usrDlg.Write_fmt_w_nl(Err_.Message_lang(exc));} + } + public void XferDir(IoEngine srcEngine, Io_url src, IoEngine trgEngine, Io_url trg, IoEngine_xrg_xferDir args) { + trgEngine.CreateDir(trg); + IoItmDir srcDir = QueryDirDeep(srcEngine, IoEngine_xrg_queryDir.new_(src).Recur_(false)); + for (Object subSrcObj : srcDir.SubDirs()) { + IoItmDir subSrc = (IoItmDir)subSrcObj; + if (!args.SubDirScanCrt().Matches(subSrc)) continue; + if (!args.MatchCrt().Matches(subSrc)) continue; + Io_url subTrg = trg.GenSubDir_nest(subSrc.Url().NameOnly()); //EX: C:\abc\def\ -> C:\123\ + def\ + if (args.Recur()) XferDir(srcEngine, subSrc.Url(), trgEngine, subTrg, args); + } + IoItmList srcFils = IoItmList.list_(src.Info().CaseSensitive()); + for (Object srcFilObj : srcDir.SubFils()) { + IoItmFil srcFil = (IoItmFil)srcFilObj; + if (args.MatchCrt().Matches(srcFil)) srcFils.Add(srcFil); + } + for (Object srcFilObj : srcFils) { + IoItmFil srcFil = (IoItmFil)srcFilObj; + Io_url srcFilPath = srcFil.Url(); + Io_url trgFilPath = trg.GenSubFil(srcFilPath.NameAndExt()); //EX: C:\abc\fil.txt -> C:\123\ + fil.txt + IoEngine_xrg_xferFil xferArgs = args.Type_move() ? IoEngine_xrg_xferFil.move_(srcFilPath, trgFilPath).Overwrite_(args.Overwrite()) : IoEngine_xrg_xferFil.copy_(srcFilPath, trgFilPath).Overwrite_(args.Overwrite()); + XferFil(srcEngine, xferArgs); + } + if (args.Type_move()) srcEngine.DeleteDirDeep(IoEngine_xrg_deleteDir.new_(src).Recur_(args.Recur()).ReadOnlyFails_(args.ReadOnlyFails()));// this.DeleteDirDeep(srcEngine, src, IoEngine_xrg_deleteItm.new_(src).Recur_(args.Recur()).ReadOnlyIgnored_(args.ReadOnlyIgnored())); + } + public void XferFil(IoEngine srcEngine, IoEngine_xrg_xferFil args) { + Io_url src = args.Src(), trg = args.Trg(); + if (String_.Eq(srcEngine.Key(), trg.Info().EngineKey())) { + if (args.Type_move()) + srcEngine.MoveFil(args); + else + srcEngine.CopyFil(args); + } + else { + TransferStream(src, trg); + if (args.Type_move()) srcEngine.DeleteFil_api(IoEngine_xrg_deleteFil.new_(src)); + } + } + public IoItmDir QueryDirDeep(IoEngine engine, IoEngine_xrg_queryDir args) { + IoItmDir rv = IoItmDir_.top_(args.Url()); + rv.Exists_set(QueryDirDeepCore(rv, args.Url(), engine, args.Recur(), args.SubDirScanCrt(), args.DirCrt(), args.FilCrt(), args.UsrDlg(), args.DirInclude())); + return rv; + } + static boolean QueryDirDeepCore(IoItmDir ownerDir, Io_url url, IoEngine engine, boolean recur, Criteria subDirScanCrt, Criteria dirCrt, Criteria filCrt, Console_adp usrDlg, boolean dirInclude) { + if (usrDlg.Canceled_chk()) return false; + if (usrDlg.Enabled()) usrDlg.Write_tmp(String_.Concat("scan: ", url.Raw())); + IoItmDir scanDir = engine.QueryDir(url); + for (Object subDirObj : scanDir.SubDirs()) { + IoItmDir subDir = (IoItmDir)subDirObj; + if (!subDirScanCrt.Matches(subDir)) continue; + if (dirCrt.Matches(subDir)) { + ownerDir.SubDirs().Add(subDir); // NOTE: always add subDir; do not use dirCrt here, else its subFils will be added to non-existent subDir + } + if (recur) + QueryDirDeepCore(subDir, subDir.Url(), engine, recur, subDirScanCrt, dirCrt, filCrt, usrDlg, dirInclude); + } + for (Object subFilObj : scanDir.SubFils()) { + IoItmFil subFil = (IoItmFil)subFilObj; + if (filCrt.Matches(subFil)) ownerDir.SubFils().Add(subFil); + } + return scanDir.Exists(); + } + void TransferStream(Io_url src, Io_url trg) { + IoStream srcStream = null; + IoStream trgStream = null; + try { + srcStream = IoEnginePool.Instance.Get_by(src.Info().EngineKey()).OpenStreamRead(src); + trgStream = IoEngine_xrg_openWrite.new_(trg).Exec(); + srcStream.Transfer(trgStream, bufferLength); + } + finally { + if (srcStream != null) srcStream.Rls(); + if (trgStream != null) trgStream.Rls(); + } + } + private static final Lru_cache Dir_cache = new Lru_cache(Bool_.Y, "gplx.ios.dir_cache", 128, 256); + public static boolean Query_read_only(IoEngine engine, Io_url url, int read_only_type) { + switch (read_only_type) { + case Io_mgr.Read_only__basic__file: + return engine.QueryFil(url).Attrib().ReadOnly(); + case Io_mgr.Read_only__basic__file_and_dirs: + if (Op_sys.Cur().Tid_is_wnt()) // only examine owner_dirs if wnt + return Query_read_only__file_and_dirs(engine, url); + else + return engine.QueryFil(url).Attrib().ReadOnly(); + case Io_mgr.Read_only__perms__file: + return engine.Query_itm_atrs(url, Io_itm_atr_req.New__read_only()).Is_read_only(); + default: + throw Err_.new_unhandled_default(read_only_type); + } + } + private static boolean Query_read_only__file_and_dirs(IoEngine engine, Io_url url) { + // if fil + if (url.Type_fil()) { + IoItmFil fil = engine.QueryFil(url); + // if read-only, return true + if (fil.ReadOnly()) + return true; + // else, set to owner dir + else + url = url.OwnerDir(); + } + + // loop until top + while (url != Io_url_.Empty) { + String dir_key = url.Raw(); + + // check cache first + IoItmDir dir = (IoItmDir)Dir_cache.Get_or_null(dir_key); + if (dir == null) { + // not in cache; query file_system + dir = engine.QueryDir(url); + Dir_cache.Set(dir_key, dir, 1); + } + + // if read-only, return true + if (dir.ReadOnly()) + return true; + // else, set to owner dir + else + url = url.OwnerDir(); + } + + return false; + } + + public static IoEngineUtl new_() {return new IoEngineUtl();} IoEngineUtl() {} +} diff --git a/100_core/src/gplx/core/ios/IoEngine_base.java b/100_core/src/gplx/core/ios/IoEngine_base.java index 140c87af6..71ba11c79 100644 --- a/100_core/src/gplx/core/ios/IoEngine_base.java +++ b/100_core/src/gplx/core/ios/IoEngine_base.java @@ -14,7 +14,8 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ package gplx.core.ios; import gplx.*; import gplx.core.*; -import gplx.core.ios.streams.*; +import gplx.core.ios.streams.*; import gplx.core.ios.atrs.*; +import gplx.core.caches.*; public abstract class IoEngine_base implements IoEngine { public abstract String Key(); public abstract boolean ExistsFil_api(Io_url url); @@ -24,6 +25,7 @@ public abstract class IoEngine_base implements IoEngine { public abstract void CopyFil(IoEngine_xrg_xferFil args); public abstract void MoveFil(IoEngine_xrg_xferFil args); public abstract IoItmFil QueryFil(Io_url url); + public abstract Io_itm_atr_req Query_itm_atrs(Io_url url, Io_itm_atr_req req); public abstract void UpdateFilAttrib(Io_url url, IoItmAttrib atr); // will fail if file does not exists public abstract void UpdateFilModifiedTime(Io_url url, DateAdp modified); public abstract IoStream OpenStreamRead(Io_url url); diff --git a/100_core/src/gplx/core/ios/IoEngine_memory.java b/100_core/src/gplx/core/ios/IoEngine_memory.java index 198388eb9..ee7df1f94 100644 --- a/100_core/src/gplx/core/ios/IoEngine_memory.java +++ b/100_core/src/gplx/core/ios/IoEngine_memory.java @@ -14,7 +14,7 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ package gplx.core.ios; import gplx.*; import gplx.core.*; -import gplx.core.ios.streams.*; +import gplx.core.ios.streams.*; import gplx.core.ios.atrs.*; public class IoEngine_memory extends IoEngine_base { @Override public String Key() {return key;} private String key = IoEngine_.MemKey; @Override public boolean ExistsFil_api(Io_url url) {return FetchFil(url) != IoItmFil_mem.Null;} @@ -190,6 +190,9 @@ public class IoEngine_memory extends IoEngine_base { byte[] bry = Bry_.new_u8(FetchFil(Io_url_.mem_fil_(xrg.Src())).Text()); return Io_stream_rdr_.New__mem(bry); } + @Override public Io_itm_atr_req Query_itm_atrs(Io_url url, Io_itm_atr_req req) { + return req; + } IoItmHash dirs = IoItmHash.new_(); IoEngineUtl utl = IoEngineUtl.new_(); diff --git a/100_core/src/gplx/core/ios/IoEngine_system.java b/100_core/src/gplx/core/ios/IoEngine_system.java index 1fb34296b..8f49c0a07 100644 --- a/100_core/src/gplx/core/ios/IoEngine_system.java +++ b/100_core/src/gplx/core/ios/IoEngine_system.java @@ -21,12 +21,8 @@ import java.net.URLConnection; import java.net.URLEncoder; import java.nio.*; import java.nio.channels.*; -import java.util.Date; - -import javax.print.FlavorException; -import javax.tools.JavaCompiler; import gplx.core.criterias.*; import gplx.core.bits.*; import gplx.core.envs.*; -import gplx.core.ios.streams.*; +import gplx.core.ios.streams.*; import gplx.core.ios.atrs.*; import gplx.core.progs.*; public class IoEngine_system extends IoEngine_base { @Override public String Key() {return IoEngine_.SysKey;} @@ -42,6 +38,9 @@ public class IoEngine_system extends IoEngine_base { if (!Fil_Exists(fil)) return; MarkFileWritable(fil, url, args.ReadOnlyFails(), "DeleteFile"); DeleteFil_lang(fil, url); + } + @Override public Io_itm_atr_req Query_itm_atrs(Io_url url, Io_itm_atr_req req) { + return Io_itm_atr_wkr.New(url).Process(req); } @Override public boolean ExistsFil_api(Io_url url) { File f = new File(url.Xto_api()); @@ -158,7 +157,9 @@ public class IoEngine_system extends IoEngine_base { && String_.Eq(url.OwnerDir().Raw(), String_.Empty) // folder is drive; EX: "C:" ) url_api = url_api + "\\"; // add "\\"; else listFiles will return working folder's files, not C:; DATE:2016-04-07 + File dirInfo = new File(url_api); + rv.ReadOnly_(!dirInfo.canWrite()); // get read-only flag for directories; ISSUE#:509; DATE:2019-07-11 if (!dirInfo.exists()) { rv.Exists_set(false); return rv; diff --git a/100_core/src/gplx/core/ios/IoItmDir.java b/100_core/src/gplx/core/ios/IoItmDir.java index 50ee535bd..79a95260f 100644 --- a/100_core/src/gplx/core/ios/IoItmDir.java +++ b/100_core/src/gplx/core/ios/IoItmDir.java @@ -17,7 +17,8 @@ package gplx.core.ios; import gplx.*; import gplx.core.*; import gplx.core.criterias.*; public class IoItmDir extends IoItm_base { public boolean Exists() {return exists;} public void Exists_set(boolean v) {exists = v;} private boolean exists = true; - @Override public int TypeId() {return Type_Dir;} @Override public boolean Type_dir() {return true;} @Override public boolean Type_fil() {return false;} public static final int Type_Dir = 1; + @Override public int TypeId() {return Type_Dir;} @Override public boolean Type_dir() {return true;} @Override public boolean Type_fil() {return false;} public static final int Type_Dir = 1; + public boolean ReadOnly() {return readOnly;} public IoItmDir ReadOnly_(boolean val) {this.readOnly = val; return this;} private boolean readOnly; @gplx.New public IoItmDir XtnProps_set(String key, Object val) {return (IoItmDir)super.XtnProps_set(key, val);} public IoItmList SubDirs() {return subDirs;} IoItmList subDirs; public IoItmList SubFils() {return subFils;} IoItmList subFils; diff --git a/100_core/src/gplx/core/ios/atrs/Io_itm_atr_req.java b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_req.java new file mode 100644 index 000000000..473ea8a6a --- /dev/null +++ b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_req.java @@ -0,0 +1,35 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012-2017 gnosygnu@gmail.com + +XOWA is licensed under the terms of the General Public License (GPL) Version 3, +or alternatively under the terms of the Apache License Version 2.0. + +You may use XOWA according to either of these licenses as is most appropriate +for your project on a case-by-case basis. + +The terms of each license can be found in the source code repository: + +GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt +Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt +*/ +package gplx.core.ios.atrs; import gplx.*; import gplx.core.*; import gplx.core.ios.*; +public class Io_itm_atr_req { + Io_itm_atr_req(boolean ignore_errors, boolean check_read_only) { + this.ignore_errors = ignore_errors; + this.check_read_only = check_read_only; + } + public boolean Check_read_only() {return check_read_only;} private final boolean check_read_only; + public boolean Is_read_only() {return is_read_only;} public void Is_read_only_(boolean v) {this.is_read_only = v;} private boolean is_read_only; + public boolean Ignore_errors() {return ignore_errors;} private final boolean ignore_errors; + public String To_str() { + Keyval[] ary = new Keyval[2]; + ary[0] = Keyval_.new_("check_read_only", check_read_only); + ary[1] = Keyval_.new_("is_read_only", is_read_only); + return Keyval_.Ary_to_str(ary); + } + + public static Io_itm_atr_req New__read_only() { + return new Io_itm_atr_req(true, true); + } +} diff --git a/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr.java b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr.java new file mode 100644 index 000000000..888c657d6 --- /dev/null +++ b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr.java @@ -0,0 +1,63 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012-2017 gnosygnu@gmail.com + +XOWA is licensed under the terms of the General Public License (GPL) Version 3, +or alternatively under the terms of the Apache License Version 2.0. + +You may use XOWA according to either of these licenses as is most appropriate +for your project on a case-by-case basis. + +The terms of each license can be found in the source code repository: + +GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt +Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt +*/ +package gplx.core.ios.atrs; import gplx.*; import gplx.core.*; import gplx.core.ios.*; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.attribute.AclFileAttributeView; +import java.nio.file.attribute.PosixFileAttributeView; +import java.util.Set; +public abstract class Io_itm_atr_wkr { + private final Path path; + public Io_itm_atr_wkr(Path path) { + this.path = path; + } + public Io_itm_atr_req Process(Io_itm_atr_req req) { + try { + if (req.Check_read_only()) + req.Is_read_only_(this.Is_read_only()); + } + catch (Exception e) { + Err err = Err_.new_wo_type("query_itm_atrs failed", "url", path.toString(), "atrs", req.To_str(), "e", Err_.Message_gplx_log(e)); + if (req.Ignore_errors()) { // https://stackoverflow.com/questions/25163174/get-generic-folder-permissions-like-generic-all-using-javas-aclfileattributev + Gfo_usr_dlg_.Instance.Warn_many("", "", err.To_str__log()); + } + else { + throw err; + } + } + return req; + } + public abstract boolean Is_read_only(); + public static Io_itm_atr_wkr New(Io_url url) { + File fil = new File(url.Xto_api()); + Path path = fil.toPath(); + Set supported_views = path.getFileSystem().supportedFileAttributeViews(); + if (supported_views.contains("posix")) { + return new Io_itm_atr_wkr__psx(path); + } + // WNT + else if (supported_views.contains("acl")) { + return new Io_itm_atr_wkr__acl(path); + } + else { + String set_string = ""; + for (String view : supported_views) { + set_string += view + ";"; + } + throw Err_.new_unhandled(set_string); + } + } +} diff --git a/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__acl.java b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__acl.java new file mode 100644 index 000000000..95dff2f8a --- /dev/null +++ b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__acl.java @@ -0,0 +1,93 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012-2017 gnosygnu@gmail.com + +XOWA is licensed under the terms of the General Public License (GPL) Version 3, +or alternatively under the terms of the Apache License Version 2.0. + +You may use XOWA according to either of these licenses as is most appropriate +for your project on a case-by-case basis. + +The terms of each license can be found in the source code repository: + +GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt +Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt +*/ +package gplx.core.ios.atrs; import gplx.*; import gplx.core.*; import gplx.core.ios.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.AclEntry; +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.nio.file.attribute.AclFileAttributeView; +import java.util.List; +import java.util.Set; + +import gplx.core.primitives.Bool_obj_val; +class Io_itm_atr_wkr__acl extends Io_itm_atr_wkr { private final AclFileAttributeView view; + public Io_itm_atr_wkr__acl(Path path) { + super(path); + this.view = Files.getFileAttributeView(path, AclFileAttributeView.class); + } + @Override public boolean Is_read_only() { + try { + // convert AclEntry to Acl_entry + List list = view.getAcl(); + int len = list.size(); + Acl_entry[] ary = new Acl_entry[len]; + for (int i = 0; i < len; i++) { + AclEntry under = list.get(i); + ary[i] = new Acl_entry(under.principal().toString(), under.type(), under.permissions()); + } + return !Is_permitted(ary, AclEntryPermission.WRITE_DATA); + } catch (Exception e) { + throw Err_.new_exc(e, "", "Is_read_only failed", "e", Err_.Message_lang(e)); + } + } + public static boolean Is_permitted(Acl_entry[] ary, AclEntryPermission permission) { + boolean rv = false; + Hash_adp principals = Hash_adp_.New(); + for (Acl_entry itm : ary) { + Set permissions = itm.Permissions(); + switch (itm.Type()) { + // If multiple ALLOW entries + // * for same principal, return false if any of them do not have permission + // * for diff principals, return true if any of them does have permissions + case ALLOW: { + // if current principal is forbidden, ignore entry; want to skip lists like Everyone:Forbidden:C:/folder;Everyone:Allowed;C:/ + Bool_obj_val forbidden = (Bool_obj_val)principals.Get_by(itm.Principal()); + if (forbidden != null) { + continue; + } + if (!permissions.contains(permission) && !rv) { + rv = false; + principals.Add(itm.Principal(), Bool_obj_val.False); + } + else { + rv = true; + } + break; + } + // If any DENY entries, return false on first entry + case DENY: { + if (permissions.contains(permission)) { + return false; + } + break; + } + } + } + return rv; + } + } +class Acl_entry { + public Acl_entry(String principal, AclEntryType type, Set permissions) { + this.principal = principal; + this.type = type; + this.permissions = permissions; + } + public String Principal() {return principal;} private final String principal; + public AclEntryType Type() {return type;} private final AclEntryType type; + public Set Permissions() {return permissions;} private final Set permissions; +} +//#} \ No newline at end of file diff --git a/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__acl__tst.java b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__acl__tst.java new file mode 100644 index 000000000..03b3a11ca --- /dev/null +++ b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__acl__tst.java @@ -0,0 +1,80 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012-2017 gnosygnu@gmail.com + +XOWA is licensed under the terms of the General Public License (GPL) Version 3, +or alternatively under the terms of the Apache License Version 2.0. + +You may use XOWA according to either of these licenses as is most appropriate +for your project on a case-by-case basis. + +The terms of each license can be found in the source code repository: + +GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt +Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt +*/ +package gplx.core.ios.atrs; import gplx.*; import gplx.core.*; import gplx.core.ios.*; +import org.junit.*; import gplx.core.tests.*; + +import java.nio.file.attribute.AclEntryPermission; +import java.nio.file.attribute.AclEntryType; +import java.util.HashSet; +import java.util.Set; +public class Io_itm_atr_wkr__acl__tst { + private final Io_itm_attrib_wkr__acl__fxt fxt = new Io_itm_attrib_wkr__acl__fxt(); + @Test public void Perm_exists() { + fxt.Test__Is_permitted + ( Bool_.Y, AclEntryPermission.WRITE_DATA + , fxt.Make__acl("Everyone", AclEntryType.ALLOW, AclEntryPermission.WRITE_DATA) + ); + } + @Test public void Perm_missing() { + fxt.Test__Is_permitted + ( Bool_.N, AclEntryPermission.WRITE_DATA + , fxt.Make__acl("Everyone", AclEntryType.ALLOW, AclEntryPermission.READ_DATA) + ); + } + @Test public void Deny_over_allow() { + fxt.Test__Is_permitted + ( Bool_.N, AclEntryPermission.WRITE_DATA + , fxt.Make__acl("Everyone", AclEntryType.ALLOW, AclEntryPermission.WRITE_DATA) + , fxt.Make__acl("Everyone", AclEntryType.DENY , AclEntryPermission.WRITE_DATA) + ); + } + @Test public void Same_principal__perm_missing_over_perm_exists() { + /* + EX: SD card wherein acl_list has 2 entries + * Entry[0] | //MACHINE/SHARE | Everyone:READ_DATA/READ_NAMED_ATTRS/EXECUTE/READ_ATTRIBUTES/READ_ACL/SYNCHRONIZE:ALLOW + * Entry[1] | DRIVE_NAME:/ | Everyone:READ_DATA/WRITE_DATA/APPEND_DATA/READ_NAMED_ATTRS/WRITE_NAMED_ATTRS/EXECUTE/DELETE_CHILD/READ_ATTRIBUTES/WRITE_ATTRIBUTES/DELETE/READ_ACL/WRITE_ACL/WRITE_OWNER/SYNCHRONIZE:ALLOW + */ + fxt.Test__Is_permitted + ( Bool_.N, AclEntryPermission.WRITE_DATA + , fxt.Make__acl("Everyone", AclEntryType.ALLOW, AclEntryPermission.READ_DATA) + , fxt.Make__acl("Everyone", AclEntryType.ALLOW, AclEntryPermission.READ_DATA, AclEntryPermission.WRITE_DATA) + ); + } + @Test public void Diff_principals__perm_exists_over_perm_missing() { + /* + EX: C drive wherein acl_list has 2 entries + * Entry[0] | C:/ | Administrators:READ_DATA/WRITE_DATA + * Entry[1] | C:/ | Everyone:READ_DATA + */ + fxt.Test__Is_permitted + ( Bool_.Y, AclEntryPermission.WRITE_DATA + , fxt.Make__acl("Administrators", AclEntryType.ALLOW, AclEntryPermission.WRITE_DATA) + , fxt.Make__acl("Users" , AclEntryType.ALLOW, AclEntryPermission.READ_DATA) + ); + } +} +class Io_itm_attrib_wkr__acl__fxt { + public void Test__Is_permitted(boolean expd, AclEntryPermission permission, Acl_entry... entries) { + boolean actl = Io_itm_atr_wkr__acl.Is_permitted(entries, permission); + Gftest.Eq__bool(expd, actl); + } + public Acl_entry Make__acl(String principal, AclEntryType type, AclEntryPermission... perms) { + Set perm_set = new HashSet(); + for (AclEntryPermission perm : perms) + perm_set.add(perm); + return new Acl_entry(principal, type, perm_set); + } +} diff --git a/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__psx.java b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__psx.java new file mode 100644 index 000000000..ff0b49809 --- /dev/null +++ b/100_core/src/gplx/core/ios/atrs/Io_itm_atr_wkr__psx.java @@ -0,0 +1,52 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012-2017 gnosygnu@gmail.com + +XOWA is licensed under the terms of the General Public License (GPL) Version 3, +or alternatively under the terms of the Apache License Version 2.0. + +You may use XOWA according to either of these licenses as is most appropriate +for your project on a case-by-case basis. + +The terms of each license can be found in the source code repository: + +GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt +Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt +*/ +package gplx.core.ios.atrs; import gplx.*; import gplx.core.*; import gplx.core.ios.*; +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFileAttributeView; +import java.nio.file.attribute.PosixFileAttributes; +import java.nio.file.attribute.PosixFilePermission; +import java.util.Set; +class Io_itm_atr_wkr__psx extends Io_itm_atr_wkr { private final PosixFileAttributeView view; + public Io_itm_atr_wkr__psx(Path path) { + super(path); + this.view = Files.getFileAttributeView(path, PosixFileAttributeView.class); + } + @Override public boolean Is_read_only() { + try { + // ASSUME:a file is read-only if it is read-only; Note that the directory may need write-access, but not handling it now; REF:https://superuser.com/a/114611 + Set perms = view.readAttributes().permissions(); + int perm_flag = Psx__permissions_to_int(perms); + return perm_flag == 0444; + } catch (Exception e) { + throw Err_.new_exc(e, "", "Is_read_only failed", "e", Err_.Message_lang(e)); + } + } + public static int Psx__permissions_to_int(Set psx_perms) { + int rv = 0; + rv |= ((psx_perms.contains(PosixFilePermission.OWNER_READ)) ? 1 << 8 : 0); + rv |= ((psx_perms.contains(PosixFilePermission.OWNER_WRITE)) ? 1 << 7 : 0); + rv |= ((psx_perms.contains(PosixFilePermission.OWNER_EXECUTE)) ? 1 << 6 : 0); + rv |= ((psx_perms.contains(PosixFilePermission.GROUP_READ)) ? 1 << 5 : 0); + rv |= ((psx_perms.contains(PosixFilePermission.GROUP_WRITE)) ? 1 << 4 : 0); + rv |= ((psx_perms.contains(PosixFilePermission.GROUP_EXECUTE)) ? 1 << 3 : 0); + rv |= ((psx_perms.contains(PosixFilePermission.OTHERS_READ)) ? 1 << 2 : 0); + rv |= ((psx_perms.contains(PosixFilePermission.OTHERS_WRITE)) ? 1 << 1 : 0); + rv |= ((psx_perms.contains(PosixFilePermission.OTHERS_EXECUTE)) ? 1 : 0); + return rv; + } + } diff --git a/100_core/src/gplx/core/ios/streams/IoStream_mock.java b/100_core/src/gplx/core/ios/streams/IoStream_mock.java index 1b8fd7ee2..5441424ad 100644 --- a/100_core/src/gplx/core/ios/streams/IoStream_mock.java +++ b/100_core/src/gplx/core/ios/streams/IoStream_mock.java @@ -24,7 +24,7 @@ public class IoStream_mock implements IoStream { if (bytes_read > read_limit) bytes_read = read_limit; // stream may limit maximum read; EX: bfr_len of 16k but only 2k will be filled int bytes_left = data_bry_len - data_bry_pos; if (bytes_read > bytes_left) bytes_read = bytes_left; // not enough bytes left in data_bry; bytes_read = whatever is left - Bry_.Copy_by_pos(data_bry, data_bry_pos, data_bry_pos + bytes_read, bfr, bfr_bgn); + Bry_.Copy_to(data_bry, data_bry_pos, data_bry_pos + bytes_read, bfr, bfr_bgn); data_bry_pos += bytes_read; return bytes_read; } diff --git a/140_dbs/src/gplx/dbs/engines/sqlite/Sqlite_engine.java b/140_dbs/src/gplx/dbs/engines/sqlite/Sqlite_engine.java index 157caa074..d54cc13cc 100644 --- a/140_dbs/src/gplx/dbs/engines/sqlite/Sqlite_engine.java +++ b/140_dbs/src/gplx/dbs/engines/sqlite/Sqlite_engine.java @@ -16,8 +16,11 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt package gplx.dbs.engines.sqlite; import gplx.*; import gplx.dbs.*; import gplx.dbs.engines.*; import java.sql.*; import gplx.core.stores.*; import gplx.dbs.engines.*; import gplx.dbs.engines.sqlite.*; import gplx.dbs.metas.*; import gplx.dbs.sqls.*; -import gplx.dbs.qrys.*; +import gplx.dbs.qrys.*; +import gplx.core.consoles.Console_adp_; +import gplx.core.consoles.Console_adp__sys; import gplx.core.ios.IoItmFil; + import org.sqlite.SQLiteConnection; public class Sqlite_engine extends Db_engine_sql_base { private final Sqlite_txn_mgr txn_mgr; private final Sqlite_schema_mgr schema_mgr; @@ -68,9 +71,14 @@ public class Sqlite_engine extends Db_engine_sql_base { // set open_mode flag if conn is read-only; needed else all SELECT queries will be very slow; DATE:2016-09-03 IoItmFil sqlite_fs_itm = Io_mgr.Instance.QueryFil(sqlite_fs_url); - Keyval[] props = sqlite_fs_itm.Exists() && sqlite_fs_itm.ReadOnly() // NOTE: must check if it exists; else missing-file will be marked as readonly connection, and missing-file will sometimes be dynamically created as read-write; DATE:2016-09-04 + boolean read_only = sqlite_fs_itm.Exists() // NOTE: must check if it exists; else missing-file will be marked as readonly connection, and missing-file will sometimes be dynamically created as read-write; DATE:2016-09-04 + && Io_mgr.Instance.Query_read_only(sqlite_fs_url.OwnerDir(), Sqlite_engine_.Read_only_detection); + Keyval[] props = read_only ? Keyval_.Ary(Keyval_.new_("open_mode", "1")) : Keyval_.Ary_empty; + if (read_only) { + Gfo_usr_dlg_.Instance.Note_many("", "", "Sqlite db opened as read-only: url=~{0}", sqlite_fs_url.Xto_api()); + } // open connection Connection rv = Conn__new_by_url_and_props(sqlite_db_url, props); diff --git a/140_dbs/src/gplx/dbs/engines/sqlite/Sqlite_engine_.java b/140_dbs/src/gplx/dbs/engines/sqlite/Sqlite_engine_.java index ad8ccee6b..2de80f79d 100644 --- a/140_dbs/src/gplx/dbs/engines/sqlite/Sqlite_engine_.java +++ b/140_dbs/src/gplx/dbs/engines/sqlite/Sqlite_engine_.java @@ -74,4 +74,5 @@ public class Sqlite_engine_ { public static String X_date_to_str(DateAdp v) {return v == Date_null ? "" : v.XtoStr_fmt_iso_8561();} public static final DateAdp Date_null = null; public static final byte Wildcard_byte = Byte_ascii.Hash; + public static int Read_only_detection = Io_mgr.Read_only__basic__file; } diff --git a/400_xowa/src/gplx/core/caches/Gfo_cache_mgr_base.java b/400_xowa/src/gplx/core/caches/Gfo_cache_mgr_base.java index bce6ab437..7acdcd386 100644 --- a/400_xowa/src/gplx/core/caches/Gfo_cache_mgr_base.java +++ b/400_xowa/src/gplx/core/caches/Gfo_cache_mgr_base.java @@ -20,11 +20,11 @@ public class Gfo_cache_mgr_base { public int Compress_to() {return compress_to;} public void Compress_to_(int v) {compress_to = v;} private int compress_to = 8; protected Object Base_get_or_null(byte[] key) { Object rv_obj = hash.Get_by(key); - return rv_obj == null ? null : ((Gfo_cache_itm)rv_obj).Val(); + return rv_obj == null ? null : ((Gfo_cache_itm_bry)rv_obj).Val(); } protected void Base_add(byte[] key, Object val) { if (hash.Count() >= compress_max) Compress(); - Gfo_cache_itm itm = new Gfo_cache_itm(key, val); + Gfo_cache_itm_bry itm = new Gfo_cache_itm_bry(key, val); hash.Add(key, itm); } protected void Base_del(byte[] key) { @@ -35,11 +35,11 @@ public class Gfo_cache_mgr_base { int del_len = hash.Count() - compress_to; List_adp del_list = List_adp_.New(); for (int i = 0; i < del_len; i++) { - Gfo_cache_itm itm = (Gfo_cache_itm)hash.Get_at(i); + Gfo_cache_itm_bry itm = (Gfo_cache_itm_bry)hash.Get_at(i); del_list.Add(itm); } for (int i = 0; i < del_len; i++) { - Gfo_cache_itm itm = (Gfo_cache_itm)del_list.Get_at(i); + Gfo_cache_itm_bry itm = (Gfo_cache_itm_bry)del_list.Get_at(i); hash.Del(itm.Key()); } } diff --git a/400_xowa/src/gplx/core/caches/Gfo_cache_mgr_bry.java b/400_xowa/src/gplx/core/caches/Gfo_cache_mgr_bry.java index 7eb90ff3d..f2511cedd 100644 --- a/400_xowa/src/gplx/core/caches/Gfo_cache_mgr_bry.java +++ b/400_xowa/src/gplx/core/caches/Gfo_cache_mgr_bry.java @@ -20,17 +20,17 @@ public class Gfo_cache_mgr_bry extends Gfo_cache_mgr_base { public void Add(byte[] key, Object val) {Base_add(key, val);} public void Del(byte[] key) {Base_del(key);} } -class Gfo_cache_itm { - public Gfo_cache_itm(Object key, Object val) {this.key = key; this.val = val; this.Touched_update();} +class Gfo_cache_itm_bry { + public Gfo_cache_itm_bry(Object key, Object val) {this.key = key; this.val = val; this.Touched_update();} public Object Key() {return key;} private Object key; public Object Val() {return val;} private Object val; public long Touched() {return touched;} private long touched; - public Gfo_cache_itm Touched_update() {touched = System_.Ticks(); return this;} + public Gfo_cache_itm_bry Touched_update() {touched = System_.Ticks(); return this;} } class Gfo_cache_itm_comparer implements gplx.core.lists.ComparerAble { public int compare(Object lhsObj, Object rhsObj) { - Gfo_cache_itm lhs = (Gfo_cache_itm)lhsObj; - Gfo_cache_itm rhs = (Gfo_cache_itm)rhsObj; + Gfo_cache_itm_bry lhs = (Gfo_cache_itm_bry)lhsObj; + Gfo_cache_itm_bry rhs = (Gfo_cache_itm_bry)rhsObj; return Long_.Compare(lhs.Touched(), rhs.Touched()); } public static final Gfo_cache_itm_comparer Touched_asc = new Gfo_cache_itm_comparer(); // TS.static diff --git a/400_xowa/src/gplx/xowa/Xoae_app.java b/400_xowa/src/gplx/xowa/Xoae_app.java index c0260be90..ab1168fd3 100644 --- a/400_xowa/src/gplx/xowa/Xoae_app.java +++ b/400_xowa/src/gplx/xowa/Xoae_app.java @@ -149,6 +149,7 @@ public class Xoae_app implements Xoa_app, Gfo_invk { stage = Xoa_stage_.Tid_init; user.Init_by_app(this); cfg.Init_by_app(this); + Ctor_dbs(); user.User_db_mgr().Cache_mgr().Init_by_app(this); misc_mgr.Init_by_app(this); user.History_mgr().Init_by_app(this); @@ -208,6 +209,18 @@ public class Xoae_app implements Xoa_app, Gfo_invk { msg_log.Clear(); wiki_mgr.Free_mem(clear_ctx); } + private void Ctor_dbs() { + if (gplx.core.envs.Env_.Mode_testing()) return; + String read_only_detection_str = cfg.Get_str_app_or("xowa.app.dbs.sqlite.read_only_detection", "basic_file"); + int read_only_detection = Io_mgr.Read_only__basic__file; + if (String_.Eq(read_only_detection_str, "basic_file")) + read_only_detection = Io_mgr.Read_only__basic__file; + else if (String_.Eq(read_only_detection_str, "basic_file_and_dirs")) + read_only_detection = Io_mgr.Read_only__basic__file_and_dirs; + else if (String_.Eq(read_only_detection_str, "perms_file")) + read_only_detection = Io_mgr.Read_only__perms__file; + gplx.dbs.engines.sqlite.Sqlite_engine_.Read_only_detection = read_only_detection; + } public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { if (ctx.Match(k, Invk_gui)) return gui_mgr; else if (ctx.Match(k, Invk_api)) return api_root; diff --git a/400_xowa/src/gplx/xowa/addons/apps/cfgs/mgrs/types/Xocfg_type_mgr.java b/400_xowa/src/gplx/xowa/addons/apps/cfgs/mgrs/types/Xocfg_type_mgr.java index 486c834e5..b7b56488e 100644 --- a/400_xowa/src/gplx/xowa/addons/apps/cfgs/mgrs/types/Xocfg_type_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/apps/cfgs/mgrs/types/Xocfg_type_mgr.java @@ -27,6 +27,7 @@ public class Xocfg_type_mgr { this.Lists__add("list:xowa.addon.http_server.file_retrieve_mode", Keyval_.new_("wait"), Keyval_.new_("skip"), Keyval_.new_("async_server", "async server")); this.Lists__add("list:xowa.addon.scribunto.engine", "luaj", "lua"); this.Lists__add("list:xowa.addon.math.renderer", Keyval_.new_("mathjax","MathJax"), Keyval_.new_("latex", "LaTeX")); + this.Lists__add("list:xowa.app.dbs.sqlite.read_only_detection", Keyval_.new_("basic_file", "Basic - File only"), Keyval_.new_("basic_file_and_dirs", "Basic - File and parent dirs"), Keyval_.new_("perms_file", "Permissions - File")); } public void Lists__add(String key, String... vals) { int len = vals.length; diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java index 6dcff029b..cdff506df 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java @@ -33,7 +33,7 @@ public class Xomp_parse_mgr { Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr()); // cache: preload tmpls and imglinks - Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates(), cfg.Page_cache_max()); + Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates(), cfg.Page_cache_min(), cfg.Page_cache_max()); wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of user cache Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr(); @@ -70,6 +70,7 @@ public class Xomp_parse_mgr { for (int i = 0; i < wkr_len; ++i) { // make wiki Xowe_wiki wkr_wiki = Xow_wiki_utl_.Clone_wiki(wiki, wiki.Fsys_mgr().Root_dir()); + Lru_cache_root.Instance.Del(wkr_wiki.Cache_mgr().Page_cache().Cache_key()); wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java index c47f2eec3..f72f87f39 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java @@ -42,7 +42,8 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk { public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100; public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10; public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70; - public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 2 * (long)Io_mgr.Len_gb; + public long Page_cache_min() {return page_cache_min;} private long page_cache_min = 1500 * Io_mgr.Len_mb_long; + public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 2000 * Io_mgr.Len_mb_long; public void Init(Xowe_wiki wiki) { if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count(); if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000; @@ -77,6 +78,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk { else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v"); else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v"); else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v"); + else if (ctx.Match(k, "page_cache_min_")) page_cache_min = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_min); else if (ctx.Match(k, "page_cache_max_")) page_cache_max = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_max); else return Gfo_invk_.Rv_unhandled; return this; diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java index 16eac1a47..1f9fc2c36 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java @@ -17,9 +17,9 @@ package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gp import gplx.dbs.*; import gplx.xowa.wikis.caches.*; public class Xomp_tmpl_cache_bldr { - public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all, long page_cache_max) { + public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all, long page_cache_min, long page_cache_max) { Xow_page_cache rv = new Xow_page_cache(wiki); - rv.Max_(page_cache_max); + rv.Min_max_(page_cache_min, page_cache_max); if (fill_all) Fill_all(rv, wiki); return rv; } diff --git a/400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java b/400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java index 4dffb6635..015c48e4d 100644 --- a/400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java +++ b/400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java @@ -14,19 +14,21 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt */ package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*; -import gplx.core.lists.caches.*; +import gplx.core.caches.*; public class Xow_page_cache { private final Object thread_lock = new Object(); // NOTE: thread-safety needed for xomp since one page-cache is shared across all wkrs private final Xowe_wiki wiki; - private final Lru_cache cache = new Lru_cache(); + private final Lru_cache cache; private long cache_tries = 0; private long cache_misses = 0; public Xow_page_cache(Xowe_wiki wiki) { this.wiki = wiki; - cache.Max_(16 * Io_mgr.Len_mb); + this.cache_key = "xowa.app.page_cache.'" + wiki.Domain_str() + "'." + wiki.hashCode(); + this.cache = new Lru_cache(Bool_.Y, cache_key, 8 * Io_mgr.Len_mb, 16 * Io_mgr.Len_mb); } + public String Cache_key() {return cache_key;} private final String cache_key; public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr; - public void Max_(long v) {cache.Max_(v);} + public void Min_max_(long min, long max) {cache.Min_max_(min, max);} public void Add_itm(String ttl_full_db, Xow_page_cache_itm itm) { synchronized (thread_lock) { @@ -61,7 +63,7 @@ public class Xow_page_cache { public void Free_mem(boolean clear_permanent_itms) { synchronized (thread_lock) { // LOCK:app-level; DATE:2016-07-06 if (clear_permanent_itms) { - cache.Clear(); + cache.Clear_all(); } } } diff --git a/res/bin/any/xowa/addon/app/cfg/xo.cfg.sqlite3 b/res/bin/any/xowa/addon/app/cfg/xo.cfg.sqlite3 index 1a6ed010598d70446bef0eefaffa42d5693e1935..f795cbc5c030947534e81e42a30f8b0c820d65bc 100644 GIT binary patch delta 6136 zcmcIod2|!kxu1LQXkW%FHej27B|a#acM&ul0Xh2p}dkN4T13PNH(!#=d?ZV zygww(+;9Kg-}l`awYDaFsCzpnhx5A^e~7jY!;h12j}xS+|`mpTH_I9p6Rh7D!0F&bN=)f z?Fm`TWD+oC1QLoWKlrNN(P%)Hn{?ouySlkb#w*OBk! z?kpqu3BrP~u5|_t=fZ-{xvd`oXF4i`{loOex9v(qxiJFU&MZ`naNuA zD4KB)pHva^2WAiBVBDQ4`-U+zSz+F%@ZWRv9wr?FB_u zWIYoXd=1zz`+l;iu;@2rK{d|6p-yn$3mwY>;kR3I!#e_L7FOn}1+SZLF3z?Wgikij z?fg8j9Yn;3n&QNrCz}u?qA7z|J+hWV_}n&>8@{$a&P)Lg@NmQ>-sS}H!hKHA%Bb=1 z!m05>EKCJYk0yyHQ14L8Crl6Z4s((@!mO~~vL3LuTk6fX&7YdX=10wOC(*ajrQ8?0rQls{6+3SO4$ z7ogYN!Tvz>vwSnx;Pdc#Y>i)RAZjxop2?DBj`i|#ljyJK-E58Mar1t*Ns#MUkI+Ck z>t=ntzd?{BQic@uQ_Z>RYyD!t>&_$j@ei_7IJTh9&({=Z*U7R^deG^tB@P1B4wu;A zto4dgORYEHR0-)Mjsm>ooRI$jPE3I#aaRy(f-y|UF8LRnOmT80-ys%3z{PplYMzxm z1ZmO7dsXxTK2lOW!O(7JIf-ouxa!p30=7Z$2IL^`U?X5Ra4o87HSgjAWUm%hmJt9V zP9#>YBb&q#%5p>!!H2}*n_V7)ek0GSJIQ)PSA8B^9gta)gjbYVx5!E&yCxvG>IoV8 z{Hm@Z`XnKh3&99i`tAH;Z;j}e171$%JuP|twlm5SvJ}3PUxVO+*HCIO#x3V1gd`0^I>0UXrT=RLg0AQ% zpL>u*^3GGci}MnwNSZ-e>KBnTspuDan|5HvExk;tBYCQ=twLd6SjYKQ7B+}(LY+FE zw0j=~>=Ka?HbmQw;4#o;6oiWJny?q)f1>nWXMJ!I2DT8uWu6ROH@li^jP39F zBH3=aj*}xP-qQ(sgla$MZ&9~QBKSu*Kq%k}kZ~sywh;u?t?3;xWTufDwaC(nIjW6I zRBfW;gNsgKEwR6t)6P zLbP!eJx*D%5WEePXQqK41AKlasESL|`V>7rJ)i}0gaE%YQfLEd#?>siBh61(%q%Rt)>!6tT7=_uvtYQkSA6R!=E3H|U zw=Dl*3FBOAQkNWzf~!jHrGksKdTOyDmFbOB8xS1TelfhaJ`;b?3>?bZr@;jX zitwUmL4LTRexma7v*1IWT6}RFi3t9AO431a!(>gPmQg3PjEy-D4l&DdLI)U?KX`99VYg$}#2N@A?|EIG4 zD4?O%{QO8X1BExt8WL{(s|JBdZJ0X+hAUm2;DnxL(x`0wmKCHL+>Jt&ELQovICvb4 zqMfKJYE=B!WaW$FAP1%}d#Mx@&BE;#y^flI$AjNI0rKfmIt0R(Hjl;T3nx6ITodu$6QIy&)Ox5zu$+eFSlEs?F&4un<8#JEhD(N54Nn^+!)!yYL8HH*KdcXFL;8jKTwRauDEx`;ab1OOi1v?~ zZ?*fig_`a3FEw47a?McsBc-DYJO}8}s1k+~FQwrNQ=kbx(1Yi{0rJRdISMDvOU5g1 zBCB%tHEKmNEP9DNDU%n9*ZW>3QgGU7Aot0WHAQEO(RvJC2jj497P8>e zr$GT`u0ky=I)^LHfHC1!e>7&)G`WeBHw+#sH3FMTMdH~ny8<%i&Z@63QB?*O{T7#> z26p`BRN8`5&Vc`+4OwU^-q#I=X(y%I(MlMud?*?FYM>bx*FXaQoO13Ba0P0tm6WC#A2|;kxUw5$J=F~= zY3ms(3-9j+LvVdJSfRczjZklgDj#=)%tWZ!h0zD(MdqNg>;o`CGlIxd$-9QFZ7J`a z9T3QvpD)yg#(bV1S#;uOE)y&wL-*QcFu~maJxeLP0=f+b{atE2olDWVSbqbo)U43| zM!!?>+yF13KD?84MMb9`qC;so@g%|c&B=6z^86Rz64LX{yo-R^q3%U)c-1LQn0?(4 zPTXIDNM7N}KV*hmUpFdazXmy0yi5oA*p^ma)j@|DKbHdIdwpUGTt3+PpHkp4yf+p4 z;vAWWz$yc783C8Z3Emn3k0bnS4xAk)xRe7+af-H9%FqRHH#A2dTAUJjU2tM? zIT(g_2VtVMKk(TlSPa5^W#(cC5MHkD5GkYjE`Gc@7D>R04YoZASK z;?zd2LnH9oMwp82E%3PVq#xPxcmBGp!huGNyDnL7%U;WBgqHEIq% z5`fd<+>xkx|1j3pLBRTbA<(2|$8AaUz&^PMW@1&2w>QCL)4BG`4;YWADm9fV7MiWZG z-MgSs`EnP0APMik4X4Jb{P{L)z{~Ey1##<#?m!-AeH)*r0uI8tdi9=5P_!x!EsgwL zH%gf|0sVQrN|Y?zQHK(-P=|_Q9Y!fbgjG0GK=E1B38(>oB%ru1Re0jZf*Ns!2gQ>x!c!77A~Y$A2i*to)dmzAK=;2ipv6j;7u`nqg&(2H0rFRW zgofzV_kuj~4--Bqp$xoRLR!39LU{v}LlS9^LlQEmd|o&}d`CiQ*e;_3gHa+YfVK{@ zuKX>4+CjA17nohLW|)yb54K$`4n915jDA5R9Y$7x=gEP5=M^ delta 3293 zcmai02~<_(7Cw9b=iYnnc(`x@nJ=PZDIkO9kcwJTPZ2RKB~5S&0~u5XN4z-Z2%U6i znn77FUyhG9`!zA^b+NiqKx74NTD{7kB%^jL- zz{-j^_ppK6V{BpS05MG(sNPb4SATIk3JBa=3SYD8^QSwfIFfq%(fT!po5bHd9IF`?3~~=!a0bMRayaYD$bOx*bC;Gc5A)R|S&b6Iq$Ks9>YHkm zI^ON53RdZjW(d-T@3+z6&az1s{e$Rme->klPs$XAp|@xxDU9N8f>U#z^fM$SH3%S- z(!PYw+iEk4+!;lujnpR&LbZlji8MD30yW!gJKq)ud8!wBx_=8qbNa)QIOVG3E3F6R zd*$!S1Vu7@U?6>=9(C7r>viS2B6B=mk=82*Ow){2#+AzcO4n$+t!au*vWHkF=Z(zI z&z+DpoyU%XO&}I`Q05(!>FDz?3i{FT@sLQJlATY=fDAx*hw5L1XpaM>z`OU7t8sfKbk+$k{3)&&L+CG)!AIXOd*;l7 z8I)CwK^{Qm>n1~?34kjF@S(3~$0Yi`90F;pOEmrq;b;qHU(|}FE`pOSSo-2tY}I18 z-h#cqq!s&d3G8ga_Ab4TaaaaSN$9K$mDCd|S68W-YNG9wt-_XW3%7n_ecxJUO}5%B zCoNkorIw);#V;P6c&f)vIG9eQ7?QR|hi6QS4dN?56{blk>7Bj-5vKnSmQ z!!CuYzkp71BpbxE`A-LE3x3lXpl5j;8{cC0{mqa6{sly#JV17Vc6#w+Wci-?8l<0o z5kx%Y1&-ATioea_0Y-3w%_)aEno26#U@#s ztXnC$9=fE|m?}-PjdzX5joesi%r&MMI~sMO>S`31lBFanI>UE{I(*mgs$sYxK!05S zo<3KfsJp8>s9T{+(*?*{M50eg1^*%^2OevWp zIu3b^Uah@WQZJz%lMYhaUWm~iKW-(*e)v_6VdGFsAL2&^Q_wxcCmetRxkB$($KCe~gbkN0K=)%Tm_L>N)*#HX0UepR|0x9wk z1Zcx*+Vg{l;5<@(Blv5F&j#vyhO}oXs8wDF!=66t8X;XPT<%YAPedK%G{RUG&iYeT zqfm*njSyx>Ng{R%o`*F&;}mq)nOuFZbkj@rF6CqB<8u)2ImeOo+d1e?jzVagf+A1Lyn;h7Uo^W8sL*@wwy0wfV zuZ|jMqd!jM)&P9T=oPxT5%>&05`mw4>o1&~OPEa6QJBbojKaQAo?z~SQ&P~Mo=L$3 zp~5evUdllo1`sGe%$$w(7Wn7RH)(B|^ARhCzJTD7=9p!u#T%HX29rI?)6M;aUmT+M;Y$ z<0Zo;*+&Le4^HU&eN{d zsPMJZu?u)KX*i{CKPO<%ObaFr6={!TD&}hC3wn7dBhHp`O)j6`iSfE%wkx zSKM{yRyu)JZo_ZFoLJmYycZJKMyV?n*B3WHyh%pH;@!o2fM&a~Bh*rr8>`Lo3z8X& zUN2e;VSMN(_^DamVAv^fr5<}govTdGckaiZW69?(I_dLfQ6)E^-V1GU?DW}7WSr%w{TL6X}yj8DdRR4w#*QVCh*Wd z@ux&=lE~7Hb&};K9q@#1>@~TvBFKBUP-Q8E>tioQvZvAW9-wW_(d%O`bY)|_-*ugM zYIpXp1aD?SmoSAy7SdX{ugzxT+p-usRcO@IsVuh*`f(~t zIcy~D&0(V+&8&ky%4KFs&1FdsRyT2h#X)Nk8tCFQHdEC8>1k{fug_zDGA`DQ0s3JE z>niZ8Ggu(S%w+F8#BoFsTiRyw<|0e>R70_o!Gf&0Wlr>Cj>}_(52G_|zpV_@Vl#SSlE5TFM5uGGr!i zj!T!Z^fu`DGL}HWud