Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Sqlite: Add option for read-only detection [#509]

Browse Source
This commit is contained in:
gnosygnu
2019-07-19 07:14:24 -04:00
parent 7f76d8128d
commit 989ccde83a
28 changed files with 669 additions and 168 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
400_xowa/src/gplx/core/caches/Gfo_cache_mgr_base.java
View File

@@ -20,11 +20,11 @@ public class Gfo_cache_mgr_base {
public int Compress_to() {return compress_to;} public void Compress_to_(int v) {compress_to = v;} private int compress_to = 8;
protected Object Base_get_or_null(byte[] key) {
Object rv_obj = hash.Get_by(key);
return rv_obj == null ? null : ((Gfo_cache_itm)rv_obj).Val();
return rv_obj == null ? null : ((Gfo_cache_itm_bry)rv_obj).Val();
}
protected void Base_add(byte[] key, Object val) {
if (hash.Count() >= compress_max) Compress();
Gfo_cache_itm itm = new Gfo_cache_itm(key, val);
Gfo_cache_itm_bry itm = new Gfo_cache_itm_bry(key, val);
hash.Add(key, itm);
}
protected void Base_del(byte[] key) {
@@ -35,11 +35,11 @@ public class Gfo_cache_mgr_base {
int del_len = hash.Count() - compress_to;
List_adp del_list = List_adp_.New();
for (int i = 0; i < del_len; i++) {
Gfo_cache_itm itm = (Gfo_cache_itm)hash.Get_at(i);
Gfo_cache_itm_bry itm = (Gfo_cache_itm_bry)hash.Get_at(i);
del_list.Add(itm);
}
for (int i = 0; i < del_len; i++) {
Gfo_cache_itm itm = (Gfo_cache_itm)del_list.Get_at(i);
Gfo_cache_itm_bry itm = (Gfo_cache_itm_bry)del_list.Get_at(i);
hash.Del(itm.Key());
}
}

10
400_xowa/src/gplx/core/caches/Gfo_cache_mgr_bry.java
View File

@@ -20,17 +20,17 @@ public class Gfo_cache_mgr_bry extends Gfo_cache_mgr_base {
public void Add(byte[] key, Object val) {Base_add(key, val);}
public void Del(byte[] key) {Base_del(key);}
}
class Gfo_cache_itm {
public Gfo_cache_itm(Object key, Object val) {this.key = key; this.val = val; this.Touched_update();}
class Gfo_cache_itm_bry {
public Gfo_cache_itm_bry(Object key, Object val) {this.key = key; this.val = val; this.Touched_update();}
public Object Key() {return key;} private Object key;
public Object Val() {return val;} private Object val;
public long Touched() {return touched;} private long touched;
public Gfo_cache_itm Touched_update() {touched = System_.Ticks(); return this;}
public Gfo_cache_itm_bry Touched_update() {touched = System_.Ticks(); return this;}
}
class Gfo_cache_itm_comparer implements gplx.core.lists.ComparerAble {
public int compare(Object lhsObj, Object rhsObj) {
Gfo_cache_itm lhs = (Gfo_cache_itm)lhsObj;
Gfo_cache_itm rhs = (Gfo_cache_itm)rhsObj;
Gfo_cache_itm_bry lhs = (Gfo_cache_itm_bry)lhsObj;
Gfo_cache_itm_bry rhs = (Gfo_cache_itm_bry)rhsObj;
return Long_.Compare(lhs.Touched(), rhs.Touched());
}
public static final Gfo_cache_itm_comparer Touched_asc = new Gfo_cache_itm_comparer(); // TS.static

117
400_xowa/src/gplx/core/lists/caches/Lru_cache.java
View File

@@ -1,117 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
public class Lru_cache {
private final Hash_adp map = Hash_adp_.New();
private long max;
private long cur = 0;
private Lru_node head;
private Lru_node tail;
private long evicts;
public void Max_(long max) {
this.max = max;
}
public long Evicts() {return evicts;}
public long Cur() {return cur;}
public Object Get_or_null(Object key) {
Lru_node nde = (Lru_node)map.Get_by(key);
if (nde == null) {
return null;
}
Del_node_from_linked_list(nde);
Add_to_tail(nde);
return nde.Val();
}
public void Set(Object key, Object val, long size) {
Lru_node nde = (Lru_node)map.Get_by(key);
if (nde != null) {
nde.Val_(val);
Del_node_from_linked_list(nde);
Add_to_tail(nde);
}
else {
while (cur + size > max) {
Del_node_from_this(head);
evicts++;
}
nde = new Lru_node(key, val, size);
Add_to_tail(nde);
map.Add(key, nde);
cur += size;
}
}
public void Del(Object key) {
Lru_node nde = (Lru_node)map.Get_by(key);
if (nde != null) {
Del_node_from_this(nde);
}
}
public void Clear() {
map.Clear();
head = null;
tail = null;
cur = 0;
}
private void Del_node_from_this(Lru_node nde) {
map.Del(nde.Key());
cur -= nde.Size();
Del_node_from_linked_list(nde);
}
private void Del_node_from_linked_list(Lru_node nde) {
if (nde.Prv() == null)
head = nde.Nxt();
else
nde.Prv().Nxt_(nde.Nxt());
if (nde.Nxt() == null)
tail = nde.Prv();
else
nde.Nxt().Prv_(nde.Prv());
}
private void Add_to_tail(Lru_node nde) {
if (tail != null)
tail.Nxt_(nde);
nde.Prv_(tail);
nde.Nxt_(null);
tail = nde;
if (head == null)
head = tail;
}
}
class Lru_node {
private final Object key;
private Object val;
private final long size;
private Lru_node prv;
private Lru_node nxt;
public Lru_node(Object key, Object val, long size) {
this.key = key;
this.val = val;
this.size = size;
}
public Object Key() {return key;}
public Object Val() {return val;} public void Val_(Object v) {this.val = v;}
public long Size() {return size;}
public Lru_node Prv() {return prv;} public void Prv_(Lru_node v) {this.prv = v;}
public Lru_node Nxt() {return nxt;} public void Nxt_(Lru_node v) {this.nxt = v;}
}

116
400_xowa/src/gplx/core/lists/caches/Lru_cache_tst.java
View File

@@ -1,116 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.core.lists.caches; import gplx.*; import gplx.core.*; import gplx.core.lists.*;
import org.junit.*; import gplx.core.tests.*;
import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.nss.*;
public class Lru_cache_tst {
private final Lru_cache_fxt fxt = new Lru_cache_fxt();
@Test public void Get_one() {
fxt.Init__max(10);
fxt.Exec__set("a", 5);
fxt.Test__get_y("a");
}
@Test public void Pop_one() {
fxt.Init__max(10);
fxt.Exec__set("a", 10);
fxt.Exec__set("b", 10);
fxt.Test__get_n("a");
fxt.Test__get_y("b");
}
@Test public void Add_many() {
fxt.Init__max(10);
fxt.Exec__set("a", 4);
fxt.Exec__set("b", 3);
fxt.Exec__set("c", 2);
fxt.Exec__set("d", 1);
fxt.Test__get_y("a", "b", "c", "d");
}
@Test public void Pop_many() {
fxt.Init__max(10);
fxt.Exec__set("a", 4);
fxt.Exec__set("b", 3);
fxt.Exec__set("c", 2);
fxt.Exec__set("d", 1);
fxt.Exec__set("e", 6);
fxt.Test__get_y("c", "d", "e");
fxt.Test__get_n("a", "b");
}
@Test public void Set_repeatedly() {
fxt.Init__max(10);
fxt.Exec__set("a", "a1", 10);
fxt.Exec__set("a", "a2", 10);
fxt.Exec__set("a", "a3", 10);
fxt.Test__get_val("a", "a3");
}
@Test public void Set_bumps_priority() {
fxt.Init__max(10);
fxt.Exec__set("a", 2);
fxt.Exec__set("b", 3);
fxt.Exec__set("c", 2);
fxt.Exec__set("a", 2);
fxt.Exec__set("d", 7);
fxt.Test__get_y("a", "d");
fxt.Test__get_n("b", "c");
}
@Test public void Del() {
fxt.Init__max(10);
fxt.Exec__set("a", 2);
fxt.Exec__set("b", 2);
fxt.Exec__del("b");
fxt.Test__get_y("a");
fxt.Test__get_n("b");
}
@Test public void Clear() {
fxt.Init__max(10);
fxt.Exec__set("a", 2);
fxt.Exec__set("b", 2);
fxt.Exec__clear();
fxt.Test__get_n("a", "b");
}
}
class Lru_cache_fxt {
private final Lru_cache cache = new Lru_cache();
public void Init__max(long max) {
cache.Max_(max);
}
public void Exec__set(String key, long size) {
cache.Set(key, key, size);
}
public void Exec__set(String key, String val, long size) {
cache.Set(key, val, size);
}
public void Exec__del(String key) {
cache.Del(key);
}
public void Exec__clear() {
cache.Clear();
}
public void Test__get_y(String... keys) {
for (String key : keys)
Test__get(key, key);
}
public void Test__get_n(String... keys) {
for (String key : keys)
Test__get(key, null);
}
public void Test__get_val(String key, String val) {
Test__get(key, val);
}
private void Test__get(String key, String expd) {
Object actl = cache.Get_or_null(key);
Gftest.Eq__obj_or_null(expd, actl);
}
}

13
400_xowa/src/gplx/xowa/Xoae_app.java
View File

@@ -149,6 +149,7 @@ public class Xoae_app implements Xoa_app, Gfo_invk {
stage = Xoa_stage_.Tid_init;
user.Init_by_app(this);
cfg.Init_by_app(this);
Ctor_dbs();
user.User_db_mgr().Cache_mgr().Init_by_app(this);
misc_mgr.Init_by_app(this);
user.History_mgr().Init_by_app(this);
@@ -208,6 +209,18 @@ public class Xoae_app implements Xoa_app, Gfo_invk {
msg_log.Clear();
wiki_mgr.Free_mem(clear_ctx);
}
private void Ctor_dbs() {
if (gplx.core.envs.Env_.Mode_testing()) return;
String read_only_detection_str = cfg.Get_str_app_or("xowa.app.dbs.sqlite.read_only_detection", "basic_file");
int read_only_detection = Io_mgr.Read_only__basic__file;
if (String_.Eq(read_only_detection_str, "basic_file"))
read_only_detection = Io_mgr.Read_only__basic__file;
else if (String_.Eq(read_only_detection_str, "basic_file_and_dirs"))
read_only_detection = Io_mgr.Read_only__basic__file_and_dirs;
else if (String_.Eq(read_only_detection_str, "perms_file"))
read_only_detection = Io_mgr.Read_only__perms__file;
gplx.dbs.engines.sqlite.Sqlite_engine_.Read_only_detection = read_only_detection;
}
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_gui)) return gui_mgr;
else if (ctx.Match(k, Invk_api)) return api_root;

1
400_xowa/src/gplx/xowa/addons/apps/cfgs/mgrs/types/Xocfg_type_mgr.java
View File

@@ -27,6 +27,7 @@ public class Xocfg_type_mgr {
this.Lists__add("list:xowa.addon.http_server.file_retrieve_mode", Keyval_.new_("wait"), Keyval_.new_("skip"), Keyval_.new_("async_server", "async server"));
this.Lists__add("list:xowa.addon.scribunto.engine", "luaj", "lua");
this.Lists__add("list:xowa.addon.math.renderer", Keyval_.new_("mathjax","MathJax"), Keyval_.new_("latex", "LaTeX"));
this.Lists__add("list:xowa.app.dbs.sqlite.read_only_detection", Keyval_.new_("basic_file", "Basic - File only"), Keyval_.new_("basic_file_and_dirs", "Basic - File and parent dirs"), Keyval_.new_("perms_file", "Permissions - File"));
}
public void Lists__add(String key, String... vals) {
int len = vals.length;

3
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr.java
View File

@@ -33,7 +33,7 @@ public class Xomp_parse_mgr {
Xomp_page_pool page_pool = new Xomp_page_pool(page_pool_loader, cfg.Num_pages_per_wkr());
// cache: preload tmpls and imglinks
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates(), cfg.Page_cache_max());
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates(), cfg.Page_cache_min(), cfg.Page_cache_max());
wiki.App().User().User_db_mgr().Cache_mgr().Enabled_n_(); // disable db lookups of user cache
Xomp_prog_mgr prog_mgr = new Xomp_prog_mgr();
@@ -70,6 +70,7 @@ public class Xomp_parse_mgr {
for (int i = 0; i < wkr_len; ++i) {
// make wiki
Xowe_wiki wkr_wiki = Xow_wiki_utl_.Clone_wiki(wiki, wiki.Fsys_mgr().Root_dir());
Lru_cache_root.Instance.Del(wkr_wiki.Cache_mgr().Page_cache().Cache_key());
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/mgrs/Xomp_parse_mgr_cfg.java
View File

@@ -42,7 +42,8 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
public long Wbase_cache_mru_size() {return wbase_cache_mru_size;} private long wbase_cache_mru_size = 100;
public long Wbase_cache_mru_weight() {return wbase_cache_mru_weight;} private long wbase_cache_mru_weight = 10;
public long Wbase_cache_mru_compress_size() {return wbase_cache_mru_compress_size;} private long wbase_cache_mru_compress_size = 70;
public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 2 * (long)Io_mgr.Len_gb;
public long Page_cache_min() {return page_cache_min;} private long page_cache_min = 1500 * Io_mgr.Len_mb_long;
public long Page_cache_max() {return page_cache_max;} private long page_cache_max = 2000 * Io_mgr.Len_mb_long;
public void Init(Xowe_wiki wiki) {
if (num_wkrs == -1) num_wkrs = gplx.core.envs.Runtime_.Cpu_count();
if (num_pages_in_pool == -1) num_pages_in_pool = num_wkrs * 1000;
@@ -77,6 +78,7 @@ public class Xomp_parse_mgr_cfg implements Gfo_invk {
else if (ctx.Match(k, "wbase_cache_mru_size_")) wbase_cache_mru_size = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_weight_")) wbase_cache_mru_weight = m.ReadLong("v");
else if (ctx.Match(k, "wbase_cache_mru_compress_size_")) wbase_cache_mru_compress_size = m.ReadLong("v");
else if (ctx.Match(k, "page_cache_min_")) page_cache_min = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_min);
else if (ctx.Match(k, "page_cache_max_")) page_cache_max = gplx.core.ios.Io_size_.parse_or(m.ReadStr("v"), page_cache_max);
else return Gfo_invk_.Rv_unhandled;
return this;

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/utls/Xomp_tmpl_cache_bldr.java
View File

@@ -17,9 +17,9 @@ package gplx.xowa.addons.bldrs.mass_parses.parses.utls; import gplx.*; import gp
import gplx.dbs.*;
import gplx.xowa.wikis.caches.*;
public class Xomp_tmpl_cache_bldr {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all, long page_cache_max) {
public static Xow_page_cache New(Xowe_wiki wiki, boolean fill_all, long page_cache_min, long page_cache_max) {
Xow_page_cache rv = new Xow_page_cache(wiki);
rv.Max_(page_cache_max);
rv.Min_max_(page_cache_min, page_cache_max);
if (fill_all) Fill_all(rv, wiki);
return rv;
}

12
400_xowa/src/gplx/xowa/wikis/caches/Xow_page_cache.java
View File

@@ -14,19 +14,21 @@ GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.wikis.caches; import gplx.*; import gplx.xowa.*; import gplx.xowa.wikis.*;
import gplx.core.lists.caches.*;
import gplx.core.caches.*;
public class Xow_page_cache {
private final Object thread_lock = new Object(); // NOTE: thread-safety needed for xomp since one page-cache is shared across all wkrs
private final Xowe_wiki wiki;
private final Lru_cache cache = new Lru_cache();
private final Lru_cache cache;
private long cache_tries = 0;
private long cache_misses = 0;
public Xow_page_cache(Xowe_wiki wiki) {
this.wiki = wiki;
cache.Max_(16 * Io_mgr.Len_mb);
this.cache_key = "xowa.app.page_cache.'" + wiki.Domain_str() + "'." + wiki.hashCode();
this.cache = new Lru_cache(Bool_.Y, cache_key, 8 * Io_mgr.Len_mb, 16 * Io_mgr.Len_mb);
}
public String Cache_key() {return cache_key;} private final String cache_key;
public void Load_wkr_(Xow_page_cache_wkr v) {this.load_wkr = v;} private Xow_page_cache_wkr load_wkr;
public void Max_(long v) {cache.Max_(v);}
public void Min_max_(long min, long max) {cache.Min_max_(min, max);}
public void Add_itm(String ttl_full_db, Xow_page_cache_itm itm) {
synchronized (thread_lock) {
@@ -61,7 +63,7 @@ public class Xow_page_cache {
public void Free_mem(boolean clear_permanent_itms) {
synchronized (thread_lock) { // LOCK:app-level; DATE:2016-07-06
if (clear_permanent_itms) {
cache.Clear();
cache.Clear_all();
}
}
}