Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2025-06-02 07:24:19 +00:00
Code Issues Projects Releases Wiki Activity

Make: Add xomp_stats to track time per page (and other attributes) [#456]

Browse Source
This commit is contained in:
gnosygnu 2019-05-08 06:35:10 -04:00
parent 5db81504fb
commit cf94f252e9
34 changed files with 336 additions and 175 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
100_core/src/gplx/GfoMsg_.java
View File

@ -72,6 +72,16 @@ public class GfoMsg_ {
rv.Add("", vals[i]);
return rv;
}
public static Hash_adp Read_str_ary_as_hash(GfoMsg m, String k) {
String[] ary = m.ReadStrAry(k, "|");
int ary_len = ary.length;
if (ary_len == 0) return Hash_adp_.Noop;
Hash_adp rv = Hash_adp_.New();
for (int i = 0; i < ary_len; i++) {
rv.Add_if_dupe_use_1st(ary[i], ary[i]);
}
return rv;
}
}
class GfoMsg_wtr extends GfoMsg_base {
@Override protected Object ReadOr(String k, Object defaultOr) {

2
400_xowa/src/gplx/xowa/Xoae_page.java
View File

@ -19,6 +19,7 @@ import gplx.xowa.guis.*; import gplx.xowa.guis.views.*;
import gplx.xowa.files.*; import gplx.xowa.files.xfers.*;
import gplx.xowa.apps.kvs.*;
import gplx.xowa.parsers.*; import gplx.xowa.wikis.pages.lnkis.*; import gplx.xowa.xtns.cites.*; import gplx.xowa.xtns.wbases.*; import gplx.xowa.xtns.wbases.pfuncs.*;
import gplx.xowa.parsers.logs.stats.*;
import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.addons.htmls.tocs.*; import gplx.xowa.htmls.modules.popups.*;
import gplx.xowa.wikis.pages.wtxts.*; import gplx.xowa.wikis.pages.dbs.*; import gplx.xowa.wikis.pages.redirects.*; import gplx.xowa.wikis.pages.hdumps.*; import gplx.xowa.wikis.pages.htmls.*;
public class Xoae_page implements Xoa_page {
@ -64,6 +65,7 @@ public class Xoae_page implements Xoa_page {
public Xopg_tab_data Tab_data() {return tab_data;} private final Xopg_tab_data tab_data = new Xopg_tab_data();
public byte Edit_mode() {return edit_mode;} private byte edit_mode; public void Edit_mode_update_() {edit_mode = Xoa_page_.Edit_mode_update;}
public Xop_root_tkn Root() {return root;} public Xoae_page Root_(Xop_root_tkn v) {root = v; return this;} private Xop_root_tkn root;
public Xop_log_stat Stat_itm() {return stat_itm;} private final Xop_log_stat stat_itm = new Xop_log_stat();
public Xoh_cmd_mgr Html_cmd_mgr() {return html_cmd_mgr;} private Xoh_cmd_mgr html_cmd_mgr = new Xoh_cmd_mgr();
public Xof_xfer_queue File_queue() {return file_queue;} private Xof_xfer_queue file_queue = new Xof_xfer_queue();

106
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/dbs/Xomp_stat_tbl.java Normal file
View File

@ -0,0 +1,106 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.bldrs.mass_parses.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*;
import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.hzips.*;
import gplx.xowa.wikis.pages.*;
import gplx.xowa.parsers.logs.stats.*;
public class Xomp_stat_tbl implements Rls_able {
private static final String tbl_name = "xomp_stats"; private static final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private static final String
fld_page_id = flds.Add_int_pkey("page_id"), fld_wkr_uid = flds.Add_int("wkr_uid")
, fld_wtxt_len = flds.Add_int("wtxt_len"), fld_html_len = flds.Add_int("html_len"), fld_zip_len = flds.Add_int("zip_len")
, fld_page_time = flds.Add_long("page_time"), fld_tidy_time = flds.Add_long("tidy_time"), fld_fulltext_time = flds.Add_long("fulltext_time")
, fld_scrib_time = flds.Add_long("scrib_time"), fld_scrib_count = flds.Add_int("scrib_count"), fld_scrib_depth = flds.Add_int("scrib_depth")
, fld_image_count = flds.Add_int("image_count"), fld_audio_count = flds.Add_int("audio_count"), fld_video_count = flds.Add_int("video_count"), fld_media_count = flds.Add_int("media_count")
, fld_lnki_count = flds.Add_int("lnki_count"), fld_lnke_count = flds.Add_int("lnke_count"), fld_hdr_count = flds.Add_int("hdr_count")
, fld_math_count = flds.Add_int("math_count"), fld_imap_count = flds.Add_int("imap_count"), fld_hiero_count = flds.Add_int("hiero_count")
, fld_gallery_count = flds.Add_int("gallery_count"), fld_gallery_packed_count = flds.Add_int("gallery_packed_count")
;
private final Db_conn conn; private Db_stmt stmt_insert;
public Xomp_stat_tbl(Db_conn conn) {
this.conn = conn;
this.Create_tbl();
conn.Stmt_delete(tbl_name).Exec_delete(); // always zap table
conn.Rls_reg(this);
}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "pkey", fld_page_id)));}
public void Rls() {
stmt_insert = Db_stmt_.Rls(stmt_insert);
}
public void Stmt_new() {
stmt_insert = conn.Stmt_insert(tbl_name, flds);
}
public void Insert_by_copy(Db_rdr rdr) {
stmt_insert.Clear()
.Val_int (fld_page_id , rdr.Read_int(fld_page_id))
.Val_int (fld_wkr_uid , rdr.Read_int(fld_wkr_uid))
.Val_int (fld_wtxt_len , rdr.Read_int(fld_wtxt_len))
.Val_int (fld_html_len , rdr.Read_int(fld_html_len))
.Val_int (fld_zip_len , rdr.Read_int(fld_zip_len))
.Val_long(fld_page_time , rdr.Read_long(fld_page_time))
.Val_long(fld_tidy_time , rdr.Read_long(fld_tidy_time))
.Val_long(fld_fulltext_time , rdr.Read_long(fld_fulltext_time))
.Val_long(fld_scrib_time , rdr.Read_long(fld_scrib_time))
.Val_int (fld_scrib_count , rdr.Read_int (fld_scrib_count))
.Val_int (fld_scrib_depth , rdr.Read_int (fld_scrib_depth))
.Val_int (fld_image_count , rdr.Read_int (fld_image_count))
.Val_int (fld_audio_count , rdr.Read_int (fld_audio_count))
.Val_int (fld_video_count , rdr.Read_int (fld_video_count))
.Val_int (fld_media_count , rdr.Read_int (fld_media_count))
.Val_int (fld_lnki_count , rdr.Read_int (fld_lnki_count))
.Val_int (fld_lnke_count , rdr.Read_int (fld_lnke_count))
.Val_int (fld_hdr_count , rdr.Read_int (fld_hdr_count))
.Val_int (fld_math_count , rdr.Read_int (fld_math_count))
.Val_int (fld_imap_count , rdr.Read_int (fld_imap_count))
.Val_int (fld_hiero_count , rdr.Read_int (fld_hiero_count))
.Val_int (fld_gallery_count , rdr.Read_int (fld_gallery_count))
.Val_int (fld_gallery_packed_count , rdr.Read_int (fld_gallery_packed_count))
.Exec_insert();
}
public void Insert(Xoae_page wpg, Xoh_page hpg, int wkr_uid, long page_time, long fulltext_time) {
Xop_log_stat stat = wpg.Stat_itm();
stmt_insert.Clear()
.Val_int(fld_page_id , hpg.Page_id())
.Val_int(fld_wkr_uid , wkr_uid)
.Val_int(fld_wtxt_len , Len_or_0(wpg.Root().Root_src()))
.Val_int(fld_html_len , Len_or_0(hpg.Db().Html().Html_bry()))
.Val_int(fld_zip_len , hpg.Db().Html().Zip_len())
.Val_long(fld_page_time , page_time)
.Val_long(fld_tidy_time , stat.Tidy_time)
.Val_long(fld_fulltext_time , fulltext_time)
.Val_long(fld_scrib_time , stat.Scrib().Time())
.Val_int (fld_scrib_count , stat.Scrib().Count())
.Val_int (fld_scrib_depth , stat.Scrib().Depth_max())
.Val_int (fld_image_count , stat.Image_count)
.Val_int (fld_audio_count , stat.Audio_count)
.Val_int (fld_video_count , stat.Video_count)
.Val_int (fld_media_count , stat.Media_count)
.Val_int (fld_lnki_count , stat.Lnki_count)
.Val_int (fld_lnke_count , stat.Lnke_count)
.Val_int (fld_hdr_count , stat.Hdr_count)
.Val_int (fld_math_count , stat.Math_count)
.Val_int (fld_imap_count , stat.Imap_count)
.Val_int (fld_hiero_count , stat.Hiero_count)
.Val_int (fld_gallery_count , stat.Gallery_count)
.Val_int (fld_gallery_packed_count , stat.Gallery_packed_count)
.Exec_insert();
}
public void Stmt_rls() {
stmt_insert = Db_stmt_.Rls(stmt_insert);
}
private static int Len_or_0(byte[] bry) {return bry == null ? 0 : bry.length;}
}

8
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd.java
View File

@ -20,8 +20,12 @@ public class Xomp_make_cmd extends Xob_cmd__base {
public Xomp_make_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
new Xomp_make_html().Exec(wiki, cfg);
new Xomp_make_lnki().Exec(wiki, cfg, 10000);
if (cfg.Mode().Has("html"))
new Xomp_make_html().Exec(wiki, cfg);
if (cfg.Mode().Has("lnki"))
new Xomp_make_lnki().Exec(wiki, cfg, 10000);
if (cfg.Mode().Has("stat"))
new Xomp_make_stat().Exec(wiki, cfg);
}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__cfg)) return cfg;

6
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_cmd_cfg.java
View File

@ -15,10 +15,12 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
public class Xomp_make_cmd_cfg implements Gfo_invk {
public Hash_adp Mode() {return mode;} private Hash_adp mode = Hash_adp_.New().Add_and_more("html", "html").Add_and_more("lnki", "lnki").Add_and_more("stat", "stat");
public boolean Delete_html_dbs() {return delete_html_dbs;} private boolean delete_html_dbs = true;
public Ordered_hash Merger_wkrs() {return merger_wkrs;} private final Ordered_hash merger_wkrs = Ordered_hash_.New();
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk__delete_html_dbs_)) delete_html_dbs = m.ReadYn("v");
if (ctx.Match(k, Invk__mode_)) mode = GfoMsg_.Read_str_ary_as_hash(m, "v");
else if (ctx.Match(k, Invk__delete_html_dbs_)) delete_html_dbs = m.ReadYn("v");
else if (ctx.Match(k, Invk__merger_wkrs_)) {
String[] ary = m.ReadStrAry("k", "|");
for (String itm : ary)
@ -27,5 +29,5 @@ public class Xomp_make_cmd_cfg implements Gfo_invk {
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk__delete_html_dbs_ = "delete_html_dbs_", Invk__merger_wkrs_ = "merger_wkrs_";
private static final String Invk__mode_ = "mode_", Invk__delete_html_dbs_ = "delete_html_dbs_", Invk__merger_wkrs_ = "merger_wkrs_";
}

51
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_make_stat.java Normal file
View File

@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.addons.bldrs.mass_parses.makes; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.core.brys.*;
import gplx.dbs.*; import gplx.xowa.htmls.core.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
class Xomp_make_stat {
public void Exec(Xowe_wiki wiki, Xomp_make_cmd_cfg cfg) {
// init mgr_db and mgr_tbl
Xomp_mgr_db mgr_db = Xomp_mgr_db.New__load(wiki);
Db_conn mgr_conn = mgr_db.Conn();
Xomp_stat_tbl mgr_tbl = new Xomp_stat_tbl(mgr_conn);
mgr_conn.Txn_bgn("xomp_stats");
mgr_tbl.Stmt_new();
// loop wkrs
String sql = String_.Format("SELECT * FROM xomp_stats;");
int wkrs_len = mgr_db.Tbl__wkr().Select_count();
for (int i = 0; i < wkrs_len; ++i) {
int count = 0;
Xomp_wkr_db wkr_db = Xomp_wkr_db.New(mgr_db.Dir(), i);
Db_rdr rdr = wkr_db.Conn().Stmt_sql(sql).Exec_select__rls_auto(); // ANSI.Y
try {
while (rdr.Move_next()) {
mgr_tbl.Insert_by_copy(rdr);
if (++count % 10000 == 0) {
Gfo_usr_dlg_.Instance.Prog_many("", "", "xomp.stat.insert: db=~{0} count=~{1}", Int_.To_str_pad_bgn_space(i, 3), Int_.To_str_pad_bgn_space(count, 8));
mgr_conn.Txn_sav();
}
}
} finally {rdr.Rls();}
}
// cleanup
mgr_tbl.Stmt_rls();
mgr_conn.Txn_end();
mgr_conn.Rls_conn();
}
}

26
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/wkrs/Xomp_parse_wkr.java
View File

@ -40,7 +40,9 @@ public class Xomp_parse_wkr implements Gfo_invk {
private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
private final int uid;
private Xomp_wkr_db wkr_db;
private Xomp_stat_tbl stat_tbl;
// indexer vars
private final Xofulltext_indexer_wkr indexer;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
@ -64,9 +66,9 @@ public class Xomp_parse_wkr implements Gfo_invk {
// wkr-specific vars
this.wiki = wiki; this.uid = uid;
this.wkr_db = Xomp_wkr_db.New(Xomp_mgr_db.New__url(wiki), uid);
this.stat_tbl = new Xomp_stat_tbl(wkr_db.Conn());
}
public void Exec() {
// init
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
// disable file download
@ -86,7 +88,7 @@ public class Xomp_parse_wkr implements Gfo_invk {
logger.Bgn();
}
// init log_mgr / property_wkr
// init log_mgr / property_wkr / stats
Xop_log_wkr_factory wkr_factory = new Xop_log_wkr_factory(wkr_db.Conn());
if (cfg.Log_math()) wiki.Parser_mgr().Math__core().Log_wkr_(wkr_factory);
@ -94,6 +96,7 @@ public class Xomp_parse_wkr implements Gfo_invk {
hdump_bldr.Enabled_(cfg.Hdump_enabled()).Hzip_enabled_(cfg.Hzip_enabled()).Hzip_diff_(cfg.Hdiff_enabled()).Zip_tid_(cfg.Zip_tid());
hdump_bldr.Init(wiki, wkr_db.Conn(), new Xob_hdump_tbl_retriever__xomp(wkr_db.Html_tbl()));
wkr_db.Conn().Txn_bgn("xomp");
stat_tbl.Stmt_new();
// set status to running
mgr_db.Tbl__wkr().Update_status(uid, Xomp_wkr_tbl.Status__running);
@ -110,8 +113,9 @@ public class Xomp_parse_wkr implements Gfo_invk {
if (ppg.Text() == null) continue; // some pages have no text; ignore them else null ref; PAGE: it.d:miercuri DATE:2015-12-05
try {
// init page
long done_bgn = gplx.core.envs.System_.Ticks();
// get ns / ttl
int cur_ns = ppg.Ns_id();
Xoa_ttl ttl = wiki.Ttl_parse(cur_ns, ppg.Ttl_bry());
// if ns changed and prv_ns is main
@ -120,10 +124,13 @@ public class Xomp_parse_wkr implements Gfo_invk {
wiki.Cache_mgr().Free_mem__all(); // NOTE: clears page and wbase cache only; needed else OutOfMemory error for en.w in 25th hour; DATE:2017-01-11
prv_ns = cur_ns;
}
// init page
Xoae_page wpg = Xoae_page.New(wiki, ttl);
wpg.Bldr__ns_ord_(ns_ord_mgr.Get_ord_by_ns_id(cur_ns)); // NOTE: must set ns_id for tier_id in lnki_temp; DATE:2016-09-19
wpg.Db().Text().Text_bry_(ppg.Text());
wpg.Db().Page().Init_by_mp(ppg.Id(), ppg.Page_score());
wpg.Stat_itm().Init(uid);
// parse page
Xop_ctx pctx = parser_mgr.Ctx();
@ -134,16 +141,22 @@ public class Xomp_parse_wkr implements Gfo_invk {
hdump_bldr.Insert(pctx, wpg);
// index
if (indexer != null) indexer.Index(wpg);
long fulltext_time = 0;
if (indexer != null) {
fulltext_time = gplx.core.envs.System_.Ticks();
indexer.Index(wpg);
fulltext_time = gplx.core.envs.System_.Ticks__elapsed_in_frac(fulltext_time);
}
// mark done for sake of progress
prog_mgr.Mark_done(ppg.Id());
// update stats
long time_cur = gplx.core.envs.System_.Ticks();
done_time += time_cur - done_bgn;
done_bgn = time_cur;
long page_time = time_cur - done_bgn;
done_time += page_time;
++done_count;
stat_tbl.Insert(wpg, hdump_bldr.Tmp_hpg(), uid, page_time, fulltext_time);
// cleanup
// ctx.App().Utl__bfr_mkr().Clear_fail_check(); // make sure all bfrs are released
@ -165,6 +178,7 @@ public class Xomp_parse_wkr implements Gfo_invk {
if (logger != null) logger.End();
wkr_db.Conn().Txn_end();
wkr_db.Conn().Rls_conn();
stat_tbl.Stmt_rls();
mgr.Wkrs_done_add_1(); // NOTE: must release latch last else thread errors
}
public void Bld_stats(Bry_bfr bfr) {

5
400_xowa/src/gplx/xowa/htmls/Xoh_page_wtr_wkr.java
View File

@ -204,8 +204,11 @@ public class Xoh_page_wtr_wkr {
// if (ns_id == Xow_ns_.Tid__category) wiki.Ctg__catpage_mgr().Write_catpage(tidy_bfr, page, hctx);
// tidy html
if (ns_id != Xow_ns_.Tid__special) // skip Special b/c
if (ns_id != Xow_ns_.Tid__special) { // skip Special b/c
long tidy_time = gplx.core.envs.System_.Ticks();
wiki.Html_mgr().Tidy_mgr().Exec_tidy(tidy_bfr, !hctx.Mode_is_hdump(), page.Url_bry_safe());
page.Stat_itm().Tidy_time = gplx.core.envs.System_.Ticks__elapsed_in_frac(tidy_time);
}
// add back to main bfr
bfr.Add_bfr_and_clear(tidy_bfr);

5
400_xowa/src/gplx/xowa/htmls/core/bldrs/Xob_hdump_bldr.java
View File

@ -22,11 +22,11 @@ import gplx.xowa.parsers.*;
public class Xob_hdump_bldr implements Gfo_invk {
private boolean enabled, hzip_enabled, hzip_diff, hzip_b256; private byte zip_tid = Byte_.Max_value_127;
private Xowe_wiki wiki; private Xob_hdump_tbl_retriever html_tbl_retriever;
private Xoh_stat_tbl stat_tbl; private Xoh_stat_itm stat_itm;
private int prv_row_len = 0;
private final Xoh_page tmp_hpg = new Xoh_page(); private final Bry_bfr tmp_bfr = Bry_bfr_.New();
private boolean op_sys_is_wnt;
private byte[] toc_label = Bry_.Empty;
public Xoh_page Tmp_hpg() {return tmp_hpg;}
public Xob_hdump_bldr Enabled_(boolean v) {this.enabled = v; return this;}
public Xob_hdump_bldr Hzip_enabled_(boolean v) {this.hzip_enabled = v; return this;}
public Xob_hdump_bldr Hzip_diff_(boolean v) {this.hzip_diff = v; return this;}
@ -36,7 +36,6 @@ public class Xob_hdump_bldr implements Gfo_invk {
if (!enabled) return false;
this.op_sys_is_wnt = gplx.core.envs.Op_sys.Cur().Tid_is_wnt();
this.wiki = wiki; this.hdump_mgr = wiki.Html__hdump_mgr(); this.html_tbl_retriever = html_tbl_retriever;
this.stat_tbl = new Xoh_stat_tbl(make_conn); this.stat_itm = hdump_mgr.Hzip_mgr().Hctx().Hzip__stat();
this.toc_label = wiki.Msg_mgr().Val_by_id(gplx.xowa.langs.msgs.Xol_msg_itm_.Id_toc);
if (zip_tid == Byte_.Max_value_127) zip_tid = Xobldr_cfg.Zip_mode__html(wiki.App());
@ -64,7 +63,7 @@ public class Xob_hdump_bldr implements Gfo_invk {
// save to db
Xowd_html_tbl html_tbl = html_tbl_retriever.Get_html_tbl(wpg.Ttl().Ns(), prv_row_len); // get html_tbl
this.prv_row_len = hdump_mgr.Save_mgr().Save(wpg, tmp_hpg.Ctor_by_hdiff(tmp_bfr, wpg, toc_label), html_tbl, true, is_wikitext); // save to db
stat_tbl.Insert(tmp_hpg, stat_itm, wpg.Root().Root_src().length, tmp_hpg.Db().Html().Html_bry().length, prv_row_len); // save stats
tmp_hpg.Db().Html().Zip_len_(prv_row_len);
// run hzip diff if enabled
if (hzip_diff && is_wikitext) {

63
400_xowa/src/gplx/xowa/htmls/core/hzips/Xoh_stat_itm.java
View File

@ -1,63 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.htmls.core.hzips; import gplx.*; import gplx.xowa.*; import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.*;
import gplx.xowa.htmls.core.wkrs.lnkes.*;
public class Xoh_stat_itm {
public void Clear() {
a_rhs = lnki_text_n = lnki_text_y = lnke__free = lnke__auto = lnke__text = 0;
hdr_1 = hdr_2 = hdr_3 = hdr_4 = hdr_5 = hdr_6 = timeline = gallery = 0;
img_full = 0;
space = 0;
Bry_.Clear(escape_bry);
}
public int A_rhs() {return a_rhs;} public void A_rhs_add() {++a_rhs;} private int a_rhs;
public int Lnki_text_n() {return lnki_text_n;} public void Lnki_text_n_add() {++lnki_text_n;} private int lnki_text_n;
public int Lnki_text_y() {return lnki_text_y;} public void Lnki_text_y_add() {++lnki_text_y;} private int lnki_text_y;
public int Lnke__free() {return lnke__free;} public void Lnke__free__add() {++lnke__free;} private int lnke__free;
public int Lnke__auto() {return lnke__auto;} public void Lnke__auto__add() {++lnke__auto;} private int lnke__auto;
public int Lnke__text() {return lnke__text;} public void Lnke__text__add() {++lnke__text;} private int lnke__text;
public int Img_full() {return img_full;} public void Img_full_add() {++img_full;} private int img_full;
public int Timeline() {return timeline;} public void Timeline_add() {++timeline;} private int timeline;
public int Gallery() {return gallery;} public void Gallery_add() {++gallery;} private int gallery;
public int Hdr_1() {return hdr_1;} private int hdr_1;
public int Hdr_2() {return hdr_2;} private int hdr_2;
public int Hdr_3() {return hdr_3;} private int hdr_3;
public int Hdr_4() {return hdr_4;} private int hdr_4;
public int Hdr_5() {return hdr_5;} private int hdr_5;
public int Hdr_6() {return hdr_6;} private int hdr_6;
public int Space() {return space;} public void Space_add(int v) {space += v;} private int space;
public byte[] Escape_bry() {return escape_bry;} public void Escape_add(byte v) {escape_bry[v] += 1;} private final byte[] escape_bry = new byte[256];
public void Hdr_add(int hdr_num) {
switch (hdr_num) {
case 1: ++hdr_1; break;
case 2: ++hdr_2; break;
case 3: ++hdr_3; break;
case 4: ++hdr_4; break;
case 5: ++hdr_5; break;
case 6: ++hdr_6; break;
default: throw Err_.new_unhandled(hdr_num);
}
}
public void Lnki_add(int orig_len, int hzip_len, int flag) {
}
public void Lnke_add(byte lnke_type) {
switch (lnke_type) {
case Xoh_lnke_dict_.Type__free: ++lnke__free; break;
case Xoh_lnke_dict_.Type__auto: ++lnke__auto; break;
case Xoh_lnke_dict_.Type__text: ++lnke__text; break;
}
}
}

73
400_xowa/src/gplx/xowa/htmls/core/hzips/Xoh_stat_tbl.java
View File

@ -1,73 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.htmls.core.hzips; import gplx.*; import gplx.xowa.*; import gplx.xowa.htmls.*; import gplx.xowa.htmls.core.*;
import gplx.dbs.*; import gplx.dbs.engines.sqlite.*; import gplx.xowa.htmls.core.hzips.*;
import gplx.xowa.wikis.pages.*;
public class Xoh_stat_tbl implements Rls_able {
private static final String tbl_name = "hdump_stats"; private static final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private static final String
fld_page_id = flds.Add_int_pkey("page_id"), fld_wtxt_len = flds.Add_int("wtxt_len"), fld_row_orig_len = flds.Add_int("row_orig_len"), fld_row_zip_len = flds.Add_int("row_zip_len")
, fld_body_len = flds.Add_int("body_len"), fld_display_ttl_len = flds.Add_int("display_ttl_len"), fld_content_sub_len = flds.Add_int("content_sub_len"), fld_sidebar_div_len = flds.Add_int("sidebar_div_len")
, fld_js_math = flds.Add_int("js_math"), fld_js_imap = flds.Add_int("js_imap"), fld_js_packed = flds.Add_int("js_packed"), fld_js_hiero = flds.Add_int("js_hiero")
, fld_a_rhs = flds.Add_int("a_rhs"), fld_lnki_text_n = flds.Add_int("lnki_text_n"), fld_lnki_text_y = flds.Add_int("lnki_text_y")
, fld_lnke_free = flds.Add_int("lnke_free"), fld_lnke_auto = flds.Add_int("lnke_auto"), fld_lnke_text = flds.Add_int("lnke_text")
, fld_hdr_1 = flds.Add_int("hdr_1"), fld_hdr_2 = flds.Add_int("hdr_2"), fld_hdr_3 = flds.Add_int("hdr_3"), fld_hdr_4 = flds.Add_int("hdr_4"), fld_hdr_5 = flds.Add_int("hdr_5"), fld_hdr_6 = flds.Add_int("hdr_6")
, fld_img_full = flds.Add_int("img_full")
;
private final Db_conn conn; private Db_stmt stmt_insert;
public Xoh_stat_tbl(Db_conn conn) {
this.conn = conn;
this.Create_tbl();
conn.Stmt_delete(tbl_name).Exec_delete(); // always zap table
conn.Rls_reg(this);
}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds, Dbmeta_idx_itm.new_unique_by_tbl(tbl_name, "pkey", fld_page_id)));}
public void Rls() {
stmt_insert = Db_stmt_.Rls(stmt_insert);
}
public void Insert(Xoh_page hpg, Xoh_stat_itm hzip, int wtxt_len, int row_orig_len, int row_zip_len) {
Xopg_module_mgr js_mgr = hpg.Head_mgr();
if (stmt_insert == null) stmt_insert = conn.Stmt_insert(tbl_name, flds);
stmt_insert.Clear()
.Val_int(fld_page_id , hpg.Page_id())
.Val_int(fld_wtxt_len , wtxt_len)
.Val_int(fld_row_orig_len , row_orig_len)
.Val_int(fld_row_zip_len , row_zip_len)
.Val_int(fld_body_len , Len_or_0(hpg.Db().Html().Html_bry()))
.Val_int(fld_display_ttl_len , Len_or_0(hpg.Display_ttl()))
.Val_int(fld_content_sub_len , Len_or_0(hpg.Content_sub()))
.Val_int(fld_sidebar_div_len , Len_or_0(hpg.Sidebar_div()))
.Val_bool_as_byte(fld_js_math , js_mgr.Math_exists())
.Val_bool_as_byte(fld_js_imap , js_mgr.Imap_exists())
.Val_bool_as_byte(fld_js_packed , js_mgr.Gallery_packed_exists())
.Val_bool_as_byte(fld_js_hiero , js_mgr.Hiero_exists())
.Val_int(fld_a_rhs , hzip.A_rhs())
.Val_int(fld_lnki_text_n , hzip.Lnki_text_n())
.Val_int(fld_lnki_text_y , hzip.Lnki_text_y())
.Val_int(fld_lnke_free , hzip.Lnke__free())
.Val_int(fld_lnke_auto , hzip.Lnke__auto())
.Val_int(fld_lnke_text , hzip.Lnke__text())
.Val_int(fld_hdr_1 , hzip.Hdr_1())
.Val_int(fld_hdr_2 , hzip.Hdr_2())
.Val_int(fld_hdr_3 , hzip.Hdr_3())
.Val_int(fld_hdr_4 , hzip.Hdr_4())
.Val_int(fld_hdr_5 , hzip.Hdr_5())
.Val_int(fld_hdr_6 , hzip.Hdr_6())
.Val_int(fld_img_full , hzip.Img_full())
.Exec_insert();
}
private int Len_or_0(byte[] bry) {return bry == null ? 0 : bry.length;}
}

2
400_xowa/src/gplx/xowa/htmls/core/wkrs/Xoh_hdoc_ctx.java
View File

@ -41,7 +41,6 @@ public class Xoh_hdoc_ctx {
public Xoh_pool_mgr__hzip Pool_mgr__hzip() {return pool_mgr__hzip;} private final Xoh_pool_mgr__hzip pool_mgr__hzip = new Xoh_pool_mgr__hzip();
public Xoh_pool_mgr__data Pool_mgr__data() {return pool_mgr__data;} private final Xoh_pool_mgr__data pool_mgr__data = new Xoh_pool_mgr__data();
public Xoh_pool_mgr__wtr Pool_mgr__wtr() {return pool_mgr__wtr;} private final Xoh_pool_mgr__wtr pool_mgr__wtr = new Xoh_pool_mgr__wtr();
public Xoh_stat_itm Hzip__stat() {return hzip__stat;} private final Xoh_stat_itm hzip__stat = new Xoh_stat_itm();
public Xohz_tag_regy Hzip__xnde__regy() {return hzip__xnde__regy;} private final Xohz_tag_regy hzip__xnde__regy = Xohz_tag_regy_.New_dflt();
public Xoh_xnde_dict_reg Hzip__xnde__dict() {return hzip__xnde__dict;} private final Xoh_xnde_dict_reg hzip__xnde__dict = new Xoh_xnde_dict_reg();
public int Uid__gly__nxt() {return ++uid__gly;} private int uid__gly;
@ -87,7 +86,6 @@ public class Xoh_hdoc_ctx {
this.Clear();
}
private void Clear() {
hzip__stat.Clear();
this.uid__gly = -1;
}

2
400_xowa/src/gplx/xowa/htmls/core/wkrs/Xoh_hdoc_wkr__hzip.java
View File

@ -17,12 +17,10 @@ package gplx.xowa.htmls.core.wkrs; import gplx.*; import gplx.xowa.*; import gpl
import gplx.langs.htmls.docs.*; import gplx.xowa.htmls.core.hzips.*;
import gplx.xowa.wikis.ttls.*;
public class Xoh_hdoc_wkr__hzip implements Xoh_hdoc_wkr {
private final Xoh_stat_itm stat_itm = new Xoh_stat_itm();
private Xoh_hzip_bfr bfr; private Xoh_hdoc_ctx hctx; private byte[] src;
private Xoh_page hpg;
public void On_page_bgn(Bry_bfr bfr, Xoh_page hpg, Xoh_hdoc_ctx hctx, byte[] src, int src_bgn, int src_end) {
this.bfr = (Xoh_hzip_bfr)bfr; this.hpg = hpg; this.hctx = hctx; this.src = src;
stat_itm.Clear();
}
public void On_page_end() {}
public void On_txt (int rng_bgn, int rng_end) {bfr.Add_mid(src, rng_bgn, rng_end);}

1
400_xowa/src/gplx/xowa/htmls/core/wkrs/addons/gallerys/Xoh_gallery_hzip.java
View File

@ -25,7 +25,6 @@ public class Xoh_gallery_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
bfr.Add_mid(src, data.Src_bgn(), data.Src_end());
hctx.Hzip__stat().Gallery_add();
return this;
}
public void Decode1(Bry_bfr bfr, Xoh_hdoc_wkr hdoc_wkr, Xoh_hdoc_ctx hctx, Xoh_page hpg, Bry_rdr rdr, byte[] src, int src_bgn, int src_end, Xoh_data_itm data_itm) {

1
400_xowa/src/gplx/xowa/htmls/core/wkrs/addons/pgbnrs/Xoh_pgbnr_hzip.java
View File

@ -25,7 +25,6 @@ public class Xoh_pgbnr_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
bfr.Add_mid(src, data.Src_bgn(), data.Src_end());
hctx.Hzip__stat().Gallery_add();
return this;
}
public void Decode1(Bry_bfr bfr, Xoh_hdoc_wkr hdoc_wkr, Xoh_hdoc_ctx hctx, Xoh_page hpg, Bry_rdr rdr, byte[] src, int src_bgn, int src_end, Xoh_data_itm data_itm) {

1
400_xowa/src/gplx/xowa/htmls/core/wkrs/addons/timelines/Xoh_timeline_hzip.java
View File

@ -26,7 +26,6 @@ public class Xoh_timeline_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
// just add the entire thing; not worth trying to compress "<pre class='xowa-timeline'>"
bfr.Add_mid(src, data.Src_bgn(), data.Src_end());
hctx.Hzip__stat().Timeline_add();
return this;
}
public void Decode1(Bry_bfr bfr, Xoh_hdoc_wkr hdoc_wkr, Xoh_hdoc_ctx hctx, Xoh_page hpg, Bry_rdr rdr, byte[] src, int src_bgn, int src_end, Xoh_data_itm data_itm) {

3
400_xowa/src/gplx/xowa/htmls/core/wkrs/escapes/Xoh_escape_hzip.java
View File

@ -17,7 +17,6 @@ package gplx.xowa.htmls.core.wkrs.escapes; import gplx.*; import gplx.xowa.*; im
import gplx.core.brys.*; import gplx.core.threads.poolables.*;
import gplx.xowa.htmls.core.hzips.*;
public class Xoh_escape_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
private byte escape_byte;
public int Tid() {return Xoh_hzip_dict_.Tid__escape;}
public byte[] Hook() {return hook;} private byte[] hook;
public String Key() {return Xoh_hzip_dict_.Key__escape;}
@ -26,7 +25,6 @@ public class Xoh_escape_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
Xoh_escape_data data = (Xoh_escape_data)data_obj;
bfr.Add(hook); // EX: 1,0
bfr.Add(data.Hook()); // EX: 2
hctx.Hzip__stat().Escape_add(escape_byte);
return this;
}
public void Decode1(Bry_bfr bfr, Xoh_hdoc_wkr hdoc_wkr, Xoh_hdoc_ctx hctx, Xoh_page hpg, Bry_rdr rdr, byte[] src, int src_bgn, int src_end, Xoh_data_itm data_itm) {
@ -37,7 +35,6 @@ public class Xoh_escape_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
Xoh_escape_hzip rv = new Xoh_escape_hzip();
rv.pool_mgr = mgr; rv.pool_idx = idx;
rv.hook = (byte[])args[0];
rv.escape_byte = rv.hook[0];
return rv;
}
}

1
400_xowa/src/gplx/xowa/htmls/core/wkrs/hdrs/Xoh_hdr_html.java
View File

@ -26,6 +26,7 @@ public class Xoh_hdr_html {
// init
int hdr_num = hdr.Num();
boolean hdr_is_valid = hdr_num > 0; // hdr_num == 0 when dangling
page.Stat_itm().Hdr_count++;
// register hdr with TOC
byte[] hdr_text_bry = Bld_hdr_html(hdr_text_bfr, wtr, page, ctx, hctx, src, hdr);

5
400_xowa/src/gplx/xowa/htmls/core/wkrs/hdrs/Xoh_hdr_hzip.java
View File

@ -24,7 +24,7 @@ public class Xoh_hdr_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
Xoh_hdr_data data = (Xoh_hdr_data)data_obj;
boolean capt_rhs_exists = flag_bldr.Set_as_bool (Flag__capt_rhs_exists , data.Capt_rhs_exists());
boolean anch_is_diff = flag_bldr.Set_as_bool (Flag__anch_is_diff , data.Anch_is_diff());
int hdr_level = flag_bldr.Set_as_int (Flag__hdr_level , data.Hdr_level());
flag_bldr.Set_as_int (Flag__hdr_level , data.Hdr_level());
bfr.Add(hook);
bfr.Add_hzip_int(1, flag_bldr.Encode());
@ -32,7 +32,6 @@ public class Xoh_hdr_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
if (anch_is_diff) bfr.Add_hzip_mid(src, data.Anch_bgn(), data.Anch_end()); // add anchor
if (capt_rhs_exists) bfr.Add_hzip_mid(src, data.Capt_rhs_bgn(), data.Capt_rhs_end());// add capt_rhs
hctx.Hzip__stat().Hdr_add(hdr_level);
return this;
}
public void Decode1(Bry_bfr bfr, Xoh_hdoc_wkr hdoc_wkr, Xoh_hdoc_ctx hctx, Xoh_page hpg, Bry_rdr rdr, byte[] src, int src_bgn, int src_end, Xoh_data_itm data_itm) {
@ -50,7 +49,7 @@ public class Xoh_hdr_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
Xoh_hdr_data data = (Xoh_hdr_data)data_itm;
data.Init_by_decode(hdr_level, anch_is_diff, anch_bgn, anch_end, capt_bgn, capt_end, capt_rhs_bgn, capt_rhs_end);
}
private final Int_flag_bldr flag_bldr = new Int_flag_bldr().Pow_ary_bld_ (1, 1, 3);
private final Int_flag_bldr flag_bldr = new Int_flag_bldr().Pow_ary_bld_ (1, 1, 3);
private static final int // SERIALIZED
Flag__capt_rhs_exists = 0
, Flag__anch_is_diff = 1

2
400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/Xoh_img_hzip.java
View File

@ -105,7 +105,7 @@ public class Xoh_img_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
if (img_src.File_page_exists()) Gfo_hzip_int_.Encode(1, bfr, Gfo_hzip_int_.Neg_1_adj + img_src.File_page());
}
else
xoimg.Encode(bfr, hctx.Hzip__stat(), src, img_xoimg);
xoimg.Encode(bfr, src, img_xoimg);
if (anch_title_exists) bfr.Add_hzip_mid(src, data.Anch_title_bgn(), data.Anch_title_end());
if (img__alt_diff_from_anch_title) bfr.Add_hzip_mid(src, data.Img_alt_bgn(), data.Img_alt_end());
if (img_cls.Other_exists()) bfr.Add_hzip_mid(src, img_cls.Other_bgn(), img_cls.Other_end());

2
400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/atrs/Xoh_img_xoimg_hzip.java
View File

@ -18,7 +18,7 @@ import gplx.core.brys.*; import gplx.core.encoders.*;
import gplx.xowa.htmls.core.hzips.*;
import gplx.xowa.parsers.lnkis.*; import gplx.xowa.files.*;
public class Xoh_img_xoimg_hzip {
public void Encode(Bry_bfr bfr, Xoh_stat_itm stat_itm, byte[] src, Xoh_img_xoimg_data arg) {
public void Encode(Bry_bfr bfr, byte[] src, Xoh_img_xoimg_data arg) {
boolean page_exists = arg.Lnki_page() != Xof_lnki_page.Null;
boolean time_exists = arg.Lnki_time() != Xof_lnki_time.Null;
boolean upright_exists = arg.Lnki_upright() != Xof_img_size.Upright_null;

1
400_xowa/src/gplx/xowa/htmls/core/wkrs/lnkes/Xoh_lnke_html.java
View File

@ -32,6 +32,7 @@ public class Xoh_lnke_html {
bfr.Add(Gfh_tag_.Div_rhs);
return;
}
ctx.Page().Stat_itm().Lnke_count++;
if (!hctx.Mode_is_alt()) { // do not write "<a ...>" if mode is alt
bfr.Add(Gfh_bldr_.Bry__a_lhs_w_href);
if (Write_href(bfr, hctx, ctx, src, lnke, href_bgn, href_end, proto_is_xowa))

3
400_xowa/src/gplx/xowa/htmls/core/wkrs/lnkes/Xoh_lnke_hzip.java
View File

@ -25,14 +25,13 @@ public class Xoh_lnke_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
boolean title_exists = flag_bldr.Set_as_bool(Flag__title_exists , data.Title_exists());
boolean auto_exists = flag_bldr.Set_as_bool(Flag__auto_exists , data.Auto_exists());
boolean capt_exists = flag_bldr.Set_as_bool(Flag__capt_exists , data.Capt_exists());
byte lnke_tid = flag_bldr.Set_as_byte(Flag__lnke_tid , data.Lnke_tid());
flag_bldr.Set_as_byte(Flag__lnke_tid , data.Lnke_tid());
bfr.Add(hook);
bfr.Add_hzip_int(1, flag_bldr.Encode()); // add flag
bfr.Add_hzip_mid(src, data.Href_bgn(), data.Href_end()); // add href
if (auto_exists) bfr.Add_hzip_int(1, data.Auto_id()); // add autonumber
if (capt_exists) bfr.Add_hzip_mid(src, data.Capt_bgn(), data.Capt_end()); // add caption
if (title_exists) bfr.Add_hzip_mid(src, data.Title_bgn(), data.Title_end()); // add title
hctx.Hzip__stat().Lnke_add(lnke_tid);
return this;
}
public void Decode1(Bry_bfr bfr, Xoh_hdoc_wkr hdoc_wkr, Xoh_hdoc_ctx hctx, Xoh_page hpg, Bry_rdr rdr, byte[] src, int src_bgn, int src_end, Xoh_data_itm data_itm) {

2
400_xowa/src/gplx/xowa/htmls/core/wkrs/lnkis/Xoh_lnki_hzip.java
View File

@ -38,7 +38,6 @@ public class Xoh_lnki_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
flag_bldr.Set_as_int(Flag__capt_cs0_tid , data.Capt_itm().Cs0_tid());
byte text_type = flag_bldr.Set_as_byte(Flag__text_type , data.Text_tid());
// Tfds.Dbg(flag_bldr.Encode(), Array_.To_str(flag_bldr.Val_ary()), text_type);
int bfr_bgn = bfr.Len();
int flag = flag_bldr.Encode();
bfr.Add(hook);
bfr.Add_hzip_int(1, flag);
@ -57,7 +56,6 @@ public class Xoh_lnki_hzip implements Xoh_hzip_wkr, Gfo_poolable_itm {
break;
}
if (title_tid == Xoh_lnki_data.Title__diff) bfr.Add_hzip_mid(src, data.Title_bgn(), data.Title_end());
hctx.Hzip__stat().Lnki_add(data.Src_end() - data.Src_bgn(), bfr.Len() - bfr_bgn, flag);
return this;
}
public void Decode1(Bry_bfr bfr, Xoh_hdoc_wkr hdoc_wkr, Xoh_hdoc_ctx hctx, Xoh_page hpg, Bry_rdr rdr, byte[] src, int src_bgn, int src_end, Xoh_data_itm data_itm) {

10
400_xowa/src/gplx/xowa/htmls/core/wkrs/lnkis/htmls/Xoh_file_wtr__basic.java
View File

@ -93,6 +93,7 @@ public class Xoh_file_wtr__basic implements Gfo_invk {
? caption_fmtr.To_bry(ctx, hctx, src, lnki.Caption_val_tkn(), Bool_.N, Xoh_lnki_text_fmtr.Null__fmt)
: lnki.Ttl().Full_db();
html_fmtr.Add_media(bfr, hctx.Mode_is_hdump(), img_orig_src, lnki_ttl_bry, media_html); // NOTE: use orig_src not view_src; DATE:2014-01-19
page.Stat_itm().Media_count++;
}
else {
// orig_is_video
@ -110,11 +111,16 @@ public class Xoh_file_wtr__basic implements Gfo_invk {
if (orig_is_video) {
xfer_itm.Html_elem_tid_(Xof_html_elem.Tid_vid); // mark as vid for js_mgr
this.Write_file_video(bfr, ctx, hctx, src, lnki, img_orig_src, uid, div_width, lnki_halign_bry, lnki_href, img_alt, lnki_ttl_bry, img_view_src, xfer_itm);
page.Stat_itm().Video_count++;
}
else if (orig_ext.Id_is_audio()) // audio
else if (orig_ext.Id_is_audio()) { // audio
this.Write_file_audio(bfr, ctx, hctx, src, lnki, img_orig_src, uid, div_width, lnki_halign_bry, lnki_href, img_alt, lnki_ttl_bry);
else // image
page.Stat_itm().Audio_count++;
}
else { // image
this.Write_file_image(bfr, ctx, hctx, src, lnki, img_orig_src, uid, div_width, lnki_halign_bry, lnki_href, img_alt, lnki_ttl_bry, img_view_src, xfer_itm, lnki_is_thumbable, lnki_halign, orig_ext);
page.Stat_itm().Image_count++;
}
}
}
private void Write_file_audio(Bry_bfr bfr, Xop_ctx ctx, Xoh_wtr_ctx hctx, byte[] src, Xop_lnki_tkn lnki, byte[] img_orig_src, int uid, int div_width, byte[] lnki_halign_bry, byte[] lnki_href, byte[] alt, byte[] lnki_ttl) {

20
400_xowa/src/gplx/xowa/htmls/core/wkrs/lnkis/htmls/Xoh_lnki_wtr.java
View File

@ -65,9 +65,22 @@ public class Xoh_lnki_wtr {
redlinks_mgr.Add(lnki);
boolean stage_is_alt = hctx.Mode_is_alt();
switch (lnki.Ns_id()) {
case Xow_ns_.Tid__media: if (!stage_is_alt) file_wtr.Write_or_queue(bfr, page, ctx, hctx, src, lnki); return; // NOTE: literal ":" has no effect; PAGE:en.w:Beethoven and [[:Media:De-Ludwig_van_Beethoven.ogg|listen]]
case Xow_ns_.Tid__file: if (!literal_link && !stage_is_alt) {file_wtr.Write_or_queue(bfr, page, ctx, hctx, src, lnki); return;} break;
case Xow_ns_.Tid__category: if (!literal_link) {page.Wtxt().Ctgs__add(lnki.Ttl()); return;} break;
case Xow_ns_.Tid__media:
if (!stage_is_alt)
file_wtr.Write_or_queue(bfr, page, ctx, hctx, src, lnki);
return; // NOTE: literal ":" has no effect; PAGE:en.w:Beethoven and [[:Media:De-Ludwig_van_Beethoven.ogg|listen]]
case Xow_ns_.Tid__file:
if (!literal_link && !stage_is_alt) {
file_wtr.Write_or_queue(bfr, page, ctx, hctx, src, lnki);
return;
}
break;
case Xow_ns_.Tid__category:
if (!literal_link) {
page.Wtxt().Ctgs__add(lnki.Ttl());
return;
}
break;
}
Write_plain_by_tkn(bfr, hctx, src, lnki, lnki_ttl);
}
@ -98,6 +111,7 @@ public class Xoh_lnki_wtr {
}
}
if (lnki.Xtn_sites_link()) return; // lnki marked for relatedSites; don't write to page
page.Stat_itm().Lnki_count++;
if (hctx.Mode_is_alt())
Write_caption(bfr, ctx, hctx, src, lnki, ttl_bry, true, caption_wkr);
else {

38
400_xowa/src/gplx/xowa/parsers/logs/stats/Xop_log_stat.java Normal file
View File

@ -0,0 +1,38 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.parsers.logs.stats; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*;
public class Xop_log_stat {
public boolean Enabled() {return enabled;} private boolean enabled;
public int Wkr_uid() {return wkr_uid;} private int wkr_uid = -1;
public Xop_log_time_count Scrib() {return scrib;} private final Xop_log_time_count scrib = new Xop_log_time_count();
public long Tidy_time;
public int Image_count;
public int Audio_count;
public int Video_count;
public int Media_count;
public int Hdr_count;
public int Lnki_count;
public int Lnke_count;
public int Math_count;
public int Imap_count;
public int Hiero_count;
public int Gallery_count;
public int Gallery_packed_count;
public void Init(int v) {
enabled = true;
wkr_uid = v;
}
}

50
400_xowa/src/gplx/xowa/parsers/logs/stats/Xop_log_time_count.java Normal file
View File

@ -0,0 +1,50 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012-2017 gnosygnu@gmail.com
XOWA is licensed under the terms of the General Public License (GPL) Version 3,
or alternatively under the terms of the Apache License Version 2.0.
You may use XOWA according to either of these licenses as is most appropriate
for your project on a case-by-case basis.
The terms of each license can be found in the source code repository:
GPLv3 License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-GPLv3.txt
Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
*/
package gplx.xowa.parsers.logs.stats; import gplx.*; import gplx.xowa.*; import gplx.xowa.parsers.*; import gplx.xowa.parsers.logs.*;
public class Xop_log_time_count {
private int count;
private long time;
private int depth;
private int depth_max;
private long time_bgn;
public int Count() {return count;}
public long Time() {return time;}
public int Depth_max() {return depth_max;}
public void Bgn() {
if (time_bgn == 0) {
time_bgn = gplx.core.envs.System_.Ticks();
}
depth++;
if (depth_max < depth)
depth_max = depth;
}
public void End() {
depth--;
if (depth == 0) {
long time_end = gplx.core.envs.System_.Ticks();
this.time += time_end - time_bgn;
this.time_bgn = 0;
}
this.count++;
}
public void Clear() {
count = 0;
time = 0;
time_bgn = 0;
depth = 0;
depth_max = 0;
}
}

2
400_xowa/src/gplx/xowa/wikis/pages/dbs/Xopg_db_html.java
View File

@ -19,11 +19,13 @@ public class Xopg_db_html {
public byte[] Html_bry() {return html_bry;} private byte[] html_bry;
public void Html_bry_(byte[] v) {this.html_bry = v;}
public int Zip_tid() {return zip_tid;} private int zip_tid;
public int Zip_len() {return zip_len;} public void Zip_len_(int v) {this.zip_len = v;} private int zip_len;
public int Hzip_tid() {return hzip_tid;} private int hzip_tid;
public void Zip_tids_(int zip_tid, int hzip_tid) {this.zip_tid = zip_tid; this.hzip_tid = hzip_tid;}
public void Clear() {
html_bry = Bry_.Empty; // NOTE: if null, will cause NullPointer exception on Special pages like Special:XowaDownloadCentral; DATE:2016-07-05
zip_tid = 0;
zip_len = 0;
hzip_tid = 0;
}
}

2
400_xowa/src/gplx/xowa/xtns/gallery/Gallery_mgr_wtr.java
View File

@ -25,6 +25,8 @@ public class Gallery_mgr_wtr {
boolean hctx_is_hdump = hctx.Mode_is_hdump();
int itm_default_w = mgr.Itm_default_w();
int itms_per_row = mgr.Itms_per_row();
page.Stat_itm().Gallery_count++;
if (Gallery_mgr_base_.Mode_is_packed(xnde.Mode())) page.Stat_itm().Gallery_packed_count++;
// write <ul> lhs
int ul_uid = page.Html_data().Xtn_gallery_next_id();

1
400_xowa/src/gplx/xowa/xtns/hieros/Hiero_xnde.java
View File

@ -38,6 +38,7 @@ public class Hiero_xnde implements Xox_xnde {
}
public static Xop_log_basic_wkr Log_wkr = Xop_log_basic_wkr.Null;
public void Xtn_write(Bry_bfr bfr, Xoae_app app, Xop_ctx ctx, Xoh_html_wtr html_wtr, Xoh_wtr_ctx hctx, Xoae_page wpg, Xop_xnde_tkn xnde, byte[] src) {
wpg.Stat_itm().Hiero_count++;
xtn_mgr.Html_wtr().Render_blocks(bfr, hctx, blocks, Hiero_html_mgr.scale, false);
}
}

1
400_xowa/src/gplx/xowa/xtns/imaps/Imap_xnde.java
View File

@ -35,6 +35,7 @@ public class Imap_xnde implements Xox_xnde {
public void Xtn_write(Bry_bfr bfr, Xoae_app app, Xop_ctx ctx, Xoh_html_wtr html_wtr, Xoh_wtr_ctx hctx, Xoae_page wpg, Xop_xnde_tkn xnde, byte[] src) {
if (imap_data.Invalid()) return;
wpg.Stat_itm().Imap_count++;
html_wtr.Write_tkn_to_html(bfr, ctx, hctx, imap_data.Img_src(), xnde, Xoh_html_wtr.Sub_idx_null, imap_data.Img().Img_link());
}
}

1
400_xowa/src/gplx/xowa/xtns/math/Xomath_xnde.java
View File

@ -26,6 +26,7 @@ public class Xomath_xnde implements Xox_xnde {
ctx.Page().Html_data().Head_mgr().Itm__mathjax().Enabled_y_();
}
public void Xtn_write(Bry_bfr bfr, Xoae_app app, Xop_ctx ctx, Xoh_html_wtr html_wtr, Xoh_wtr_ctx hctx, Xoae_page wpg, Xop_xnde_tkn xnde, byte[] src) {
wpg.Stat_itm().Math_count++;
ctx.Wiki().Parser_mgr().Math__core().Write(bfr, ctx, xnde, src);
}
public static Xop_log_basic_wkr Log_wkr = Xop_log_basic_wkr.Null;

5
400_xowa/src/gplx/xowa/xtns/scribunto/Scrib_invoke_func.java
View File

@ -25,6 +25,8 @@ public class Scrib_invoke_func extends Pf_func_base {
@Override public int Id() {return Xol_kwd_grp_.Id_invoke;}
@Override public Pf_func New(int id, byte[] name) {return new Scrib_invoke_func().Name_(name);}
@Override public void Func_evaluate(Bry_bfr bfr, Xop_ctx ctx, Xot_invk caller, Xot_invk self, byte[] src) {// {{#invoke:mod_name|prc_name|prc_args...}}
boolean stat_enabled = ctx.Page().Stat_itm().Enabled();
if (stat_enabled) ctx.Page().Stat_itm().Scrib().Bgn();
Xowe_wiki wiki = ctx.Wiki();
byte[] mod_name = Eval_argx(ctx, src, caller, self);
if (Bry_.Len_eq_0(mod_name)) {Error(bfr, wiki.Msg_mgr(), Err_mod_missing); return;} // EX: "{{#invoke:}}"
@ -65,11 +67,12 @@ public class Scrib_invoke_func extends Pf_func_base {
Error(bfr, wiki.Msg_mgr(), err);
Scrib_err_filter_mgr err_filter_mgr = invoke_wkr == null ? null : invoke_wkr.Err_filter_mgr();
if ( err_filter_mgr == null // no err_filter_mgr defined;
|| err_filter_mgr.Count_eq_0( ) // err_filter_mgr exists, but no definitions
|| err_filter_mgr.Count_eq_0() // err_filter_mgr exists, but no definitions
|| !err_filter_mgr.Match(String_.new_u8(mod_name), String_.new_u8(fnc_name), err.To_str__msg_only())) // NOTE: must be To_str__msg_only; err_filter_mgr has defintion and it doesn't match current; print warn; DATE:2015-07-24
ctx.App().Usr_dlg().Warn_many("", "", "invoke failed: ~{0} ~{1} ~{2}", ctx.Page().Ttl().Raw(), Bry_.Replace_nl_w_tab(src, self.Src_bgn(), self.Src_end()), err.To_str__log());
wiki.Parser_mgr().Scrib().Terminate_when_page_changes_y_(); // NOTE: terminate core when page changes; not terminating now, else page with many errors will be very slow due to multiple remakes of core; PAGE:th.d:all; DATE:2014-10-03
}
if (stat_enabled) ctx.Page().Stat_itm().Scrib().End();
}
public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, Err err) {Error(bfr, msg_mgr, Err_.Cast_or_make(err).To_str__top_wo_args());}// NOTE: must use "short" error message to show in wikitext; DATE:2015-07-27
public static void Error(Bry_bfr bfr, Xow_msg_mgr msg_mgr, String error) {