Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

HTML Databases: Fix loading of plain-text pages; Normalize some behavior in page-sync code [#320]

Browse Source
This commit is contained in:
gnosygnu
2019-01-05 11:47:24 -05:00
parent 9e5a13f54f
commit 2b4320b302
22 changed files with 158 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
400_xowa/src/gplx/xowa/htmls/core/Xow_hdump_mgr.java
View File

@@ -27,6 +27,9 @@ public class Xow_hdump_mgr {
public Xow_hdump_mgr__save Save_mgr() {return save_mgr;} private Xow_hdump_mgr__save save_mgr;
public Xow_hdump_mgr__load Load_mgr() {return load_mgr;} private Xow_hdump_mgr__load load_mgr;
public Xoh_hzip_mgr Hzip_mgr() {return hzip_mgr;} private final Xoh_hzip_mgr hzip_mgr = new Xoh_hzip_mgr();
public void Init_by_app(Xoae_app app) {
save_mgr.Init_by_app(app);
}
public void Init_by_db(Xow_wiki wiki) {
byte dflt_zip_tid = gplx.core.ios.streams.Io_stream_tid_.Tid__raw;
boolean dflt_hzip_enable = false;
@@ -41,7 +44,7 @@ public class Xow_hdump_mgr {
load_mgr.Init_by_wiki(wiki);
}
public void Init_by_db(byte dflt_zip_tid, boolean dflt_hzip_enable, boolean mode_is_b256) {
int dflt_hzip_tid = dflt_hzip_enable ? Xoh_hzip_dict_.Hzip__v1 : Xoh_hzip_dict_.Hzip__none;
int dflt_hzip_tid = dflt_hzip_enable ? Xoh_hzip_dict_.Hzip__v1 : Xoh_hzip_dict_.Hzip__plain;
save_mgr.Init_by_db(dflt_zip_tid, dflt_hzip_tid, Bool_.N);
}
}

23
400_xowa/src/gplx/xowa/htmls/core/Xow_hdump_mgr__save.java
View File

@@ -16,13 +16,18 @@ Apache License: https://github.com/gnosygnu/xowa/blob/master/LICENSE-APACHE2.txt
package gplx.xowa.htmls.core; import gplx.*; import gplx.xowa.*; import gplx.xowa.htmls.*;
import gplx.xowa.htmls.core.htmls.*; import gplx.xowa.htmls.core.wkrs.*; import gplx.xowa.htmls.core.hzips.*; import gplx.xowa.htmls.heads.*; import gplx.xowa.htmls.core.dbs.*;
import gplx.core.ios.*; import gplx.core.primitives.*; import gplx.xowa.wikis.data.*; import gplx.xowa.wikis.pages.*;
import gplx.xowa.addons.wikis.pages.syncs.core.*;
public class Xow_hdump_mgr__save {
private final Xow_wiki wiki; private final Xoh_hzip_mgr hzip_mgr; private final Io_stream_zip_mgr zip_mgr;
private final Xosync_update_mgr update_mgr = new Xosync_update_mgr();
private final Xoh_page tmp_hpg; private final Xoh_hzip_bfr tmp_bfr = Xoh_hzip_bfr.New_txt(32); private Bool_obj_ref html_db_is_new = Bool_obj_ref.n_();
private int dflt_zip_tid, dflt_hzip_tid;
public Xow_hdump_mgr__save(Xow_wiki wiki, Xoh_hzip_mgr hzip_mgr, Io_stream_zip_mgr zip_mgr, Xoh_page tmp_hpg) {
this.wiki = wiki; this.hzip_mgr = hzip_mgr; this.zip_mgr = zip_mgr; this.tmp_hpg = tmp_hpg;
}
public void Init_by_app(Xoae_app app) {
update_mgr.Init_by_app(app);
}
public void Init_by_db(int dflt_zip_tid, int dflt_hzip_tid, boolean mode_is_b256) {
this.dflt_zip_tid = dflt_zip_tid; this.dflt_hzip_tid = dflt_hzip_tid; tmp_bfr.Mode_is_b256_(mode_is_b256);
}
@@ -32,12 +37,12 @@ public class Xow_hdump_mgr__save {
Bld_hdump(page);
tmp_hpg.Ctor_by_hdiff(tmp_bfr, page, page.Wikie().Msg_mgr().Val_by_id(gplx.xowa.langs.msgs.Xol_msg_itm_.Id_toc));
Xow_db_file html_db = Get_html_db(wiki, page, html_db_is_new.Val_n_());
return Save(tmp_hpg, html_db.Tbl__html(), html_db_is_new.Val(), true);
return Save(page, tmp_hpg, html_db.Tbl__html(), html_db_is_new.Val(), true);
}
}
public int Save(Xoh_page hpg, Xowd_html_tbl html_tbl, boolean insert, boolean use_hzip_dflt) {
public int Save(Xoae_page page, Xoh_page hpg, Xowd_html_tbl html_tbl, boolean insert, boolean use_hzip_dflt) {
int hzip_tid = use_hzip_dflt ? dflt_hzip_tid : Xoh_hzip_dict_.Hzip__none;
byte[] db_body = Write(tmp_bfr, wiki, hpg, hzip_mgr, zip_mgr, dflt_zip_tid, hzip_tid, hpg.Db().Html().Html_bry());
byte[] db_body = Write(tmp_bfr, wiki, page, hpg, hzip_mgr, zip_mgr, dflt_zip_tid, hzip_tid, hpg.Db().Html().Html_bry());
if (insert) html_tbl.Insert(hpg, dflt_zip_tid, dflt_hzip_tid, db_body);
else html_tbl.Update(hpg, dflt_zip_tid, dflt_hzip_tid, db_body);
return db_body.length;
@@ -47,8 +52,16 @@ public class Xow_hdump_mgr__save {
wiki.Html__wtr_mgr().Wkr(Xopg_page_.Tid_read).Write_body(tmp_bfr, page.Wikie().Parser_mgr().Ctx(), Xoh_wtr_ctx.Hdump, page); // save as hdump_fmt
page.Db().Html().Html_bry_(tmp_bfr.To_bry_and_clear());
}
private byte[] Write(Xoh_hzip_bfr bfr, Xow_wiki wiki, Xoh_page hpg, Xoh_hzip_mgr hzip_mgr, Io_stream_zip_mgr zip_mgr, int zip_tid, int hzip_tid, byte[] src) {
if (hzip_tid != Xoh_hzip_dict_.Hzip__none) src = hzip_mgr.Encode_as_bry((Xoh_hzip_bfr)bfr.Clear(), wiki, hpg, src);
private byte[] Write(Xoh_hzip_bfr bfr, Xow_wiki wiki, Xoae_page page, Xoh_page hpg, Xoh_hzip_mgr hzip_mgr, Io_stream_zip_mgr zip_mgr, int zip_tid, int hzip_tid, byte[] src) {
switch (hzip_tid) {
case Xoh_hzip_dict_.Hzip__none:
update_mgr.Parse(hpg, wiki, page.Url_bry_safe(), src);
src = hpg.Db().Html().Html_bry();
break;
case Xoh_hzip_dict_.Hzip__v1:
src = hzip_mgr.Encode_as_bry((Xoh_hzip_bfr)bfr.Clear(), wiki, hpg, src);
break;
}
src_as_hzip = src;
if (zip_tid > gplx.core.ios.streams.Io_stream_tid_.Tid__raw)
src = zip_mgr.Zip((byte)zip_tid, src);

3
400_xowa/src/gplx/xowa/htmls/core/bldrs/Xob_hdump_bldr.java
View File

@@ -40,6 +40,7 @@ public class Xob_hdump_bldr implements Gfo_invk {
this.toc_label = wiki.Msg_mgr().Val_by_id(gplx.xowa.langs.msgs.Xol_msg_itm_.Id_toc);
if (zip_tid == Byte_.Max_value_127) zip_tid = Xobldr_cfg.Zip_mode__html(wiki.App());
hdump_mgr.Init_by_app(wiki.Appe());
hdump_mgr.Init_by_db(zip_tid, hzip_enabled, hzip_b256);
return true;
}
@@ -63,7 +64,7 @@ public class Xob_hdump_bldr implements Gfo_invk {
// save to db
Xowd_html_tbl html_tbl = html_tbl_retriever.Get_html_tbl(wpg.Ttl().Ns(), prv_row_len); // get html_tbl
this.prv_row_len = hdump_mgr.Save_mgr().Save(tmp_hpg.Ctor_by_hdiff(tmp_bfr, wpg, toc_label), html_tbl, true, is_wikitext); // save to db
this.prv_row_len = hdump_mgr.Save_mgr().Save(wpg, tmp_hpg.Ctor_by_hdiff(tmp_bfr, wpg, toc_label), html_tbl, true, is_wikitext); // save to db
stat_tbl.Insert(tmp_hpg, stat_itm, wpg.Root().Root_src().length, tmp_hpg.Db().Html().Html_bry().length, prv_row_len); // save stats
// run hzip diff if enabled

10
400_xowa/src/gplx/xowa/htmls/core/hzips/Xoh_hzip_dict_.java
View File

@@ -51,9 +51,9 @@ public class Xoh_hzip_dict_ {
, Key__pgbnr = "pgbnr"
, Key__media = "media"
;
public static final int // SERIALIZED
Hzip__none = 0
, Hzip__v1 = 1
, Hzip__plain = 2 // used for page_sync
;
public static final int // SERIALIZED:html_db.html.body_flag
Hzip__none = 0
, Hzip__v1 = 1
, Hzip__plain = 2 // used for page_sync
;
}

26
400_xowa/src/gplx/xowa/htmls/core/wkrs/Xoh_hdoc_wkr__make.java
View File

@@ -20,8 +20,8 @@ import gplx.xowa.htmls.core.hzips.*;
import gplx.xowa.htmls.core.wkrs.hdrs.*; import gplx.xowa.htmls.core.wkrs.imgs.*;
public class Xoh_hdoc_wkr__make implements Xoh_hdoc_wkr {
private Xoh_hzip_bfr bfr; private Xoh_page hpg; private Xoh_hdoc_ctx hctx; private byte[] src;
private final Xoh_hdr_wtr wkr__hdr = new Xoh_hdr_wtr();
private final Xoh_img_wtr wkr__img = new Xoh_img_wtr();
private final Xoh_hdr_wtr wkr__hdr = new Xoh_hdr_wtr();
private final Xoh_img_wtr wkr__img = new Xoh_img_wtr();
public void On_new_page(Xoh_hzip_bfr bfr, Xoh_page hpg, Xoh_hdoc_ctx hctx, byte[] src, int src_bgn, int src_end) {
this.bfr = bfr; this.hpg = hpg; this.hctx = hctx; this.src = src;
}
@@ -29,16 +29,24 @@ public class Xoh_hdoc_wkr__make implements Xoh_hdoc_wkr {
public void On_escape (gplx.xowa.htmls.core.wkrs.escapes.Xoh_escape_data data) {bfr.Add(data.Hook());}
public void On_xnde (gplx.xowa.htmls.core.wkrs.xndes.Xoh_xnde_parser data) {bfr.Add_mid(src, data.Src_bgn(), data.Src_end());}
public void On_lnki (gplx.xowa.htmls.core.wkrs.lnkis.Xoh_lnki_data data) {bfr.Add_mid(src, data.Src_bgn(), data.Src_end());}
public void On_thm (gplx.xowa.htmls.core.wkrs.thms.Xoh_thm_data data) {bfr.Add_mid(src, data.Src_bgn(), data.Src_end());}
public void On_gly (gplx.xowa.htmls.core.wkrs.glys.Xoh_gly_grp_data data) {}
public void On_thm (gplx.xowa.htmls.core.wkrs.thms.Xoh_thm_data data) {
Xoh_img_data img_data = (gplx.xowa.htmls.core.wkrs.imgs.Xoh_img_data)data.Img_data();
bfr.Add_mid(src, data.Src_bgn(), img_data.Src_bgn());
wkr__img.Init_by_parse(bfr, hpg, hctx, src, img_data);
bfr.Add_mid(src, img_data.Src_end(), data.Src_end());
}
public void On_gly (gplx.xowa.htmls.core.wkrs.glys.Xoh_gly_grp_data data) {
bfr.Add_mid(src, data.Src_bgn(), data.Src_end());
}
public boolean Process_parse(Xoh_data_itm data) {
switch (data.Tid()) {
case Xoh_hzip_dict_.Tid__img: wkr__img.Init_by_parse(bfr, hpg, hctx, src, (gplx.xowa.htmls.core.wkrs.imgs.Xoh_img_data)data); return true;
case Xoh_hzip_dict_.Tid__hdr: wkr__hdr.Init_by_parse(bfr, hpg, hctx, src, (gplx.xowa.htmls.core.wkrs.hdrs.Xoh_hdr_data)data); return true;
case Xoh_hzip_dict_.Tid__lnke: break;
default: throw Err_.new_unhandled(data.Tid());
case Xoh_hzip_dict_.Tid__img: return wkr__img.Init_by_parse(bfr, hpg, hctx, src, (gplx.xowa.htmls.core.wkrs.imgs.Xoh_img_data)data);
case Xoh_hzip_dict_.Tid__hdr: return wkr__hdr.Init_by_parse(bfr, hpg, hctx, src, (gplx.xowa.htmls.core.wkrs.hdrs.Xoh_hdr_data)data);
case Xoh_hzip_dict_.Tid__lnke:
default:
bfr.Add_mid(src, data.Src_bgn(), data.Src_end());
break;
}
bfr.Add_mid(src, data.Src_bgn(), data.Src_end());
return true;
}
}

4
400_xowa/src/gplx/xowa/htmls/core/wkrs/Xoh_itm_parser_fxt.java
View File

@@ -41,7 +41,7 @@ public abstract class Xoh_itm_parser_fxt {
Xoh_page hpg = new Xoh_page(); // NOTE: no need to pass url and ttl now
hctx.Init_by_page(wiki, hpg);
err_wkr.Init_by_page(Xoa_page_.Main_page_str, src);
Exec_parse_hook(err_wkr, hctx, 0, src_len);
Exec_parse_hook(err_wkr, hctx, src, 0, src_len);
}
public abstract void Exec_parse_hook(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, int src_bgn, int src_end);
public abstract void Exec_parse_hook(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, byte[] src, int src_bgn, int src_end);
}

5
400_xowa/src/gplx/xowa/htmls/core/wkrs/hdrs/Xoh_hdr_wtr.java
View File

@@ -20,9 +20,10 @@ import gplx.xowa.htmls.sections.*; import gplx.xowa.htmls.core.hzips.*;
public class Xoh_hdr_wtr implements gplx.core.brys.Bfr_arg, Xoh_wtr_itm {
private int hdr_num; private byte[] hdr_id, hdr_content, hdr_capt_rhs;
private Xoh_page hpg;
public void Init_by_parse(Bry_bfr bfr, Xoh_page hpg, Xoh_hdoc_ctx hctx, byte[] src, Xoh_hdr_data data) {
Init_by_decode(hpg, hctx, src, data);
public boolean Init_by_parse(Bry_bfr bfr, Xoh_page hpg, Xoh_hdoc_ctx hctx, byte[] src, Xoh_hdr_data data) {
if (!Init_by_decode(hpg, hctx, src, data)) return false;
this.Bfr_arg__add(bfr);
return true;
}
public boolean Init_by_decode(Xoh_page hpg, Xoh_hdoc_ctx hctx, byte[] src, Xoh_data_itm data_itm) {
this.hpg = hpg;

10
400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/Xoh_img_wtr.java
View File

@@ -56,9 +56,10 @@ public class Xoh_img_wtr implements Bfr_arg, Xoh_wtr_itm {
}
public Xoh_img_wtr Anch_cls_(byte[] v) {anch_cls.Set_by_bry(v); return this;}
public Xoh_img_wtr Img_id_(int uid) {img_id.Set(uid); return this;}
public void Init_by_parse(Bry_bfr bfr, Xoh_page hpg, Xoh_hdoc_ctx hctx, byte[] src, Xoh_img_data data) {
Init_by_decode(hpg, hctx, src, data);
public boolean Init_by_parse(Bry_bfr bfr, Xoh_page hpg, Xoh_hdoc_ctx hctx, byte[] src, Xoh_img_data data) {
if (!Init_by_decode(hpg, hctx, src, data)) return false;
this.Bfr_arg__add(bfr);
return true;
}
public void Init_html(int html_w, int html_h, byte[] src_bry) {
img_w.Set_by_int(html_w);
@@ -98,12 +99,13 @@ public class Xoh_img_wtr implements Bfr_arg, Xoh_wtr_itm {
else if (data.Img_w() != -1) { // orig exists or some hard-coded image (hiero)
Xoh_img_src_data img_src_data = data.Img_src();
this.Init_html(data.Img_w(), data.Img_h(), img_src_data.Src_bry());
byte[] img_src_bry = img_src_data.Src_mid();
this.Init_html(data.Img_w(), data.Img_h(), img_src_bry);
int file_w = data.Img_src().File_w();
// NOTE: init lnki with "64|file_w|-1|-1|-1|-1"; DATE:2016-08-10
fsdb_itm.Init_at_lnki(Xof_exec_tid.Tid_wiki_page, hctx.Cache__wiki_abrv(img_src_data.Repo_is_commons()), lnki_ttl, Xop_lnki_type.Tid_orig_known, Xop_lnki_tkn.Upright_null, file_w, Xof_img_size.Null, data.Img_src().File_time(), data.Img_src().File_page(), Xof_patch_upright_tid_.Tid_all);
fsdb_itm.Init_at_gallery_bgn(data.Img_w(), data.Img_h(), file_w);
fsdb_itm.Html_view_url_(Io_url_.New__http_or_fail(img_src_data.Src_bry()));
fsdb_itm.Html_view_url_(Io_url_.New__http_or_fail(img_src_bry));
fsdb_itm.File_is_orig_(data.Img_src().File_is_orig());
// ASSUME: if file_w != img_w, then page has packed gallery; PAGE:en.w:Mexico; DATE:2016-08-14

1
400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/atrs/Xoh_img_cls_data.java
View File

@@ -45,6 +45,7 @@ public class Xoh_img_cls_data implements Bfr_arg_clearable {
if (pos < src_end && src[pos] == Byte_ascii.Space)
++pos;
if (cls_tid == Xoh_img_cls_.Tid__manual || pos < src_end) {
this.src = atr.Src(); // set src, else NPE when trying to write to bfr; DATE:2019-01-04
this.other_bgn = pos;
this.other_end = src_end;
}

10
400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/atrs/Xoh_img_src_data.java
View File

@@ -21,9 +21,10 @@ import gplx.xowa.wikis.domains.*;
public class Xoh_img_src_data implements Bfr_arg_clearable, Xoh_itm_parser {
private final Bry_rdr rdr = new Bry_rdr().Dflt_dlm_(Byte_ascii.Slash);
private boolean fail_throws_err;
public byte[] Src_bry() {return src_bry;} private byte[] src_bry;
private byte[] src_bry;
public int Src_bgn() {return src_bgn;} private int src_bgn;
public int Src_end() {return src_end;} private int src_end;
public byte[] Src_mid() {return src_mid;} private byte[] src_mid;
public int Repo_bgn() {return repo_bgn;} private int repo_bgn;
public int Repo_end() {return repo_end;} private int repo_end;
public boolean Repo_is_commons() {return repo_is_commons;} private boolean repo_is_commons;
@@ -49,13 +50,14 @@ public class Xoh_img_src_data implements Bfr_arg_clearable, Xoh_itm_parser {
public boolean Parse(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, byte[] domain_bry, Gfh_tag tag) {
this.Clear();
Gfh_atr atr = tag.Atrs__get_by_or_empty(Gfh_atr_.Bry__src);
return Parse(err_wkr, hctx, domain_bry, atr.Val_bgn(), atr.Val_end());
return Parse(err_wkr, hctx, domain_bry, atr.Src(), atr.Val_bgn(), atr.Val_end());
}
public boolean Parse(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, byte[] domain_bry, int src_bgn, int src_end) { // EX: src="file:///C:/xowa/file/commons.wikimedia.org/thumb/7/0/1/2/A.png/220px.png"
public boolean Parse(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, byte[] domain_bry, byte[] src_bry, int src_bgn, int src_end) { // EX: src="file:///C:/xowa/file/commons.wikimedia.org/thumb/7/0/1/2/A.png/220px.png"
this.Clear();
this.src_bry = err_wkr.Src();
this.src_bry = src_bry;
this.src_bgn = src_bgn; this.src_end = src_end;
if (src_end == src_bgn) return true; // empty src; just return true;
this.src_mid = Bry_.Mid(src_bry, src_bgn, src_end);
// get repo_bgn; note that some <img> may be hiero / enlarge / magnify and should exit
rdr.Init_by_wkr(err_wkr, "img.src.xowa", src_bgn, src_end).Fail_throws_err_(fail_throws_err);

4
400_xowa/src/gplx/xowa/htmls/core/wkrs/imgs/atrs/Xoh_img_src_data_tst.java
View File

@@ -59,8 +59,8 @@ class Xoh_img_src_data_fxt extends Xoh_itm_parser_fxt { private final Xoh_im
Tfds.Eq_double(expd_time, parser.File_time());
Tfds.Eq_int(expd_page, parser.File_page());
}
@Override public void Exec_parse_hook(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, int src_bgn, int src_end) {
@Override public void Exec_parse_hook(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, byte[] src, int src_bgn, int src_end) {
parser.Fail_throws_err_(true);
parser.Parse(err_wkr, new Xoh_hdoc_ctx(), Xow_domain_itm_.Bry__enwiki, src_bgn, src_end);
parser.Parse(err_wkr, new Xoh_hdoc_ctx(), Xow_domain_itm_.Bry__enwiki, src, src_bgn, src_end);
}
}

4
400_xowa/src/gplx/xowa/htmls/core/wkrs/lnkis/anchs/Xoh_anch_href_data_tst.java
View File

@@ -54,7 +54,7 @@ class Xoh_anch_href_data_fxt extends Xoh_itm_parser_fxt { private final Xoh_
Tfds.Eq_str(expd_site, parser.Site_bgn() == -1 ? "" : String_.new_u8(src, parser.Site_bgn(), parser.Site_end()));
Tfds.Eq_str(expd_page, String_.new_u8(src, parser.Ttl_bgn(), parser.Ttl_end()));
}
@Override public void Exec_parse_hook(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, int src_bgn, int src_end) {
parser.Parse(err_wkr, hctx, err_wkr.Src(), src_bgn, src_end);
@Override public void Exec_parse_hook(Bry_err_wkr err_wkr, Xoh_hdoc_ctx hctx, byte[] src, int src_bgn, int src_end) {
parser.Parse(err_wkr, hctx, src, src_bgn, src_end);
}
}