Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

Wiki: Support renamed folders (fix)

Browse Source
This commit is contained in:
gnosygnu
2017-02-06 22:12:56 -05:00
parent 6f9e92afff
commit 938beac9f9
4379 changed files with 0 additions and 327818 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
400_xowa/src/gplx/xowa/bldrs/xmls/Xob_import_cfg.java
View File

@@ -1,64 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.ios.*; import gplx.core.ios.streams.*; import gplx.core.envs.*;
import gplx.xowa.addons.wikis.ctgs.*; import gplx.xowa.wikis.tdbs.*;
import gplx.xowa.bldrs.wkrs.*;
public class Xob_import_cfg {
public Xob_import_cfg(Xowe_wiki wiki) {this.wiki = wiki;} private Xowe_wiki wiki; private boolean src_fil_is_bz2 = true;
public byte Category_version() {return category_version;} public Xob_import_cfg Category_version_(byte v) {category_version = v; return this;} private byte category_version = Xoa_ctg_mgr.Version_1;
public long Src_rdr_len() {return src_rdr_len;} private long src_rdr_len;
public Io_url Src_fil_xml() {return src_fil_xml;}
public Io_url Src_fil() {return src_fil;} private Io_url src_fil;
public Xob_import_cfg Src_fil_xml_(Io_url v) {src_fil_xml = v; src_fil_is_bz2 = Bool_.N; return this;} private Io_url src_fil_xml;
public Xob_import_cfg Src_fil_bz2_(Io_url v) {src_fil_bz2 = v; src_fil_is_bz2 = Bool_.Y; return this;} private Io_url src_fil_bz2;
public Io_url Src_dir() {
if (src_fil_xml == null && src_fil_bz2 == null) return wiki.Fsys_mgr().Root_dir();
else if (src_fil_xml != null) return src_fil_xml.OwnerDir();
else if (src_fil_bz2 != null) return src_fil_bz2.OwnerDir();
else throw Err_.new_wo_type("unknown src dir");
}
public Io_stream_rdr Src_rdr() {
if (src_fil_xml == null && src_fil_bz2 == null) { // will usually be null; non-null when user specifies src through command-line
Io_url url = Xob_io_utl_.Find_nth_by_wildcard_or_null(wiki.Fsys_mgr().Root_dir(), Xob_io_utl_.Pattern__wilcard, ".xml", ".bz2");
if (url == null) throw Err_.new_wo_type("could not find any .xml or .bz2 file", "dir", wiki.Fsys_mgr().Root_dir().Raw());
if (String_.Eq(url.Ext(), ".xml")) Src_fil_xml_(url);
else Src_fil_bz2_(url);
}
if (src_fil_is_bz2) {
Chk_file_ext(wiki.Appe(), src_fil_bz2, ".bz2", "xml");
src_fil = src_fil_bz2; src_rdr_len = Io_mgr.Instance.QueryFil(src_fil_bz2).Size();
Xoae_app app = wiki.Appe();
if (gplx.xowa.bldrs.installs.Xoi_dump_mgr.Import_bz2_by_stdout(app)) {
Process_adp process = app.Prog_mgr().App_decompress_bz2_by_stdout();
return Io_stream_rdr_process.new_(process.Exe_url(), src_fil_bz2, process.Xto_process_bldr_args(src_fil_bz2.Raw()));
}
else
return Io_stream_rdr_.New__bzip2(src_fil_bz2);
}
else {
Chk_file_ext(wiki.Appe(), src_fil_xml, ".xml", "bz2");
src_fil = src_fil_xml; src_rdr_len = Io_mgr.Instance.QueryFil(src_fil_xml).Size();
return Io_stream_rdr_.New__raw(src_fil_xml);
}
}
private static void Chk_file_ext(Xoae_app app, Io_url fil, String expd_ext, String alt_ext) {
if (!String_.Eq(fil.Ext(), expd_ext))
app.Usr_dlg().Warn_many("", "", "File extension is not " + expd_ext + ". Please use '.src_" + alt_ext + "_fil_' instead; file=~{0}", fil.Raw());
}
}

49
400_xowa/src/gplx/xowa/bldrs/xmls/Xob_import_marker.java
View File

@@ -1,49 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.gfui.*; import gplx.gfui.kits.core.*; import gplx.xowa.bldrs.cmds.utils.*;
public class Xob_import_marker {
private final Hash_adp_bry in_progress_hash = Hash_adp_bry.cs();
public void Bgn(Xowe_wiki wiki) {
in_progress_hash.Add_as_key_and_val(wiki.Domain_bry());
Io_mgr.Instance.SaveFilStr(url_(wiki), "XOWA has created this file to indicate that an import is in progress. This file will be deleted once the import is completed.");
}
public void End(Xowe_wiki wiki) {
in_progress_hash.Del(wiki.Domain_bry());
Io_mgr.Instance.DeleteFil_args(url_(wiki)).MissingFails_off().Exec();
}
public boolean Chk(Xowe_wiki wiki) {
if (!wiki.App().Mode().Tid_is_gui()) return true; // NOTE: ignore during Server / Console modes; DATE:2015-04-01
if (in_progress_hash.Has(wiki.Domain_bry())) return true; // NOTE: ignore if currently building; different bldr commands call wiki.Init_assert() which may lead to fals checks;
Io_url url = url_(wiki);
if (!Io_mgr.Instance.ExistsFil(url)) return true;
Xoae_app app = wiki.Appe();
app.Usr_dlg().Log_many("", "", "import.marker: marker found: url=~{0}", url.Raw());
byte[] incompete_msg_bry = app.Usere().Msg_mgr().Val_by_key_args(Bry_.new_a7("api-xowa.import.core.incomplete"), wiki.Domain_str());
int rslt = app.Gui_mgr().Kit().Ask_yes_no_cancel("", "", String_.new_u8(incompete_msg_bry));
switch (rslt) {
case Gfui_dlg_msg_.Btn_yes: Xob_cleanup_cmd.Delete_wiki_sql(wiki); End(wiki); return false; // delete wiki
case Gfui_dlg_msg_.Btn_no: End(wiki); return true; // delete marker
case Gfui_dlg_msg_.Btn_cancel: return true; // noop
default: throw Err_.new_unhandled(rslt);
}
}
private static Io_url url_(Xowe_wiki wiki) {
return wiki.Fsys_mgr().Root_dir().GenSubFil(wiki.Domain_str() + "-import.lock");
}
}

98
400_xowa/src/gplx/xowa/bldrs/xmls/Xob_xml_dumper.java
View File

@@ -1,98 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.langs.xmls.*;
import gplx.xowa.wikis.nss.*;
import gplx.xowa.wikis.domains.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_xml_dumper {
private final Gfo_xml_wtr wtr = new Gfo_xml_wtr();
public String Bld_str() {return wtr.Bld_str();}
public void Write_root_bgn(Xow_ns_mgr ns_mgr, Xow_domain_itm domain, String wiki_abrv, String main_page, String ns_case, String app_version) {
wtr.Nde_lhs_bgn_grp("mediawiki");
wtr.Atr_kv_str_a7("xmlns" , "http://www.mediawiki.org/xml/export-0.10/");
wtr.Atr_kv_str_a7("xmlns:xsi" , "http://www.w3.org/2001/XMLSchema-instance");
wtr.Atr_kv_str_a7("xsi:schemaLocation" , "http://www.mediawiki.org/xml/export-0.10/ http://www.mediawiki.org/xml/export-0.10.xsd");
wtr.Atr_kv_str_a7("version" , "0.10");
wtr.Atr_kv_str_a7("xml:lang" , "en");
wtr.Nde_lhs_end();
Write_siteinfo(domain, wiki_abrv, main_page, ns_case, app_version);
Write_ns_mgr(ns_mgr);
}
public void Write_root_end() {
wtr.Nde_rhs();
}
private void Write_siteinfo(Xow_domain_itm domain, String wiki_abrv, String main_page, String ns_case, String app_version) {
wtr.Nde_lhs("siteinfo");
wtr.Nde_txt_bry("sitename" , Xow_domain_tid_.Get_type_as_bry(domain.Domain_type_id()));
wtr.Nde_txt_str("dbname" , wiki_abrv);
wtr.Nde_txt_str("base" , main_page);
wtr.Nde_txt_str("generator" , app_version);
wtr.Nde_txt_str("case" , ns_case);
wtr.Nde_rhs();
}
private void Write_ns_mgr(Xow_ns_mgr ns_mgr) {
wtr.Nde_lhs("namespaces");
int len = ns_mgr.Ords_len();
for (int i = 0; i < len; ++i) {
Xow_ns ns = ns_mgr.Ords_get_at(i);
Write_ns(ns);
}
wtr.Nde_rhs();
}
private void Write_ns(Xow_ns ns) {
wtr.Nde_lhs_bgn_itm("namespace");
wtr.Atr_kv_int("key" , ns.Id());
wtr.Atr_kv_str_a7("case" , Xow_ns_case_.To_str(ns.Case_match()));
wtr.Nde_lhs_end();
wtr.Txt_bry(ns.Name_db());
wtr.Nde_rhs();
}
public void Write_page(Xowd_page_itm page) {
wtr.Nde_lhs("page");
wtr.Nde_txt_bry("title" , page.Ttl_full_db());
wtr.Nde_txt_int("id" , page.Id());
Write_revision(page);
wtr.Nde_rhs();
}
private void Write_revision(Xowd_page_itm page) {
wtr.Nde_lhs("revision");
wtr.Nde_txt_int("id" , -1);
wtr.Nde_txt_int("parent" , -1);
wtr.Nde_txt_str("timestamp" , page.Modified_on().XtoStr_fmt_iso_8561());
Write_revision_contributor(page);
wtr.Nde_txt_str("comment" , "");
wtr.Nde_txt_str("model" , "wikitext");
wtr.Nde_txt_str("format" , "text/x-wiki");
Write_revision_text(page);
wtr.Nde_txt_str("sha1" , "");
wtr.Nde_rhs();
}
private void Write_revision_contributor(Xowd_page_itm page) {
wtr.Nde_lhs("contributor");
wtr.Nde_txt_str("username" , "");
wtr.Nde_txt_int("id" , -1);
wtr.Nde_rhs();
}
private void Write_revision_text(Xowd_page_itm page) {
wtr.Nde_lhs_bgn_itm("text");
wtr.Atr_kv_str_a7("xml:space", "preserve");
wtr.Nde_lhs_end();
wtr.Txt_bry(page.Text());
wtr.Nde_rhs();
}
}

102
400_xowa/src/gplx/xowa/bldrs/xmls/Xob_xml_dumper_tst.java
View File

@@ -1,102 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*; import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.nss.*;
public class Xob_xml_dumper_tst {
private final Xob_xml_dumper_fxt fxt = new Xob_xml_dumper_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Basic() {
fxt.Test_page(fxt.Make_ary(fxt.Make_page(1, Xow_ns_.Tid__main, "A", "A_text")), String_.Concat_lines_nl_skip_last
( "<mediawiki xmlns='http://www.mediawiki.org/xml/export-0.10/' xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance' xsi:schemaLocation='http://www.mediawiki.org/xml/export-0.10/ http://www.mediawiki.org/xml/export-0.10.xsd' version='0.10' xml:lang='en'>"
, " <siteinfo>"
, " <sitename>other</sitename>"
, " <dbname></dbname>"
, " <base>Main_Page</base>"
, " <generator>XOWA 2.5.2.2</generator>"
, " <case>first-letter</case>"
, " </siteinfo>"
, " <namespaces>"
, " <namespace key='-2' case='first-letter'>Media</namespace>"
, " <namespace key='-1' case='first-letter'>Special</namespace>"
, " <namespace key='0' case='first-letter'></namespace>"
, " <namespace key='1' case='first-letter'>Talk</namespace>"
, " <namespace key='2' case='first-letter'>User</namespace>"
, " <namespace key='3' case='first-letter'>User_talk</namespace>"
, " <namespace key='4' case='first-letter'>Wikipedia</namespace>"
, " <namespace key='5' case='first-letter'>Wikipedia_talk</namespace>"
, " <namespace key='6' case='first-letter'>File</namespace>"
, " <namespace key='7' case='first-letter'>File_talk</namespace>"
, " <namespace key='8' case='first-letter'>MediaWiki</namespace>"
, " <namespace key='9' case='first-letter'>MediaWiki_talk</namespace>"
, " <namespace key='10' case='first-letter'>Template</namespace>"
, " <namespace key='11' case='first-letter'>Template_talk</namespace>"
, " <namespace key='12' case='first-letter'>Help</namespace>"
, " <namespace key='13' case='first-letter'>Help_talk</namespace>"
, " <namespace key='14' case='first-letter'>Category</namespace>"
, " <namespace key='15' case='first-letter'>Category_talk</namespace>"
, " <namespace key='100' case='first-letter'>Portal</namespace>"
, " <namespace key='101' case='first-letter'>Portal_talk</namespace>"
, " <namespace key='108' case='first-letter'>Book</namespace>"
, " <namespace key='109' case='first-letter'>Book_talk</namespace>"
, " <namespace key='828' case='first-letter'>Module</namespace>"
, " <namespace key='829' case='first-letter'>Module_talk</namespace>"
, " </namespaces>"
, " <page>"
, " <title>A</title>"
, " <id>1</id>"
, " <revision>"
, " <id>-1</id>"
, " <parent>-1</parent>"
, " <timestamp>0001-01-01 00:00:00</timestamp>"
, " <contributor>"
, " <username></username>"
, " <id>-1</id>"
, " </contributor>"
, " <comment></comment>"
, " <model>wikitext</model>"
, " <format>text/x-wiki</format>"
, " <text xml:space='preserve'>A_text</text>"
, " <sha1></sha1>"
, " </revision>"
, " </page>"
, "</mediawiki>"
));
}
}
class Xob_xml_dumper_fxt {
private Xowe_wiki wiki;
private final Xob_xml_dumper export_wtr = new Xob_xml_dumper();
public void Clear() {
Xoae_app app = Xoa_app_fxt.Make__app__edit();
this.wiki = Xoa_app_fxt.Make__wiki__edit(app, "enwiki");
}
public Xowd_page_itm[] Make_ary(Xowd_page_itm... ary) {return ary;}
public Xowd_page_itm Make_page(int id, int ns_id, String ttl_str, String text) {
Xoa_ttl ttl = wiki.Ttl_parse(ns_id, Bry_.new_u8(ttl_str));
return new Xowd_page_itm().Id_(id).Ns_id_(ns_id).Ttl_(ttl).Text_(Bry_.new_u8(text));
}
public void Test_page(Xowd_page_itm[] ary, String expd) {
export_wtr.Write_root_bgn(wiki.Ns_mgr(), wiki.Domain_itm(), "", String_.new_u8(wiki.Props().Main_page()), "first-letter", "XOWA 2.5.2.2");
int len = ary.length;
for (int i = 0; i < len; ++i)
export_wtr.Write_page(ary[i]);
export_wtr.Write_root_end();
String actl = export_wtr.Bld_str();
Tfds.Eq_str_lines(expd, actl);
}
}

80
400_xowa/src/gplx/xowa/bldrs/xmls/Xob_xml_page_bldr.java
View File

@@ -1,80 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.ios.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_xml_page_bldr {
public byte[] Xto_bry() {return bfr.To_bry_and_clear();}
public Io_buffer_rdr XtoByteStreamRdr() {return XtoByteStreamRdr(Io_mgr.Len_kb);}
public Io_buffer_rdr XtoByteStreamRdr(int bfr_len) {
Io_url url = Io_url_.mem_fil_("mem/byteStreamRdr.txt");
Io_mgr.Instance.SaveFilBry(url, bfr.To_bry_and_clear());
return Io_buffer_rdr.new_(gplx.core.ios.streams.Io_stream_rdr_.New__raw(url), bfr_len);
}
public Bry_bfr Bfr() {return bfr;} Bry_bfr bfr = Bry_bfr_.New();
public Xob_xml_page_bldr Upd(String find, String repl) {
String all = bfr.To_str_and_clear();
all = String_.Replace(all, find, repl);
bfr.Add_str_u8(all);
return this;
}
public Xob_xml_page_bldr Add_ary(Xowd_page_itm... ary) {
for (Xowd_page_itm doc : ary)
Add(doc);
return this;
}
public Xob_xml_page_bldr Add(Xowd_page_itm doc) {
bfr.Add(Indent_2).Add(Xob_xml_parser_.Bry_page_bgn).Add_byte_nl();
bfr.Add(Indent_4).Add(Xob_xml_parser_.Bry_title_bgn).Add(doc.Ttl_full_db()).Add(Xob_xml_parser_.Bry_title_end).Add_byte_nl();
bfr.Add(Indent_4).Add(Xob_xml_parser_.Bry_id_bgn).Add_int_variable(doc.Id()).Add(Xob_xml_parser_.Bry_id_end).Add_byte_nl();
bfr.Add(Indent_4).Add(Xob_xml_parser_.Bry_redirect_bgn_frag).Add(Nde_inline).Add_byte_nl();
bfr.Add(Indent_4).Add(Xob_xml_parser_.Bry_revision_bgn).Add_byte_nl();
bfr.Add(Indent_6).Add(Xob_xml_parser_.Bry_id_bgn).Add_int_variable(Revision_id).Add(Xob_xml_parser_.Bry_id_end).Add_byte_nl();
bfr.Add(Indent_6).Add(Xob_xml_parser_.Bry_timestamp_bgn).Add_dte(doc.Modified_on()).Add(Xob_xml_parser_.Bry_timestamp_end).Add_byte_nl();
bfr.Add(Indent_6).Add(Xob_xml_parser_.Bry_contributor_bgn).Add_byte_nl();
bfr.Add(Indent_8).Add(Xob_xml_parser_.Bry_username_bgn).Add(Contributor_username).Add(Xob_xml_parser_.Bry_username_end).Add_byte_nl();
bfr.Add(Indent_8).Add(Xob_xml_parser_.Bry_id_bgn).Add_int_variable(Contributor_id).Add(Xob_xml_parser_.Bry_id_end).Add_byte_nl();
bfr.Add(Indent_6).Add(Xob_xml_parser_.Bry_contributor_end).Add_byte_nl();
bfr.Add(Indent_6).Add(Xob_xml_parser_.Bry_minor_bgn_frag).Add(Nde_inline).Add_byte_nl();
bfr.Add(Indent_6).Add(Xob_xml_parser_.Bry_comment_bgn).Add(Revision_comment).Add(Xob_xml_parser_.Bry_comment_end).Add_byte_nl();
bfr.Add(Indent_6).Add(Xob_xml_parser_.Bry_text_bgn).Add(doc.Text()).Add(Xob_xml_parser_.Bry_text_end).Add_byte_nl();
bfr.Add(Indent_4).Add(Xob_xml_parser_.Bry_revision_end).Add_byte_nl();
bfr.Add(Indent_2).Add(Xob_xml_parser_.Bry_page_end).Add_byte_nl();
return this;
}
private static final byte[] Nde_inline = Bry_.new_a7(" />"), Indent_2 = Bry_.Repeat_space(2), Indent_4 = Bry_.Repeat_space(4), Indent_6 = Bry_.Repeat_space(6), Indent_8 = Bry_.Repeat_space(8);
private static final int Revision_id = 1234, Contributor_id = 9876;
private static final byte[] Contributor_username = Bry_.new_a7("contributor_username"), Revision_comment = Bry_.new_a7("revision_comment");
}
/*
<page>
<title>AccessibleComputing</title>
<id>10</id>
<redirect />
<revision>
<id>381202555</id>
<timestamp>2010-08-26T22:38:36Z</timestamp>
<contributor>
<username>OlEnglish</username>
<id>7181920</id>
</contributor>
<minor />
<comment>[[Help:Reverting|Reverted]] edits by [[Special:Contributions/76.28.186.133|76.28.186.133]] ([[User talk:76.28.186.133|talk]]) to last version by Gurch</comment>
<text xml:space="preserve">#REDIRECT [[Computer accessibility]] {{R from CamelCase}}</text>
</revision>
</page>
*/

129
400_xowa/src/gplx/xowa/bldrs/xmls/Xob_xml_parser.java
View File

@@ -1,129 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.btries.*; import gplx.core.ios.*; import gplx.core.times.*;
import gplx.xowa.wikis.data.tbls.*; import gplx.xowa.wikis.nss.*;
public class Xob_xml_parser {
Btrie_fast_mgr trie = Xob_xml_parser_.trie_(); Bry_bfr data_bfr = Bry_bfr_.New(); DateAdp_parser date_parser = DateAdp_parser.new_();
public Xob_xml_parser Tag_len_max_(int v) {tag_len_max = v; return this;} private int tag_len_max = 255; // max size of any (a) xml tag, (b) int or (c) date; everything else goes into a data_bfr
public Xob_xml_parser Data_bfr_len_(int v) {data_bfr.Resize(v); return this;} // PERF: resize data_bfr once to large size, rather than grow incremently to it
public Xob_xml_parser Trie_tab_del_() {trie.Del(Xob_xml_parser_.Bry_tab); return this;}
public int Parse_page(Xowd_page_itm rv, Gfo_usr_dlg usr_dlg, Io_buffer_rdr fil, byte[] src, int pos, Xow_ns_mgr ns_mgr) {
rv.Clear();
int src_len = fil.Bfr_len(), data_bgn = -1, page_bgn = -1;
boolean data_bfr_add = false, page_id_needed = true, title_needed = true, reading = true;
int[] modified_on_ary = new int[7];
while (reading) {
if ( pos + tag_len_max > src_len // near end of src
&& !fil.Fil_eof()) { // not at fil end
int refill_pos = 0;
if (page_bgn == -1) { // keep page in same data_bfr; NOTE: needed else timestamp/id may fail
refill_pos = pos;
pos = 0;
}
else {
refill_pos = page_bgn;
data_bgn -= page_bgn;
pos -= page_bgn;
}
fil.Bfr_load_from(refill_pos); // refill src from pos;
src_len = fil.Bfr_len();
}
if (pos >= src_len) return Bry_find_.Not_found; // no more src left; should only happen at end of file
byte b = src[pos];
Object o = trie.Match_bgn_w_byte(b, src, pos, src_len);
if (o == null) { // text_data; not an xml_nde (<id>), xml_escape (&lt;), or tab
if (data_bfr_add) data_bfr.Add_byte(b); // add to src if data_bfr_add is on (only happens for <title>, <text>)
++pos;
}
else { // is xml_nde, xml_escape, or tab
Xob_xml_parser_itm itm = (Xob_xml_parser_itm)o;
int hook_bgn = pos; // mark old pos
pos += itm.Hook_len(); // calc new pos
switch (itm.Tid()) {
case Xob_xml_parser_.Id_page_bgn: page_bgn = hook_bgn; break;
case Xob_xml_parser_.Id_id_bgn: if (page_id_needed) data_bgn = pos; break; // only flag if first <id>; note that 1st <id> always belongs to <page>;
case Xob_xml_parser_.Id_id_end:
if (page_id_needed) {
int page_id = Bry_.To_int_or(src, data_bgn, hook_bgn, -1); if (page_id == -1) usr_dlg.Warn_many(GRP_KEY, "page_id_invalid", "page_id_is_invalid: ~{0}", String_.new_u8(src, data_bgn, hook_bgn));
rv.Id_(page_id);
page_id_needed = false; // turn off for other <id> tags (<contributor>; <revision>)
}
break;
case Xob_xml_parser_.Id_timestamp_bgn: data_bgn = pos; break;
case Xob_xml_parser_.Id_timestamp_end:
date_parser.Parse_iso8651_like(modified_on_ary, src, data_bgn, hook_bgn);
rv.Modified_on_(DateAdp_.seg_(modified_on_ary));
break;
case Xob_xml_parser_.Id_title_bgn: if (title_needed) data_bfr_add = true; break;
case Xob_xml_parser_.Id_text_bgn: data_bfr_add = true; break;
case Xob_xml_parser_.Id_title_end:
if (title_needed) {
data_bfr_add = false;
byte[] ttl = data_bfr.To_bry_and_clear();
Bry_.Replace_reuse(ttl, Byte_ascii.Space, Byte_ascii.Underline);
rv.Ttl_(ttl, ns_mgr);
title_needed = false;
}
break;
case Xob_xml_parser_.Id_text_end: data_bfr_add = false; rv.Text_(data_bfr.To_bry_and_clear()); break;
case Xob_xml_parser_.Id_amp: case Xob_xml_parser_.Id_quot: case Xob_xml_parser_.Id_lt: case Xob_xml_parser_.Id_gt:
case Xob_xml_parser_.Id_cr_nl: case Xob_xml_parser_.Id_cr:
if (data_bfr_add) data_bfr.Add_byte(itm.Subst_byte());
break;
case Xob_xml_parser_.Id_tab:
if (data_bfr_add) data_bfr.Add(itm.Subst_ary()); // NOTE: tab can exist in xml; see en.wiktionary.org_20120109: "<page>\n <title>Thread:User talk:Yair rand/newentrywiz.js/quiashed</title>\n <id>2578382</id>\n<DiscussionThreading>\n\t<ThreadSubject>
break;
case Xob_xml_parser_.Id_page_end: reading = false; page_bgn = -1; break;
case Xob_xml_parser_.Id_page_bgn_frag:
pos = Find_gt(src, src_len, pos);
/*warn*/
break;
case Xob_xml_parser_.Id_title_bgn_frag:
if (title_needed) {
pos = Find_gt(src, src_len, pos) + 1; // +1 to get next character
if (!gt_was_inline) data_bfr_add = true;
}
break;
case Xob_xml_parser_.Id_text_bgn_frag:
pos = Find_gt(src, src_len, pos) + 1; // +1 to get next character
if (!gt_was_inline) data_bfr_add = true;
break;
case Xob_xml_parser_.Id_id_bgn_frag: case Xob_xml_parser_.Id_timestamp_bgn_frag:
data_bgn = pos; /*warn*/
break;
default: throw Err_.new_unhandled(itm.Tid()); // shouldn't happen
}
}
}
return pos;
}
int Find_gt(byte[] src, int src_len, int src_pos) {
gt_was_inline = false;
while (src_pos < src_len) {
switch (src[src_pos]) {
case Byte_ascii.Slash: gt_was_inline = true; break;
case Byte_ascii.Gt: return src_pos;
default: gt_was_inline = false; break;
}
++src_pos;
}
return -1;
} boolean gt_was_inline = false;
static final String GRP_KEY = "xowa.bldrs.xmls.xml_parser";
}

73
400_xowa/src/gplx/xowa/bldrs/xmls/Xob_xml_parser_.java
View File

@@ -1,73 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import gplx.core.btries.*;
public class Xob_xml_parser_ {
public static Btrie_fast_mgr trie_() {
Btrie_fast_mgr rv = Btrie_fast_mgr.cs();
trie_add(rv, Bry_page_bgn, Id_page_bgn); trie_add(rv, Bry_page_bgn_frag, Id_page_bgn_frag); trie_add(rv, Bry_page_end, Id_page_end);
trie_add(rv, Bry_id_bgn, Id_id_bgn); trie_add(rv, Bry_id_bgn_frag, Id_id_bgn_frag); trie_add(rv, Bry_id_end, Id_id_end);
trie_add(rv, Bry_title_bgn, Id_title_bgn); trie_add(rv, Bry_title_bgn_frag, Id_title_bgn_frag); trie_add(rv, Bry_title_end, Id_title_end);
trie_add(rv, Bry_timestamp_bgn, Id_timestamp_bgn); trie_add(rv, Bry_timestamp_bgn_frag, Id_timestamp_bgn_frag); trie_add(rv, Bry_timestamp_end, Id_timestamp_end);
trie_add(rv, Bry_text_bgn, Id_text_bgn); trie_add(rv, Bry_text_bgn_frag, Id_text_bgn_frag); trie_add(rv, Bry_text_end, Id_text_end);
trie_add(rv, Bry_amp, Id_amp, Byte_ascii.Amp); trie_add(rv, Bry_quot, Id_quot, Byte_ascii.Quote); trie_add(rv, Bry_gt, Id_gt, Byte_ascii.Gt); trie_add(rv, Bry_lt, Id_lt, Byte_ascii.Lt);
trie_add(rv, Bry_tab, Id_tab, Bry_tab_ent); trie_add(rv, Bry_cr_nl, Id_cr_nl, Byte_ascii.Nl); trie_add(rv, Bry_cr, Id_cr, Byte_ascii.Nl);
return rv;
}
public static final byte[]
Bry_page_bgn = Bry_.new_a7("<page>"), Bry_page_bgn_frag = Bry_.new_a7("<page"), Bry_page_end = Bry_.new_a7("</page>")
, Bry_title_bgn = Bry_.new_a7("<title>"), Bry_title_bgn_frag = Bry_.new_a7("<title"), Bry_title_end = Bry_.new_a7("</title>")
, Bry_id_bgn = Bry_.new_a7("<id>"), Bry_id_bgn_frag = Bry_.new_a7("<id"), Bry_id_end = Bry_.new_a7("</id>")
, Bry_redirect_bgn = Bry_.new_a7("<redirect>"), Bry_redirect_bgn_frag = Bry_.new_a7("<redirect"), Bry_redirect_end = Bry_.new_a7("</redirect>")
, Bry_revision_bgn = Bry_.new_a7("<revision>"), Bry_revision_bgn_frag = Bry_.new_a7("<revision"), Bry_revision_end = Bry_.new_a7("</revision>")
, Bry_timestamp_bgn = Bry_.new_a7("<timestamp>"), Bry_timestamp_bgn_frag = Bry_.new_a7("<timestamp"), Bry_timestamp_end = Bry_.new_a7("</timestamp>")
, Bry_contributor_bgn = Bry_.new_a7("<contributor>"), Bry_contributor_bgn_frag = Bry_.new_a7("<contributor"), Bry_contributor_end = Bry_.new_a7("</contributor>")
, Bry_username_bgn = Bry_.new_a7("<username>"), Bry_username_bgn_frag = Bry_.new_a7("<username"), Bry_username_end = Bry_.new_a7("</username>")
, Bry_minor_bgn = Bry_.new_a7("<minor>"), Bry_minor_bgn_frag = Bry_.new_a7("<minor"), Bry_minor_end = Bry_.new_a7("</minor>")
, Bry_comment_bgn = Bry_.new_a7("<comment>"), Bry_comment_bgn_frag = Bry_.new_a7("<comment"), Bry_comment_end = Bry_.new_a7("</comment>")
, Bry_text_bgn = Bry_.new_a7("<text>"), Bry_text_bgn_frag = Bry_.new_a7("<text"), Bry_text_end = Bry_.new_a7("</text>")
, Bry_amp = Bry_.new_a7("&amp;"), Bry_quot = Bry_.new_a7("&quot;"), Bry_gt = Bry_.new_a7("&gt;"), Bry_lt = Bry_.new_a7("&lt;")
, Bry_tab_ent = Bry_.new_a7("&#09;"), Bry_tab = Bry_.new_a7("\t"), Bry_cr_nl = Bry_.new_a7("\r\n"), Bry_cr = Bry_.new_a7("\r")
;
public static final byte
Id_page_bgn = 0, Id_page_bgn_frag = 1, Id_page_end = 2
, Id_title_bgn = 3, Id_title_bgn_frag = 4, Id_title_end = 5
, Id_id_bgn = 6, Id_id_bgn_frag = 7, Id_id_end = 8
, Id_redirect_bgn = 9, Id_redirect_bgn_frag = 10, Id_redirect_end = 11
, Id_revision_bgn = 12, Id_revision_bgn_frag = 13, Id_revision_end = 14
, Id_timestamp_bgn = 15, Id_timestamp_bgn_frag = 16, Id_timestamp_end = 17
, Id_contributor_bgn = 18, Id_contributor_bgn_frag = 19, Id_contributor_end = 20
, Id_username_bgn = 21, Id_username_bgn_frag = 22, Id_username_end = 23
, Id_minor_bgn = 24, Id_minor_bgn_frag = 25, Id_minor_end = 26
, Id_comment_bgn = 27, Id_comment_bgn_frag = 28, Id_comment_end = 29
, Id_text_bgn = 30, Id_text_bgn_frag = 31, Id_text_end = 32
, Id_amp = 33, Id_quot = 34, Id_gt = 35, Id_lt = 36
, Id_tab = 37, Id_cr_nl = 38, Id_cr = 39
;
private static void trie_add(Btrie_fast_mgr rv, byte[] hook, byte id) {rv.Add(hook, new Xob_xml_parser_itm(hook, id, Byte_.Zero , Bry_.Empty));}
private static void trie_add(Btrie_fast_mgr rv, byte[] hook, byte id, byte subst_byte) {rv.Add(hook, new Xob_xml_parser_itm(hook, id, subst_byte , Bry_.Empty));}
private static void trie_add(Btrie_fast_mgr rv, byte[] hook, byte id, byte[] subst_ary) {rv.Add(hook, new Xob_xml_parser_itm(hook, id, Byte_.Zero , subst_ary));}
}
class Xob_xml_parser_itm {
public Xob_xml_parser_itm(byte[] hook, byte tid, byte subst_byte, byte[] subst_ary) {this.hook = hook; this.hook_len = hook.length; this.tid = tid; this.subst_byte = subst_byte; this.subst_ary = subst_ary;}
public byte Tid() {return tid;} private byte tid;
public byte[] Hook() {return hook;} private byte[] hook;
public int Hook_len() {return hook_len;} private int hook_len;
public byte Subst_byte() {return subst_byte;} private byte subst_byte;
public byte[] Subst_ary() {return subst_ary;} private byte[] subst_ary;
}

148
400_xowa/src/gplx/xowa/bldrs/xmls/Xob_xml_parser_tst.java
View File

@@ -1,148 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.bldrs.xmls; import gplx.*; import gplx.xowa.*; import gplx.xowa.bldrs.*;
import org.junit.*; import gplx.core.ios.*; import gplx.core.times.*;
import gplx.xowa.wikis.nss.*; import gplx.xowa.wikis.data.tbls.*;
public class Xob_xml_parser_tst {
@Before public void init() {
Io_mgr.Instance.InitEngine_mem();
Xoae_app app = Xoa_app_fxt.Make__app__edit();
bldr = new Xob_bldr(app);
} private Xow_ns_mgr ns_mgr = Xow_ns_mgr_.default_(gplx.xowa.langs.cases.Xol_case_mgr_.A7());
@Test public void Basic_docs_1() {
Xowd_page_itm doc = doc_(1, "a", "a a", Date_1);
fil = page_bldr.Add(doc).XtoByteStreamRdr();
tst_parse(fil, doc, 0);
}
@Test public void Basic_docs_2() {
Xowd_page_itm doc1 = doc_(1, "a", "a a", Date_1);
Xowd_page_itm doc2 = doc_(2, "b", "b b", Date_2);
fil = page_bldr.Add_ary(doc1, doc2).XtoByteStreamRdr();
int pos = tst_parse(fil, doc1, 0);
tst_parse(fil, doc2, pos);
}
@Test public void Basic_space() {
Xowd_page_itm doc1 = doc_(1, "a_b", "abc", Date_1);
fil = page_bldr.Add_ary(doc1).Upd("a_b", "a b").XtoByteStreamRdr();
tst_parse(fil, doc1, 0);
}
@Test public void Xml() {
Xowd_page_itm doc = doc_(1, "a", "&quot;a &amp; b &lt;&gt; a | b&quot;", Date_1);
fil = page_bldr.Add(doc).XtoByteStreamRdr();
tst_parse(fil, doc.Text_(Bry_.new_a7("\"a & b <> a | b\"")), 0);
}
@Test public void Tab() {
Xowd_page_itm doc = doc_(1, "a", "a \t b", Date_1);
fil = page_bldr.Add(doc).XtoByteStreamRdr();
tst_parse(fil, doc.Text_(Bry_.new_a7("a &#09; b")), 0);
}
@Test public void Tab_disable() {
Xowd_page_itm doc = doc_(1, "a", "a \t b", Date_1);
page_parser.Trie_tab_del_();
fil = page_bldr.Add(doc).XtoByteStreamRdr();
tst_parse(fil, doc.Text_(Bry_.new_a7("a \t b")), 0);
}
@Test public void Cr_nl() {
Xowd_page_itm doc = doc_(1, "a", "a \r\n b", Date_1);
fil = page_bldr.Add(doc).XtoByteStreamRdr();
tst_parse(fil, doc.Text_(Bry_.new_a7("a \n b")), 0);
}
@Test public void Cr() {
Xowd_page_itm doc = doc_(1, "a", "a \r b", Date_1);
fil = page_bldr.Add(doc).XtoByteStreamRdr();
tst_parse(fil, doc.Text_(Bry_.new_a7("a \n b")), 0);
}
@Test public void Text_long() {
String s = String_.Repeat("a", 1024);
Xowd_page_itm doc = doc_(1, "a", s, Date_1);
page_parser.Tag_len_max_(32);
fil = page_bldr.Add(doc).XtoByteStreamRdr(512);
tst_parse(fil, doc, 0);
}
@Test public void Text_empty() {
Xowd_page_itm doc = doc_(1, "a", "", Date_1);
fil = page_bldr.Add(doc).Upd("<text></text>", "<text />").XtoByteStreamRdr();
tst_parse(fil, doc, 0);
}
@Test public void Text_frag() {
Xowd_page_itm doc = doc_(1, "a", "a", Date_1);
fil = page_bldr.Add(doc).Upd("<text>a</text>", "<text xml:space=\"preserve\">a</text>").XtoByteStreamRdr();
tst_parse(fil, doc, 0);
}
@Test public void Ns_file() {
Xowd_page_itm doc = doc_(1, "File:a", "a", Date_1);
Tfds.Eq(Xow_ns_.Tid__file, doc.Ns_id());
Tfds.Eq("a", String_.new_u8(doc.Ttl_page_db()));
}
@Test public void Ns_main() {
Xowd_page_itm doc = doc_(1, "a", "a", Date_1);
Tfds.Eq(Xow_ns_.Tid__main, doc.Ns_id());
Tfds.Eq("a", String_.new_u8(doc.Ttl_page_db()));
}
@Test public void Ns_main_book() {
Xowd_page_itm doc = doc_(1, "Book", "a", Date_1);
Tfds.Eq(Xow_ns_.Tid__main, doc.Ns_id());
Tfds.Eq("Book", String_.new_u8(doc.Ttl_page_db()));
}
@Test public void XmlEntities() {
Xowd_page_itm orig = doc_(1, "A&amp;b", "a", Date_1);
Xowd_page_itm actl = new Xowd_page_itm();
fil = page_bldr.Add(orig).XtoByteStreamRdr();
page_parser.Parse_page(actl, usr_dlg, fil, fil.Bfr(), 0, ns_mgr);
Tfds.Eq("A&b", String_.new_u8(actl.Ttl_full_db()));
}
@Test public void Root() {
Xowd_page_itm doc = doc_(1, "a", "a", Date_1);
page_bldr.Bfr().Add_str_a7("<root>\n");
page_bldr.Add(doc);
page_bldr.Bfr().Add_str_a7("</root>");
fil = page_bldr.XtoByteStreamRdr();
tst_parse(fil, doc, 0);
}
private static final String Date_1 = "2012-01-01T01:01:01Z", Date_2 = "2012-02-02T02:02:02Z"; DateAdp_parser dateParser = DateAdp_parser.new_();
Bry_bfr bfr = Bry_bfr_.New();
Xob_xml_page_bldr page_bldr = new Xob_xml_page_bldr(); Io_buffer_rdr fil; Xob_xml_parser page_parser = new Xob_xml_parser(); Xob_bldr bldr;
Gfo_usr_dlg usr_dlg = Gfo_usr_dlg_.Test();
int tst_parse(Io_buffer_rdr fil, Xowd_page_itm expd, int cur_pos) {
Xowd_page_itm actl = new Xowd_page_itm();
int rv = page_parser.Parse_page(actl, usr_dlg, fil, fil.Bfr(), cur_pos, ns_mgr);
Tfds.Eq(expd.Id(), actl.Id(), "id");
Tfds.Eq(String_.new_u8(expd.Ttl_full_db()), String_.new_u8(actl.Ttl_full_db()), "title");
Tfds.Eq(String_.new_u8(expd.Text()), String_.new_u8(actl.Text()), "text");
Tfds.Eq_date(expd.Modified_on(), actl.Modified_on(), "timestamp");
return rv;
}
Xowd_page_itm doc_(int id, String title, String text, String date) {
Xowd_page_itm rv = new Xowd_page_itm().Id_(id).Ttl_(Bry_.new_a7(title), ns_mgr).Text_(Bry_.new_a7(text));
int[] modified_on = new int[7];
dateParser.Parse_iso8651_like(modified_on, date);
rv.Modified_on_(DateAdp_.seg_(modified_on));
return rv;
}
}
class Xob_xml_parser_fxt {
// private final Xob_xml_parser page_parser = new Xob_xml_parser();
// public void Test__parse(Io_buffer_rdr fil, Xowd_page_itm expd, int cur_pos) {
// Xowd_page_itm actl = new Xowd_page_itm();
// int rv = page_parser.Parse_page(actl, usr_dlg, fil, fil.Bfr(), cur_pos, ns_mgr);
// Tfds.Eq(expd.Id(), actl.Id(), "id");
// Tfds.Eq(String_.new_u8(expd.Ttl_full_db()), String_.new_u8(actl.Ttl_full_db()), "title");
// Tfds.Eq(String_.new_u8(expd.Text()), String_.new_u8(actl.Text()), "text");
// Tfds.Eq_date(expd.Modified_on(), actl.Modified_on(), "timestamp");
// }
}