Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v2.4.1.1

Browse Source
This commit is contained in:
gnosygnu
2015-04-05 21:00:31 -04:00
parent 80b9928b5c
commit 18dcd3f89e
615 changed files with 9826 additions and 8619 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
400_xowa/src_200_bldr/gplx/xowa/Xob_bldr.java
View File

@@ -16,30 +16,31 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.xowa.bldrs.*; import gplx.xowa.apps.*; import gplx.xowa.bldrs.xmls.*;
import gplx.xowa.bldrs.cfgs.*; import gplx.xowa.bldrs.langs.*;
import gplx.xowa.apps.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.xmls.*; import gplx.xowa.bldrs.cfgs.*; import gplx.xowa.bldrs.langs.*;
public class Xob_bldr implements GfoInvkAble {
private boolean pause_at_end = false;
private boolean pause_at_end = false; private long prv_prog_time; private Xob_xml_parser dump_parser;
public Xob_bldr(Xoae_app app) {
this.app = app;
cmd_mgr = new Xob_cmd_mgr(this);
wiki_cfg_bldr = new Xob_wiki_cfg_bldr(this);
this.cmd_mgr = new Xob_cmd_mgr(this);
this.import_marker = new Xob_import_marker();
this.wiki_cfg_bldr = new Xob_wiki_cfg_bldr(this);
}
public Xoae_app App() {return app;} private Xoae_app app;
public Xoae_app App() {return app;} private final Xoae_app app;
public Xob_cmd_mgr Cmd_mgr() {return cmd_mgr;} private final Xob_cmd_mgr cmd_mgr;
public Gfo_usr_dlg Usr_dlg() {return app.Usr_dlg();}
public Xob_cmd_mgr Cmd_mgr() {return cmd_mgr;} private Xob_cmd_mgr cmd_mgr;
public int Sort_mem_len() {return sort_mem_len;} public Xob_bldr Sort_mem_len_(int v) {sort_mem_len = v; return this;} private int sort_mem_len = 16 * Io_mgr.Len_mb;
public int Dump_fil_len() {return dump_fil_len;} public Xob_bldr Dump_fil_len_(int v) {dump_fil_len = v; return this;} private int dump_fil_len = 1 * Io_mgr.Len_mb;
public int Make_fil_len() {return make_fil_len;} public Xob_bldr Make_fil_len_(int v) {make_fil_len = v; return this;} private int make_fil_len = 64 * Io_mgr.Len_kb;
public Xob_xml_parser Parser() {if (parser == null) parser = new Xob_xml_parser(); return parser;} private Xob_xml_parser parser;
public Xob_xml_parser Dump_parser() {if (dump_parser == null) this.dump_parser = new Xob_xml_parser(); return dump_parser;}
public Xob_import_marker Import_marker() {return import_marker;} private Xob_import_marker import_marker;
public Xob_wiki_cfg_bldr Wiki_cfg_bldr() {return wiki_cfg_bldr;} private Xob_wiki_cfg_bldr wiki_cfg_bldr;
public void StatusMgr_prog_fmt(long cur, long end, int pct_idx, String fmt, Object... ary) {
long now = Env_.TickCount(); if (now - StatusMgr_prog_prv < 100) return;
StatusMgr_prog_prv = now;
public void Pause_at_end_(boolean v) {this.pause_at_end = v;}
public void Print_prog_msg(long cur, long end, int pct_idx, String fmt, Object... ary) {
long now = Env_.TickCount(); if (now - prv_prog_time < 100) return;
this.prv_prog_time = now;
if (pct_idx > -1) ary[pct_idx] = DecimalAdp_.CalcPctStr(cur, end, "00.00");
app.Usr_dlg().Prog_many(GRP_KEY, "prog", fmt, ary);
} long StatusMgr_prog_prv;
public void Pause_at_end_(boolean v) {pause_at_end = v;}
app.Usr_dlg().Prog_many("", "", fmt, ary);
}
public void Run() {
try {
app.Launch(); // HACK: bldr will be called by a gfs file which embeds "bldr.run" inside it; need to call Launch though before Run; DATE:2013-03-23
@@ -47,28 +48,28 @@ public class Xob_bldr implements GfoInvkAble {
int cmd_mgr_len = cmd_mgr.Len();
for (int i = 0; i < cmd_mgr_len; i++) {
Xob_cmd cmd = cmd_mgr.Get_at(i);
cmd.Cmd_ini(this);
cmd.Cmd_init(this);
}
cmd_mgr_len = cmd_mgr.Len(); // NOTE: refresh len b/c other cmds may have other added
cmd_mgr_len = cmd_mgr.Len(); // NOTE: refresh len b/c other cmds may have added new ones in Cmd_init
for (int i = 0; i < cmd_mgr_len; i++) {
Xob_cmd cmd = cmd_mgr.Get_at(i);
app.Usr_dlg().Note_many(GRP_KEY, "cmd_bgn", "cmd bgn: ~{0}", cmd.Cmd_key());
app.Usr_dlg().Note_many("", "", "cmd bgn: ~{0}", cmd.Cmd_key());
long time_cur = Env_.TickCount();
cmd.Cmd_bgn(this);
cmd.Cmd_run();
cmd.Cmd_end();
Env_.GarbageCollect();
app.Usr_dlg().Note_many(GRP_KEY, "cmd_end", "cmd end: ~{0} ~{1}", cmd.Cmd_key(), TimeSpanAdp_.from_(time_cur).XtoStrUiAbbrv());
app.Usr_dlg().Note_many("", "", "cmd end: ~{0} ~{1}", cmd.Cmd_key(), TimeSpanAdp_.from_(time_cur).XtoStrUiAbbrv());
}
for (int i = 0; i < cmd_mgr_len; i++)
cmd_mgr.Get_at(i).Cmd_print();
app.Usr_dlg().Note_many(GRP_KEY, "bldr_done", "bldr done: ~{0}", TimeSpanAdp_.from_(time_bgn).XtoStrUiAbbrv());
for (int i = 0; i < cmd_mgr_len; i++) {
Xob_cmd cmd = cmd_mgr.Get_at(i);
cmd.Cmd_term();
}
app.Usr_dlg().Note_many("", "", "bldr done: ~{0}", TimeSpanAdp_.from_(time_bgn).XtoStrUiAbbrv());
cmd_mgr.Clear();
if (pause_at_end && !Env_.Mode_testing()) {ConsoleAdp._.ReadLine("press enter to continue");}
}
catch (Exception e) {
throw Err_.err_(e, "error during build: ~{0}", Err_.Message_gplx(e));
}
catch (Exception e) {throw Err_.err_(e, "error during build: ~{0}", Err_.Message_gplx(e));}
}
private void Cancel() {
int cmd_mgr_len = cmd_mgr.Len();
@@ -94,7 +95,6 @@ public class Xob_bldr implements GfoInvkAble {
, Invk_pause_at_end_ = "pause_at_end_", Invk_sort_mem_len_ = "sort_mem_len_", Invk_dump_fil_len_ = "dump_fil_len_", Invk_make_fil_len_ = "make_fil_len_"
, Invk_run = "run", Invk_cancel = "cancel"
;
static final String GRP_KEY = "xowa.bldr";
}
/*
. make_fil_len: max size of made file; EX: /id/..../0000000001.csv will have max len of 64 KB

5
400_xowa/src_200_bldr/gplx/xowa/Xob_page_cmd.java
View File

@@ -16,9 +16,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
interface Xodb_page_cmd {
import gplx.xowa.wikis.data.tbls.*;
interface Xowd_page_cmd {
String Cmd_key();
void Cmd_bgn(Xob_bldr bldr);
void Cmd_run(Xodb_page page);
void Cmd_run(Xowd_page_itm page);
void Cmd_end();
}

2
400_xowa/src_200_bldr/gplx/xowa/Xob_stat_mgr.java
View File

@@ -38,7 +38,7 @@ public class Xob_stat_mgr {
for (int i = 0; i < regy.Count(); i++) {
Xob_stat_type typ = (Xob_stat_type)regy.FetchAt(i);
Xob_stat_itm itm = (Xob_stat_itm)typ.GetOrNew(ns);
sb.Add(Int_.Xto_str_pad_bgn(itm.Fils, 5)).Add(" ");
sb.Add(Int_.Xto_str_pad_bgn_zero(itm.Fils, 5)).Add(" ");
}
sb.Add_str_w_crlf(ns);
}

2
400_xowa/src_200_bldr/gplx/xowa/Xob_xdat_file_wtr.java
View File

@@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.xowa.tdbs.*;
public class Xob_xdat_file_wtr {
public static Xob_xdat_file_wtr new_file_(int fil_max, Io_url root_dir) {return new Xob_xdat_file_wtr(fil_max, root_dir, Io_stream_.Tid_file);}
public static Xob_xdat_file_wtr new_file_(int fil_max, Io_url root_dir) {return new Xob_xdat_file_wtr(fil_max, root_dir, Io_stream_.Tid_raw);}
public static Xob_xdat_file_wtr new_by_tid_(int fil_max, Io_url root_dir, byte dir_tid, byte tid) {return new Xob_xdat_file_wtr(fil_max, root_dir.GenSubDir(Xotdb_dir_info_.Tid_name(dir_tid) + Xotdb_dir_info.Wtr_dir(tid)), tid);}
Xob_xdat_file_wtr(int fil_max, Io_url root_dir, byte wtr_tid) {
this.fil_max = fil_max;

6
400_xowa/src_200_bldr/gplx/xowa/Xobd_parser.java
View File

@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.btries.*;
import gplx.core.btries.*; import gplx.xowa.wikis.data.tbls.*;
public class Xobd_parser implements Xobd_wkr {
private Btrie_slim_mgr trie = Btrie_slim_mgr.ci_ascii_(); // NOTE:ci.ascii:MW_const.en; ctg.v1 assumes [[Category:
private ListAdp wkr_list = ListAdp_.new_();
@@ -35,8 +35,8 @@ public class Xobd_parser implements Xobd_wkr {
}
}
}
public void Wkr_run(Xodb_page page) {
byte[] src = page.Wtxt(); int src_len = src.length;
public void Wkr_run(Xowd_page_itm page) {
byte[] src = page.Text(); int src_len = src.length;
int pos = 0;
while (true) {
if (pos == src_len) break;

3
400_xowa/src_200_bldr/gplx/xowa/Xobd_parser_wkr.java
View File

@@ -16,9 +16,10 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.xowa.wikis.data.tbls.*;
public interface Xobd_parser_wkr extends GfoInvkAble {
OrderedHash Wkr_hooks();
void Wkr_bgn(Xob_bldr bldr);
int Wkr_run(Xodb_page page, byte[] src, int src_len, int bgn, int end);
int Wkr_run(Xowd_page_itm page, byte[] src, int src_len, int bgn, int end);
void Wkr_end();
}

15
400_xowa/src_200_bldr/gplx/xowa/Xobd_rdr.java
View File

@@ -16,11 +16,12 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.xmls.*;
import gplx.ios.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.xmls.*; import gplx.xowa.wikis.data.tbls.*;
public class Xobd_rdr implements Xob_cmd {
public Xobd_rdr(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki;} private Xob_bldr bldr; Xowe_wiki wiki;
private Xob_bldr bldr; private Xowe_wiki wiki;
public Xobd_rdr(Xob_bldr bldr, Xowe_wiki wiki) {this.bldr = bldr; this.wiki = wiki;}
public String Cmd_key() {return KEY;} public static final String KEY = "dump_mgr";
public void Cmd_ini(Xob_bldr bldr) {
public void Cmd_init(Xob_bldr bldr) {
Xobd_wkr[] wkr_ary = (Xobd_wkr[])wkrs.Xto_ary(Xobd_wkr.class); int wkr_ary_len = wkr_ary.length;
for (int i = 0; i < wkr_ary_len; i++)
wkr_ary[i].Wkr_ini(bldr);
@@ -31,8 +32,8 @@ public class Xobd_rdr implements Xob_cmd {
Xobd_wkr[] wkr_ary = (Xobd_wkr[])wkrs.Xto_ary(Xobd_wkr.class); int wkr_ary_len = wkr_ary.length;
for (int i = 0; i < wkr_ary_len; i++)
wkr_ary[i].Wkr_bgn(bldr);
Io_buffer_rdr fil = Io_buffer_rdr.Null; Xodb_page page = new Xodb_page(); Xow_ns_mgr ns_mgr = wiki.Ns_mgr();
Xob_xml_parser parser = bldr.Parser().Data_bfr_len_(Io_mgr.Len_mb);
Io_buffer_rdr fil = Io_buffer_rdr.Null; Xowd_page_itm page = new Xowd_page_itm(); Xow_ns_mgr ns_mgr = wiki.Ns_mgr();
Xob_xml_parser parser = bldr.Dump_parser().Data_bfr_len_(Io_mgr.Len_mb);
long fil_len = 0;
Gfo_usr_dlg usr_dlg = bldr.App().Usr_dlg();
try {
@@ -45,7 +46,7 @@ public class Xobd_rdr implements Xob_cmd {
while (true) {
int cur_pos = parser.Parse_page(page, usr_dlg, fil, fil.Bfr(), prv_pos, ns_mgr); if (cur_pos == Bry_.NotFound) break;
if (cur_pos < prv_pos)
bldr.StatusMgr_prog_fmt(fil.Fil_pos(), fil_len, 1, optRdrFillFmt, Int_.Xto_str_pad_bgn((int)(fil.Fil_pos() / Io_mgr.Len_mb), Int_.DigitCount((int)(fil.Fil_len() / Io_mgr.Len_mb))), "", String_.new_utf8_(page.Ttl_full_db()));
bldr.Print_prog_msg(fil.Fil_pos(), fil_len, 1, optRdrFillFmt, Int_.Xto_str_pad_bgn_zero((int)(fil.Fil_pos() / Io_mgr.Len_mb), Int_.DigitCount((int)(fil.Fil_len() / Io_mgr.Len_mb))), "", String_.new_utf8_(page.Ttl_full_db()));
prv_pos = cur_pos;
try {
for (int i = 0; i < wkr_ary_len; i++)
@@ -72,7 +73,7 @@ public class Xobd_rdr implements Xob_cmd {
for (int i = 0; i < wkr_ary_len; i++)
wkr_ary[i].Wkr_end();
}
public void Cmd_print() {
public void Cmd_term() {
Xobd_wkr[] wkr_ary = (Xobd_wkr[])wkrs.Xto_ary(Xobd_wkr.class); int wkr_ary_len = wkr_ary.length;
for (int i = 0; i < wkr_ary_len; i++)
wkr_ary[i].Wkr_print();

3
400_xowa/src_200_bldr/gplx/xowa/Xobd_wkr.java
View File

@@ -16,11 +16,12 @@ You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.xowa.wikis.data.tbls.*;
public interface Xobd_wkr extends GfoInvkAble {
String Wkr_key();
void Wkr_ini(Xob_bldr bldr);
void Wkr_bgn(Xob_bldr bldr);
void Wkr_run(Xodb_page page);
void Wkr_run(Xowd_page_itm page);
void Wkr_end();
void Wkr_print();
}

58
400_xowa/src_200_bldr/gplx/xowa/Xodb_page_raw_parser.java
View File

@@ -1,58 +0,0 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.core.brys.*; import gplx.ios.*;
public class Xodb_page_raw_parser {
public void Init(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, int load_len) {
this.wiki = wiki; ns_mgr = wiki.Ns_mgr();
rdr = new Io_line_rdr(usr_dlg, new Io_url[1]);
rdr.Line_dlm_(Byte_ascii.Tab).Load_len_(load_len).Key_gen_(Io_line_rdr_key_gen_all._);
}
public void Init_ns(Xow_ns ns_itm) {this.ns_itm = ns_itm;}
public void Reset_one(Io_url url) {
rdr.Reset_one(url);
}
public void Load(Gfo_usr_dlg usr_dlg, Xowe_wiki wiki, Xow_ns ns_itm, Io_url[] urls, int load_len) {
this.wiki = wiki; ns_mgr = wiki.Ns_mgr(); this.ns_itm = ns_itm;
rdr = new Io_line_rdr(usr_dlg, urls);
rdr.Line_dlm_(Byte_ascii.Tab).Load_len_(load_len).Key_gen_(Io_line_rdr_key_gen_all._);
} Io_line_rdr rdr; Xowe_wiki wiki; Xow_ns_mgr ns_mgr; Xow_ns ns_itm;
public void Skip_first_line() {
rdr.Read_next();
int pos = Bry_finder.Find_fwd(rdr.Bfr(), Byte_ascii.NewLine);
// rdr.Move(pos + 1);
rdr.Truncate(pos + 1);
}
public boolean Read(Xodb_page page) {
boolean read = false;
read = rdr.Read_next(); if (!read) return false;
int id = Base85_utl.XtoIntByAry(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 2);
page.Id_(id);
read = rdr.Read_next(); if (!read) throw Err_.new_("could not read timestamp");
int timestamp = Base85_utl.XtoIntByAry(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 1);
page.Modified_on_(Bit_.Xto_date_short(timestamp));
read = rdr.Read_next(); if (!read) throw Err_.new_("could not read ttl");
byte[] ttl = Bry_.Mid(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 1);
page.Ttl_(ttl, ns_mgr);
read = rdr.Read_next(); if (!read) throw Err_.new_("could not read text");
byte[] text = Bry_.Mid(rdr.Bfr(), rdr.Key_pos_bgn(), rdr.Key_pos_end() - 1);
page.Wtxt_(text);
rdr.Bfr_last_read_add(1);
return true;
}
}