Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

v1.6.5.1

Browse Source
This commit is contained in:
gnosygnu
2014-06-30 00:04:32 -04:00
parent 85594d3cdd
commit bae88e739c
2482 changed files with 198730 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

51
400_xowa/src_220_bldr_file/gplx/xowa/Xob_xnde_wkr.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*;
public interface Xob_xnde_wkr {
void Wkr_run(Xop_ctx ctx, Xop_root_tkn root, Xop_xnde_tkn xnde);
}
class Xobc_xnde_math_dump extends Xob_itm_dump_base implements Xob_xnde_wkr { // note: similar to img_dump, except for Wkr_run
public Xobc_xnde_math_dump(Xob_bldr bldr, Xow_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = 100 * Io_mgr.Len_kb;}
public static final String KEY = "math.text";
public Io_url_gen Make_url_gen() {return make_url_gen;} public Xobc_xnde_math_dump Make_url_gen_(Io_url_gen v) {make_url_gen = v; return this;} Io_url_gen make_url_gen;
public void Wkr_bgn(Xob_bldr bldr) {
this.Init_dump(KEY);
this.fld_wtr = Gfo_fld_wtr.xowa_().Bfr_(dump_bfr);
sort_dir = temp_dir.GenSubDir("sort");
make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
} Io_url sort_dir; Gfo_fld_wtr fld_wtr; Bry_bfr url_decoder_bfr = Bry_bfr.new_();
public void Wkr_run(Xop_ctx ctx, Xop_root_tkn root, Xop_xnde_tkn xnde) {
if (xnde.CloseMode() == Xop_xnde_tkn.CloseMode_inline) return; // ignore <math/>; EX:FOSD origami
byte[] math = Bry_.Mid(root.Root_src(), xnde.Src_bgn() + 6, xnde.Src_end() - 7); // 6=<math>; 7=</math>
byte[] ttl = ctx.Cur_page().Ttl().Full_txt();
int entry_len = ttl.length + math.length + 3; // 3=| + | + \n
if (dump_bfr.Len() + entry_len > dump_fil_len) Flush();
fld_wtr.Write_bry_escape_fld(math);
fld_wtr.Write_bry_escape_row(ttl);
}
public void Wkr_end() {
Flush();
Io_sort_cmd_img img_cmd = new Io_sort_cmd_img().Make_url_gen_(make_url_gen).Make_fil_max_(make_fil_len);
Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, sort_dir, sort_mem_len, Io_line_rdr_key_gen_.first_pipe, img_cmd);
}
public void Flush() {
Io_mgr._.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
dump_bfr.Reset_if_gt(Io_mgr.Len_mb);
}
}

78
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_base_fxt.java Normal file
View File

@@ -0,0 +1,78 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.xowa.bldrs.*;
public class Xobc_base_fxt {
public Xobc_base_fxt Clear() {
if (app == null) {
app = Xoa_app_fxt.app_();
wiki = Xoa_app_fxt.wiki_tst_(app);
bldr = Xoa_app_fxt.bldr_(app);
}
this.Init_(bldr, wiki);
Clear_hook();
return this;
}
@gplx.Virtual public void Clear_hook() {}
public Xobc_base_fxt Init_(Xob_bldr bldr, Xow_wiki wiki) {this.bldr = bldr; this.wiki = wiki; return this;}
public Xoa_app App() {return app;} private Xoa_app app;
public Xob_bldr Bldr() {return bldr;} private Xob_bldr bldr;
public Xow_wiki Wiki() {return wiki;} private Xow_wiki wiki;
public GfoInvkAble Bldr_itm() {return bldr_itm;} GfoInvkAble bldr_itm;
public Xodb_page page_(String ttl) {return page_(ttl, "");}
public Xodb_page page_(String ttl, String text) {return new Xodb_page().Ttl_(Bry_.new_utf8_(ttl), wiki.Ns_mgr()).Text_(Bry_.new_utf8_(text));}
public Io_fil_chkr meta_(String url, String data) {return new Io_fil_chkr(Io_url_.mem_fil_(url), data);}
public void Init_fxts(Xob_bldr bldr, Xow_wiki wiki, Xobc_base_fxt... fxt_ary) {
int fxt_ary_len = fxt_ary.length;
for (int i = 0; i < fxt_ary_len; i++)
fxt_ary[i].Init_(bldr, wiki);
}
public Xobc_base_fxt Init_fil(String url, String raw) {return Init_fil(Io_url_.new_fil_(url), raw);}
public Xobc_base_fxt Init_fil(Io_url url, String raw) {Io_mgr._.SaveFilStr(url, raw); return this;}
public Xobc_base_fxt Exec_cmd(String cmd_key, GfoMsg... msgs) {
Xob_cmd cmd = (Xob_cmd)bldr.Cmd_mgr().Add_cmd(wiki, cmd_key);
this.bldr_itm = cmd;
int len = msgs.length;
GfsCtx ctx = GfsCtx.new_();
for (int i = 0; i < len; i++) {
GfoMsg msg = msgs[i];
cmd.Invk(ctx, GfsCtx.Ikey_null, msg.Key(), msg);
}
Run_cmd(bldr, cmd);
return this;
}
public Xobc_base_fxt Test_fil(String url, String expd) {return Test_fil(Io_url_.new_fil_(url), expd);}
public Xobc_base_fxt Test_fil(Io_url url, String expd) {
Tfds.Eq_str_lines(expd, Io_mgr._.LoadFilStr(url));
return this;
}
public static void Run_cmd(Xob_bldr bldr, Xob_cmd cmd) {
cmd.Cmd_bgn(bldr);
cmd.Cmd_run();
cmd.Cmd_end();
}
public static void Run_wkr(Xob_bldr bldr, Xobd_wkr wkr, Xodb_page[] page_ary) {
wkr.Wkr_bgn(bldr);
int page_ary_len = page_ary.length;
for (int i = 0; i < page_ary_len; i++) {
Xodb_page page = page_ary[i];
wkr.Wkr_run(page);
}
wkr.Wkr_end();
}
}

47
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_img_dump_ttl.java Normal file
View File

@@ -0,0 +1,47 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*;
public class Xobc_img_dump_ttl extends Xob_itm_dump_base implements Xobd_wkr, GfoInvkAble {
public Xobc_img_dump_ttl(Xob_bldr bldr, Xow_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Wkr_key() {return KEY;} public static final String KEY = "img.dump_ttl";
@gplx.Internal protected Io_url_gen Make_url_gen() {return make_url_gen;} Io_url_gen make_url_gen;
public void Wkr_ini(Xob_bldr bldr) {}
public void Wkr_bgn(Xob_bldr bldr) {
this.Init_dump(KEY);
make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
fld_wtr = Gfo_fld_wtr.xowa_().Bfr_(dump_bfr);
redirect_mgr = wiki.Redirect_mgr();
} Gfo_fld_wtr fld_wtr; Xop_redirect_mgr redirect_mgr;
public void Wkr_run(Xodb_page page) {
if (page.Ns_id() != Xow_ns_.Id_file) return;
Xoa_ttl ttl = Xoa_ttl.parse_(wiki, page.Ttl_w_ns());
byte[] ttl_bry = ttl.Page_db();
byte[] src = page.Text();
Xoa_ttl redirect_ttl = redirect_mgr.Extract_redirect(src, src.length);
byte[] redirect_bry = redirect_ttl == null ? Bry_.Empty : redirect_ttl.Page_db();
if (dump_bfr.Len() + ttl_bry.length + redirect_bry.length + Fixed_row_len > dump_fil_len) super.Flush_dump();
fld_wtr .Write_bry_escape_fld(ttl_bry)
.Write_bry_escape_row(redirect_bry);
} private static final int Fixed_row_len = 3; // 2=| 1=\n
public void Wkr_end() {
super.Flush_dump();
Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, temp_dir.GenSubDir("sort"), sort_mem_len, Io_line_rdr_key_gen_all._, new Io_sort_fil_basic(bldr.Usr_dlg(), make_url_gen, dump_fil_len));
}
public void Wkr_print() {}
}

48
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_img_dump_ttl_tst.java Normal file
View File

@@ -0,0 +1,48 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import org.junit.*;
import gplx.ios.*;
public class Xobc_img_dump_ttl_tst {
Xobc_img_dump_ttl_fxt fxt = new Xobc_img_dump_ttl_fxt();
@Before public void init() {fxt.Clear();}
@Test public void Basic() {
fxt.Run
( fxt.page_("File:B", "a")
, fxt.page_("Image:A", "#REDIRECT [[File:B]]")
);
fxt.Tst(String_.Concat_lines_nl
( "A|B"
, "B|"
));
}
}
class Xobc_img_dump_ttl_fxt extends Xobc_base_fxt {
Xobc_img_dump_ttl cmd;
public Xobc_img_dump_ttl Run(Xodb_page... page_ary) {return Run(new Xobc_img_dump_ttl(this.Bldr(), this.Wiki()), page_ary);}
public Xobc_img_dump_ttl Run(Xobc_img_dump_ttl rv, Xodb_page[] page_ary) {
this.cmd = rv;
Run_wkr(this.Bldr(), rv, page_ary);
return rv;
}
public void Tst(String expd) {
Io_url trg_fil = cmd.Make_url_gen().Prv_urls()[0];
String actl = Io_mgr._.LoadFilStr(trg_fil);
Tfds.Eq_str_lines(expd, actl);
}
}

92
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_img_merge_ttl_sql.java Normal file
View File

@@ -0,0 +1,92 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wikis.images.*;
public class Xobc_img_merge_ttl_sql extends Xob_itm_dump_base implements Xob_cmd, GfoInvkAble {
public Xobc_img_merge_ttl_sql(Xob_bldr bldr, Xow_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
public String Cmd_key() {return KEY;} public static final String KEY = "img.merge_ttl";
public Io_url Ttl_name_dir() {return ttl_name_dir;} Io_url ttl_name_dir;
public Io_url Sql_size_dir() {return sql_size_dir;} Io_url sql_size_dir;
public Io_url Make_url_dir() {return make_url_dir;} Io_url make_url_dir;
public Io_url_gen Make_url_gen() {return make_url_gen;} Io_url_gen make_url_gen;
public void Cmd_ini(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
this.Init_dump(KEY);
sql_size_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xob_wiki_image_sql.KEY, "make");
ttl_name_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xobc_img_dump_ttl.KEY, "make");
redirect_name_dir = temp_dir.GenSubDir("redirect_dump");
redirect_sort_dir = temp_dir.GenSubDir("redirect_sort");
redirect_make_dir = temp_dir.GenSubDir("redirect_make");
make_url_dir = temp_dir.GenSubDir("make");
standard_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("standard_dump"));
redirect_url_gen = Io_url_gen_.dir_(redirect_name_dir);
make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
} Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_(); Gfo_fld_rdr fld_rdr = Gfo_fld_rdr.xowa_(); Io_url redirect_name_dir, redirect_sort_dir, redirect_make_dir;
Io_line_rdr rdr_(Io_url dir) {
Io_url[] fils = Io_mgr._.QueryDir_fils(dir);
return new Io_line_rdr(bldr.Usr_dlg(), fils).Key_gen_(Io_line_rdr_key_gen_.first_pipe);
}
public void Cmd_end() {}
public void Cmd_print() {}
public void Cmd_run() {
Io_line_rdr name_rdr = rdr_(ttl_name_dir);
Io_line_rdr size_rdr = rdr_(sql_size_dir);
Merge(true, name_rdr, size_rdr);
Flush(standard_dump_bfr, standard_url_gen);
Flush(redirect_dump_bfr, redirect_url_gen);
Xobdc_merger.Basic(bldr.Usr_dlg(), redirect_url_gen, redirect_sort_dir, sort_mem_len, Io_line_rdr_key_gen_all._, new Io_sort_fil_basic(bldr.Usr_dlg(), Io_url_gen_.dir_(redirect_make_dir), make_fil_len));
name_rdr = rdr_(redirect_make_dir);
size_rdr = rdr_(sql_size_dir);
Merge(false, name_rdr, size_rdr);
Flush(standard_dump_bfr, standard_url_gen);
Xobdc_merger.Basic(bldr.Usr_dlg(), standard_url_gen, temp_dir.GenSubDir("sort"), sort_mem_len, Io_line_rdr_key_gen_all._, new Io_sort_fil_basic(bldr.Usr_dlg(), make_url_gen, make_fil_len));
}
private void Merge(boolean pass_0, Io_line_rdr name_rdr, Io_line_rdr size_rdr) {
while (name_rdr.Read_next()) {
Xofo_file itm = new Xofo_file();
itm.Load_by_name_rdr(fld_rdr, name_rdr);
byte[] name_lower = Xof_ext_.Lower_ext(Bry_.Copy(itm.Name()));
boolean found = size_rdr.Match(name_lower);
if (found) itm.Load_by_size_rdr(fld_rdr, size_rdr);
Write(pass_0, itm);
}
}
private void Write(boolean pass_0, Xofo_file itm) {
if (pass_0) {
if (itm.Redirect_exists())
Write(redirect_dump_bfr, redirect_url_gen, itm, true);
else
Write(standard_dump_bfr, standard_url_gen, itm, false);
}
else
Write(standard_dump_bfr, standard_url_gen, itm, true); // reverse name/redirect from pass_0
}
private void Write(Bry_bfr bfr, Io_url_gen url_gen, Xofo_file itm, boolean reverse) {
if (bfr.Len() > dump_fil_len) Flush(bfr, url_gen);
fld_wtr.Bfr_(bfr);
itm.Write(reverse, fld_wtr);
}
private void Flush(Bry_bfr bfr, Io_url_gen url_gen) {Io_mgr._.SaveFilBfr(url_gen.Nxt_url(), bfr);}
Sql_file_parser parser = new Sql_file_parser();
Bry_bfr standard_dump_bfr = Bry_bfr.new_(), redirect_dump_bfr = Bry_bfr.new_();
Io_url_gen standard_url_gen, redirect_url_gen;
Bry_bfr noop = Bry_bfr.new_();
}

123
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_img_prep_xfer.java Normal file
View File

@@ -0,0 +1,123 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.xowa.bldrs.*; import gplx.xowa.parsers.lnkis.redlinks.*;
public class Xobc_img_prep_xfer extends Xob_itm_basic_base implements Xob_cmd, GfoInvkAble {
public Xobc_img_prep_xfer(Xob_bldr bldr, Xow_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return KEY;} public static final String KEY = "img.prep_xfer";
public Io_url Src_fil() {return src_fil;} public Xobc_img_prep_xfer Src_fil_(Io_url v) {src_fil = v; return this;} Io_url src_fil;
public Io_url Temp_dir() {return temp_dir;}
public Io_url Link_dir() {return link_dir;} Io_url link_dir;
public Io_url Wiki_1_dir() {return wiki_1_dir;} Io_url wiki_1_dir;
public Io_url Wiki_0_dir() {return wiki_0_dir;} Io_url wiki_0_dir;
public Io_url_gen Dump_url_gen() {return dump_url_gen;} public Xobc_img_prep_xfer Dump_url_gen_(Io_url_gen v) {dump_url_gen = v; return this;} Io_url_gen dump_url_gen;
public void Commons_url_(Io_url v) {commons_url = v;} Io_url commons_url;
public void Cmd_ini(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
this.bldr = bldr; this.fld_wtr = Gfo_fld_wtr.xowa_().Bfr_(dump_bfr); fld_rdr = Gfo_fld_rdr.xowa_();
temp_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir(KEY);
Io_mgr._.DeleteDirDeep(temp_dir);
if (commons_url == null) commons_url = wiki.Fsys_mgr().Root_dir().OwnerDir().GenSubDir_nest(Xow_wiki_.Domain_commons_str);
link_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xobc_lnki_wkr_file.KEY, "make");
wiki_1_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xobc_img_merge_ttl_sql.KEY, "make");
wiki_0_dir = commons_url.GenSubDir_nest("tmp", Xobc_img_merge_ttl_sql.KEY, "make");
dump_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("dump"));
} Io_line_rdr img_link_rdr; Gfo_fld_wtr fld_wtr; Gfo_fld_rdr fld_rdr;
Io_line_rdr rdr_(Io_url dir) {
Io_url[] fils = Io_mgr._.QueryDir_fils(dir);
return new Io_line_rdr(bldr.Usr_dlg(), fils).Key_gen_(Io_line_rdr_key_gen_.first_pipe);
}
private void Write(Xofo_file xfer, ListAdp link_list) {
xfer.Links_(link_list);
if (dump_bfr.Len() > dump_fil_len) {Io_mgr._.AppendFilByt(dump_url_gen.Nxt_url(), dump_bfr.Bfr(), dump_bfr.Len()); dump_bfr.Clear();}
xfer.Write_bldr(fld_wtr);
} private Xofo_lnki_parser lnki_parser = new Xofo_lnki_parser();
public void Cmd_end() {}
public void Cmd_print() {}
public void Cmd_run() {
img_link_rdr = rdr_(link_dir);
Xofo_file xfer = null;
byte[] prv_name = Bry_.Empty;
Io_line_rdr wiki_1_rdr = rdr_(wiki_1_dir), wiki_0_rdr = rdr_(wiki_0_dir);
ListAdp link_list = ListAdp_.new_();
while (img_link_rdr.Read_next()) {
Xofo_lnki link = new Xofo_lnki().Load_link_rdr(fld_rdr, img_link_rdr, lnki_parser);
byte[] cur_name = link.Name();
if (cur_name == null) continue;
if (!Bry_.Eq(cur_name, prv_name)) {
if (xfer != null) Write(xfer, link_list);
xfer = new Xofo_file();
prv_name = cur_name;
boolean exists = wiki_0_rdr.Match(cur_name);
if (exists)
xfer.Load_by_merge_rdr(fld_rdr, wiki_0_rdr).Repo_id_(0); // NOTE: 0 is an ordinal (denotes 1st rdr)
else {
exists = wiki_1_rdr.Match(cur_name);
if (exists)
xfer.Load_by_merge_rdr(fld_rdr, wiki_1_rdr).Repo_id_(1); // NOTE: 0 is an ordinal (denotes 2nd rdr)
else
xfer.Name_(cur_name);
}
total_count++;
}
link_list.Add(link);
total_size += xfer.Orig_size();
}
Write(xfer, link_list);
Io_mgr._.AppendFilByt(dump_url_gen.Nxt_url(), dump_bfr.Bfr(), dump_bfr.Len()); dump_bfr.Clear();
}
public int Total_count() {return total_count;} private int total_count;
public long Total_size() {return total_size;} long total_size;
Sql_file_parser parser = new Sql_file_parser(); Io_url temp_dir;
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_src_fil_)) {src_fil = m.ReadIoUrl("url");}
else return super.Invk(ctx, ikey, k, m);
return this;
} private static final String Invk_src_fil_ = "src_fil_";
Bry_bfr dump_bfr = Bry_bfr.new_(); int dump_fil_len = 100 * Io_mgr.Len_kb;
}
class Xobc_lnki_wkr_file extends Xob_itm_dump_base implements Xop_lnki_logger {
public static final String KEY = "img.dump_link";
public Xobc_lnki_wkr_file(Xob_bldr bldr, Xow_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = 1 * Io_mgr.Len_mb;}
public Io_url_gen Make_url_gen() {return make_url_gen;} public Xobc_lnki_wkr_file Make_url_gen_(Io_url_gen v) {make_url_gen = v; return this;} Io_url_gen make_url_gen;
public void Wkr_bgn(Xob_bldr bldr) {
Io_url temp_dir2 = wiki.Fsys_mgr().Tmp_dir().GenSubDir(KEY);
this.Init_dump(KEY, temp_dir2.GenSubDir("make"));
this.lnki_wtr = Gfo_fld_wtr.xowa_().Bfr_(dump_bfr);
Io_mgr._.DeleteDirDeep_ary(temp_dir);
dump_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("dump"));
sort_dir = temp_dir.GenSubDir("sort");
make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
} Io_url sort_dir; Gfo_fld_wtr lnki_wtr;
public void Wkr_exec(Xop_ctx ctx, byte[] src, Xop_lnki_tkn lnki, byte lnki_src_tid) {
byte[] ttl = lnki.Ttl().Page_db();
ttl = encoder.Decode_lax(lnki.Ttl().Page_db());
Xofo_lnki.Write(lnki_wtr, ttl, lnki);
if (dump_bfr.Len() > dump_fil_len) Flush(); // NOTE: doing this after; in order to do before, need to precalc len of width/height which is simply not worth it
} Url_encoder encoder = Url_encoder.new_file_();
public void Wkr_end() {
Flush();
Io_sort_cmd_img img_cmd = new Io_sort_cmd_img().Make_url_gen_(make_url_gen).Make_fil_max_(make_fil_len);
Xobdc_merger.Basic(bldr.Usr_dlg(), dump_url_gen, sort_dir, sort_mem_len, Io_line_rdr_key_gen_all_wo_nl._, img_cmd);
}
public void Flush() {
Io_mgr._.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
dump_bfr.Reset_if_gt(Io_mgr.Len_mb);
}
}

68
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_img_prep_xfer_tst.java Normal file
View File

@@ -0,0 +1,68 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import org.junit.*;
public class Xobc_img_prep_xfer_tst {
Xobc_img_xfer_bldr_fxt fxt = new Xobc_img_xfer_bldr_fxt();
@Test public void Img_download_bldr() {
fxt.Img_link_
( "A.png|0,100,200,upright=0.8"
, "A.png|1,101,201,upright=1"
, "B.png|1,102,202,upright=1"
, ""
)
.Name_1_
( "A.png|||10000|100|200|8|"
)
.Name_0_
( "B.png|C.png||10001|101|201|4|"
)
.Expd_
( "A.png|||10000|100|200|8|1||0,100,200,upright=0.8;1,101,201,upright=1"
, "B.png|C.png||10001|101|201|4|0||1,102,202,upright=1"
)
.Run()
;
}
}
class Xobc_img_xfer_bldr_fxt {
public Xobc_img_xfer_bldr_fxt Img_link_(String... v) {img_link = String_.Concat_lines_nl_skip_last(v); return this;} private String img_link;
public Xobc_img_xfer_bldr_fxt Name_1_(String... v) {name_wiki_1 = String_.Concat_lines_nl_skip_last(v); return this;} private String name_wiki_1;
public Xobc_img_xfer_bldr_fxt Name_0_(String... v) {name_wiki_0 = String_.Concat_lines_nl_skip_last(v); return this;} private String name_wiki_0;
public Xobc_img_xfer_bldr_fxt Expd_(String... v) {expd = String_.Concat_lines_nl_skip_last(v); return this;} private String expd;
public void Run() {
Io_mgr._.InitEngine_mem();
Xoa_app app = Xoa_app_fxt.app_();
Xow_wiki wiki = Xoa_app_fxt.wiki_tst_(app);
Xob_bldr bldr = Xoa_app_fxt.bldr_(app);
Xobc_img_prep_xfer cmd = new Xobc_img_prep_xfer(bldr, wiki);
cmd.Commons_url_(Io_url_.mem_dir_("mem/commons"));
cmd.Cmd_bgn(bldr);
Io_mgr._.SaveFilStr(cmd.Link_dir().GenSubFil("dump.csv"), img_link);
Io_mgr._.SaveFilStr(cmd.Wiki_0_dir().GenSubFil("dump.csv"), name_wiki_0);
Io_mgr._.SaveFilStr(cmd.Wiki_1_dir().GenSubFil("dump.csv"), name_wiki_1);
cmd.Cmd_run();
Io_url[] dump_urls = cmd.Dump_url_gen().Prv_urls();
Io_url dump_url = dump_urls[0];
String actl = Io_mgr._.LoadFilStr(dump_url);
Tfds.Eq_str_lines(expd, actl);
}
}

156
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_img_run_xfer.java Normal file
View File

@@ -0,0 +1,156 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.gfui.*; import gplx.xowa.bldrs.*; import gplx.xowa.files.*;
public class Xobc_img_run_xfer extends Xob_itm_basic_base implements Xob_cmd, GfoInvkAble {
public Xobc_img_run_xfer(Xob_bldr bldr, Xow_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return KEY;} public static final String KEY = "img.run_xfer";
public Io_url Rdr_dir() {return rdr_dir;} Io_url rdr_dir;
public void Cmd_ini(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
this.bldr = bldr; fld_wtr = Gfo_fld_wtr.xowa_(); fld_rdr = Gfo_fld_rdr.xowa_();
tmp_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir(KEY);
if (rdr_dir == null) rdr_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xobc_img_prep_xfer.KEY, "dump");
Io_url errors_dir = tmp_dir.GenSubDir("errors");
// Io_mgr._.DeleteDirDeep_ary(errors_dir);
dump_url_gen = Io_url_gen_.dir_(errors_dir);
dump_bfr = Bry_bfr.new_(dump_fil_len);
file_mgr = bldr.App().File_mgr();
wiki.App().File_mgr().Download_mgr().Enabled_(true);
wiki.File_mgr().Cfg_download().Enabled_(true);
// BLOCK: init ext_stats
int ext_stats_len = Xof_ext_.Id_ogv + 1;
ext_stats = new Xof_file_stat[ext_stats_len];
for (int i = 0; i < ext_stats_len; i++)
ext_stats[i] = new Xof_file_stat(i);
} Io_url tmp_dir; Bry_bfr url_bfr = Bry_bfr.new_(); Xof_file_stat[] ext_stats; int fail_count; Xof_file_mgr file_mgr;
byte[] Redirect(Xofo_file row) {return row.Redirect_exists() ? row.Name() : Bry_.Empty;} // handle redirect rows which are reversed (Redirect|Name)
public Xof_xfer_itm Xfer_itm() {return xfer_itm;} private Xof_xfer_itm xfer_itm = new Xof_xfer_itm();
public void Cmd_run() {
Io_line_rdr rdr = new Io_line_rdr(bldr.Usr_dlg(), Io_mgr._.QueryDir_fils(rdr_dir));
ListAdp link_list = ListAdp_.new_();
int count = 0; fail_count = 0;
Xof_xfer_mgr xfer = new Xof_xfer_mgr(file_mgr).Check_file_exists_before_xfer_n_(); // turn off check for mass download
int url_idx = rdr.Url_idx();
Io_url dump_url = dump_url_gen.Nxt_url(); // gen one url
time_bgn = Env_.TickCount();
xfer_itm.Clear();
Xofo_file row = new Xofo_file();
Xow_repo_mgr wiki_repo_mgr = wiki.File_mgr().Repo_mgr();
Xof_repo_itm src_repo, trg_repo;
Xofo_lnki_parser lnki_parser = new Xofo_lnki_parser();
Xof_meta_mgr trg_meta_mgr = wiki.File_mgr().Meta_mgr();
fld_wtr.Bfr_(dump_bfr);
while (rdr.Read_next()) {
try {
if (url_idx != rdr.Url_idx()) {
this.Flush(rdr, dump_url, url_idx, time_bgn, count);
url_idx = rdr.Url_idx();
count = 0;
fail_count = 0;
time_bgn = Env_.TickCount();
}
row.Load_by_xfer_rdr(fld_rdr, lnki_parser, rdr, link_list);
int repo_id = row.Repo_id();
if (repo_id == -1) repo_id = 0; // HACK: xfer_itm not found in either File ns; assume 0 (which should be commons)
Xof_repo_pair repo_pair = wiki_repo_mgr.Repos_get_at(repo_id);
src_repo = repo_pair.Src(); trg_repo = repo_pair.Trg();
xfer_itm.Atrs_by_orig(row.Orig_w(), row.Orig_h(), row.Orig_size());
xfer_itm.Atrs_by_ttl(row.Name(), row.Redirect());
xfer_itm.Trg_repo_idx_(repo_id);
xfer_itm.Atrs_by_meta(trg_meta_mgr.Get_itm_or_new(xfer_itm.Lnki_ttl(), xfer_itm.Lnki_md5()), trg_repo, wiki.Html_mgr().Img_thumb_width());
xfer_itm.Meta_itm().Load_orig_(xfer_itm.Orig_w(), xfer_itm.Orig_h());
if (row.Redirect_exists()) { // write redirect row; note that rdr has name/redirect reversed
byte[] redirect_ttl = row.Name();
Xof_meta_itm redirect_itm = trg_meta_mgr.Get_itm_or_new(redirect_ttl, Xof_xfer_itm_.Md5_(redirect_ttl));
redirect_itm.Ptr_ttl_(row.Redirect()).Vrtl_repo_(row.Repo_id());
}
Xofo_lnki[] lnks = row.Links(); int lnks_len = lnks.length;
ext_stats[xfer_itm.Lnki_ext().Id()].Update_file(row, lnks_len);
boolean thumb_pass = false, skip = false; //byte orig_pass = Bool_.__byte;
for (int i = 0; i < lnks_len; i++) {
Xofo_lnki lnk = lnks[i];
xfer_itm.Atrs_by_lnki(lnk.Lnki_type(), lnk.Lnki_w(), lnk.Lnki_h(), lnk.Lnki_upright(), lnk.Lnki_thumbtime(), Xof_doc_page.Null);
xfer_itm.Atrs_calc_for_html();
xfer.Atrs_by_itm(xfer_itm, src_repo, trg_repo);
if (!xfer.Download_allowed_by_ext() || xfer_itm.Meta_itm().Orig_exists() == Xof_meta_itm.Exists_n) {skip = true; break;}
thumb_pass = xfer.Make_file(wiki);
if (!thumb_pass) break;
}
if (skip) continue;
if (thumb_pass)
bldr.Usr_dlg().Prog_many(GRP_KEY, "xfer_passed", "pass|~{0}|~{1}", Int_.XtoStr_PadBgn_space(count++, 7), String_.new_utf8_(row.Name()));
else {
bldr.Usr_dlg().Note_many(GRP_KEY, "xfer_failed", "fail|~{0}|~{1}|~{2}|~{3}", Int_.XtoStr_PadBgn_space(count++, 7), xfer.Rslt().Err_msg(), String_.new_utf8_(xfer_itm.Lnki_md5(), 0, 2) , String_.new_utf8_(row.Name()));
if (dump_bfr.Len() > dump_fil_len) Io_mgr._.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
// if (orig_pass == Bool_.Y_byte && thumb_pass) mode = Xof_meta_itm.Mode_both_exists;
// else if (orig_pass == Bool_.Y_byte) mode = Xof_meta_itm.Mode_orig_exists;
// else if (thumb_pass) mode = Xof_meta_itm.Mode_thumb_exists;
xfer.Meta_itm().Orig_exists_(Xof_meta_itm.Exists_unknown);
xfer.Meta_itm().Vrtl_repo_(Xof_meta_itm.Repo_missing);
// xfer.Regy_itm().Mode_set(xfer_itm.File_type().Id_is_video() ? Xof_meta_itm.Mode_thumb_missing : Xof_meta_itm.Mode_orig_missing);
row.Status_msg_(Bry_.new_utf8_(xfer.Rslt().Err_msg()));
row.Write_bldr(fld_wtr);
fail_count++;
}
}
catch (Exception exc) {
bldr.Usr_dlg().Note_many(GRP_KEY, "xfer_error", "fail|~{0}|~{1}|~{2}", Int_.XtoStr_PadBgn_space(count++, 7), String_.new_utf8_(xfer_itm.Lnki_md5(), 0, 2) + "|" + String_.new_utf8_(row.Name()), Err_.Message_lang(exc));
if (dump_bfr.Len() > dump_fil_len) Io_mgr._.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
row.Write_bldr(fld_wtr);
fail_count++;
}
}
this.Flush(rdr, dump_url, url_idx, time_bgn, count);
} long files_sum = 0, elapsed_sum = 0; long time_bgn;
private void Flush(Io_line_rdr rdr, Io_url dump_url, int url_idx, long time_bgn, int fil_count) {
wiki.File_mgr().Meta_mgr().Save();
// int dirty_meta_mgrs_len = dirty_meta_mgrs.Count();
// for (int i = 0; i < dirty_meta_mgrs_len; i++) {
// Xof_meta_mgr dirty_meta_mgr = (Xof_meta_mgr)dirty_meta_mgrs.FetchAt(i);
// dirty_meta_mgr.Save();
// }
Io_mgr._.AppendFilBfr(dump_url, dump_bfr);
Io_url cur = rdr.Urls()[url_idx];
int seconds = Env_.TickCount_elapsed_in_sec(time_bgn);
double rate = Math_.Div_safe_as_double(fil_count * 3600, seconds);
files_sum += fil_count;
elapsed_sum += seconds;
String txt = bldr.Usr_dlg().Note_many(GRP_KEY, "flush", "name=~{0} files=~{1} fails=~{2} seconds=~{3} files/hour=~{4} files_total=~{5} seconds_total=~{6}", cur.NameOnly(), fil_count, fail_count, seconds, rate, files_sum, elapsed_sum);
Io_mgr._.AppendFilStr(tmp_dir.GenSubFil("progress.txt"), txt + "\n");
}
public void Cmd_end() {
Bry_bfr bfr = Bry_bfr.new_();
int ext_stats_len = ext_stats.length;
for (int i = 0; i < ext_stats_len; i++)
ext_stats[i].Bld(bfr);
bldr.Usr_dlg().Note_many(GRP_KEY, "end", "~{0}", bfr.XtoStrAndClear());
}
public void Cmd_print() {}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_rdr_dir_)) {rdr_dir = Io_url_.new_dir_(m.ReadStr("v"));}
else return super.Invk(ctx, ikey, k, m);
return this;
} public static final String Invk_rdr_dir_ = "rdr_dir_";
Bry_bfr dump_bfr = Bry_bfr.new_(); Gfo_fld_wtr fld_wtr; Gfo_fld_rdr fld_rdr; Bry_bfr tmp = Bry_bfr.new_();
Io_url_gen dump_url_gen; int dump_fil_len = 4 * Io_mgr.Len_mb;
static final String GRP_KEY = "xowa.bldr.img_run_xfer";
}

125
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_img_xfer_base_fxt.java Normal file
View File

@@ -0,0 +1,125 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.xowa.files.cnvs.*;
class Xobc_img_xfer_base_fxt {
public Xow_wiki En_wiki() {return en_wiki;} private Xow_wiki en_wiki;
public Xow_wiki Commons() {return commons;} private Xow_wiki commons;
Xofw_wiki_wkr_mock mock_wkr = new Xofw_wiki_wkr_mock();
public void Init_page_create(Xow_wiki wiki, String ttl) {Init_page_create(wiki, ttl, "");}
public void Init_page_create(Xow_wiki wiki, String ttl, String txt) {
Xoa_ttl page_ttl = Xoa_ttl.parse_(wiki, Bry_.new_utf8_(ttl));
byte[] page_raw = Bry_.new_utf8_(txt);
wiki.Db_mgr().Save_mgr().Data_create(page_ttl, page_raw);
}
public void Ini(boolean src_local) {
Io_mgr._.InitEngine_mem();
Xoa_app app = Xoa_app_fxt.app_();
en_wiki = Xoa_app_fxt.wiki_tst_(app);
commons = Xoa_app_fxt.wiki_(app, Xow_wiki_.Domain_commons_str);
mock_wkr.Clear_commons(); // assume all files are in repo 0
en_wiki.File_mgr().Repo_mgr().Page_finder_(mock_wkr);
en_wiki.Db_mgr().Load_mgr().Clear();
commons.Db_mgr().Load_mgr().Clear();
app.Wiki_mgr().Add(commons);
Xof_file_mgr file_mgr = app.File_mgr();
file_mgr.Img_mgr().Wkr_resize_img_(Xof_img_wkr_resize_img_mok._);
file_mgr.Img_mgr().Wkr_query_img_size_(new Xof_img_wkr_query_img_size_test());
byte[] src_commons = Bry_.new_ascii_("src_commons");
byte[] src_en_wiki = Bry_.new_ascii_("src_en_wiki");
byte[] trg_commons = Bry_.new_ascii_("trg_commons");
byte[] trg_en_wiki = Bry_.new_ascii_("trg_en_wiki");
App_repo_add(file_mgr, src_commons, "mem/src/commons.wikimedia.org/", Xow_wiki_.Domain_commons_str, false);
App_repo_add(file_mgr, src_en_wiki, "mem/src/en.wikipedia.org/", Xow_wiki_.Domain_enwiki_str, false);
App_repo_add(file_mgr, trg_commons, "mem/trg/commons.wikimedia.org/", Xow_wiki_.Domain_commons_str, true);
App_repo_add(file_mgr, trg_en_wiki, "mem/trg/en.wikipedia.org/", Xow_wiki_.Domain_enwiki_str, true);
Xow_repo_mgr wiki_repo_mgr = en_wiki.File_mgr().Repo_mgr();
Xof_repo_pair pair = null;
pair = wiki_repo_mgr.Add_repo(src_commons, trg_commons);
pair.Src().Fsys_is_wnt_(true).Wmf_fsys_(!src_local);
pair.Trg().Fsys_is_wnt_(true);
pair = wiki_repo_mgr.Add_repo(src_en_wiki, trg_en_wiki);
pair.Src().Fsys_is_wnt_(true).Wmf_fsys_(!src_local);
pair.Trg().Fsys_is_wnt_(true);
src_fils = trg_fils = null;
this.Ini_hook(app, en_wiki);
html_src = null;
}
@gplx.Virtual public void Ini_hook(Xoa_app app, Xow_wiki wiki) {}
private void App_repo_add(Xof_file_mgr file_mgr, byte[] key, String root, String wiki, boolean trg) {
Xof_repo_itm repo = file_mgr.Repo_mgr().Set(String_.new_utf8_(key), root, wiki).Ext_rules_(Xoft_rule_grp.Grp_app_default);
if (trg) {
byte[][] ary = repo.Mode_names();
ary[0] = Bry_.new_ascii_("raw");
ary[1] = Bry_.new_ascii_("fit");
}
}
public Xobc_img_xfer_base_fxt Src_base(Io_fil... v) {src_fils = v; return this;} Io_fil[] src_fils;
public Xobc_img_xfer_base_fxt Trg_base(Io_fil... v) {trg_fils = v; return this;} Io_fil[] trg_fils;
public String Html_view_src() {return html_src;} protected Xobc_img_xfer_base_fxt Html_src_base_(String v) {html_src = v; return this;} private String html_src;
public void ini_src_fils() {
if (src_fils != null) {
for (int i = 0; i < src_fils.length; i++) {
Io_fil src_fil = src_fils[i];
Io_mgr._.SaveFilStr(src_fil.Url(), src_fil.Data());
}
}
}
public void tst_trg_fils() {
for (int i = 0; i < trg_fils.length; i++) {
Io_fil trg_fil = trg_fils[i];
String data = Io_mgr._.LoadFilStr(trg_fil.Url());
Tfds.Eq_str_lines(trg_fil.Data(), data, trg_fil.Url().Raw());
}
}
public void save_(Io_fil v) {Io_mgr._.SaveFilStr(v.Url(), v.Data());}
public Io_fil reg_(String url, String... v) {return new Io_fil(Io_url_.mem_fil_(url), String_.Concat_lines_nl(v));}
public Io_fil img_(String url_str, int w, int h) {return file_(url_str, file_img(w, h));}
public Io_fil svg_(String url_str, int w, int h) {return file_(url_str, file_svg(w, h));}
public Io_fil ogg_(String url_str) {return file_(url_str, "");}
Io_fil file_(String url_str, String data) {return new Io_fil(Io_url_.mem_fil_(url_str), data);}
String file_img(int w, int h) {return String_.Format("{0},{1}", w, h);}
String file_svg(int w, int h) {return String_.Format("<svg width=\"{0}\" height=\"{1}\" />", w, h);}
}
class Xobc_img_run_xfer_fxt extends Xobc_img_xfer_base_fxt {
public Xobc_img_run_xfer_fxt Rdr(String... v) {rdr = String_.Concat_lines_nl(v); return this;} private String rdr;
public Xobc_img_run_xfer_fxt Src(Io_fil... v) {return (Xobc_img_run_xfer_fxt)this.Src_base(v);}
public Xobc_img_run_xfer_fxt Trg(Io_fil... v) {return (Xobc_img_run_xfer_fxt)this.Trg_base(v);}
@Override public void Ini_hook(Xoa_app app, Xow_wiki wiki) {
bldr = Xoa_app_fxt.bldr_(app);
wkr = new Xobc_img_run_xfer(bldr, wiki);
GfoInvkAble_.InvkCmd_val(wkr, Xobc_img_run_xfer.Invk_rdr_dir_, "mem/rdr/");
} private Xob_bldr bldr; Xobc_img_run_xfer wkr;
public void tst() {
ini_src_fils();
Io_mgr._.SaveFilStr(wkr.Rdr_dir().GenSubFil("0000.csv"), rdr);
wkr.Cmd_bgn(bldr);
wkr.Cmd_run();
wkr.Cmd_end();
tst_trg_fils();
}
public void tst_html_src(String expd) {
boolean found = wkr.Xfer_itm().Atrs_calc_for_html();
Tfds.Eq(expd, String_.new_utf8_(wkr.Xfer_itm().Html_view_src()));
Tfds.Eq(true, found, "img not found");
}
}

40
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_lnki_wkr_ctg.java Normal file
View File

@@ -0,0 +1,40 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.xowa.parsers.lnkis.redlinks.*;
class Xobc_lnki_wkr_ctg extends Xob_itm_dump_base implements Xop_lnki_logger {
public static final String KEY = "dump.ctg";
public Xobc_lnki_wkr_ctg(Xob_bldr bldr, Xow_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = 1 * Io_mgr.Len_mb;}
public Io_url_gen Make_url_gen() {return make_url_gen;} Io_url_gen make_url_gen;
public void Wkr_bgn(Xob_bldr bldr) {
this.Init_dump(KEY, wiki.Fsys_mgr().Tmp_dir().GenSubDir(KEY).GenSubDir("make"));
this.lnki_wtr = Gfo_fld_wtr.xowa_().Bfr_(dump_bfr);
Io_mgr._.DeleteDirDeep_ary(temp_dir);
dump_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("dump"));
sort_dir = temp_dir.GenSubDir("sort");
make_url_gen = Io_url_gen_.dir_(temp_dir.GenSubDir("make"));
fld_wtr.Bfr_(dump_bfr);
} Io_url sort_dir; Gfo_fld_wtr lnki_wtr; private Gfo_fld_wtr fld_wtr = Gfo_fld_wtr.xowa_();
public void Wkr_exec(Xop_ctx ctx, byte[] src, Xop_lnki_tkn lnki, byte lnki_src_tid) {
gplx.xowa.bldrs.imports.ctgs.Xob_ctg_v1_base.Process_ctg_row(fld_wtr, dump_fil_len, dump_url_gen, ctx.Cur_page().Revision_data().Id(), src, src.length, lnki.Src_bgn(), lnki.Src_end());
}
public void Wkr_end() {
this.Term_dump(new Xob_make_cmd_site(bldr.Usr_dlg(), make_dir, make_fil_len));
if (delete_temp) Io_mgr._.DeleteDirDeep(temp_dir);
}
}

90
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_math_dir_list_tst.java Normal file
View File

@@ -0,0 +1,90 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import org.junit.*;
public class Xobc_math_dir_list_tst {
Xobc_math_dir_list_fxt fxt = new Xobc_math_dir_list_fxt();
@Before public void init() {
fxt.Reset().Root_dir_(Io_url_.mem_dir_("mem/xowa/file/math/")).Temp_dir_(Io_url_.mem_dir_("mem/xowa/temp/"));
}
@Test public void Basic() {
fxt.Create_fils("0/0/0", "000a.png", "000b.png");
fxt.Create_fils("1/0/0", "100a.png", "100b.png");
fxt.Create_fils("1/1/0", "110b.png", "110a.png"); // "unsorted"
fxt.tst("0.txt", "000a", "000b");
fxt.tst("1.txt", "100a", "100b", "110a", "110b"); // "sorted"; "combined"
}
}
class Xobc_math_dir_list_fxt {
public Xobc_math_dir_list_fxt Reset() {Io_mgr._.InitEngine_mem(); return this;}
public Xobc_math_dir_list_fxt Root_dir_(Io_url v) {root_dir = v; return this;} Io_url root_dir;
public Xobc_math_dir_list_fxt Temp_dir_(Io_url v) {temp_dir = v; return this;} Io_url temp_dir;
public Xobc_math_dir_list_fxt Create_fils(String dir_rel_path, String... names) {
Io_url dir = root_dir.GenSubDir(dir_rel_path);
int names_len = names.length;
for (int i = 0; i < names_len; i++) {
Io_url fil = dir.GenSubFil(names[i]);
Io_mgr._.SaveFilStr(fil, "");
}
return this;
}
public Xobc_math_dir_list_fxt tst(String fil_name, String... md5s) {
Xobc_math_dir_manifest wkr = new Xobc_math_dir_manifest();
wkr.Bld(root_dir, temp_dir);
String fil_txt = Io_mgr._.LoadFilStr(temp_dir.GenSubFil(fil_name));
String[] actl_names = String_.Split(fil_txt, '\n');
Tfds.Eq_ary_str(md5s, actl_names);
return this;
}
}
class Xobc_math_dir_manifest {
public void Bld(Io_url math_dir, Io_url gen_dir) {
this.gen_dir = gen_dir;
Bry_bfr dir_path = Bry_bfr.new_();
Bld_dir(math_dir, dir_path, 0);
}
private void Bld_dir(Io_url dir, Bry_bfr dir_path, int depth) {
Io_url[] subs = Io_mgr._.QueryDir_args(dir).DirInclude_().ExecAsUrlAry();
Array_.Sort(subs); // NOTE: file system should have sorted these urls already, but sort again, just in case; sorting is "marginal" cost since it should be magnitudes faster than reading from the actual file system
int subs_len = subs.length;
for (int i = 0; i < subs_len; i++) {
Io_url sub = subs[i];
if (sub.Type_dir()) {
byte[] name_bry = Bry_.new_ascii_(sub.NameOnly());
if (name_bry.length != 1) throw Err_.new_fmt_("invalid math dir name; should be 1 char wide; {0}", dir.Raw());
dir_path.Add_byte(name_bry[0]);
Bld_dir(sub, dir_path, depth + 1);
dir_path.Del_by_1();
}
else
Write(sub);
}
if (depth == flush_depth) Flush(dir_path);
}
private void Write(Io_url fil) {
byte[] md5 = Bry_.new_ascii_(fil.NameOnly());
bfr.Add(md5).Add_byte_nl();
}
private void Flush(Bry_bfr dir_path) {
Io_url fil = gen_dir.GenSubFil(dir_path.XtoStr() + ".txt");
Io_mgr._.SaveFilBfr(fil, bfr);
}
Bry_bfr bfr = Bry_bfr.new_();
Io_url gen_dir = null;
int flush_depth = 1;
}

96
400_xowa/src_220_bldr_file/gplx/xowa/Xobc_math_run.java Normal file
View File

@@ -0,0 +1,96 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
import gplx.ios.*; import gplx.xowa.bldrs.*; import gplx.xowa.xtns.math.*;
public class Xobc_math_run extends Xob_itm_basic_base implements Xob_cmd, GfoInvkAble {
public Xobc_math_run(Xob_bldr bldr, Xow_wiki wiki) {this.Cmd_ctor(bldr, wiki);}
public String Cmd_key() {return KEY;} public static final String KEY = "math.run";
Xof_file_mgr file_mgr;
public Io_url Rdr_dir() {return rdr_dir;} Io_url rdr_dir;
public void Cmd_ini(Xob_bldr bldr) {}
public void Cmd_bgn(Xob_bldr bldr) {
this.bldr = bldr; fld_rdr = Gfo_fld_rdr.xowa_();
lnk_parser = (Gfo_fld_rdr)Gfo_fld_rdr.xowa_().Row_dlm_(Byte_ascii.Pipe).Fld_dlm_(Byte_ascii.Comma);
tmp_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir(KEY);
dump_bfr = Bry_bfr.new_(dump_fil_len); fld_wtr = Gfo_fld_wtr.xowa_().Bfr_(dump_bfr);
file_mgr = bldr.App().File_mgr(); math_mgr = file_mgr.Math_mgr(); math_mgr.Init(bldr.App());
if (rdr_dir == null) rdr_dir = wiki.Fsys_mgr().Tmp_dir().GenSubDir_nest(Xobc_xnde_math_dump.KEY, "make");
Io_url errors_dir = tmp_dir.GenSubDir("errors");
dump_url_gen = Io_url_gen_.dir_(errors_dir);
Io_mgr._.DeleteDirDeep_ary(errors_dir);
} Io_url tmp_dir; Bry_bfr url_bfr = Bry_bfr.new_(); private Xof_math_mgr math_mgr;
public void Cmd_run() {
Io_line_rdr rdr = new Io_line_rdr(bldr.Usr_dlg(), Io_mgr._.QueryDir_fils(rdr_dir));
int count = 0;
int url_idx = rdr.Url_idx();
Io_url dump_url = dump_url_gen.Nxt_url(); // gen one url
long time_bgn = Env_.TickCount();
byte[] math = Bry_.Empty;
Xof_math_itm math_itm = new Xof_math_itm();
while (rdr.Read_next()) {
try {
if (url_idx != rdr.Url_idx()) {
this.Flush(rdr, dump_url, url_idx, time_bgn, count);
url_idx = rdr.Url_idx();
count = 0;
time_bgn = Env_.TickCount();
}
fld_rdr.Ini(rdr.Bfr(), rdr.Itm_pos_bgn());
math = fld_rdr.Read_bry_escape();
math_mgr.Make_itm(math_itm, wiki.Domain_str(), math);
boolean pass = math_mgr.MakePng(math_itm.Math(), math_itm.Hash(), math_itm.Png_url(), ""); // NOTE: no progress needed for batch
if (pass)
bldr.Usr_dlg().Note_many(GRP_KEY, "run.pass", "pass|~{0}|~{1}", Int_.XtoStr_PadBgn_space(count++, 7), String_.new_utf8_(math_itm.Math()));
else {
bldr.Usr_dlg().Note_many(GRP_KEY, "run.fail", "fail|~{0}|~{1}", Int_.XtoStr_PadBgn_space(count++, 7), String_.new_utf8_(math_itm.Math()));
Write_err(math);
}
math = Bry_.Empty;
}
catch (Exception exc) {
bldr.Usr_dlg().Warn_many(GRP_KEY, "fail", "error: ~{0}", Err_.Message_lang(exc));
Write_err(math);
}
}
this.Flush(rdr, dump_url, url_idx, time_bgn, count);
} long files_sum = 0, elapsed_sum = 0;
private void Write_err(byte[] math) {
if (dump_bfr.Len() > dump_fil_len) Io_mgr._.AppendFilBfr(dump_url_gen.Nxt_url(), dump_bfr);
fld_wtr.Write_bry_escape_row(math);
}
private void Flush(Io_line_rdr rdr, Io_url dump_url, int url_idx, long time_bgn, int fil_count) {
Io_mgr._.AppendFilBfr(dump_url, dump_bfr);
Io_url cur = rdr.Urls()[url_idx];
int seconds = Env_.TickCount_elapsed_in_sec(time_bgn);
double rate = Math_.Div_safe_as_double(fil_count * 3600, seconds);
files_sum += fil_count;
elapsed_sum += seconds;
String txt = bldr.Usr_dlg().Note_many(GRP_KEY, "flush", "name=~{0} files=~{1} seconds=~{2} files/hour=~{3} files_total=~{4} seconds_total=~{5}", cur.NameOnly(), fil_count, seconds, rate, files_sum, elapsed_sum);
Io_mgr._.AppendFilStr(tmp_dir.GenSubFil("progress.txt"), txt + "\n");
}
public void Cmd_end() {}
public void Cmd_print() {}
@Override public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
if (ctx.Match(k, Invk_rdr_dir_)) {rdr_dir = Io_url_.new_dir_(m.ReadStr("v"));}
else return super.Invk(ctx, ikey, k, m);
return this;
} private static final String Invk_rdr_dir_ = "rdr_dir_";
Bry_bfr dump_bfr = Bry_bfr.new_(); Gfo_fld_wtr fld_wtr; Gfo_fld_rdr fld_rdr, lnk_parser; Bry_bfr tmp = Bry_bfr.new_();
Io_url_gen dump_url_gen; int dump_fil_len = 100 * Io_mgr.Len_kb;
static final String GRP_KEY = "xowa.bldr.math";
}

96
400_xowa/src_220_bldr_file/gplx/xowa/Xof_file_stat.java Normal file
View File

@@ -0,0 +1,96 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa; import gplx.*;
class Xof_file_stat {
public Xof_file_stat(int tid) {file_type = Xof_ext_.new_(tid, Xof_ext_.Bry__ary[tid]);} private Xof_ext file_type;
public int Count_file() {return count_file;} private int count_file;
public int Count_lnki() {return count_lnki;} private int count_lnki;
public long Size_orig() {return size_orig;} long size_orig;
public long Size_300() {return size_300;} long size_300;
public long Size_250() {return size_250;} long size_250;
public long Size_220() {return size_220;} long size_220;
public long Size_200() {return size_200;} long size_200;
public long Size_180() {return size_180;} long size_180;
public long Size_150() {return size_150;} long size_150;
public long Size_120() {return size_120;} long size_120;
public void Bld(Bry_bfr bfr) {
bfr.Add(Xof_ext_.Bry__ary[file_type.Id()]).Add_byte(Byte_ascii.Pipe);
bfr.Add_int_variable(count_file).Add_byte(Byte_ascii.Pipe);
bfr.Add_long_variable(size_orig).Add_byte(Byte_ascii.Pipe);
bfr.Add_long_variable(size_300).Add_byte(Byte_ascii.Pipe);
bfr.Add_long_variable(size_250).Add_byte(Byte_ascii.Pipe);
bfr.Add_long_variable(size_220).Add_byte(Byte_ascii.Pipe);
bfr.Add_long_variable(size_200).Add_byte(Byte_ascii.Pipe);
bfr.Add_long_variable(size_180).Add_byte(Byte_ascii.Pipe);
bfr.Add_long_variable(size_150).Add_byte(Byte_ascii.Pipe);
bfr.Add_long_variable(size_120).Add_byte(Byte_ascii.Pipe);
bfr.Add_byte_nl();
}
public void Update_file(Xofo_file file, int lnki_len) {
count_file++;
size_orig += file.Orig_size();
count_lnki += lnki_len;
for (int i = 0; i < lnki_len; i++) {
size_300 += Calc_thumb_size(file, 300);
size_250 += Calc_thumb_size(file, 250);
size_220 += Calc_thumb_size(file, 220);
size_200 += Calc_thumb_size(file, 200);
size_180 += Calc_thumb_size(file, 180);
size_150 += Calc_thumb_size(file, 150);
size_120 += Calc_thumb_size(file, 120);
if (!file_type.Id_is_thumbable_img()) break;
}
}
int Calc_thumb_size(Xofo_file file, int thumb_w) {
int file_size = file.Orig_size(), orig_w = file.Orig_w(), orig_h = file.Orig_h();
switch (file_type.Id()) {
case Xof_ext_.Id_png:
case Xof_ext_.Id_jpg:
case Xof_ext_.Id_jpeg:
case Xof_ext_.Id_gif:
case Xof_ext_.Id_tif:
case Xof_ext_.Id_tiff:
case Xof_ext_.Id_bmp:
case Xof_ext_.Id_xcf:
{
if (file_size == 0 || orig_w == 0) return 0;
int bits = file.Bits();
if (bits == 0) bits = 8;
double compression = (double)file_size / (double)((double)orig_w * (double)orig_h * (double)bits);
int thumb_h = (thumb_w * orig_h) / orig_w;
int rv = (int)((thumb_w * thumb_h * file.Bits()) * compression);
return rv;
}
case Xof_ext_.Id_svg: {
if (file.Orig_w() == 0) return 0;
int thumb_h = (thumb_w * orig_h) / orig_w;
int rv = (int)((thumb_w * thumb_h * 8) * .03);
return rv;
}
case Xof_ext_.Id_djvu:
case Xof_ext_.Id_pdf:
case Xof_ext_.Id_mid:
case Xof_ext_.Id_ogg:
case Xof_ext_.Id_oga:
case Xof_ext_.Id_ogv:
case Xof_ext_.Id_flac:
default:
return file_size;
}
}
}