Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

'v3.8.2.1'

Browse Source
This commit is contained in:
gnosygnu
2016-08-07 21:36:50 -04:00
parent b0fdf78a41
commit e4a2af026b
165 changed files with 2534 additions and 1247 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__fsdb_db__create_data.java
View File

@@ -37,7 +37,7 @@ public class Xobldr__fsdb_db__create_data extends Xob_cmd__base implements Xob_c
public Xobldr__fsdb_db__create_data(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);
if (bldr != null) {
this.poll_mgr = new Xobu_poll_mgr(bldr.App());
wiki.File__fsdb_mode().Tid_v2_bld_y_();
wiki.File__fsdb_mode().Tid__v2__bld__y_();
this.src_bin_mgr = new Xof_bin_mgr(new Fsm_mnt_mgr(), wiki.File__repo_mgr(), app.File__img_mgr().Wkr_resize_img(), app.Wmf_mgr().Download_wkr().Download_xrg().Download_fmt());
}
}

2
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__fsdb_db__create_orig.java
View File

@@ -24,7 +24,7 @@ public class Xobldr__fsdb_db__create_orig extends Xob_cmd__base {
public Xobldr__fsdb_db__create_orig(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_bgn(Xob_bldr bldr) {
Xof_fsdb_mode fsdb_mode = wiki.File__fsdb_mode();
fsdb_mode.Tid_v2_bld_y_();
fsdb_mode.Tid__v2__bld__y_();
wiki.Init_assert();
Fsdb_db_mgr db_core_mgr = Fsdb_db_mgr_.new_detect(wiki, wiki.Fsys_mgr().Root_dir(), wiki.Fsys_mgr().File_dir());
this.schema_1 = db_core_mgr.File__schema_is_1();

4
400_xowa/src/gplx/xowa/addons/bldrs/files/cmds/Xobldr__lnki_temp__create.java
View File

@@ -81,13 +81,13 @@ public class Xobldr__lnki_temp__create extends Xob_dump_mgr_base implements gplx
// init fsdb
Xof_fsdb_mgr__sql trg_fsdb_mgr = new Xof_fsdb_mgr__sql();
wiki.File__fsdb_mode().Tid_v2_bld_y_();
wiki.File__fsdb_mode().Tid__v2__bld__y_();
Fsdb_db_mgr__v2 fsdb_core = Fsdb_db_mgr__v2_bldr.Get_or_make(wiki, Bool_.Y);
trg_fsdb_mgr.Init_by_wiki(wiki);
Fsm_mnt_mgr trg_mnt_mgr = trg_fsdb_mgr.Mnt_mgr();
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__orig_mgr().Wkrs_del(gplx.xowa.files.origs.Xof_orig_wkr_.Tid_wmf_api);
wiki.File__orig_mgr().Wkrs__del(gplx.xowa.files.origs.Xof_orig_wkr_.Tid_wmf_api);
trg_mnt_mgr = new Fsm_mnt_mgr(); trg_mnt_mgr.Ctor_by_load(fsdb_core);
trg_mnt_mgr.Mnts__get_insert_idx_(Fsm_mnt_mgr.Mnt_idx_main);

4
400_xowa/src/gplx/xowa/addons/bldrs/hdumps/diffs/Dumpdiff_log_tbl.java
View File

@@ -45,7 +45,9 @@ class Dumpdiff_log_tbl implements Db_tbl {
conn.Txn_end();
stmt__insert.Rls();
}
public void Rls() {}
public void Rls() {
stmt__insert = Db_stmt_.Rls(stmt__insert);
}
public static Dumpdiff_log_tbl New(Xowe_wiki wiki) {
Db_conn conn = Db_conn_bldr.Instance.Get_or_autocreate(true, wiki.Fsys_mgr().Root_dir().GenSubFil("xowa.diff.sqlite3"));

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/makes/Xomp_html_db_rdr.java
View File

@@ -35,7 +35,9 @@ class Xomp_html_db_rdr {
src_tbl.Select_as_row(rv, page_id);
}
public void Rls() {
for (Xowd_html_tbl src_tbl : src_tbls)
for (Xowd_html_tbl src_tbl : src_tbls) {
if (src_tbl == null) continue; // can be null if fsys has dirs, but db does not have any wkr_ids
src_tbl.Conn().Rls_conn();
}
}
}

4
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_mgr.java
View File

@@ -20,6 +20,7 @@ import gplx.core.threads.*; import gplx.core.threads.utils.*; import gplx.core.c
import gplx.xowa.langs.*; import gplx.xowa.langs.cases.*;
import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.wikis.caches.*;
import gplx.xowa.addons.bldrs.wmdumps.imglinks.*;
class Xomp_parse_mgr {
private Gfo_countdown_latch latch;
public Xomp_parse_mgr_cfg Cfg() {return cfg;} private final Xomp_parse_mgr_cfg cfg = new Xomp_parse_mgr_cfg();
@@ -37,6 +38,7 @@ class Xomp_parse_mgr {
Xow_page_cache page_cache = Xomp_tmpl_cache_bldr.New(wiki, cfg.Load_all_templates());
Gfo_cache_mgr commons_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
Gfo_cache_mgr ifexist_cache = new Gfo_cache_mgr().Max_size_(Int_.Max_value).Reduce_by_(Int_.Max_value);
Xof_orig_wkr__img_links orig_wkr = Xof_orig_wkr__img_links_.Load_all(wiki);
// load_wkr: init and start
// Xomp_load_wkr load_wkr = new Xomp_load_wkr(wiki, db_core.Mgr_db().Conn(), cfg.Num_pages_in_pool(), cfg.Num_wkrs());
@@ -48,7 +50,7 @@ class Xomp_parse_mgr {
Xomp_parse_wkr[] wkrs = new Xomp_parse_wkr[wkr_len];
for (int i = 0; i < wkr_len; ++i) {
Xowe_wiki wkr_wiki = Xow_wiki_utl_.Clone_wiki(wiki, wiki.Fsys_mgr().Root_dir());
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, wkr_wiki, page_pool, i, cfg, cfg.Cleanup_interval(), cfg.Progress_interval(), cfg.Log_file_lnkis());
Xomp_parse_wkr wkr = new Xomp_parse_wkr(this, wkr_wiki, orig_wkr, page_pool, i, cfg, cfg.Cleanup_interval(), cfg.Progress_interval(), cfg.Log_file_lnkis());
wkr_wiki.Cache_mgr().Page_cache_(page_cache).Commons_cache_(commons_cache).Ifexist_cache_(ifexist_cache);
// remove wmf wkr, else will try to download images during parsing
if (wkr_wiki.File__bin_mgr() != null)

10
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xomp_parse_wkr.java
View File

@@ -17,19 +17,21 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.mass_parses.parses; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.mass_parses.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.mass_parses.dbs.*;
import gplx.xowa.files.origs.*;
import gplx.xowa.htmls.core.bldrs.*;
import gplx.xowa.parsers.*;
class Xomp_parse_wkr implements Gfo_invk {
private final Xomp_parse_mgr mgr;
private final Xomp_page_pool page_pool;
private final Xof_orig_wkr orig_wkr;
private final int idx;
private final List_adp list = List_adp_.New(); private int list_idx = 0, list_len = 0;
private final Xomp_parse_mgr_cfg cfg;
private int done_count; private long done_time;
private Xomp_wkr_db wkr_db; private int cleanup_interval, commit_interval;
private boolean log_file_lnkis;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xomp_page_pool page_pool, int idx, Xomp_parse_mgr_cfg cfg, int cleanup_interval, int commit_interval, boolean log_file_lnkis) {
this.mgr = mgr; this.wiki = wiki;
public Xomp_parse_wkr(Xomp_parse_mgr mgr, Xowe_wiki wiki, Xof_orig_wkr orig_wkr, Xomp_page_pool page_pool, int idx, Xomp_parse_mgr_cfg cfg, int cleanup_interval, int commit_interval, boolean log_file_lnkis) {
this.mgr = mgr; this.wiki = wiki; this.orig_wkr = orig_wkr;
this.page_pool = page_pool;
this.idx = idx;
this.wkr_db = mgr.Db_core().Wkr_db(Bool_.Y, idx); // NOTE: must go in ctor, or else thread issues
@@ -39,7 +41,7 @@ class Xomp_parse_wkr implements Gfo_invk {
this.log_file_lnkis = log_file_lnkis;
}
public Xowe_wiki Wiki() {return wiki;} private final Xowe_wiki wiki;
public Xob_hdump_bldr Hdump_bldr() {return hdump_bldr;} private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
public Xob_hdump_bldr Hdump_bldr() {return hdump_bldr;} private final Xob_hdump_bldr hdump_bldr = new Xob_hdump_bldr();
public void Exec() {
// init
Xow_parser_mgr parser_mgr = wiki.Parser_mgr();
@@ -47,7 +49,7 @@ class Xomp_parse_wkr implements Gfo_invk {
// disable file download
wiki.File_mgr().Init_file_mgr_by_load(wiki); // must happen after fsdb.make
wiki.File__bin_mgr().Wkrs__del(gplx.xowa.files.bins.Xof_bin_wkr_.Key_http_wmf); // must happen after init_file_mgr_by_load; remove wmf wkr, else will try to download images during parsing
wiki.File__orig_mgr().Wkrs_del(gplx.xowa.files.origs.Xof_orig_wkr_.Tid_wmf_api);
wiki.File__orig_mgr().Wkrs__set(orig_wkr);
// disable categories else progress messages written (also for PERF)
wiki.Html_mgr().Page_wtr_mgr().Wkr(gplx.xowa.wikis.pages.Xopg_page_.Tid_read).Ctgs_enabled_(false);

2
400_xowa/src/gplx/xowa/addons/bldrs/mass_parses/parses/Xow_wiki_utl_.java
View File

@@ -23,6 +23,8 @@ public class Xow_wiki_utl_ {
Xol_lang_itm_.Lang_init(lang);
Xowe_wiki rv = new Xowe_wiki(wiki.Appe(), lang, gplx.xowa.wikis.nss.Xow_ns_mgr_.default_(lang.Case_mgr()), wiki.Domain_itm(), wiki_dir);
rv.Init_by_wiki();
rv.File_mgr().Repo_mgr().Clone(wiki.File_mgr().Repo_mgr());
rv.File__fsdb_mode().Tid__v2__bld__y_();
return rv;
}
}

77
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Db_bulk_cmd_base.java Normal file
View File

@@ -0,0 +1,77 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.dbs.*;
public abstract class Db_bulk_cmd_base {
public void Exec() {
int uid_max = Get_uid_max();
int uid_rng = Get_uid_rng();
int uid_bgn = -1, uid_end = uid_rng;
this.Bulk_bgn();
while (uid_bgn <= uid_max) {
Bulk_run(uid_bgn, uid_end);
uid_bgn += uid_rng;
uid_end += uid_rng;
}
this.Bulk_end();
}
protected abstract int Get_uid_max();
protected abstract int Get_uid_rng();
protected abstract void Bulk_bgn();
protected abstract void Bulk_end();
protected abstract void Bulk_run(int uid_bgn, int uid_end);
}
class Imglnk_bulk_cmd__img_id extends Db_bulk_cmd_base {
private final Db_conn conn;
private final Db_attach_mgr attach_mgr;
private final int img_wiki;
private String sql;
public Imglnk_bulk_cmd__img_id(Db_conn conn, boolean wiki_is_local, Xowe_wiki wiki) {
this.conn = conn;
this.attach_mgr = new Db_attach_mgr(conn, new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Tbl__page().Conn()));
this.img_wiki = wiki_is_local ? 0 : 1;
// xowa.wiki.image.sqlite3
// INSERT INTO img_link (img_name, img_wiki) SELECT img_name, Count(img_name) FROM img_link_tmp GROUP BY img_name
}
@Override protected int Get_uid_max() {
return conn.Exec_select_as_int("SELECT Max(img_uid) FROM img_link_tmp", -1);
}
@Override protected int Get_uid_rng() {return 10000;}
@Override protected void Bulk_bgn() {
sql = String_.Concat_lines_nl_skip_last // ANSI.Y
( "UPDATE img_link_tmp"
, "SET img_wiki = {0}"
, ", img_id = (SELECT p.page_id FROM <page_db>page p WHERE p.page_namespace = 6 AND p.page_title = img_link_tmp.img_name)"
, "WHERE img_uid > {1} AND img_uid <= {2}"
, "AND img_name IN (SELECT p.page_title FROM <page_db>page p WHERE p.page_namespace = 6 AND p.page_title = img_link_tmp.img_name)"
);
sql = attach_mgr.Resolve_sql(sql);
attach_mgr.Attach();
conn.Txn_bgn("imglnk.bulk");
}
@Override protected void Bulk_end() {
conn.Txn_end();
attach_mgr.Detach();
}
@Override protected void Bulk_run(int uid_bgn, int uid_end) {
conn.Exec_sql(String_.Format("updating img_link_tmp; wiki={0} uid={1}", img_wiki, uid_bgn), String_.Format(sql, img_wiki, uid_bgn, uid_end));
}
public static void Bulk_exec(Db_conn conn, boolean wiki_is_local, Xowe_wiki wiki) {
new Imglnk_bulk_cmd__img_id(conn, wiki_is_local, wiki).Exec();
}
}

29
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_addon.java Normal file
View File

@@ -0,0 +1,29 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.wmdumps.pagelinks.bldrs.*;
public class Imglnk_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
public Xob_cmd[] Bldr_cmds() {
return new Xob_cmd[]
{ Imglnk_bldr_cmd.Prototype
};
}
public String Addon__key() {return "xowa.builds.imglinks";}
}

51
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_cmd.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
public class Imglnk_bldr_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd {
private Imglnk_bldr_mgr mgr;
private int tmp_page_id;
private int rows = 0;
public Imglnk_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {this.Cmd_ctor(bldr, wiki); this.make_fil_len = Io_mgr.Len_mb;}
@Override public String Sql_file_name() {return "imagelinks";}
@Override public void Cmd_bgn_hook(Xob_bldr bldr, Sql_file_parser parser) {
parser.Fld_cmd_(this).Flds_req_idx_(3, 0, 1);
mgr = new Imglnk_bldr_mgr(wiki);
}
public void Exec(byte[] src, byte[] fld_key, int fld_idx, int fld_bgn, int fld_end, Bry_bfr file_bfr, Sql_file_parser_data data) {
switch (fld_idx) {
case Fld__il_from: this.tmp_page_id = Bry_.To_int_or(src, fld_bgn, fld_end, -1); break;
case Fld__il_to:
mgr.Tmp_tbl().Insert_by_batch(tmp_page_id, Bry_.Mid(src, fld_bgn, fld_end));
if (++rows % 100000 == 0) usr_dlg.Prog_many("", "", "reading row ~{0}", Int_.To_str_fmt(rows, "#,##0"));
break;
}
} private static final byte Fld__il_from = 0, Fld__il_to = 1;
@Override public void Cmd_end() {
if (fail) return;
mgr.On_cmd_end();
}
public static final String BLDR_CMD_KEY = "wiki.imagelinks";
@Override public String Cmd_key() {return BLDR_CMD_KEY;}
public static final Xob_cmd Prototype = new Imglnk_bldr_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Imglnk_bldr_cmd(bldr, wiki);}
}

51
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_bldr_mgr.java Normal file
View File

@@ -0,0 +1,51 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.files.repos.*;
class Imglnk_bldr_mgr {
private final Db_conn conn;
private final Xowe_wiki wiki;
public Imglnk_tmp_tbl Tmp_tbl() {return tmp_tbl;} private Imglnk_tmp_tbl tmp_tbl;
public Imglnk_bldr_mgr(Xowe_wiki wiki) {
wiki.Init_assert();
this.wiki = wiki;
this.conn = Xob_db_file.New__img_link(wiki).Conn();
this.tmp_tbl = new Imglnk_tmp_tbl(conn);
conn.Meta_tbl_remake(tmp_tbl);
tmp_tbl.Create_tbl();
tmp_tbl.Insert_bgn();
}
public void On_cmd_end() {
// finalize txn; create idx
tmp_tbl.Insert_end();
tmp_tbl.Create_idx__img_ttl();
// create reg_tbl
Imglnk_reg_tbl reg_tbl = new Imglnk_reg_tbl(conn);
conn.Meta_tbl_remake(reg_tbl);
reg_tbl.Create_idx__src_ttl();
reg_tbl.Insert(conn, Xof_repo_itm_.Repo_local , wiki);
reg_tbl.Insert(conn, Xof_repo_itm_.Repo_remote, wiki.Appe().Wiki_mgr().Wiki_commons());
reg_tbl.Create_idx__trg_ttl();
// Imglnk_bulk_cmd__img_id.Bulk_exec(conn, Bool_.Y, wiki);
// Imglnk_bulk_cmd__img_id.Bulk_exec(conn, Bool_.N, wiki.Appe().Wiki_mgr().Wiki_commons());
}
}

69
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_reg_tbl.java Normal file
View File

@@ -0,0 +1,69 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.*;
public class Imglnk_reg_tbl implements Db_tbl {
private final String tbl_name = "imglnk_reg"; private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private final String fld__img_src, fld__img_trg, fld__img_repo;
private final Db_conn conn;
public Imglnk_reg_tbl(Db_conn conn) {
this.conn = conn;
fld__img_src = flds.Add_str("img_src", 255);
fld__img_trg = flds.Add_str("img_trg", 255);
fld__img_repo = flds.Add_byte("img_repo");
flds.Add_int("img_count");
conn.Rls_reg(this);
}
public Db_conn Conn() {return conn;}
public String Tbl_name() {return tbl_name;}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx__src_ttl() {conn.Meta_idx_create(tbl_name, fld__img_src, fld__img_src, fld__img_repo);}
public void Create_idx__trg_ttl() {conn.Meta_idx_create(tbl_name, fld__img_trg, fld__img_trg, fld__img_repo);}
public void Insert(Db_conn conn, byte repo_id, Xowe_wiki wiki) {
String repo_id_str = Byte_.To_str(repo_id);
Db_attach_mgr attach_mgr = new Db_attach_mgr(conn);
String sql = "";
Xob_db_file redirect_db = Xob_db_file.New__wiki_redirect(wiki.Fsys_mgr().Root_dir());
attach_mgr.Conn_links_(new Db_attach_itm("redirect_db", redirect_db.Conn()));
sql = String_.Concat_lines_nl_skip_last // ANSI.Y
( "INSERT INTO imglnk_reg (img_src, img_trg, img_repo, img_count)"
, "SELECT ilt.img_name, r.trg_ttl, " + repo_id_str + ", Count(ilt.img_name)"
, "FROM imglnk_tmp ilt"
, " JOIN <redirect_db>redirect r ON ilt.img_name = r.src_ttl"
, " LEFT JOIN imglnk_reg il ON il.img_src = ilt.img_name"
, "WHERE il.img_src IS NULL"
, "GROUP BY ilt.img_name"
);
attach_mgr.Exec_sql_w_msg("imglnk_reg.insert.redirect: repo=" + repo_id_str, sql);
attach_mgr.Conn_links_(new Db_attach_itm("page_db", wiki.Data__core_mgr().Db__core().Tbl__page().Conn()));
sql = String_.Concat_lines_nl_skip_last // ANSI.Y
( "INSERT INTO imglnk_reg (img_src, img_trg, img_repo, img_count)"
, "SELECT ilt.img_name, ilt.img_name, " + repo_id_str + ", Count(ilt.img_name)"
, "FROM imglnk_tmp ilt"
, " JOIN <page_db>page p ON p.page_namespace = 6 AND p.page_title = ilt.img_name"
, " LEFT JOIN imglnk_reg il ON il.img_src = ilt.img_name"
, "WHERE il.img_src IS NULL"
, "GROUP BY ilt.img_name"
);
attach_mgr.Exec_sql_w_msg("imglnk_reg.insert.direct: repo=" + repo_id_str, sql);
}
public void Rls() {}
}

54
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Imglnk_tmp_tbl.java Normal file
View File

@@ -0,0 +1,54 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.dbs.*;
public class Imglnk_tmp_tbl implements Db_tbl {
private final String tbl_name = "imglnk_tmp"; private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private final String fld__page_id, fld__img_name;
private final Db_conn conn;
private Db_stmt stmt__insert;
public Imglnk_tmp_tbl(Db_conn conn) {
this.conn = conn;
flds.Add_int_pkey_autonum("img_uid");
this.fld__page_id = flds.Add_int("page_id");
this.fld__img_name = flds.Add_str("img_name", 255);
flds.Add_int_dflt("img_wiki", -1);
flds.Add_int_dflt("img_id", -1);
conn.Rls_reg(this);
}
public Db_conn Conn() {return conn;}
public String Tbl_name() {return tbl_name;}
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Insert_bgn() {
stmt__insert = conn.Stmt_insert(tbl_name, fld__page_id, fld__img_name);
conn.Txn_bgn(tbl_name);
}
public void Insert_by_batch(int page_id, byte[] img_name) {
stmt__insert.Clear().Val_int(fld__page_id, page_id).Val_bry_as_str(fld__img_name, img_name).Exec_insert();
}
public void Insert_end() {
conn.Txn_end();
stmt__insert.Rls();
}
public void Create_idx__img_ttl() {
conn.Meta_idx_create(tbl_name, fld__img_name, fld__img_name);
}
public void Rls() {
stmt__insert = Db_stmt_.Rls(stmt__insert);
}
}

32
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Xof_orig_wkr__img_links.java Normal file
View File

@@ -0,0 +1,32 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.xowa.files.origs.*;
public class Xof_orig_wkr__img_links implements Xof_orig_wkr {
private final Ordered_hash hash = Ordered_hash_.New_bry();
public byte Tid() {return Xof_orig_wkr_.Tid_xowa_img_links;}
public Xof_orig_itm Find_as_itm(byte[] ttl, int list_idx, int list_len) {return (Xof_orig_itm)hash.Get_by(ttl);}
public void Find_by_list(Ordered_hash rv, List_adp itms) {throw Err_.new_unimplemented();}
public boolean Add_orig(byte repo, byte[] page, int ext_id, int w, int h, byte[] redirect) {return false;}
public void Db_txn_save() {}
public void Db_rls() {}
public void Add_by_db(Xof_orig_itm itm) {
hash.Add(itm.Ttl(), itm);
}
}

63
400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/imglinks/Xof_orig_wkr__img_links_.java Normal file
View File

@@ -0,0 +1,63 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.wmdumps.imglinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.dbs.*;
import gplx.xowa.bldrs.*;
import gplx.xowa.files.repos.*; import gplx.xowa.files.origs.*;
public class Xof_orig_wkr__img_links_ {
public static Xof_orig_wkr__img_links Load_all(Xowe_wiki wiki) {
Xof_orig_wkr__img_links rv = new Xof_orig_wkr__img_links();
Db_conn conn = Xob_db_file.New__img_link(wiki).Conn();
Load_by_wiki(rv, conn, Xof_repo_itm_.Repo_local , wiki);
Load_by_wiki(rv, conn, Xof_repo_itm_.Repo_remote, wiki.Appe().Wiki_mgr().Wiki_commons());
return rv;
}
private static void Load_by_wiki(Xof_orig_wkr__img_links rv, Db_conn conn, byte repo_id, Xowe_wiki wiki) {
String sql = String_.Concat_lines_nl_skip_last // ANSI.Y
( "SELECT ilr.img_repo, ilr.img_src, i.img_media_type, i.img_minor_mime, i.img_size, i.img_width, i.img_height, i.img_bits, i.img_ext_id, i.img_timestamp, ilr.img_trg AS img_redirect"
, "FROM imglnk_reg ilr"
, " JOIN <img_db>image i ON ilr.img_trg = i.img_name"
, "WHERE ilr.img_repo = " + repo_id
);
Xob_db_file img_db = Xob_db_file.New__wiki_image(wiki.Fsys_mgr().Root_dir());
Db_attach_mgr attach_mgr = new Db_attach_mgr(conn, new Db_attach_itm("img_db", img_db.Conn()));
sql = attach_mgr.Resolve_sql(sql);
attach_mgr.Attach();
int count = 0;
Db_rdr rdr = conn.Stmt_sql(sql).Exec_select__rls_auto();
try {
while (rdr.Move_next()) {
rv.Add_by_db(new Xof_orig_itm
( rdr.Read_byte("img_repo")
, rdr.Read_bry_by_str("img_src")
, rdr.Read_int("img_ext_id")
, rdr.Read_int("img_width")
, rdr.Read_int("img_height")
, rdr.Read_bry_by_str("img_redirect")
));
if ((++count % 10000) == 0)
Gfo_usr_dlg_.Instance.Prog_many("", "", "loading img_links.orig: rows=~{0}", count);
}
} finally {rdr.Rls();}
attach_mgr.Detach();
}
}

4
400_xowa/src/gplx/xowa/addons/bldrs/pagelinks/Xoax_builds_pagelinks_addon.java → 400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/pagelinks/Xoax_builds_pagelinks_addon.java
View File

@@ -15,9 +15,9 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.pagelinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*;
package gplx.xowa.addons.bldrs.wmdumps.pagelinks; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*;
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.pagelinks.bldrs.*;
import gplx.xowa.addons.bldrs.wmdumps.pagelinks.bldrs.*;
public class Xoax_builds_pagelinks_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
public Xob_cmd[] Bldr_cmds() {
return new Xob_cmd[]

4
400_xowa/src/gplx/xowa/addons/bldrs/pagelinks/bldrs/Pglnk_bldr_cmd.java → 400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/pagelinks/bldrs/Pglnk_bldr_cmd.java
View File

@@ -15,9 +15,9 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.pagelinks.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.pagelinks.*;
package gplx.xowa.addons.bldrs.wmdumps.pagelinks.bldrs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*; import gplx.xowa.addons.bldrs.wmdumps.pagelinks.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; import gplx.xowa.bldrs.sqls.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.*; import gplx.xowa.addons.bldrs.pagelinks.dbs.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.data.*; import gplx.xowa.addons.bldrs.wmdumps.pagelinks.dbs.*;
public class Pglnk_bldr_cmd extends Xob_sql_dump_base implements Sql_file_parser_cmd {
private Db_conn conn;
private Pglnk_page_link_temp_tbl temp_tbl;

2
400_xowa/src/gplx/xowa/addons/bldrs/pagelinks/dbs/Pglnk_page_link_tbl.java → 400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/pagelinks/dbs/Pglnk_page_link_tbl.java
View File

@@ -15,7 +15,7 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.pagelinks.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.pagelinks.*;
package gplx.xowa.addons.bldrs.wmdumps.pagelinks.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*; import gplx.xowa.addons.bldrs.wmdumps.pagelinks.*;
import gplx.core.ios.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.dbs.cfgs.*;
public class Pglnk_page_link_tbl implements Rls_able {
private final String tbl_name = "page_link"; private final Dbmeta_fld_list flds = new Dbmeta_fld_list();

2
400_xowa/src/gplx/xowa/addons/bldrs/pagelinks/dbs/Pglnk_page_link_temp_tbl.java → 400_xowa/src/gplx/xowa/addons/bldrs/wmdumps/pagelinks/dbs/Pglnk_page_link_temp_tbl.java
View File

@@ -15,7 +15,7 @@ GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.pagelinks.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.pagelinks.*;
package gplx.xowa.addons.bldrs.wmdumps.pagelinks.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.wmdumps.*; import gplx.xowa.addons.bldrs.wmdumps.pagelinks.*;
import gplx.core.ios.*; import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.wikis.dbs.*; import gplx.dbs.cfgs.*;
public class Pglnk_page_link_temp_tbl implements Rls_able {
private final String tbl_name = "page_link_temp"; private final Dbmeta_fld_list flds = new Dbmeta_fld_list();