Archives/gnosygnu_xowa
1
0
Fork 0
mirror of https://github.com/gnosygnu/xowa.git synced 2026-03-02 03:49:30 +00:00
Code Issues Projects Releases Wiki Activity

uca category support and other changes

Browse Source
This commit is contained in:
gnosygnu
2016-10-12 08:57:22 -04:00
parent e3b393650d
commit 3fc2e0741f
187 changed files with 3486 additions and 2984 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
400_xowa/src/gplx/xowa/addons/bldrs/files/Xoax_builds_files_addon.java
View File

@@ -19,6 +19,8 @@ package gplx.xowa.addons.bldrs.files; import gplx.*; import gplx.xowa.*; import
import gplx.xowa.bldrs.wkrs.*;
import gplx.xowa.addons.bldrs.files.cmds.*;
import gplx.xowa.addons.bldrs.mass_parses.inits.*; import gplx.xowa.addons.bldrs.mass_parses.parses.*; import gplx.xowa.addons.bldrs.mass_parses.makes.*;
import gplx.xowa.addons.bldrs.files.cksums.*;
import gplx.xowa.addons.bldrs.app_cfgs.wm_server_cfgs.*;
public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__bldr {
public Xob_cmd[] Bldr_cmds() {
return new Xob_cmd[]
@@ -43,6 +45,9 @@ public class Xoax_builds_files_addon implements Xoax_addon_itm, Xoax_addon_itm__
, Xomp_init_cmd.Prototype
, Xomp_parse_cmd.Prototype
, Xomp_make_cmd.Prototype
, Xocksum_calc_cmd.Prototype
, Xowm_server_cfg_cmd.Prototype
};
}

30
400_xowa/src/gplx/xowa/addons/bldrs/files/cksums/Xocksum_calc_cmd.java Normal file
View File

@@ -0,0 +1,30 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Xocksum_calc_cmd extends Xob_cmd__base {
public Xocksum_calc_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
wiki.Init_assert();
new Xocksum_calc_mgr().Exec(wiki);
}
@Override public String Cmd_key() {return BLDR_CMD_KEY;} private static final String BLDR_CMD_KEY = "fsdb.cksums.calc";
public static final Xob_cmd Prototype = new Xocksum_calc_cmd(null, null);
@Override public Xob_cmd Cmd_clone(Xob_bldr bldr, Xowe_wiki wiki) {return new Xocksum_calc_cmd(bldr, wiki);}
}

79
400_xowa/src/gplx/xowa/addons/bldrs/files/cksums/Xocksum_calc_mgr.java Normal file
View File

@@ -0,0 +1,79 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*;
import gplx.core.ios.streams.*; import gplx.core.security.*;
import gplx.dbs.*; import gplx.xowa.addons.bldrs.files.cksums.dbs.*;
import gplx.xowa.files.*; import gplx.fsdb.*; import gplx.fsdb.data.*;
public class Xocksum_calc_mgr {
public void Exec(Xowe_wiki wiki) {
// get conn variables
Xocksum_cksum_db db = Xocksum_cksum_db.Get(wiki);
Db_conn conn = db.Conn();
Xocksum_cksum_tbl tbl = db.Tbl__cksum();
conn.Meta_tbl_assert(tbl);
// insert missing items
tbl.Insert_missing();
tbl.Create_idx();
// get updates
int count = 0;
Hash_algo md5_algo = Hash_algo_.New__md5();
List_adp updates = List_adp_.New();
String cur_date = Datetime_now.Get().XtoStr_gplx();
Db_stmt select_stmt = tbl.Select_samples_stmt(10000);
while (true) {
// get cksum_rows
Xocksum_cksum_row[] rows = tbl.Select_samples(select_stmt);
// loop cksum_rows and (a) get bin_data; (b) if md5 diff, then add to updates
int len = rows.length; if (len == 0) break;
for (int i = 0; i < len; ++i) {
Xocksum_cksum_row row = rows[i];
byte[] bin_bry = Get_bin(wiki, row);
if (bin_bry == null) {
Gfo_usr_dlg_.Instance.Prog_many("", "", "null; fil_id=~{0} thm_id=~{1}", row.Fil_id(), row.Thm_id());
bin_bry = Bry_.Empty;
}
row.Bin_size_(bin_bry.length);
byte[] md5 = md5_algo.Hash_bry_as_bry(bin_bry);
if (!Bry_.Eq(md5, row.Cksum_val())) {
row.Cksum_val_(md5);
updates.Add(row);
}
}
// run updates
conn.Txn_bgn("cksum_update");
len = updates.Len();
for (int i = 0; i < len; ++i) {
Xocksum_cksum_row row = (Xocksum_cksum_row)updates.Get_at(i);
tbl.Update(row.Fil_id(), row.Thm_id(), row.Bin_db_id(), row.Bin_size(), row.Cksum_tid(), 0, row.Cksum_val(), cur_date);
if (++count % 2000 == 0) Gfo_usr_dlg_.Instance.Prog_many("", "", "updating; rows=~{0}", count);
}
updates.Clear();
conn.Txn_end();
}
select_stmt.Rls();
}
private byte[] Get_bin(Xowe_wiki wiki, Xocksum_cksum_row row) {
int bin_id = row.Thm_id() == -1 ? row.Fil_id() : row.Thm_id();
Fsd_bin_itm bin_itm = wiki.File__mnt_mgr().Mnts__get_main().Bin_mgr().Dbs__get_at(row.Bin_db_id()).Select_as_itm(bin_id);
return bin_itm.Bin_data();
}
}

31
400_xowa/src/gplx/xowa/addons/bldrs/files/cksums/dbs/Xocksum_cksum_db.java Normal file
View File

@@ -0,0 +1,31 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.bldrs.files.cksums.*;
import gplx.dbs.*; import gplx.fsdb.meta.*;
public class Xocksum_cksum_db {
public Xocksum_cksum_db(Db_conn conn) {
this.conn = conn;
this.tbl__cksum = new Xocksum_cksum_tbl(conn);
}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public Xocksum_cksum_tbl Tbl__cksum() {return tbl__cksum;} private final Xocksum_cksum_tbl tbl__cksum;
public static Xocksum_cksum_db Get(Xowe_wiki wiki) {
return new Xocksum_cksum_db(wiki.File__fsdb_core().File__abc_file__at(Fsm_mnt_mgr.Mnt_idx_main).Conn());
}
}

41
400_xowa/src/gplx/xowa/addons/bldrs/files/cksums/dbs/Xocksum_cksum_row.java Normal file
View File

@@ -0,0 +1,41 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.bldrs.files.cksums.*;
public class Xocksum_cksum_row {
public Xocksum_cksum_row(int fil_id, int thm_id, int bin_db_id, long bin_size, byte cksum_tid, int cksum_count, byte[] cksum_val, String cksum_date) {
this.fil_id = fil_id;
this.thm_id = thm_id;
this.bin_db_id = bin_db_id;
this.bin_size = bin_size;
this.cksum_tid = cksum_tid;
this.cksum_count = cksum_count;
this.cksum_val = cksum_val;
this.cksum_date = cksum_date;
}
public int Fil_id() {return fil_id;} private final int fil_id;
public int Thm_id() {return thm_id;} private final int thm_id;
public int Bin_db_id() {return bin_db_id;} private final int bin_db_id;
public long Bin_size() {return bin_size;} private long bin_size;
public byte Cksum_tid() {return cksum_tid;} private final byte cksum_tid;
public int Cksum_count() {return cksum_count;} private final int cksum_count;
public byte[] Cksum_val() {return cksum_val;} private byte[] cksum_val;
public String Cksum_date() {return cksum_date;} private final String cksum_date;
public void Bin_size_(long v) {this.bin_size = v;}
public void Cksum_val_(byte[] v) {this.cksum_val = v;}
}

107
400_xowa/src/gplx/xowa/addons/bldrs/files/cksums/dbs/Xocksum_cksum_tbl.java Normal file
View File

@@ -0,0 +1,107 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.files.cksums.dbs; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.files.*; import gplx.xowa.addons.bldrs.files.cksums.*;
import gplx.dbs.*; import gplx.dbs.qrys.*; import gplx.xowa.addons.wikis.ctgs.*;
public class Xocksum_cksum_tbl implements Db_tbl {
private final Dbmeta_fld_list flds = new Dbmeta_fld_list();
private final String fld__fil_id, fld__thm_id, fld__bin_db_id, fld__bin_len, fld__cksum_tid, fld__cksum_count, fld__cksum_val, fld__cksum_date;
private Db_stmt stmt__update;
public Xocksum_cksum_tbl(Db_conn conn) {
this.conn = conn;
this.tbl_name = "fsdb_cksum";
this.fld__fil_id = flds.Add_int("fil_id");
this.fld__thm_id = flds.Add_int("thm_id");
this.fld__bin_db_id = flds.Add_int("bin_db_id");
this.fld__bin_len = flds.Add_long("bin_size");
this.fld__cksum_tid = flds.Add_byte("cksum_tid");
this.fld__cksum_count = flds.Add_int("cksum_count");
this.fld__cksum_val = flds.Add_str("cksum_val", 255);
this.fld__cksum_date = flds.Add_str("cksum_date", 16);
conn.Rls_reg(this);
}
public Db_conn Conn() {return conn;} private final Db_conn conn;
public String Tbl_name() {return tbl_name;} private final String tbl_name;
public void Create_tbl() {conn.Meta_tbl_create(Dbmeta_tbl_itm.New(tbl_name, flds));}
public void Create_idx() {
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, "fil_id__thm_id", fld__fil_id, fld__thm_id));
conn.Meta_idx_create(Dbmeta_idx_itm.new_normal_by_tbl(tbl_name, fld__cksum_val, fld__cksum_val));
}
public void Insert_missing() {
// insert from fsdb_fil
conn.Exec_sql(Db_sql_.Make_by_fmt(String_.Ary
( "INSERT INTO fsdb_cksum (fil_id, thm_id, bin_db_id, bin_size, cksum_tid, cksum_count, cksum_val, cksum_date)"
, "SELECT f.fil_id, -1, f.fil_bin_db_id, f.fil_size, {0}, 0, '', ''"
, "FROM fsdb_fil f"
, " LEFT JOIN fsdb_cksum c ON c.fil_id = f.fil_id AND c.thm_id = -1"
, "WHERE c.fil_id IS NULL"
, "AND f.fil_bin_db_id != -1"
), Cksum_tid__md5));
// insert from fsdb_fil
conn.Exec_sql(Db_sql_.Make_by_fmt(String_.Ary
( "INSERT INTO fsdb_cksum (fil_id, thm_id, bin_db_id, bin_size, cksum_tid, cksum_count, cksum_val, cksum_date)"
, "SELECT t.thm_owner_id, t.thm_id, t.thm_bin_db_id, t.thm_size, {0}, 0, '', ''"
, "FROM fsdb_thm t"
, " LEFT JOIN fsdb_cksum c ON c.fil_id = t.thm_owner_id AND c.thm_id = t.thm_id"
, "WHERE c.fil_id IS NULL"
), Cksum_tid__md5));
}
public Db_stmt Select_samples_stmt(int count) {
return conn.Stmt_sql(Db_sql_.Make_by_fmt(String_.Ary
( "SELECT *"
, "FROM fsdb_cksum"
, "WHERE cksum_val = ''"
// , "ORDER BY cksum_count, cksum_date"
, "LIMIT {0}"
), count));
}
public Xocksum_cksum_row[] Select_samples(Db_stmt stmt) {
List_adp rv = List_adp_.New();
Db_rdr rdr = stmt.Exec_select__rls_manual();
try {
while (rdr.Move_next()) {
rv.Add(new Xocksum_cksum_row
( rdr.Read_int("fil_id")
, rdr.Read_int("thm_id")
, rdr.Read_int("bin_db_id")
, rdr.Read_long("bin_size")
, rdr.Read_byte("cksum_tid")
, rdr.Read_int("cksum_count")
, rdr.Read_bry_by_str("cksum_val")
, rdr.Read_str("cksum_date")
));
}
} finally {rdr.Rls();}
return (Xocksum_cksum_row[])rv.To_ary_and_clear(Xocksum_cksum_row.class);
}
public void Update(int fil_id, int thm_id, int bin_db_id, long bin_size, byte cksum_tid, int cksum_count, byte[] cksum_val, String cksum_date) {
if (stmt__update == null) stmt__update = conn.Stmt_update_exclude(tbl_name, flds, fld__fil_id, fld__thm_id);
stmt__update.Clear()
.Val_int(fld__bin_db_id, bin_db_id).Val_long(fld__bin_len, bin_size)
.Val_byte(fld__cksum_tid, cksum_tid).Val_int(fld__cksum_count, cksum_count)
.Val_bry_as_str(fld__cksum_val, cksum_val).Val_str(fld__cksum_date, cksum_date)
.Crt_int(fld__fil_id, fil_id).Crt_int(fld__thm_id, thm_id)
.Exec_update();
}
public void Rls() {
this.stmt__update = Db_stmt_.Rls(stmt__update);
}
public static final byte Cksum_tid__md5 = 1;
}