Add option to include patch files

v3.3.4
gnosygnu 8 years ago
parent af8306ae9b
commit 37a72e6ad4

@ -28,5 +28,6 @@ public class Xobc_import_type {
, Tid__file__data = 7
, Tid__fsdb__delete = 8
, Tid__wiki__ctg = 9
, Tid__misc = 10
;
}

@ -18,7 +18,7 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Pack_file_bldr_cmd extends Xob_cmd__base {
private final Pack_file_bldr_cfg cfg = new Pack_file_bldr_cfg();
private final Pack_file_cfg cfg = new Pack_file_cfg();
public Pack_file_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);}
@Override public void Cmd_run() {
new Pack_file_mgr().Exec(wiki, cfg);

@ -17,12 +17,16 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*;
public class Pack_file_bldr_cfg implements Gfo_invk {
public class Pack_file_cfg implements Gfo_invk {
public Io_url Deploy_dir() {return deploy_dir;} private Io_url deploy_dir;
public boolean Pack_text() {return pack_text;} private boolean pack_text = false;
public boolean Pack_html() {return pack_html;} private boolean pack_html = true;
public boolean Pack_file() {return pack_file;} private boolean pack_file = true;
public boolean Pack_fsdb_delete() {return pack_fsdb_delete;} private boolean pack_fsdb_delete;
public boolean Pack_custom() {return pack_custom_files != null;}
public String Pack_custom_files() {return pack_custom_files;} private String pack_custom_files;
public String Pack_custom_types() {return pack_custom_types;} private String pack_custom_types;
public String Pack_custom_name() {return pack_custom_name;} private String pack_custom_name;
public DateAdp Pack_file_cutoff() {return pack_file_cutoff;} private DateAdp pack_file_cutoff = null;
public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) {
@ -32,11 +36,15 @@ public class Pack_file_bldr_cfg implements Gfo_invk {
else if (ctx.Match(k, Invk__pack_file_)) pack_file = m.ReadYn("v");
else if (ctx.Match(k, Invk__pack_file_cutoff_)) pack_file_cutoff = m.ReadDate("v");
else if (ctx.Match(k, Invk__pack_fsdb_delete_)) pack_fsdb_delete = m.ReadYn("v");
else if (ctx.Match(k, Invk__pack_custom_name_)) pack_custom_name = m.ReadStr("v");
else if (ctx.Match(k, Invk__pack_custom_files_)) pack_custom_files = m.ReadStr("v"); // pack_custom {files='en.wikipedia.org-core.xowa|en.wikipedia.org-html-ns.008.xowa'}}
else if (ctx.Match(k, Invk__pack_custom_types_)) pack_custom_types = m.ReadStr("v"); // pack_custom {types='core|srch|html';}
else return Gfo_invk_.Rv_unhandled;
return this;
}
private static final String Invk__deploy_dir_ = "deploy_dir_"
, Invk__pack_text_ = "pack_text_", Invk__pack_html_ = "pack_html_", Invk__pack_file_ = "pack_file_", Invk__pack_file_cutoff_ = "pack_file_cutoff_"
, Invk__pack_fsdb_delete_ = "pack_fsdb_delete_"
, Invk__pack_custom_name_ = "pack_custom_name_", Invk__pack_custom_files_ = "pack_custom_files_", Invk__pack_custom_types_ = "pack_custom_types_"
;
}

@ -20,19 +20,20 @@ import gplx.core.progs.*; import gplx.core.ios.zips.*; import gplx.core.ios.stre
import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.fsdb.*; import gplx.fsdb.meta.*;
import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.hosts.*;
public class Pack_file_mgr {
public void Exec(Xowe_wiki wiki, Pack_file_bldr_cfg cfg) {
public void Exec(Xowe_wiki wiki, Pack_file_cfg cfg) {
// init
wiki.Init_assert();
Io_url wiki_dir = wiki.Fsys_mgr().Root_dir();
Io_url pack_dir = wiki_dir.GenSubDir_nest("tmp", "pack");
Io_mgr.Instance.DeleteDirDeep(pack_dir); Io_mgr.Instance.CreateDirIfAbsent(pack_dir);
String wiki_date = wiki.Props().Modified_latest().XtoStr_fmt("yyyy.MM");
Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg.Pack_text(), cfg.Pack_html(), cfg.Pack_file(), cfg.Pack_file_cutoff(), cfg.Pack_fsdb_delete());
Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg);
// get import_tbl
byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo();
Xobc_data_db bc_db = Xobc_data_db.New(wiki.App().Fsys_mgr());
Db_conn bc_conn = bc_db.Conn();
if (!cfg.Pack_custom()) // only delete files if not custom
bc_db.Delete_by_import(wiki_abrv, wiki_date);
bc_conn.Txn_bgn("xobc_import_insert");
@ -54,6 +55,8 @@ public class Pack_file_mgr {
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html, Xobc_import_type.Tid__wiki__ctg);
if (cfg.Pack_file())
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data); // , Xobc_import_type.Tid__fsdb__delete
if (cfg.Pack_custom())
Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, cfg.Pack_custom_name(), Xobc_import_type.Tid__misc);
bc_conn.Txn_end();
// deploy
@ -168,79 +171,3 @@ public class Pack_file_mgr {
return String_.Split(task_key, "|");
}
}
class Pack_hash_bldr {
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_text, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff, boolean pack_fsdb_delete) {
Pack_hash rv = new Pack_hash();
Pack_zip_name_bldr zip_name_bldr = new Pack_zip_name_bldr(pack_dir, wiki.Domain_str(), String_.new_a7(wiki.Domain_itm().Abrv_wm()), wiki_date);
Xow_db_mgr db_mgr = wiki.Data__core_mgr();
// bld html pack
if (pack_html) {
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; ++i) {
Xow_db_file file = db_mgr.Dbs__get_at(i);
int pack_tid = Get_pack_tid(file.Tid());
if (pack_tid == Xobc_import_type.Tid__ignore) continue;
rv.Add(zip_name_bldr, pack_tid, file.Url());
}
rv.Consolidate(Xobc_import_type.Tid__wiki__srch);
}
// bld text pack
if (pack_text) {
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; ++i) {
Xow_db_file file = db_mgr.Dbs__get_at(i);
int pack_tid = Get_pack_tid(file.Tid());
if (pack_tid == Xobc_import_type.Tid__ignore) continue;
rv.Add(zip_name_bldr, pack_tid, file.Url());
}
}
// bld file pack
if (pack_file) {
Fsm_mnt_itm mnt_itm = wiki.File__mnt_mgr().Mnts__get_at(Fsm_mnt_mgr.Mnt_idx_main);
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__core, wiki_dir.GenSubFil(mnt_itm.Atr_mgr().Db__core().Url_rel()));
if (db_mgr.Props().Layout_file().Tid_is_lot()) {
Fsm_bin_mgr bin_mgr = mnt_itm.Bin_mgr();
int bin_len = bin_mgr.Dbs__len();
for (int i = 0; i < bin_len; ++i) {
Fsm_bin_fil bin_fil = bin_mgr.Dbs__get_at(i);
Io_url bin_fil_url = bin_fil.Url();
// ignore if bin_fil is earlier than cutoff
if (pack_file_cutoff != null) {
DateAdp bin_fil_date = Io_mgr.Instance.QueryFil(bin_fil_url).ModifiedTime();
if (bin_fil_date.Timestamp_unix() < pack_file_cutoff.Timestamp_unix()) continue;
}
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__data, bin_fil_url);
}
}
}
// bld pack_fsdb_delete
if (pack_fsdb_delete) {
gplx.xowa.bldrs.Xob_db_file fsdb_deletion_db = gplx.xowa.bldrs.Xob_db_file.New__deletion_db(wiki);
if (!Io_mgr.Instance.ExistsFil(fsdb_deletion_db.Url())) throw Err_.new_wo_type("deletion db does not exists: url=" + fsdb_deletion_db.Url().Raw());
rv.Add(zip_name_bldr, Xobc_import_type.Tid__fsdb__delete, fsdb_deletion_db.Url());
}
return rv;
}
private static int Get_pack_tid(byte db_file_tid) {
switch (db_file_tid) {
case Xow_db_file_.Tid__core: return Xobc_import_type.Tid__wiki__core;
case Xow_db_file_.Tid__search_core:
case Xow_db_file_.Tid__search_link: return Xobc_import_type.Tid__wiki__srch;
case Xow_db_file_.Tid__html_solo:
case Xow_db_file_.Tid__html_data: return Xobc_import_type.Tid__wiki__html;
case Xow_db_file_.Tid__cat:
case Xow_db_file_.Tid__cat_core:
case Xow_db_file_.Tid__cat_link: return Xobc_import_type.Tid__wiki__ctg;
case Xow_db_file_.Tid__file_core: return Xobc_import_type.Tid__file__core;
case Xow_db_file_.Tid__file_solo:
case Xow_db_file_.Tid__file_data: return Xobc_import_type.Tid__file__data;
case Xow_db_file_.Tid__text: return Xobc_import_type.Tid__wiki__text;
default: return Xobc_import_type.Tid__ignore;
}
}
}

@ -0,0 +1,111 @@
/*
XOWA: the XOWA Offline Wiki Application
Copyright (C) 2012 gnosygnu@gmail.com
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*;
import gplx.fsdb.meta.*;
import gplx.xowa.wikis.data.*;
import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*;
class Pack_hash_bldr {
public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, Pack_file_cfg cfg) {
// boolean pack_text, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff, boolean pack_fsdb_delete
Pack_hash rv = new Pack_hash();
Pack_zip_name_bldr zip_name_bldr = new Pack_zip_name_bldr(pack_dir, wiki.Domain_str(), String_.new_a7(wiki.Domain_itm().Abrv_wm()), wiki_date, cfg.Pack_custom_name());
Xow_db_mgr db_mgr = wiki.Data__core_mgr();
// bld custom_files
if (cfg.Pack_custom())
return Bld_custom_files(rv, wiki, wiki_dir, zip_name_bldr, cfg.Pack_custom_files());
// bld html pack
if (cfg.Pack_html()) {
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; ++i) {
Xow_db_file file = db_mgr.Dbs__get_at(i);
int pack_tid = Get_pack_tid(file.Tid());
if (pack_tid == Xobc_import_type.Tid__ignore) continue;
rv.Add(zip_name_bldr, pack_tid, file.Url());
}
rv.Consolidate(Xobc_import_type.Tid__wiki__srch);
}
// bld text pack
if (cfg.Pack_text()) {
int len = db_mgr.Dbs__len();
for (int i = 0; i < len; ++i) {
Xow_db_file file = db_mgr.Dbs__get_at(i);
int pack_tid = Get_pack_tid(file.Tid());
if (pack_tid == Xobc_import_type.Tid__ignore) continue;
rv.Add(zip_name_bldr, pack_tid, file.Url());
}
}
// bld file pack
if (cfg.Pack_file()) {
Fsm_mnt_itm mnt_itm = wiki.File__mnt_mgr().Mnts__get_at(Fsm_mnt_mgr.Mnt_idx_main);
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__core, wiki_dir.GenSubFil(mnt_itm.Atr_mgr().Db__core().Url_rel()));
if (db_mgr.Props().Layout_file().Tid_is_lot()) {
Fsm_bin_mgr bin_mgr = mnt_itm.Bin_mgr();
int bin_len = bin_mgr.Dbs__len();
for (int i = 0; i < bin_len; ++i) {
Fsm_bin_fil bin_fil = bin_mgr.Dbs__get_at(i);
Io_url bin_fil_url = bin_fil.Url();
// ignore if bin_fil is earlier than cutoff
if (cfg.Pack_file_cutoff() != null) {
DateAdp bin_fil_date = Io_mgr.Instance.QueryFil(bin_fil_url).ModifiedTime();
if (bin_fil_date.Timestamp_unix() < cfg.Pack_file_cutoff().Timestamp_unix()) continue;
}
rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__data, bin_fil_url);
}
}
}
// bld pack_fsdb_delete
if (cfg.Pack_fsdb_delete()) {
gplx.xowa.bldrs.Xob_db_file fsdb_deletion_db = gplx.xowa.bldrs.Xob_db_file.New__deletion_db(wiki);
if (!Io_mgr.Instance.ExistsFil(fsdb_deletion_db.Url())) throw Err_.new_wo_type("deletion db does not exists: url=" + fsdb_deletion_db.Url().Raw());
rv.Add(zip_name_bldr, Xobc_import_type.Tid__fsdb__delete, fsdb_deletion_db.Url());
}
return rv;
}
private static Pack_hash Bld_custom_files(Pack_hash rv, Xow_wiki wiki, Io_url wiki_dir, Pack_zip_name_bldr zip_name_bldr, String custom_files_blob) {
String[] custom_files = String_.Split(custom_files_blob, "|");
int len = custom_files.length;
for (int i = 0; i < len; ++i) {
Io_url file_url = wiki_dir.GenSubFil(custom_files[i]);
rv.Add(zip_name_bldr, Xobc_import_type.Tid__misc, file_url);
}
return rv;
}
private static int Get_pack_tid(byte db_file_tid) {
switch (db_file_tid) {
case Xow_db_file_.Tid__core: return Xobc_import_type.Tid__wiki__core;
case Xow_db_file_.Tid__search_core:
case Xow_db_file_.Tid__search_link: return Xobc_import_type.Tid__wiki__srch;
case Xow_db_file_.Tid__html_solo:
case Xow_db_file_.Tid__html_data: return Xobc_import_type.Tid__wiki__html;
case Xow_db_file_.Tid__cat:
case Xow_db_file_.Tid__cat_core:
case Xow_db_file_.Tid__cat_link: return Xobc_import_type.Tid__wiki__ctg;
case Xow_db_file_.Tid__file_core: return Xobc_import_type.Tid__file__core;
case Xow_db_file_.Tid__file_solo:
case Xow_db_file_.Tid__file_data: return Xobc_import_type.Tid__file__data;
case Xow_db_file_.Tid__text: return Xobc_import_type.Tid__wiki__text;
default: return Xobc_import_type.Tid__ignore;
}
}
}

@ -20,10 +20,12 @@ public class Pack_zip_name_bldr { // en.wikipedia.org-file-ns.000-db.001.xowa ->
private final Io_url pack_dir;
private final byte[] wiki_domain, zip_name_prefix;
public Pack_zip_name_bldr(Io_url pack_dir, String wiki_domain_str, String wiki_abrv, String wiki_date) {
public Pack_zip_name_bldr(Io_url pack_dir, String wiki_domain_str, String wiki_abrv, String wiki_date, String custom_name) {
this.pack_dir = pack_dir;
this.wiki_domain = Bry_.new_u8(wiki_domain_str);
this.zip_name_prefix = Bry_.new_u8("Xowa_" + wiki_abrv + "_" + String_.Replace(wiki_date, ".", "-"));
String zip_name_suffix = custom_name == null ? String_.Replace(wiki_date, ".", "-") : custom_name; // "Xowa_enwiki_2016-10_core.zip" vs "Xowa_enwiki_custom_core.zip"
this.zip_name_prefix = Bry_.new_u8("Xowa_" + wiki_abrv + "_" + zip_name_suffix);
}
public Io_url Bld(Io_url orig_url) {
String orig_str = orig_url.NameOnly() + ".zip";

Loading…
Cancel
Save