From 37a72e6ad4185fceb9675c9e61dd7150e83a3be0 Mon Sep 17 00:00:00 2001 From: gnosygnu Date: Wed, 19 Oct 2016 12:44:24 -0400 Subject: [PATCH] Add option to include patch files --- .../dbs/datas/imports/Xobc_import_type.java | 21 ++-- .../packs/files/Pack_file_bldr_cmd.java | 2 +- ..._file_bldr_cfg.java => Pack_file_cfg.java} | 10 +- .../exports/packs/files/Pack_file_mgr.java | 85 +------------- .../exports/packs/files/Pack_hash_bldr.java | 111 ++++++++++++++++++ .../packs/files/Pack_zip_name_bldr.java | 6 +- 6 files changed, 142 insertions(+), 93 deletions(-) rename 400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/{Pack_file_bldr_cfg.java => Pack_file_cfg.java} (71%) create mode 100644 400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_hash_bldr.java diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java index 8aefafaa2..0a10ecee1 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/centrals/dbs/datas/imports/Xobc_import_type.java @@ -18,15 +18,16 @@ along with this program. If not, see . package gplx.xowa.addons.bldrs.centrals.dbs.datas.imports; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.centrals.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; public class Xobc_import_type { public static final int // SERIALIZED: bc_db; import_step - Tid__ignore = 0 - , Tid__pack = 1 - , Tid__wiki__core = 2 - , Tid__wiki__srch = 3 - , Tid__wiki__html = 4 - , Tid__wiki__text = 5 - , Tid__file__core = 6 - , Tid__file__data = 7 - , Tid__fsdb__delete = 8 - , Tid__wiki__ctg = 9 + Tid__ignore = 0 + , Tid__pack = 1 + , Tid__wiki__core = 2 + , Tid__wiki__srch = 3 + , Tid__wiki__html = 4 + , Tid__wiki__text = 5 + , Tid__file__core = 6 + , Tid__file__data = 7 + , Tid__fsdb__delete = 8 + , Tid__wiki__ctg = 9 + , Tid__misc = 10 ; } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cmd.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cmd.java index 06874dfbb..8adf1c5bf 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cmd.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cmd.java @@ -18,7 +18,7 @@ along with this program. If not, see . package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; public class Pack_file_bldr_cmd extends Xob_cmd__base { - private final Pack_file_bldr_cfg cfg = new Pack_file_bldr_cfg(); + private final Pack_file_cfg cfg = new Pack_file_cfg(); public Pack_file_bldr_cmd(Xob_bldr bldr, Xowe_wiki wiki) {super(bldr, wiki);} @Override public void Cmd_run() { new Pack_file_mgr().Exec(wiki, cfg); diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_cfg.java similarity index 71% rename from 400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java rename to 400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_cfg.java index 8bdb8cdcf..bc7e42155 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_bldr_cfg.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_cfg.java @@ -17,12 +17,16 @@ along with this program. If not, see . */ package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*; import gplx.xowa.bldrs.*; import gplx.xowa.bldrs.wkrs.*; -public class Pack_file_bldr_cfg implements Gfo_invk { +public class Pack_file_cfg implements Gfo_invk { public Io_url Deploy_dir() {return deploy_dir;} private Io_url deploy_dir; public boolean Pack_text() {return pack_text;} private boolean pack_text = false; public boolean Pack_html() {return pack_html;} private boolean pack_html = true; public boolean Pack_file() {return pack_file;} private boolean pack_file = true; public boolean Pack_fsdb_delete() {return pack_fsdb_delete;} private boolean pack_fsdb_delete; + public boolean Pack_custom() {return pack_custom_files != null;} + public String Pack_custom_files() {return pack_custom_files;} private String pack_custom_files; + public String Pack_custom_types() {return pack_custom_types;} private String pack_custom_types; + public String Pack_custom_name() {return pack_custom_name;} private String pack_custom_name; public DateAdp Pack_file_cutoff() {return pack_file_cutoff;} private DateAdp pack_file_cutoff = null; public Object Invk(GfsCtx ctx, int ikey, String k, GfoMsg m) { @@ -32,11 +36,15 @@ public class Pack_file_bldr_cfg implements Gfo_invk { else if (ctx.Match(k, Invk__pack_file_)) pack_file = m.ReadYn("v"); else if (ctx.Match(k, Invk__pack_file_cutoff_)) pack_file_cutoff = m.ReadDate("v"); else if (ctx.Match(k, Invk__pack_fsdb_delete_)) pack_fsdb_delete = m.ReadYn("v"); + else if (ctx.Match(k, Invk__pack_custom_name_)) pack_custom_name = m.ReadStr("v"); + else if (ctx.Match(k, Invk__pack_custom_files_)) pack_custom_files = m.ReadStr("v"); // pack_custom {files='en.wikipedia.org-core.xowa|en.wikipedia.org-html-ns.008.xowa'}} + else if (ctx.Match(k, Invk__pack_custom_types_)) pack_custom_types = m.ReadStr("v"); // pack_custom {types='core|srch|html';} else return Gfo_invk_.Rv_unhandled; return this; } private static final String Invk__deploy_dir_ = "deploy_dir_" , Invk__pack_text_ = "pack_text_", Invk__pack_html_ = "pack_html_", Invk__pack_file_ = "pack_file_", Invk__pack_file_cutoff_ = "pack_file_cutoff_" , Invk__pack_fsdb_delete_ = "pack_fsdb_delete_" + , Invk__pack_custom_name_ = "pack_custom_name_", Invk__pack_custom_files_ = "pack_custom_files_", Invk__pack_custom_types_ = "pack_custom_types_" ; } diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java index 7d01c9f3a..af0c4514a 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_file_mgr.java @@ -20,20 +20,21 @@ import gplx.core.progs.*; import gplx.core.ios.zips.*; import gplx.core.ios.stre import gplx.dbs.*; import gplx.xowa.wikis.data.*; import gplx.fsdb.*; import gplx.fsdb.meta.*; import gplx.xowa.addons.bldrs.centrals.dbs.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.*; import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; import gplx.xowa.addons.bldrs.centrals.steps.*; import gplx.xowa.addons.bldrs.centrals.hosts.*; public class Pack_file_mgr { - public void Exec(Xowe_wiki wiki, Pack_file_bldr_cfg cfg) { + public void Exec(Xowe_wiki wiki, Pack_file_cfg cfg) { // init wiki.Init_assert(); Io_url wiki_dir = wiki.Fsys_mgr().Root_dir(); Io_url pack_dir = wiki_dir.GenSubDir_nest("tmp", "pack"); Io_mgr.Instance.DeleteDirDeep(pack_dir); Io_mgr.Instance.CreateDirIfAbsent(pack_dir); String wiki_date = wiki.Props().Modified_latest().XtoStr_fmt("yyyy.MM"); - Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg.Pack_text(), cfg.Pack_html(), cfg.Pack_file(), cfg.Pack_file_cutoff(), cfg.Pack_fsdb_delete()); + Pack_hash hash = Pack_hash_bldr.Bld(wiki, wiki_dir, pack_dir, wiki_date, cfg); // get import_tbl byte[] wiki_abrv = wiki.Domain_itm().Abrv_xo(); Xobc_data_db bc_db = Xobc_data_db.New(wiki.App().Fsys_mgr()); Db_conn bc_conn = bc_db.Conn(); - bc_db.Delete_by_import(wiki_abrv, wiki_date); + if (!cfg.Pack_custom()) // only delete files if not custom + bc_db.Delete_by_import(wiki_abrv, wiki_date); bc_conn.Txn_bgn("xobc_import_insert"); // build zip packs @@ -54,6 +55,8 @@ public class Pack_file_mgr { Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "html", Xobc_import_type.Tid__wiki__core, Xobc_import_type.Tid__wiki__srch, Xobc_import_type.Tid__wiki__html, Xobc_import_type.Tid__wiki__ctg); if (cfg.Pack_file()) Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, "file", Xobc_import_type.Tid__file__core, Xobc_import_type.Tid__file__data); // , Xobc_import_type.Tid__fsdb__delete + if (cfg.Pack_custom()) + Make_task(tmp_bfr, wiki, wiki_date, bc_db, hash, cfg.Pack_custom_name(), Xobc_import_type.Tid__misc); bc_conn.Txn_end(); // deploy @@ -168,79 +171,3 @@ public class Pack_file_mgr { return String_.Split(task_key, "|"); } } -class Pack_hash_bldr { - public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, boolean pack_text, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff, boolean pack_fsdb_delete) { - Pack_hash rv = new Pack_hash(); - Pack_zip_name_bldr zip_name_bldr = new Pack_zip_name_bldr(pack_dir, wiki.Domain_str(), String_.new_a7(wiki.Domain_itm().Abrv_wm()), wiki_date); - Xow_db_mgr db_mgr = wiki.Data__core_mgr(); - - // bld html pack - if (pack_html) { - int len = db_mgr.Dbs__len(); - for (int i = 0; i < len; ++i) { - Xow_db_file file = db_mgr.Dbs__get_at(i); - int pack_tid = Get_pack_tid(file.Tid()); - if (pack_tid == Xobc_import_type.Tid__ignore) continue; - rv.Add(zip_name_bldr, pack_tid, file.Url()); - } - rv.Consolidate(Xobc_import_type.Tid__wiki__srch); - } - - // bld text pack - if (pack_text) { - int len = db_mgr.Dbs__len(); - for (int i = 0; i < len; ++i) { - Xow_db_file file = db_mgr.Dbs__get_at(i); - int pack_tid = Get_pack_tid(file.Tid()); - if (pack_tid == Xobc_import_type.Tid__ignore) continue; - rv.Add(zip_name_bldr, pack_tid, file.Url()); - } - } - - // bld file pack - if (pack_file) { - Fsm_mnt_itm mnt_itm = wiki.File__mnt_mgr().Mnts__get_at(Fsm_mnt_mgr.Mnt_idx_main); - rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__core, wiki_dir.GenSubFil(mnt_itm.Atr_mgr().Db__core().Url_rel())); - if (db_mgr.Props().Layout_file().Tid_is_lot()) { - Fsm_bin_mgr bin_mgr = mnt_itm.Bin_mgr(); - int bin_len = bin_mgr.Dbs__len(); - for (int i = 0; i < bin_len; ++i) { - Fsm_bin_fil bin_fil = bin_mgr.Dbs__get_at(i); - Io_url bin_fil_url = bin_fil.Url(); - - // ignore if bin_fil is earlier than cutoff - if (pack_file_cutoff != null) { - DateAdp bin_fil_date = Io_mgr.Instance.QueryFil(bin_fil_url).ModifiedTime(); - if (bin_fil_date.Timestamp_unix() < pack_file_cutoff.Timestamp_unix()) continue; - } - rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__data, bin_fil_url); - } - } - } - - // bld pack_fsdb_delete - if (pack_fsdb_delete) { - gplx.xowa.bldrs.Xob_db_file fsdb_deletion_db = gplx.xowa.bldrs.Xob_db_file.New__deletion_db(wiki); - if (!Io_mgr.Instance.ExistsFil(fsdb_deletion_db.Url())) throw Err_.new_wo_type("deletion db does not exists: url=" + fsdb_deletion_db.Url().Raw()); - rv.Add(zip_name_bldr, Xobc_import_type.Tid__fsdb__delete, fsdb_deletion_db.Url()); - } - return rv; - } - private static int Get_pack_tid(byte db_file_tid) { - switch (db_file_tid) { - case Xow_db_file_.Tid__core: return Xobc_import_type.Tid__wiki__core; - case Xow_db_file_.Tid__search_core: - case Xow_db_file_.Tid__search_link: return Xobc_import_type.Tid__wiki__srch; - case Xow_db_file_.Tid__html_solo: - case Xow_db_file_.Tid__html_data: return Xobc_import_type.Tid__wiki__html; - case Xow_db_file_.Tid__cat: - case Xow_db_file_.Tid__cat_core: - case Xow_db_file_.Tid__cat_link: return Xobc_import_type.Tid__wiki__ctg; - case Xow_db_file_.Tid__file_core: return Xobc_import_type.Tid__file__core; - case Xow_db_file_.Tid__file_solo: - case Xow_db_file_.Tid__file_data: return Xobc_import_type.Tid__file__data; - case Xow_db_file_.Tid__text: return Xobc_import_type.Tid__wiki__text; - default: return Xobc_import_type.Tid__ignore; - } - } -} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_hash_bldr.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_hash_bldr.java new file mode 100644 index 000000000..b49c8f127 --- /dev/null +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_hash_bldr.java @@ -0,0 +1,111 @@ +/* +XOWA: the XOWA Offline Wiki Application +Copyright (C) 2012 gnosygnu@gmail.com + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as +published by the Free Software Foundation, either version 3 of the +License, or (at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package gplx.xowa.addons.bldrs.exports.packs.files; import gplx.*; import gplx.xowa.*; import gplx.xowa.addons.*; import gplx.xowa.addons.bldrs.*; import gplx.xowa.addons.bldrs.exports.*; import gplx.xowa.addons.bldrs.exports.packs.*; +import gplx.fsdb.meta.*; +import gplx.xowa.wikis.data.*; +import gplx.xowa.addons.bldrs.centrals.dbs.datas.imports.*; +class Pack_hash_bldr { + public static Pack_hash Bld(Xow_wiki wiki, Io_url wiki_dir, Io_url pack_dir, String wiki_date, Pack_file_cfg cfg) { + // boolean pack_text, boolean pack_html, boolean pack_file, DateAdp pack_file_cutoff, boolean pack_fsdb_delete + Pack_hash rv = new Pack_hash(); + Pack_zip_name_bldr zip_name_bldr = new Pack_zip_name_bldr(pack_dir, wiki.Domain_str(), String_.new_a7(wiki.Domain_itm().Abrv_wm()), wiki_date, cfg.Pack_custom_name()); + Xow_db_mgr db_mgr = wiki.Data__core_mgr(); + + // bld custom_files + if (cfg.Pack_custom()) + return Bld_custom_files(rv, wiki, wiki_dir, zip_name_bldr, cfg.Pack_custom_files()); + + // bld html pack + if (cfg.Pack_html()) { + int len = db_mgr.Dbs__len(); + for (int i = 0; i < len; ++i) { + Xow_db_file file = db_mgr.Dbs__get_at(i); + int pack_tid = Get_pack_tid(file.Tid()); + if (pack_tid == Xobc_import_type.Tid__ignore) continue; + rv.Add(zip_name_bldr, pack_tid, file.Url()); + } + rv.Consolidate(Xobc_import_type.Tid__wiki__srch); + } + + // bld text pack + if (cfg.Pack_text()) { + int len = db_mgr.Dbs__len(); + for (int i = 0; i < len; ++i) { + Xow_db_file file = db_mgr.Dbs__get_at(i); + int pack_tid = Get_pack_tid(file.Tid()); + if (pack_tid == Xobc_import_type.Tid__ignore) continue; + rv.Add(zip_name_bldr, pack_tid, file.Url()); + } + } + + // bld file pack + if (cfg.Pack_file()) { + Fsm_mnt_itm mnt_itm = wiki.File__mnt_mgr().Mnts__get_at(Fsm_mnt_mgr.Mnt_idx_main); + rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__core, wiki_dir.GenSubFil(mnt_itm.Atr_mgr().Db__core().Url_rel())); + if (db_mgr.Props().Layout_file().Tid_is_lot()) { + Fsm_bin_mgr bin_mgr = mnt_itm.Bin_mgr(); + int bin_len = bin_mgr.Dbs__len(); + for (int i = 0; i < bin_len; ++i) { + Fsm_bin_fil bin_fil = bin_mgr.Dbs__get_at(i); + Io_url bin_fil_url = bin_fil.Url(); + + // ignore if bin_fil is earlier than cutoff + if (cfg.Pack_file_cutoff() != null) { + DateAdp bin_fil_date = Io_mgr.Instance.QueryFil(bin_fil_url).ModifiedTime(); + if (bin_fil_date.Timestamp_unix() < cfg.Pack_file_cutoff().Timestamp_unix()) continue; + } + rv.Add(zip_name_bldr, Xobc_import_type.Tid__file__data, bin_fil_url); + } + } + } + + // bld pack_fsdb_delete + if (cfg.Pack_fsdb_delete()) { + gplx.xowa.bldrs.Xob_db_file fsdb_deletion_db = gplx.xowa.bldrs.Xob_db_file.New__deletion_db(wiki); + if (!Io_mgr.Instance.ExistsFil(fsdb_deletion_db.Url())) throw Err_.new_wo_type("deletion db does not exists: url=" + fsdb_deletion_db.Url().Raw()); + rv.Add(zip_name_bldr, Xobc_import_type.Tid__fsdb__delete, fsdb_deletion_db.Url()); + } + return rv; + } + private static Pack_hash Bld_custom_files(Pack_hash rv, Xow_wiki wiki, Io_url wiki_dir, Pack_zip_name_bldr zip_name_bldr, String custom_files_blob) { + String[] custom_files = String_.Split(custom_files_blob, "|"); + int len = custom_files.length; + for (int i = 0; i < len; ++i) { + Io_url file_url = wiki_dir.GenSubFil(custom_files[i]); + rv.Add(zip_name_bldr, Xobc_import_type.Tid__misc, file_url); + } + return rv; + } + private static int Get_pack_tid(byte db_file_tid) { + switch (db_file_tid) { + case Xow_db_file_.Tid__core: return Xobc_import_type.Tid__wiki__core; + case Xow_db_file_.Tid__search_core: + case Xow_db_file_.Tid__search_link: return Xobc_import_type.Tid__wiki__srch; + case Xow_db_file_.Tid__html_solo: + case Xow_db_file_.Tid__html_data: return Xobc_import_type.Tid__wiki__html; + case Xow_db_file_.Tid__cat: + case Xow_db_file_.Tid__cat_core: + case Xow_db_file_.Tid__cat_link: return Xobc_import_type.Tid__wiki__ctg; + case Xow_db_file_.Tid__file_core: return Xobc_import_type.Tid__file__core; + case Xow_db_file_.Tid__file_solo: + case Xow_db_file_.Tid__file_data: return Xobc_import_type.Tid__file__data; + case Xow_db_file_.Tid__text: return Xobc_import_type.Tid__wiki__text; + default: return Xobc_import_type.Tid__ignore; + } + } +} diff --git a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr.java b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr.java index 577b4b043..7c75846a9 100644 --- a/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr.java +++ b/400_xowa/src/gplx/xowa/addons/bldrs/exports/packs/files/Pack_zip_name_bldr.java @@ -20,10 +20,12 @@ public class Pack_zip_name_bldr { // en.wikipedia.org-file-ns.000-db.001.xowa -> private final Io_url pack_dir; private final byte[] wiki_domain, zip_name_prefix; - public Pack_zip_name_bldr(Io_url pack_dir, String wiki_domain_str, String wiki_abrv, String wiki_date) { + public Pack_zip_name_bldr(Io_url pack_dir, String wiki_domain_str, String wiki_abrv, String wiki_date, String custom_name) { this.pack_dir = pack_dir; this.wiki_domain = Bry_.new_u8(wiki_domain_str); - this.zip_name_prefix = Bry_.new_u8("Xowa_" + wiki_abrv + "_" + String_.Replace(wiki_date, ".", "-")); + + String zip_name_suffix = custom_name == null ? String_.Replace(wiki_date, ".", "-") : custom_name; // "Xowa_enwiki_2016-10_core.zip" vs "Xowa_enwiki_custom_core.zip" + this.zip_name_prefix = Bry_.new_u8("Xowa_" + wiki_abrv + "_" + zip_name_suffix); } public Io_url Bld(Io_url orig_url) { String orig_str = orig_url.NameOnly() + ".zip";