Category: Support addition / deletion in existing WMF wikis

400_xowa/src/gplx/xowa/addons/wikis/ctgs/dbs/Xodb_cat_link_tbl.java

@@ -42,7 +42,14 @@ public class Xodb_cat_link_tbl implements Db_tbl {
 	public void Insert_bgn() {conn.Txn_bgn("cl__insert"); stmt_insert = conn.Stmt_insert(tbl_name, flds);}
 	public void Insert_end() {conn.Txn_end(); stmt_insert = Db_stmt_.Rls(stmt_insert);}
 	public void Insert_cmd_by_batch(int from, int to_id, byte type_id, long timestamp_unix, byte[] sortkey, byte[] sortkey_prefix) {
-		stmt_insert.Clear()
+		this.Insert_cmd_by_batch(stmt_insert, from, to_id, type_id, timestamp_unix, sortkey, sortkey_prefix);
+	}
+	public void Insert_(int from, int to_id, byte type_id, long timestamp_unix, byte[] sortkey, byte[] sortkey_prefix) {
+		Db_stmt stmt = conn.Stmt_insert(tbl_name, flds);
+		this.Insert_cmd_by_batch(stmt, from, to_id, type_id, timestamp_unix, sortkey, sortkey_prefix);
+	}
+	private void Insert_cmd_by_batch(Db_stmt stmt, int from, int to_id, byte type_id, long timestamp_unix, byte[] sortkey, byte[] sortkey_prefix) {
+		stmt.Clear()
 			.Val_int(fld__from					, from)
 			.Val_int(fld__to_id					, to_id)
 			.Val_byte(fld__type_id				, type_id)
@@ -98,34 +105,8 @@ public class Xodb_cat_link_tbl implements Db_tbl {
 	public void Rls() {
 		stmt_insert = Db_stmt_.Rls(stmt_insert);
 	}
+
 	public static final String TBL_NAME = "cat_link", FLD__cl_sortkey_prefix = "cl_sortkey_prefix";
-	public static Xodb_cat_link_tbl[] Get_catlink_tbls(Xow_db_mgr db_mgr) {
-		List_adp rv = List_adp_.New();
-
-		boolean layout_is_lot = db_mgr.Props().Layout_text().Tid_is_lot();
-
-		// loop all dbs
-		int len = db_mgr.Dbs__len();
-		for (int i = 0; i < len; i++) {
-			Xow_db_file link_db = db_mgr.Dbs__get_at(i);
-			switch (link_db.Tid()) {
-				// if core, add if all or few; skip if lot
-				case Xow_db_file_.Tid__core:
-					if (layout_is_lot)
-						continue;
-					break;
-				// if cat_link, add
-				case Xow_db_file_.Tid__cat_link:
-					break;
-				// else, skip
-				default:
-					continue;
-			}
-			rv.Add(new Xodb_cat_link_tbl(link_db.Conn()));
-		}
-
-		return (Xodb_cat_link_tbl[])rv.To_ary_and_clear(Xodb_cat_link_tbl.class);
-	}
 }
 /*
 NOTE_1: categorylinks row size: 34 + 20 + 12 + (cat_sortkey.length * 2)

400_xowa/src/gplx/xowa/addons/wikis/ctgs/edits/Xoctg_edit_mgr.java

@@ -23,82 +23,89 @@ import gplx.xowa.addons.wikis.ctgs.dbs.*; import gplx.xowa.addons.wikis.ctgs.htm
 import gplx.xowa.addons.wikis.directorys.specials.items.bldrs.*;
 public class Xoctg_edit_mgr {
 	public static void Update(Xowe_wiki wiki, byte[] ttl_bry, int page_id, Xoa_ttl[] ctg_ttls) {
-		// only apply to home or other wiki
+		// get ttl, page, ns_id
-		if (!(   wiki.Domain_tid() == Xow_domain_tid_.Tid__other
-			||   wiki.Domain_tid() == Xow_domain_tid_.Tid__home))
-			return;
-
-		// get page
 		Xoa_ttl ttl = wiki.Ttl_parse(ttl_bry);
 		int ns_id = ttl.Ns().Id();
 		Xoae_page wpg = Xoae_page.New_edit(wiki, ttl);
 		wpg.Db().Page().Id_(page_id);
 
+		// delete old categories
+		Delete(wiki, ns_id, page_id);
+
+		// insert new categories
 		// get page_tbl
 		Xow_db_mgr db_mgr = wiki.Data__core_mgr();
-		Xowd_page_tbl page_tbl = db_mgr.Db__core().Tbl__page();
+		Xow_db_file core_db = db_mgr.Db__core();
+		Xowd_page_tbl page_tbl = core_db.Tbl__page();
 
 		// get cat_core_tbl
 		Xowd_cat_core_tbl cat_core_tbl = Xodb_cat_db_.Get_cat_core_or_fail(db_mgr);
 
-		// get cat_link_tbl
+		// get last cat_link_tbl; note that
-		Xodb_cat_link_tbl cat_link_tbl = new Xodb_cat_link_tbl(cat_core_tbl.Conn());
+		// * cat_link tbls are organized by page
-		Xowd_page_itm tmp_page = new Xowd_page_itm();
+		// * all old cat_links have been deleted
-		db_mgr.Tbl__page().Select_by_id(tmp_page, page_id);
+		// * new catlinks will go into last db
+		Xow_db_file last_cat_link_db = db_mgr.Dbs__get_for_create(Xow_db_file_.Tid__cat_link, ns_id);
+		Xodb_cat_link_tbl last_cat_link_tbl = new Xodb_cat_link_tbl(last_cat_link_db.Conn());
 
-		// delete old categories
+		// get last text tbl for new categories; EX: [[Category:New]] will create entry in page_tb with text_db_id set to last_text_db
-		Delete(wiki, ns_id, page_id);
+		Xow_db_file last_text_db = db_mgr.Dbs__get_for_create(Xow_db_file_.Tid__text, ns_id);
+		Xowd_text_tbl last_text_tbl = new Xowd_text_tbl(last_text_db.Conn(), db_mgr.Props().Schema_is_1(), db_mgr.Props().Zip_tid_text());
 
-		// get some variables
+		// get some variables for creating cat_link rows
 		int timestamp = (int)Datetime_now.Get().Timestamp_unix();
 		Xoctg_collation_mgr collation_mgr = wiki.Ctg__catpage_mgr().Collation_mgr();
-		Xow_db_file core_db = db_mgr.Db__core();
+		Xowd_page_itm tmp_page = new Xowd_page_itm();
 
-		// cat_core:update; for each category, add one to count_page, count_file, or count_subcs
+		// loop over each category listed on page
-		int ctg_ttls_len = ctg_ttls.length;
+		for (Xoa_ttl ctg_ttl : ctg_ttls) {
-		cat_link_tbl.Insert_bgn();
+			// get page_tbl data for sub_cat
-		for (int i = 0; i < ctg_ttls_len; i++) {
+			boolean exists = page_tbl.Select_by_ttl(tmp_page, ctg_ttl);
-			// get cat_core itm for sub_cat
+			int ctg_id = tmp_page.Id();
-			Xoa_ttl sub_ttl = ctg_ttls[i];
-			boolean exists = page_tbl.Select_by_ttl(tmp_page, sub_ttl);
 
 			// create category if it doesn't exist
-			int sub_id = tmp_page.Id();
 			if (!exists) {
-				sub_id = Xopg_db_mgr.Create
+				// create [[Category]] page
-					( core_db.Tbl__page(), core_db.Tbl__text(), core_db.Tbl__ns(), core_db.Tbl__cfg()
+				ctg_id = Xopg_db_mgr.Create
-					, gplx.xowa.wikis.nss.Xow_ns_.Tid__category, sub_ttl.Page_db(), Bry_.Empty, -1);
+					( page_tbl, last_text_tbl, last_text_db.Id(), core_db.Tbl__ns(), core_db.Tbl__cfg()
+					, gplx.xowa.wikis.nss.Xow_ns_.Tid__category, ctg_ttl.Page_db(), Bry_.Empty
+					, last_cat_link_db.Id()); // NOTE: new categories go into last cat_link_db
+
+				// create cat_core row
 				cat_core_tbl.Insert_bgn();
-				cat_core_tbl.Insert_cmd_by_batch(sub_id, 0, 0, 0, Bool_.N_byte, -1);
+				cat_core_tbl.Insert_cmd_by_batch(ctg_id, 0, 0, 0, Bool_.N_byte, -1);
 				cat_core_tbl.Insert_end();
 			}
 
-			Xowd_category_itm sub_core_itm = cat_core_tbl.Select(sub_id);
+			// get cat_core_itm
+			Xowd_category_itm cat_core_itm = cat_core_tbl.Select(ctg_id);
 
 			// adjust it and save it
-			sub_core_itm.Adjust(ns_id, 1);
+			cat_core_itm.Adjust(ns_id, 1);
-			cat_core_tbl.Update(sub_core_itm);
+			cat_core_tbl.Update(cat_core_itm);
 
-			// cat_link:add
+			// add to cat_link tbl
-			cat_link_tbl.Insert_cmd_by_batch(page_id, sub_id, Xoa_ctg_mgr.To_tid_by_ns(ns_id), timestamp, collation_mgr.Get_sortkey(wpg.Ttl().Page_db()), wpg.Ttl().Page_db());
+			last_cat_link_tbl.Insert_(page_id, ctg_id, Xoa_ctg_mgr.To_tid_by_ns(ns_id), timestamp, collation_mgr.Get_sortkey(wpg.Ttl().Page_db()), wpg.Ttl().Page_db());
 		}
-		cat_link_tbl.Insert_end();
 
 		// update page.cat_db_id
-		page_tbl.Update__cat_db_id(page_id, core_db.Id());
+		page_tbl.Update__cat_db_id(page_id, last_cat_link_db.Id());
 	}
 	public static void Delete(Xowe_wiki wiki, int ns_id, int page_id) {
+		Xow_db_mgr db_mgr = wiki.Data__core_mgr();
 		boolean ns_id_is_category = ns_id == Xow_ns_.Tid__category;
 
 		// get cat_core_tbl
-		Xowd_cat_core_tbl cat_core_tbl = Xodb_cat_db_.Get_cat_core_or_fail(wiki.Data__core_mgr());
+		Xowd_cat_core_tbl cat_core_tbl = Xodb_cat_db_.Get_cat_core_or_fail(db_mgr);
 
 		// get cat_link_tbls
-		Xodb_cat_link_tbl[] cat_link_tbls = Xodb_cat_link_tbl.Get_catlink_tbls(wiki.Data__core_mgr());
+		Xow_db_file[] cat_link_dbs = db_mgr.Dbs__get_ary(Xow_db_file_.Tid__cat_link, ns_id);
 
 		// loop cat_link tbls to find linked categories
-		for (Xodb_cat_link_tbl cat_link_tbl : cat_link_tbls) {
+		for (Xow_db_file cat_link_db : cat_link_dbs) {
+			Xodb_cat_link_tbl cat_link_tbl = new Xodb_cat_link_tbl(cat_link_db.Conn());
 			Xodb_cat_link_row[] cat_link_rows = cat_link_tbl.Select_by_page_id(page_id);
+
 			// loop linked categories
 			for (Xodb_cat_link_row cat_link_row : cat_link_rows) {
 				// get cat_core_itm
@@ -119,15 +126,20 @@ public class Xoctg_edit_mgr {
 		if (ns_id_is_category) {
 			cat_core_tbl.Delete(page_id);
 		}
+
+		// set cat_db_id to -1
+		db_mgr.Tbl__page().Update__cat_db_id(page_id, -1);
 	}
 	public static void Update_page_id(Xow_db_mgr db_mgr, int ns_id, int old_id, int new_id) {
 		boolean ns_id_is_category = ns_id == Xow_ns_.Tid__category;
 
 		// get cat_link_tbls
-		Xodb_cat_link_tbl[] cat_link_tbls = Xodb_cat_link_tbl.Get_catlink_tbls(db_mgr);
+		Xow_db_file[] cat_link_dbs = db_mgr.Dbs__get_ary(Xow_db_file_.Tid__cat_link, ns_id);
 
 		// loop cat_link tbls to find linked categories
-		for (Xodb_cat_link_tbl cat_link_tbl : cat_link_tbls) {
+		for (Xow_db_file cat_link_db : cat_link_dbs) {
+			Xodb_cat_link_tbl cat_link_tbl = new Xodb_cat_link_tbl(cat_link_db.Conn());
+
 			// delete cat_links
 			cat_link_tbl.Update_page_id_for_pages(old_id, new_id);
 			if (ns_id_is_category)

400_xowa/src/gplx/xowa/addons/wikis/ctgs/htmls/pageboxs/Xoctg_pagebox_wtr.java

@@ -47,6 +47,7 @@ public class Xoctg_pagebox_wtr implements Gfo_invk {
 		Xoctg_pagebox_loader select_cbk = new Xoctg_pagebox_loader(hash, page.Url_bry_safe());
 
 		// get cat_db_id from page
+		tmp_page_itm.Clear();
 		boolean exists = wiki.Data__core_mgr().Tbl__page().Select_by_ttl(tmp_page_itm, page.Ttl().Ns(), page.Ttl().Page_db());
 		int cat_db_id = tmp_page_itm.Cat_db_id();
 		if (exists && cat_db_id != -1) {// note that wtxt_dbs can have 0 ctgs but will have cat_db_id == -1

400_xowa/src/gplx/xowa/addons/wikis/directorys/specials/items/bldrs/Xopg_db_mgr.java

@@ -24,10 +24,21 @@ import gplx.xowa.addons.wikis.directorys.specials.items.bldrs.*;
 import gplx.xowa.addons.wikis.searchs.*; import gplx.xowa.addons.wikis.searchs.dbs.*;
 public class Xopg_db_mgr {
 	public static int Create
-		( Xowd_page_tbl page_tbl, Xowd_text_tbl text_tbl, Xowd_site_ns_tbl ns_tbl, Db_cfg_tbl cfg_tbl
+		( Xowd_page_tbl page_tbl, Xowd_text_tbl text_tbl, int text_db_id, Xowd_site_ns_tbl ns_tbl, Db_cfg_tbl cfg_tbl
 		, int ns_id, byte[] ttl_page_db, byte[] text_raw, int cat_db_id) {
 		// get next page_id			
 		int page_id = cfg_tbl.Select_int_or(Xowd_cfg_key_.Grp__db, Xowd_cfg_key_.Key__wiki__page__id_next, 1);
+
+		// check if page_id is unique; needed for existing WMF .xowa dbs which don't set Xowd_cfg_key_.Key__wiki__page__id_next; DATE:2017-02-19
+		if (page_tbl.Select_by_id_or_null(page_id) != null) {
+			int max_page_id = page_tbl.Conn().Exec_select_max_as_int(page_tbl.Tbl_name(), page_tbl.Fld_page_id(), Xowd_page_tbl.INVALID_PAGE_ID);
+			if (max_page_id == Xowd_page_tbl.INVALID_PAGE_ID) {
+				throw Err_.new_wo_type("no max found in page_tbl even though page_id was not unique?: db=~{0} page_id=~{1}", page_tbl.Conn().Conn_info().Db_api(), page_id);
+			}
+			page_id = max_page_id + 1;
+		}
+
+		// update it
 		cfg_tbl.Upsert_int(Xowd_cfg_key_.Grp__db, Xowd_cfg_key_.Key__wiki__page__id_next, page_id + 1);
 
 		// zip if needed
@@ -41,7 +52,7 @@ public class Xopg_db_mgr {
 		text_tbl.Insert_bgn();
 		int ns_count = ns_tbl.Select_ns_count(ns_id) + 1;
 		try {
-			page_tbl.Insert_cmd_by_batch(page_id, ns_id, ttl_page_db, redirect, Datetime_now.Get(), text_raw.length, ns_count, 0, -1, cat_db_id);
+			page_tbl.Insert_cmd_by_batch(page_id, ns_id, ttl_page_db, redirect, Datetime_now.Get(), text_raw.length, ns_count, text_db_id, -1, cat_db_id);
 			text_tbl.Insert_cmd_by_batch(page_id, text_zip);
 			ns_tbl.Update_ns_count(ns_id, ns_count);
 		} finally {

400_xowa/src/gplx/xowa/addons/wikis/directorys/specials/items/bldrs/Xow_db_mkr.java

@@ -65,7 +65,7 @@ public class Xow_db_mkr {
 		// insert data: page
 		Xopg_db_mgr.Create
 			( Xowd_page_tbl.Get_by_key(core_db)
-			, Xowd_text_tbl.Get_by_key(core_db)
+			, Xowd_text_tbl.Get_by_key(core_db), Xow_db_file_.Uid__core
 			, Xowd_site_ns_tbl.Get_by_key(core_db)
 			, Db_cfg_tbl.Get_by_key(core_db, Xowd_cfg_tbl_.Tbl_name)
 			, Xow_ns_.Tid__main, mainpage_name, mainpage_text

400_xowa/src/gplx/xowa/wikis/data/Xow_db_mgr.java

@@ -130,6 +130,53 @@ public class Xow_db_mgr {
 		// call init again to regen list of dbs
 		this.Init_by_load(db__core.Url());
 	}
+	public Xow_db_file Dbs__get_for_create(byte tid, int ns_id) {
+		Xow_db_file[] ary = Dbs__get_ary(tid, ns_id);
+		if (ary.length == 0) throw Err_.new_wo_type("no dbs exist; wiki=~{0} type=~{1}", domain_str, tid);
+		return ary[ary.length - 1];
+	}
+	public Xow_db_file[] Dbs__get_ary(byte tid, int ns_id) {
+		List_adp rv = List_adp_.New();
+
+		// loop all dbs
+		int len = this.Dbs__len();
+		for (int i = 0; i < len; i++) {
+			boolean add = false;
+			Xow_db_file db = this.Dbs__get_at(i);
+			switch (tid) {
+				// cat_link requested
+				case Xow_db_file_.Tid__cat_link:
+					switch (db.Tid()) {
+						case Xow_db_file_.Tid__core:
+							add = props.Layout_text().Tid_is_all_or_few(); // cat_link will be in "core" if "all" or "few" (-text, -html, -file)
+							break;
+						case Xow_db_file_.Tid__cat_link:
+							add = true;
+							break;
+					}
+					break;
+				// text requested
+				case Xow_db_file_.Tid__text:
+					switch (db.Tid()) {
+						case Xow_db_file_.Tid__core:
+							add = props.Layout_text().Tid_is_all(); // text will be in "core" if "all"
+							break;
+						case Xow_db_file_.Tid__text_solo:  // EX: "en.wikipedia.org-text.xowa"
+							add = true;                    // text will be in db if solo; 
+							break;
+						case Xow_db_file_.Tid__text:       // EX: "en.wikipedia.org-text-ns.000.xowa"
+							int db_ns_id = Int_.parse(db.Ns_ids());
+							add = db_ns_id == ns_id;       // text will be in db if ns matches; EX: en.wikipedia.org-text-ns.014.xowa
+							break;
+					}
+					break;
+			}
+			if (add)
+				rv.Add(db);
+		}
+
+		return (Xow_db_file[])rv.To_ary_and_clear(Xow_db_file.class);
+	}
 	public void Create_page(Xowd_page_tbl core_tbl, Xowd_text_tbl text_tbl, int page_id, int ns_id, byte[] ttl_wo_ns, boolean redirect, DateAdp modified_on, byte[] text_zip_data, int text_raw_len, int random_int, int text_db_id, int html_db_id) {
 		core_tbl.Insert_cmd_by_batch(page_id, ns_id, ttl_wo_ns, redirect, modified_on, text_raw_len, random_int, text_db_id, html_db_id, -1);
 		text_tbl.Insert_cmd_by_batch(page_id, text_zip_data);

400_xowa/src/gplx/xowa/wikis/data/tbls/Xowd_page_itm.java

@@ -29,7 +29,7 @@ public class Xowd_page_itm {
 	public int			Text_len() {return text_len;} public Xowd_page_itm Text_len_(int v) {text_len = v; return this;} private int text_len;
 	public int			Text_db_id() {return text_db_id;} public Xowd_page_itm Text_db_id_(int v) {text_db_id = v; return this;} private int text_db_id;
 	public int			Html_db_id() {return html_db_id;} private int html_db_id;
-	public int			Cat_db_id()  {return cat_db_id;} private int cat_db_id;
+	public int			Cat_db_id()  {return cat_db_id;} private int cat_db_id = -1; // NOTE: cannot be 0, else catpage loader will try to load cat_links from db.id=0
 	public byte[]		Text() {return text;} public Xowd_page_itm Text_(byte[] v) {text = v; if (v != null) text_len = v.length; return this;} private byte[] text;
 	public int			Random_int() {return random_int;} private int random_int;
 	public int			Redirect_id() {return redirect_id;} private int redirect_id;

400_xowa/src/gplx/xowa/wikis/data/tbls/Xowd_page_tbl.java

@@ -400,4 +400,5 @@ public class Xowd_page_tbl implements Db_tbl {
 	public static final    String Page_touched_fmt = "yyyyMMddHHmmss";
 	public static final String TBL_NAME = "page", FLD__page_cat_db_id = "page_cat_db_id";
 	public static Xowd_page_tbl Get_by_key(Db_tbl_owner owner) {return (Xowd_page_tbl)owner.Tbls__get_by_key(TBL_NAME);}
+	public static final int INVALID_PAGE_ID = -1;
 }