From 8daf05754d6efc217caa367d3be01b20ae44ede6 Mon Sep 17 00:00:00 2001 From: gnosygnu Date: Fri, 2 Sep 2016 15:39:48 -0400 Subject: [PATCH] $version_number --- home/wiki/Blog/2016-01.html | 2 +- home/wiki/Blog/2016-06.html | 2 +- home/wiki/Change_log/2016.html | 2 +- home/wiki/Dashboard/Wiki_maintenance.html | 202 ++++++++++++++++++++-- home/wiki/Dev/Command-line/Dumps.html | 27 ++- home/wiki/Dev/Command-line/Thumbs.html | 27 ++- home/wiki/Help/Contents.html | 4 +- home/wiki/Options/Files.html | 6 +- home/wiki/Options/Menus.html | 14 +- 9 files changed, 257 insertions(+), 29 deletions(-) diff --git a/home/wiki/Blog/2016-01.html b/home/wiki/Blog/2016-01.html index 446082ac4..a1d9c76b8 100644 --- a/home/wiki/Blog/2016-01.html +++ b/home/wiki/Blog/2016-01.html @@ -399,7 +399,7 @@ The Android app is a major release. It has a Random feature, shows more images, and adds CSS-tweaks.

- Documentation for html-dump script. See Dev/Command-line/Dumps + Documentation for html-dump script. See Dev/Command-line/Dumps

This item is self-explanatory. The XOWA Android app is getting more stable, so I felt it would be time to document the generation of the HTML databases. diff --git a/home/wiki/Blog/2016-06.html b/home/wiki/Blog/2016-06.html index 1e72c0691..fe05a5b7b 100644 --- a/home/wiki/Blog/2016-06.html +++ b/home/wiki/Blog/2016-06.html @@ -420,7 +420,7 @@

diff --git a/home/wiki/Change_log/2016.html b/home/wiki/Change_log/2016.html index 6cd033b9d..946e4c9ef 100644 --- a/home/wiki/Change_log/2016.html +++ b/home/wiki/Change_log/2016.html @@ -908,7 +908,7 @@

diff --git a/home/wiki/Dashboard/Wiki_maintenance.html b/home/wiki/Dashboard/Wiki_maintenance.html index e4a26bac1..0e9195101 100644 --- a/home/wiki/Dashboard/Wiki_maintenance.html +++ b/home/wiki/Dashboard/Wiki_maintenance.html @@ -150,6 +150,76 @@ + + + cs.wikibooks.org + + + n + + + 2016-08-29 + + + 2015-08-05 + + + Dump complete + + + sqlite3 + + + v2 + + + v1 + + + + + + + + + + + + + + cs.wikinews.org + + + n + + + 2016-09-01 + + + 2015-08-06 + + + Dump complete + + + sqlite3 + + + v2 + + + v1 + + + + + + + + + + + cs.wikipedia.org @@ -158,7 +228,7 @@ n - 2016-08-01 + 2016-09-01 2015-08-06 @@ -187,36 +257,142 @@ - cs.wikipedia_old.org + cs.wikiquote.org n - 1969-12-31 + 2016-08-31 + 2015-08-05 - <<NULL>> + Dump complete - xdat - - - v1 + sqlite3 v2 - + v1 - + - + + + + + + + + + cs.wikisource.org + + + n + + + 2016-09-01 + + + 2015-08-05 + + + Dump complete + + + sqlite3 + + + v2 + + + v1 + + + + + + + + + + + + + + cs.wikiversity.org + + + n + + + 2016-08-31 + + + 2015-08-06 + + + Dump complete + + + sqlite3 + + + v2 + + + v1 + + + + + + + + + + + + + + cs.wiktionary.org + + + n + + + 2016-09-01 + + + 2015-08-05 + + + Dump complete + + + sqlite3 + + + v2 + + + v1 + + + + + + + + + @@ -716,7 +892,7 @@ n - 2016-06-01 + 2016-09-01 2015-08-06 @@ -925,7 +1101,7 @@ n - 2016-08-02 + 2016-08-21 2015-08-05 diff --git a/home/wiki/Dev/Command-line/Dumps.html b/home/wiki/Dev/Command-line/Dumps.html index 56e6e04a7..fbeba4d5f 100644 --- a/home/wiki/Dev/Command-line/Dumps.html +++ b/home/wiki/Dev/Command-line/Dumps.html @@ -343,6 +343,7 @@ app.bldr.cmds { add ('simple.wikipedia.org' , 'util.download') {dump_type = 'page_props';} add ('simple.wikipedia.org' , 'util.download') {dump_type = 'image';} add ('simple.wikipedia.org' , 'util.download') {dump_type = 'pagelinks';} + add ('simple.wikipedia.org' , 'util.download') {dump_type = 'imagelinks';} add ('simple.wikipedia.org' , 'text.init'); add ('simple.wikipedia.org' , 'text.page') { // calculate redirect_id for #REDIRECT pages. needed for html databases @@ -370,6 +371,9 @@ app.bldr.cmds { // create an "image" table to get the metadata for all files in the current wiki add ('simple.wikipedia.org' , 'wiki.image'); + // create an "imagelinks" table to find out which images are used for the wiki (performance optimization) + add ('simple.wikipedia.org' , 'wiki.imagelinks'); + // parse all page-to-page links add ('simple.wikipedia.org' , 'wiki.page_link'); @@ -384,7 +388,8 @@ app.bldr.cmds { // cleanup all downloaded files as well as temporary files add ('simple.wikipedia.org' , 'util.cleanup') {delete_tmp = 'y'; delete_by_match('*.xml|*.sql|*.bz2|*.gz');} - + + // v1 html generator // parse every page in the listed namespace and gather data on their lnkis. // this step will take the longest amount of time. add ('simple.wikipedia.org' , 'file.lnki_temp') { @@ -419,6 +424,24 @@ app.bldr.cmds { hzip_diff = 'y'; } } + + // v2 html generator; allows for multi-threaded / multi-machine builds + /* + add ('simple.wikipedia.org' , 'wiki.mass_parse.init') {cfg {ns_ids = '0|4|14';}} + add ('simple.wikipedia.org' , 'wiki.mass_parse.exec') { + cfg { + num_wkrs = 8; load_all_templates = 'y'; cleanup_interval = 50; hzip_enabled = 'y'; hdiff_enabled ='y'; manual_now = '2016-08-01 01:02:03'; + + // uncomment the following 3 lines if using the build script as a "worker" helping a "server" + // num_pages_in_pool = 32000; + // mgr_url = '\\server_machine_name\xowa\wiki\en.wikipedia.org\tmp\xomp\'; + // wkr_machine_name = 'worker_machine_1' + } + } + + // note that if multi-machine mode is enabled, all worker directories must be manually copied to the server directory (a build command will be added later) + add ('simple.wikipedia.org' , 'wiki.mass_parse.make'); + */ // aggregate the lnkis add ('simple.wikipedia.org' , 'file.lnki_regy'); @@ -511,6 +534,7 @@ app.bldr.cmds { add ('en.wikipedia.org' , 'util.download') {dump_type = 'page_props';} add ('en.wikipedia.org' , 'util.download') {dump_type = 'image';} add ('en.wikipedia.org' , 'util.download') {dump_type = 'pagelinks';} + add ('en.wikipedia.org' , 'util.download') {dump_type = 'imagelinks';} */ /* add ('en.wikipedia.org' , 'text.init'); @@ -525,6 +549,7 @@ app.bldr.cmds { add ('en.wikipedia.org' , 'wiki.image'); add ('en.wikipedia.org' , 'wiki.page_dump.make'); add ('en.wikipedia.org' , 'wiki.page_link'); + add ('en.wikipedia.org' , 'wiki.imagelinks'); add ('en.wikipedia.org' , 'search.page__page_score') {iteration_max = 100;} add ('en.wikipedia.org' , 'search.link__link_score') {page_rank_enabled = 'y'; score_adjustment_mgr { diff --git a/home/wiki/Dev/Command-line/Thumbs.html b/home/wiki/Dev/Command-line/Thumbs.html index 901134d85..0be802eaa 100644 --- a/home/wiki/Dev/Command-line/Thumbs.html +++ b/home/wiki/Dev/Command-line/Thumbs.html @@ -343,6 +343,7 @@ app.bldr.cmds { add ('simple.wikipedia.org' , 'util.download') {dump_type = 'page_props';} add ('simple.wikipedia.org' , 'util.download') {dump_type = 'image';} add ('simple.wikipedia.org' , 'util.download') {dump_type = 'pagelinks';} + add ('simple.wikipedia.org' , 'util.download') {dump_type = 'imagelinks';} add ('simple.wikipedia.org' , 'text.init'); add ('simple.wikipedia.org' , 'text.page') { // calculate redirect_id for #REDIRECT pages. needed for html databases @@ -370,6 +371,9 @@ app.bldr.cmds { // create an "image" table to get the metadata for all files in the current wiki add ('simple.wikipedia.org' , 'wiki.image'); + // create an "imagelinks" table to find out which images are used for the wiki (performance optimization) + add ('simple.wikipedia.org' , 'wiki.imagelinks'); + // parse all page-to-page links add ('simple.wikipedia.org' , 'wiki.page_link'); @@ -384,7 +388,8 @@ app.bldr.cmds { // cleanup all downloaded files as well as temporary files add ('simple.wikipedia.org' , 'util.cleanup') {delete_tmp = 'y'; delete_by_match('*.xml|*.sql|*.bz2|*.gz');} - + + // v1 html generator // parse every page in the listed namespace and gather data on their lnkis. // this step will take the longest amount of time. add ('simple.wikipedia.org' , 'file.lnki_temp') { @@ -419,6 +424,24 @@ app.bldr.cmds { hzip_diff = 'y'; } } + + // v2 html generator; allows for multi-threaded / multi-machine builds + /* + add ('simple.wikipedia.org' , 'wiki.mass_parse.init') {cfg {ns_ids = '0|4|14';}} + add ('simple.wikipedia.org' , 'wiki.mass_parse.exec') { + cfg { + num_wkrs = 8; load_all_templates = 'y'; cleanup_interval = 50; hzip_enabled = 'y'; hdiff_enabled ='y'; manual_now = '2016-08-01 01:02:03'; + + // uncomment the following 3 lines if using the build script as a "worker" helping a "server" + // num_pages_in_pool = 32000; + // mgr_url = '\\server_machine_name\xowa\wiki\en.wikipedia.org\tmp\xomp\'; + // wkr_machine_name = 'worker_machine_1' + } + } + + // note that if multi-machine mode is enabled, all worker directories must be manually copied to the server directory (a build command will be added later) + add ('simple.wikipedia.org' , 'wiki.mass_parse.make'); + */ // aggregate the lnkis add ('simple.wikipedia.org' , 'file.lnki_regy'); @@ -511,6 +534,7 @@ app.bldr.cmds { add ('en.wikipedia.org' , 'util.download') {dump_type = 'page_props';} add ('en.wikipedia.org' , 'util.download') {dump_type = 'image';} add ('en.wikipedia.org' , 'util.download') {dump_type = 'pagelinks';} + add ('en.wikipedia.org' , 'util.download') {dump_type = 'imagelinks';} */ /* add ('en.wikipedia.org' , 'text.init'); @@ -525,6 +549,7 @@ app.bldr.cmds { add ('en.wikipedia.org' , 'wiki.image'); add ('en.wikipedia.org' , 'wiki.page_dump.make'); add ('en.wikipedia.org' , 'wiki.page_link'); + add ('en.wikipedia.org' , 'wiki.imagelinks'); add ('en.wikipedia.org' , 'search.page__page_score') {iteration_max = 100;} add ('en.wikipedia.org' , 'search.link__link_score') {page_rank_enabled = 'y'; score_adjustment_mgr { diff --git a/home/wiki/Help/Contents.html b/home/wiki/Help/Contents.html index 5d47305aa..597ab54a0 100644 --- a/home/wiki/Help/Contents.html +++ b/home/wiki/Help/Contents.html @@ -181,7 +181,7 @@ Overview

  • - Image dumps + Image dumps
  • Wikidata @@ -203,7 +203,7 @@ Language files
  • - Menu configuration + Menu configuration
    • diff --git a/home/wiki/Options/Files.html b/home/wiki/Options/Files.html index 6b95b1309..e4ebbf76f 100644 --- a/home/wiki/Options/Files.html +++ b/home/wiki/Options/Files.html @@ -360,9 +360,9 @@ li.active a, li.active a:hover diff --git a/home/wiki/Options/Menus.html b/home/wiki/Options/Menus.html index e57b11739..ceffd8fb9 100644 --- a/home/wiki/Options/Menus.html +++ b/home/wiki/Options/Menus.html @@ -379,6 +379,8 @@ add_grp_default('xowa.gui.menus.group.tools') { add_spr; add_btn_default('xowa.nav.setup.maintenance'); add_btn_default('xowa.nav.setup.download'); + add_spr; + add_btn('gnosygnu.random', 'Random Page', 'e', '', 'app.api.nav.goto("Special:Random");'); } add_grp_default('xowa.gui.menus.group.help') { add_btn_default('xowa.nav.help.help'); @@ -811,7 +813,7 @@ add_btn_default('xowa.gui.browser.prog_log.show');
  • At the top of the window.
    • - For more information, see App/UI/Menus + For more information, see App/UI/Menus
  • ^ Specify the source for the tab button popup menu. This menu shows when: @@ -819,7 +821,7 @@ add_btn_default('xowa.gui.browser.prog_log.show');
  • Right-clicking on a tab button or in the blank area in the tab bar
    • - For more information, see App/UI/Menus + For more information, see App/UI/Menus
  • ^ Specify the source for the HTML box's popup menu. This menu shows when: @@ -827,7 +829,7 @@ add_btn_default('xowa.gui.browser.prog_log.show');
  • Right-clicking anywhere in the HTML area and nothing is selected
    • - For more information, see App/UI/Menus + For more information, see App/UI/Menus
  • ^ Specify the source for the HTML box's link menu. This menu shows when: @@ -835,7 +837,7 @@ add_btn_default('xowa.gui.browser.prog_log.show');
  • Right-clicking on a link or selected text in the HTML area
    • - For more information, see App/UI/Menus + For more information, see App/UI/Menus
  • ^ Specify the source for the HTML box's file menu. This menu shows when: @@ -843,7 +845,7 @@ add_btn_default('xowa.gui.browser.prog_log.show');
  • Right-clicking on an image or any other file in the HTML area
    • - For more information, see App/UI/Menus + For more information, see App/UI/Menus
  • ^ Specify the source for the status bar popup menu. This menu shows when: @@ -851,7 +853,7 @@ add_btn_default('xowa.gui.browser.prog_log.show');
  • Right-clicking on the status bar at the bottom of the screen
    • - For more information, see App/UI/Menus + For more information, see App/UI/Menus