diff --git a/home/wiki/App/Import/English_Wikipedia.html b/home/wiki/App/Import/English_Wikipedia.html index 67681f95d..b1cf35ff5 100644 --- a/home/wiki/App/Import/English_Wikipedia.html +++ b/home/wiki/App/Import/English_Wikipedia.html @@ -138,7 +138,7 @@ Detailed start

- See Wiki_setup/English_wikis + See Wiki_setup/English_wikis

diff --git a/home/wiki/App/Wiki_types/Wikia.com.html b/home/wiki/App/Wiki_types/Wikia.com.html index bd9870c5c..5666e6cb1 100644 --- a/home/wiki/App/Wiki_types/Wikia.com.html +++ b/home/wiki/App/Wiki_types/Wikia.com.html @@ -170,7 +170,12 @@
- Links: Wiki_setup/English_wikis + Links: Wiki_setup/English_wikis
- Links: Dev/Command-line/Dumps + Links: Dev/Command-line/Dumps
- Links: Wiki_setup/English_wikis + Links: Wiki_setup/English_wikis
- Links: Wiki_setup/English_wikis + Links: Wiki_setup/English_wikis
- Links: Wiki_setup/English_wikis + Links: Wiki_setup/English_wikis

diff --git a/home/wiki/Change_log/v3.9.2.1.html b/home/wiki/Change_log/v3.9.2.1.html index ef566107e..1809feecd 100644 --- a/home/wiki/Change_log/v3.9.2.1.html +++ b/home/wiki/Change_log/v3.9.2.1.html @@ -279,7 +279,7 @@
- Links: Dev/Command-line/Dumps + Links: Dev/Command-line/Dumps

- commons.wikimedia.org (thum + commons.wikimedia.org

You will need the latest version of commons.wikimedia.org. Note that if you have an older version, you will have missing images or wrong size information. @@ -304,6 +304,10 @@ 

 app.bldr.pause_at_end_('n');
 app.scripts.run_file_by_type('xowa_cfg_app');
+app.cfg.set_temp('app', 'xowa.app.web.enabled', 'y');
+app.cfg.set_temp('app', 'xowa.bldr.db.layout_size.text', '0');
+app.cfg.set_temp('app', 'xowa.bldr.db.layout_size.html', '0');
+app.cfg.set_temp('app', 'xowa.bldr.db.layout_size.file', '0');
 app.bldr.cmds {
   // build commons database; this only needs to be done once, whenever commons is updated
   add     ('commons.wikimedia.org' , 'util.cleanup')          {delete_all = 'y';}
@@ -391,9 +395,11 @@ app.bldr.cmds {
   // cleanup all downloaded files as well as temporary files
   add     ('simple.wikipedia.org' , 'util.cleanup')          {delete_tmp = 'y'; delete_by_match('*.xml|*.sql|*.bz2|*.gz');}
 
+  // OBSOLETE: use v2
   // v1 html generator
   // parse every page in the listed namespace and gather data on their lnkis.
   // this step will take the longest amount of time.
+/*
   add     ('simple.wikipedia.org' , 'file.lnki_temp') {
     // save data every # of pages
     commit_interval = 10000; 
@@ -426,13 +432,14 @@ app.bldr.cmds {
       hzip_diff = 'y';
     }
   }
-
+*/
   // v2 html generator; allows for multi-threaded / multi-machine builds
-  /*
-  add     ('simple.wikipedia.org' , 'wiki.mass_parse.init') {cfg {ns_ids = '0|4|14';}}
+  add     ('simple.wikipedia.org' , 'wiki.mass_parse.init') {cfg {ns_ids = '0|4|14|8';}}
+
   add     ('simple.wikipedia.org' , 'wiki.mass_parse.exec') {
     cfg {
       num_wkrs = 8; load_all_templates = 'y'; cleanup_interval = 50; hzip_enabled = 'y'; hdiff_enabled ='y'; manual_now = '2016-08-01 01:02:03';
+      load_all_imglinks = 'y'; 
       
       // uncomment the following 3 lines if using the build script as a "worker" helping a "server"
       // num_pages_in_pool = 32000; 
@@ -443,8 +450,7 @@ app.bldr.cmds {
 
   // note that if multi-machine mode is enabled, all worker directories must be manually copied to the server directory (a build command will be added later)
   add     ('simple.wikipedia.org' , 'wiki.mass_parse.make');
-  */
-  
+
   // aggregate the lnkis
   add     ('simple.wikipedia.org' , 'file.lnki_regy');
   
@@ -492,7 +498,10 @@ app.bldr.run;
 
 app.bldr.pause_at_end_('n');
 app.scripts.run_file_by_type('xowa_cfg_app');
-app.cfgs.get('app.user.cfg.security.web_access_enabled', 'app').val = 'y';
+app.cfg.set_temp('app', 'xowa.app.web.enabled', 'y');
+app.cfg.set_temp('app', 'xowa.bldr.db.layout_size.text', '0');
+app.cfg.set_temp('app', 'xowa.bldr.db.layout_size.html', '0');
+app.cfg.set_temp('app', 'xowa.bldr.db.layout_size.file', '0');
 app.bldr.cmds {
 /*
   add     ('www.wikidata.org' , 'util.cleanup')          {delete_all = 'y';}
@@ -502,56 +511,47 @@ app.bldr.cmds {
   add     ('www.wikidata.org' , 'util.download')         {dump_type = 'image';}
   add     ('www.wikidata.org' , 'text.init');
   add     ('www.wikidata.org' , 'text.page');
-  add     ('www.wikidata.org' , 'text.cat.core');
-  add     ('www.wikidata.org' , 'text.cat.link');
-  add     ('www.wikidata.org' , 'text.cat.hidden');
   add     ('www.wikidata.org' , 'text.term');
   add     ('www.wikidata.org' , 'text.css');
   add     ('www.wikidata.org' , 'wiki.image');
-  add     ('www.wikidata.org' , 'file.page_regy')        {build_commons = 'y'}
   add     ('www.wikidata.org' , 'wiki.page_dump.make');
+  add     ('www.wikidata.org' , 'wiki.page_props');
+  add     ('www.wikidata.org' , 'wiki.categorylinks');
   add     ('www.wikidata.org' , 'wiki.redirect')         {commit_interval = 1000; progress_interval = 100; cleanup_interval = 100;}
-  add     ('www.wikidata.org' , 'util.cleanup')          {delete_tmp = 'y'; delete_by_match('*.xml|*.sql|*.bz2|*.gz');}
+//  add     ('www.wikidata.org' , 'util.cleanup')          {delete_tmp = 'y'; delete_by_match('*.xml|*.sql|*.bz2|*.gz');}
   add     ('commons.wikimedia.org' , 'util.cleanup')          {delete_all = 'y';}
   add     ('commons.wikimedia.org' , 'util.download')         {dump_type = 'pages-articles';}
+  add     ('commons.wikimedia.org' , 'util.download')         {dump_type = 'image';}
   add     ('commons.wikimedia.org' , 'util.download')         {dump_type = 'categorylinks';}
   add     ('commons.wikimedia.org' , 'util.download')         {dump_type = 'page_props';}
-  add     ('commons.wikimedia.org' , 'util.download')         {dump_type = 'image';}
   add     ('commons.wikimedia.org' , 'text.init');
   add     ('commons.wikimedia.org' , 'text.page');
-  add     ('commons.wikimedia.org' , 'text.cat.core');
-  add     ('commons.wikimedia.org' , 'text.cat.link');
-  add     ('commons.wikimedia.org' , 'text.cat.hidden');
   add     ('commons.wikimedia.org' , 'text.term');
   add     ('commons.wikimedia.org' , 'text.css');
   add     ('commons.wikimedia.org' , 'wiki.image');
   add     ('commons.wikimedia.org' , 'file.page_regy')        {build_commons = 'y'}
   add     ('commons.wikimedia.org' , 'wiki.page_dump.make');
   add     ('commons.wikimedia.org' , 'wiki.redirect')         {commit_interval = 1000; progress_interval = 100; cleanup_interval = 100;}
-*/
-/*
-  // en.wikipedia.org
+//  add     ('commons.wikimedia.org' , 'util.cleanup')          {delete_tmp = 'y'; delete_by_match('*.xml|*.sql|*.bz2|*.gz');}
   add     ('en.wikipedia.org' , 'util.download')         {dump_type = 'pages-articles';}
+  add     ('en.wikipedia.org' , 'util.download')         {dump_type = 'pagelinks';}
   add     ('en.wikipedia.org' , 'util.download')         {dump_type = 'categorylinks';}
   add     ('en.wikipedia.org' , 'util.download')         {dump_type = 'page_props';}
   add     ('en.wikipedia.org' , 'util.download')         {dump_type = 'image';}
-  add     ('en.wikipedia.org' , 'util.download')         {dump_type = 'pagelinks';}
   add     ('en.wikipedia.org' , 'util.download')         {dump_type = 'imagelinks';}
 */
 /*
+  // en.wikipedia.org
   add     ('en.wikipedia.org' , 'text.init');
   add     ('en.wikipedia.org' , 'text.page')             {redirect_id_enabled = 'y';}
   add     ('en.wikipedia.org' , 'text.search');
   add     ('en.wikipedia.org' , 'text.css');
-  add     ('en.wikipedia.org' , 'text.cat.core');
-  add     ('en.wikipedia.org' , 'text.cat.link');
-  add     ('en.wikipedia.org' , 'text.cat.hidden');
   add     ('en.wikipedia.org' , 'text.term');
-//  add     ('en.wikipedia.org' , 'wiki.redirect')         {commit_interval = 1000; progress_interval = 100; cleanup_interval = 100;}
   add     ('en.wikipedia.org' , 'wiki.image');
-  add     ('en.wikipedia.org' , 'wiki.page_dump.make');
-  add     ('en.wikipedia.org' , 'wiki.page_link');
   add     ('en.wikipedia.org' , 'wiki.imagelinks');
+  add     ('en.wikipedia.org' , 'wiki.page_dump.make');
+  add     ('en.wikipedia.org' , 'wiki.redirect')         {commit_interval = 1000; progress_interval = 100; cleanup_interval = 100;}
+  add     ('en.wikipedia.org' , 'wiki.page_link');
   add     ('en.wikipedia.org' , 'search.page__page_score') {iteration_max = 100;}
   add     ('en.wikipedia.org' , 'search.link__link_score') {page_rank_enabled = 'y';
     score_adjustment_mgr {
@@ -566,32 +566,43 @@ app.bldr.cmds {
     }
   }
   add     ('en.wikipedia.org' , 'search.word__link_count')
-/*
-  // SELECT * FROM xowa_cfg WHERE cfg_key = 'props.modified_latest';
-  add     ('en.wikipedia.org' , 'file.lnki_temp') {
-    commit_interval = 10000; progress_interval = 50; cleanup_interval = 50; select_size = 25;
-    ns_ids = '0|4|14|100';
-    hdump_bldr {enabled = 'y'; hzip_enabled = 'y'; hzip_diff = 'y';}
-  }  
-  add     ('en.wikipedia.org' , 'file.lnki_regy');
-  add     ('commons.wikimedia.org' , 'file.page_regy')        {build_commons = 'y'}
-  add     ('en.wikipedia.org'      , 'file.page_regy')        {build_commons = 'n';}
-  add     ('en.wikipedia.org' , 'wiki.image');
-  add     ('en.wikipedia.org' , 'file.orig_regy');
-  // SELECT * FROM orig_regy WHERE lnki_ttl = 'BSicon_CONTr.svg';
-  // SELECT * FROM page_regy WHERE src_ttl = 'BSicon_CONTr.svg';
-  add     ('en.wikipedia.org' , 'file.xfer_temp.thumb');
-  // SELECT Count(*) FROM xfer_regy WHERE xfer_status = 0;
-  // SELECT * FROM xfer_regy WHERE xfer_status = 0 AND lnki_page_id = 372692; --en.w:Featured_picture_candidates
-  add     ('en.wikipedia.org' , 'file.xfer_regy');
+  add     ('en.wikipedia.org' , 'wiki.page_props');
+  add     ('en.wikipedia.org' , 'wiki.categorylinks');
 */
 /*
+  add     ('en.wikipedia.org' , 'file.page_regy')        {build_commons = 'n'}
+  add     ('en.wikipedia.org' , 'wiki.mass_parse.init') {cfg {ns_ids = '0|4|100|14|8';}}
+  // add     ('en.wikipedia.org' , 'wiki.mass_parse.resume');
+  add     ('en.wikipedia.org' , 'wiki.mass_parse.exec') {cfg {
+    num_wkrs = 8; load_all_templates = 'y'; load_ifexists_ns = '*'; cleanup_interval = 25; hzip_enabled = 'y'; hdiff_enabled ='y'; manual_now = '2017-01-01 01:02:03';}
+//    num_wkrs = 1; load_all_templates = 'n'; load_all_imglnks = 'n'; cleanup_interval = 50; hzip_enabled = 'y'; hdiff_enabled ='y'; manual_now = '2016-07-28 01:02:03';}
+  }
+  add     ('en.wikipedia.org' , 'wiki.mass_parse.make');
+*/
+/*
+  add     ('en.wikipedia.org' , 'file.lnki_temp') {
+    commit_interval = 10000; progress_interval = 50; cleanup_interval = 50; select_size = 25;
+    ns_ids = '0|4|14|100|12|8|6|10|828|108|118|446|710|2300|2302|2600';
+    hdump_bldr {enabled = 'y'; hzip_enabled = 'y'; hzip_diff = 'y';}
+  }  
+*/
+/*
+  add     ('commons.wikimedia.org' , 'file.page_regy')        {build_commons = 'y'}
+  add     ('en.wikipedia.org'      , 'file.page_regy')        {build_commons = 'n';}
+  add     ('en.wikipedia.org' , 'file.lnki_regy');
+//  add     ('en.wikipedia.org' , 'wiki.image');
+  add     ('en.wikipedia.org' , 'file.orig_regy');
+  add     ('en.wikipedia.org' , 'file.xfer_temp.thumb');
+
+  add     ('en.wikipedia.org' , 'file.xfer_regy');
   add     ('en.wikipedia.org' , 'file.xfer_regy_update');
+*/
+/*
   add     ('en.wikipedia.org' , 'file.fsdb_make') {
     commit_interval = 1000; progress_interval = 200; select_interval = 10000;
-    ns_ids = '0|4|14|100';
+    ns_ids = '0|4|100|14|8';
 //    // specify whether original wiki databases are v1 (.sqlite3) or v2 (.xowa)
-//    // src_bin_mgr__fsdb_version = 'v2';
+//    src_bin_mgr__fsdb_version = 'v2';
 
 //   trg_bin_mgr__fsdb_version = 'v1';
 
@@ -603,6 +614,7 @@ app.bldr.cmds {
   }
   add     ('en.wikipedia.org' , 'file.orig_reg');
   add     ('en.wikipedia.org' , 'wiki.page_dump.drop');
+  add     ('en.wikipedia.org' , 'file.page_file_map.create');
 */
 }
 app.bldr.run;
@@ -614,6 +626,9 @@ app.bldr.run;
       
  • 
         2016-10-12: explicitly set web_access_enabled to y
       
    • 
+      
  • 
+        2017-02-02: updated script for multi-threaded version and new options
+      
    • 
     
   
     
diff --git a/home/wiki/Dev/Command-line/Thumbs.html b/home/wiki/Dev/Command-line/Thumbs.html
index 7c0c93db2..8d3ac65a0 100644
--- a/home/wiki/Dev/Command-line/Thumbs.html
+++ b/home/wiki/Dev/Command-line/Thumbs.html
@@ -68,7 +68,7 @@
           2 Requirements 
           
   
     
diff --git a/home/wiki/Help/Contents.html b/home/wiki/Help/Contents.html
index 82995a209..a5214bf54 100644
--- a/home/wiki/Help/Contents.html
+++ b/home/wiki/Help/Contents.html
@@ -181,7 +181,7 @@
               Overview
             
             
  • 
-              Image dumps
+              Image dumps
             
    • 
             
  • 
               Wikidata
diff --git a/home/wiki/Options.html b/home/wiki/Options.html
index 5d68284bf..3d11decb1 100644
--- a/home/wiki/Options.html
+++ b/home/wiki/Options.html
@@ -278,17 +278,17 @@
           
           
    
              
           
    
           
    
 
           
     
         
diff --git a/home/wiki/Options/Bookmarks.html b/home/wiki/Options/Bookmarks.html
index a8ef10f61..4faeb9178 100644
--- a/home/wiki/Options/Bookmarks.html
+++ b/home/wiki/Options/Bookmarks.html
@@ -264,7 +264,7 @@ li.active a, li.active a:hover
           
         
         
    
-          gplx.Gfo_invk__noop@7c584ba
+          gplx.Gfo_invk__noop@70b02e04
         
    
       
     
diff --git a/home/wiki/Options/Dev.html b/home/wiki/Options/Dev.html
index e8cca165b..6283fdd20 100644
--- a/home/wiki/Options/Dev.html
+++ b/home/wiki/Options/Dev.html
@@ -312,8 +312,7 @@
           
    
 
           
     
         
diff --git a/home/wiki/Options/Files.html b/home/wiki/Options/Files.html
index 5d6c816b9..906e89aa9 100644
--- a/home/wiki/Options/Files.html
+++ b/home/wiki/Options/Files.html
@@ -278,17 +278,17 @@
           
           
    
              
           
    
           
    
 
           
     
         
diff --git a/home/wiki/Options/Firefox_addon.html b/home/wiki/Options/Firefox_addon.html
index 53e345025..f2e665f6c 100644
--- a/home/wiki/Options/Firefox_addon.html
+++ b/home/wiki/Options/Firefox_addon.html
@@ -276,7 +276,7 @@ li.active a, li.active a:hover
           
         
         
    
-          gplx.Gfo_invk__noop@7c584ba
+          gplx.Gfo_invk__noop@70b02e04
         
    
       
     
diff --git a/home/wiki/Options/HTML_databases.html b/home/wiki/Options/HTML_databases.html
index 8fad9891f..16724d3d3 100644
--- a/home/wiki/Options/HTML_databases.html
+++ b/home/wiki/Options/HTML_databases.html
@@ -126,11 +126,11 @@
           
    
             Prefer HTML Databases for Read tab
           
    
-          
    
+          
    
              
           
    
           
    
-            
+            
           
     
         
    
         
    
diff --git a/home/wiki/Options/XOWA_viewer.html b/home/wiki/Options/XOWA_viewer.html
index 2e26612d4..4ad492fa2 100644
--- a/home/wiki/Options/XOWA_viewer.html
+++ b/home/wiki/Options/XOWA_viewer.html
@@ -276,7 +276,7 @@ li.active a, li.active a:hover
           
    
         
         
    
-          gplx.Gfo_invk__noop@7c584ba
+          gplx.Gfo_invk__noop@70b02e04