* Moved content from liveCmdLine.inc into commandLine.inc, obsoleting the former.

* Put some option handling code in commandLine.inc which is untested and unused (for the moment).
* Converted all existing command line scripts to use the standard header and argument array.
* Did a quick test of compressOld.php, rebuildall.php and rebuildMessages.php to check for breakage.
* rebuildall.php was broken due to the unmaintained rebuildlinks.php, so I converted it to use refreshLinks instead. Required splitting into refreshLinks.inc and refreshLinks.php

17 files changed, 198 insertions, 214 deletions

diff --git a/maintenance/DiffLanguage.php b/maintenance/DiffLanguage.php
index 89955c53c864..0d97d22911a7 100644
--- a/maintenance/DiffLanguage.php
+++ b/maintenance/DiffLanguage.php
@@ -40,8 +40,8 @@ require_once( "commandLine.inc" );
 $wgLanguageCode = strtoupper(substr($wgLanguageCode,0,1)).strtolower(substr($wgLanguageCode,1));
 
 # read command line argument
-if ( isset($argv[1]) ) {
-	$lang = $argv[1];
+if ( isset($args[0]) ) {
+	$lang = $args[0];
 
 # or prompt a simple menu
 } else {
diff --git a/maintenance/attribute.php b/maintenance/attribute.php
index 56c2d2acb3ec..a6dcf4a67346 100644
--- a/maintenance/attribute.php
+++ b/maintenance/attribute.php
@@ -1,31 +1,20 @@
 <?php
+# Script for re-attributing edits
+require_once( "commandLine.inc" );
 
 # Parameters
-
-if ($argc < 4) {
+if ( count( $args ) < 2 ) {
 	print "Not enough parameters\n";
-	print "Usage: php attribute.php <lang> <source> <destination>\n";
+	if ( $wgWikiFarm ) {
+		print "Usage: php attribute.php <language> <site> <source> <destination>\n";
+	} else {
+		print "Usage: php attribute.php <source> <destination>\n";
+	}
 	exit;
 }
 
-$lang = $argv[1];
-$source = $argv[2];
-$dest = $argv[3];
-
-# Initialisation
-
-$wgCommandLineMode = true;
-$DP = "../includes";
-
-$sep = strchr( $include_path = ini_get( "include_path" ), ";" ) ? ";" : ":";
-ini_set( "include_path", "$IP$sep$include_path" );
-
-require_once( "/apache/htdocs/$lang/w/LocalSettings.php" );
-require_once( "Setup.php" );
-
-$wgTitle = Title::newFromText( "Changing attribution script" );
-set_time_limit(0);
-$wgCommandLineMode = true;
+$source = $args[0];
+$dest = $args[1];
 
 $eSource = wfStrencode( $source );
 $eDest = wfStrencode( $dest );
diff --git a/maintenance/commandLine.inc b/maintenance/commandLine.inc
index b29c43f12c3c..ee1d78e863e2 100644
--- a/maintenance/commandLine.inc
+++ b/maintenance/commandLine.inc
@@ -1,47 +1,118 @@
 <?php
-if ( strpos( `hostname -a`, "wikimedia.org" ) !== false ) {
-	require_once( "liveCmdLine.inc" );
-} else {
-	# Turn off output buffering if it's on
-	@ob_end_flush();
-	
-	if ( isset( $_SERVER ) && array_key_exists( 'REQUEST_METHOD', $_SERVER ) ) {
-		print "This script must be run from the command line\n";
-		exit();
+
+# Abort if called from a web server
+if ( isset( $_SERVER ) && array_key_exists( 'REQUEST_METHOD', $_SERVER ) ) {
+	print "This script must be run from the command line\n";
+	exit();
+}
+
+# Process command line arguments
+# $options becomes an array with keys set to the option names
+# $optionsWithArgs is an array of GNU-style options that take an argument. The arguments are returned
+# in the values of $options.
+
+if ( !isset( $optionsWithArgs ) ) {
+	$optionsWithArgs = array();
+}
+
+$self = array_shift( $argv );
+$IP = realpath( dirname( $self ) . "/.." );
+chdir( $IP );
+
+$options = array();
+$args = array();
+
+for( $arg = reset( $argv ); $arg !== false; $arg = next( $argv ) ) {
+	if ( substr( $arg, 0, 2 ) == '--' ) {
+		# Long options
+		$option = substr( $arg, 2 );
+		if ( in_array( $option, $optionsWithArgs ) ) {
+			$param = next( $argv );
+			if ( $param === false ) {
+				die( "$arg needs an value after it\n" );
+			}
+			$options[$option] = $param;
+		} else {
+			$options[$option] = 1;
+		}
+	} elseif ( $arg{0} == '-' ) {
+		# Short options
+		for ( $p=1; $p<strlen( $arg ); $p++ ) {
+			$option = $arg{$p};
+			if ( in_array( $option, $optionsWithArgs ) ) {
+				$param = next( $argv );
+				if ( $param === false ) {
+					die( "$arg needs an value after it\n" );
+				}
+				$options[$option] = $param;
+			} else {
+				$options[$option] = 1;
+			}
+		}
+	} else {
+		$args[] = $arg;
 	}
+}
 
-	$wgCommandLineMode = true;
+# General initialisation
+
+$wgCommandLineMode = true;
+# Turn off output buffering if it's on
+@ob_end_flush();
+$sep = strchr( $include_path = ini_get( "include_path" ), ";" ) ? ";" : ":";
 
-	$sep = strchr( $include_path = ini_get( "include_path" ), ";" ) ? ";" : ":";
-	if ( @$argv[1] && @$argv[1] != "-" ) {
-		$self = array_shift( $argv );
-		$lang = array_shift( $argv );
-		putenv( "wikilang=$lang");
-		$settingsFile = "/apache/htdocs/{$argv[1]}/w/LocalSettings.php";
-		$newpath = "/apache/common/php$sep";
+if ( $sep == ":" && strpos( `hostname -a`, "wikimedia.org" ) !== false ) {
+	$wgWikiFarm = true;
+	if ( isset( $args[0] ) ) {
+		$lang = array_shift( $args );
 	} else {
-		$settingsFile = "../LocalSettings.php";
-		$newpath = "";
+		$lang = "aa";
 	}
+	if ( isset( $args[0] ) ) {
+		$site = array_shift( $args );
+	} else {
+		$site = "wikipedia";
+	}
+
+	# This is for the IRC scripts, which now run as the apache user
+	# The apache user doesn't have access to the wikiadmin_pass command
+	if ( $_ENV['USER'] != "apache" ) {
+		$wgDBadminuser = "wikiadmin";
+		$wgDBadminpassword = trim(`wikiadmin_pass`);
+	}
+
+	putenv( "wikilang=$lang");
+
+	$DP = $IP;
+	ini_set( "include_path", ".:$IP:$IP/includes:$IP/languages:$IP/maintenance" );
+
+	require_once( "/home/wikipedia/common/php-new/CommonSettings.php" );
+} else {
+	$wgWikiFarm = false;
+	$settingsFile = "$IP/LocalSettings.php";
 
 	if ( ! is_readable( $settingsFile ) ) {
 		print "A copy of your installation's LocalSettings.php\n" .
 		  "must exist in the source directory.\n";
 		exit();
 	}
-
-
 	$wgCommandLineMode = true;
-	$DP = "../includes";
+	$DP = $IP;
 	include_once( $settingsFile );
-	ini_set( "include_path", "../includes$sep../languages$sep$newpath$IP$sep$include_path" );
-
-	$wgUsePHPTal = false;
-	define("MEDIAWIKI",true);
-	include_once( "Setup.php" );
-	include_once( "../install-utils.inc" );
-	$wgTitle = Title::newFromText( "Command line script" );
-	$wgCommandLineMode = true;
-	set_time_limit(0);
+	ini_set( "include_path", ".$sep$IP$sep$IP/includes$sep$IP/languages$sep$IP/maintenance" );
+	include_once( "$IP/AdminSettings.php" );
 }
+
+# Turn off output buffering again, it might have been turned on in the settings files
+@ob_end_flush();
+# Same with this one
+$wgCommandLineMode = true;
+
+$wgUsePHPTal = false;
+define("MEDIAWIKI",true);
+require_once( "Setup.php" );
+require_once( "install-utils.inc" );
+$wgTitle = Title::newFromText( "Command line script" );
+set_time_limit(0);
+
 ?>
diff --git a/maintenance/compressOld.inc b/maintenance/compressOld.inc
index ff47e15f96d7..8da4272571c2 100644
--- a/maintenance/compressOld.inc
+++ b/maintenance/compressOld.inc
@@ -1,7 +1,5 @@
 <?php
 
-include_once( "Article.php" );
-
 function compressOldPages( $start = 0 ) {
 	$chunksize = 50;
 	print "Starting from old_id $start...\n";
diff --git a/maintenance/compressOld.php b/maintenance/compressOld.php
index d3b88ddf9927..0d3531376efa 100644
--- a/maintenance/compressOld.php
+++ b/maintenance/compressOld.php
@@ -1,29 +1,9 @@
 <?php
 
-# Rebuild search index table from scratch.  This takes several
-# hours, depending on the database size and server configuration.
+# Compress the old table, old_flags=gzip
 
-if ( ! is_readable( "../LocalSettings.php" ) ) {
-	print "A copy of your installation's LocalSettings.php\n" .
-	  "must exist in the source directory.\n";
-	exit();
-}
-
-$wgCommandLineMode = true;
-$DP = "../includes";
-require_once( "../LocalSettings.php" );
-require_once( "../AdminSettings.php" );
-
-$sep = strchr( $include_path = ini_get( "include_path" ), ";" ) ? ";" : ":";
-ini_set( "include_path", "$IP$sep$include_path" );
-
-require_once( "Setup.php" );
-require_once( "./compressOld.inc" );
-$wgTitle = Title::newFromText( "Compress old pages script" );
-set_time_limit(0);
-
-$wgDBuser			= $wgDBadminuser;
-$wgDBpassword		= $wgDBadminpassword;
+require_once( "commandLine.inc" );
+require_once( "compressOld.inc" );
 
 if( !function_exists( "gzdeflate" ) ) {
 	print "You must enable zlib support in PHP to compress old revisions!\n";
diff --git a/maintenance/convertLinks.php b/maintenance/convertLinks.php
index 62b41eb630ec..3511e407d5ce 100644
--- a/maintenance/convertLinks.php
+++ b/maintenance/convertLinks.php
@@ -3,8 +3,6 @@
 # The wiki should be put into read-only mode while this script executes
 
 require_once( "commandLine.inc" );
-# the below should probably be moved into commandLine.inc at some point
-require_once( "../AdminSettings.php" );
 require_once( "convertLinks.inc" );
 
 convertLinks();
diff --git a/maintenance/rebuildMessages.php b/maintenance/rebuildMessages.php
index 068ac1f84cba..eb549b463032 100755
--- a/maintenance/rebuildMessages.php
+++ b/maintenance/rebuildMessages.php
@@ -1,11 +1,11 @@
 <?php
 require_once( "commandLine.inc" );
-include_once( "./InitialiseMessages.inc" );
+include_once( "InitialiseMessages.inc" );
 
 $wgTitle = Title::newFromText( "Rebuild messages script" );
 
-if ( isset( $argv[0] ) ) {
-	$response = array_shift( $argv );
+if ( isset( $args[0] ) ) {
+	$response = array_shift( $args );
 	if ( $response == "update" ) {
 		$response = 1;
 	} elseif ( $response == "rebuild" ) {
@@ -14,8 +14,8 @@ if ( isset( $argv[0] ) ) {
 } else {
 	$response = 0;
 }
-if ( isset( $argv[0] ) ) {
-	$messages = loadLanguageFile( array_shift( $argv ) );
+if ( isset( $args[0] ) ) {
+	$messages = loadLanguageFile( array_shift( $args ) );
 } else {
 	$messages = false;
 }
diff --git a/maintenance/rebuildall.php b/maintenance/rebuildall.php
index a9d0a9760430..d9ec307c06c0 100644
--- a/maintenance/rebuildall.php
+++ b/maintenance/rebuildall.php
@@ -3,31 +3,21 @@
 # Rebuild link tracking tables from scratch.  This takes several
 # hours, depending on the database size and server configuration.
 
-if ( ! is_readable( "../LocalSettings.php" ) ) {
-	print "A copy of your installation's LocalSettings.php\n" .
-	  "must exist in the source directory.\n";
-	exit();
-}
-
-$wgCommandLineMode = true;
-$DP = "../includes";
-require_once( "../LocalSettings.php" );
-require_once( "../AdminSettings.php" );
-
-$sep = strchr( $include_path = ini_get( "include_path" ), ";" ) ? ";" : ":";
-ini_set( "include_path", "$IP$sep$include_path" );
-
-require_once( "Setup.php" );
-require_once( "./rebuildlinks.inc" );
-require_once( "./rebuildtextindex.inc" );
-require_once( "./rebuildrecentchanges.inc" );
-$wgTitle = Title::newFromText( "Rebuild links script" );
-set_time_limit(0);
+require_once( "commandLine.inc" );
+
+#require_once( "rebuildlinks.inc" );
+require_once( "refreshlinks.inc" );
+require_once( "rebuildtextindex.inc" );
+require_once( "rebuildrecentchanges.inc" );
 
 $wgDBuser			= $wgDBadminuser;
 $wgDBpassword		= $wgDBadminpassword;
 
-rebuildLinkTables();
+# Doesn't work anymore
+# rebuildLinkTables();
+
+# Use the slow incomplete one instead. It's designed to work in the background
+#refreshLinks( 1 );
 
 dropTextIndex();
 rebuildTextIndex();
diff --git a/maintenance/rebuildlinks.php b/maintenance/rebuildlinks.php
index bc4a291f976c..f47b922dbe3b 100644
--- a/maintenance/rebuildlinks.php
+++ b/maintenance/rebuildlinks.php
@@ -3,26 +3,10 @@
 # Rebuild link tracking tables from scratch.  This takes several
 # hours, depending on the database size and server configuration.
 
-if ( ! is_readable( "../LocalSettings.php" ) ) {
-	print "A copy of your installation's LocalSettings.php\n" .
-	  "must exist in the source directory.\n";
-	exit();
-}
-
-$wgCommandLineMode = true;
-ini_set("implicit_flush", 1);
-
-$DP = "../includes";
-require_once( "../LocalSettings.php" );
-require_once( "../AdminSettings.php" );
-
-$sep = strchr( $include_path = ini_get( "include_path" ), ";" ) ? ";" : ":";
-ini_set( "include_path", "$IP$sep$include_path" );
-
-require_once( "Setup.php" );
+require_once( "commandLine.inc" );
 require_once( "./rebuildlinks.inc" );
+
 $wgTitle = Title::newFromText( "Rebuild links script" );
-set_time_limit(0);
 
 $wgDBuser			= $wgDBadminuser;
 $wgDBpassword		= $wgDBadminpassword;
diff --git a/maintenance/rebuildrecentchanges.php b/maintenance/rebuildrecentchanges.php
index 3ea838b613c7..6e342b2b1609 100644
--- a/maintenance/rebuildrecentchanges.php
+++ b/maintenance/rebuildrecentchanges.php
@@ -3,24 +3,9 @@
 # Rebuild link tracking tables from scratch.  This takes several
 # hours, depending on the database size and server configuration.
 
-if ( ! is_readable( "../LocalSettings.php" ) ) {
-	print "A copy of your installation's LocalSettings.php\n" .
-	  "must exist in the source directory.\n";
-	exit();
-}
-
-$wgCommandLineMode = true;
-$DP = "../includes";
-require_once( "../LocalSettings.php" );
-require_once( "../AdminSettings.php" );
-
-$sep = strchr( $include_path = ini_get( "include_path" ), ";" ) ? ";" : ":";
-ini_set( "include_path", "$IP$sep$include_path" );
-
-require_once( "Setup.php" );
+require_once( "commandLine.inc" );
 require_once( "./rebuildrecentchanges.inc" );
 $wgTitle = Title::newFromText( "Rebuild recent changes script" );
-set_time_limit(0);
 
 $wgDBuser			= $wgDBadminuser;
 $wgDBpassword		= $wgDBadminpassword;
diff --git a/maintenance/rebuildtextindex.php b/maintenance/rebuildtextindex.php
index d99cea25e8aa..83a4ea94e58d 100644
--- a/maintenance/rebuildtextindex.php
+++ b/maintenance/rebuildtextindex.php
@@ -1,26 +1,10 @@
 <?php
-define("MEDIAWIKI",true);
 # Rebuild search index table from scratch.  This takes several
 # hours, depending on the database size and server configuration.
 
-if ( ! is_readable( "../LocalSettings.php" ) ) {
-	print "A copy of your installation's LocalSettings.php\n" .
-	  "must exist in the source directory.\n";
-	exit();
-}
-
-$wgCommandLineMode = true;
-$DP = "../includes";
-require_once( "../LocalSettings.php" );
-require_once( "../AdminSettings.php" );
-
-$sep = strchr( $include_path = ini_get( "include_path" ), ";" ) ? ";" : ":";
-ini_set( "include_path", "$IP$sep$include_path" );
-
-require_once( "Setup.php" );
+require_once( "commandLine.inc" );
 require_once( "./rebuildtextindex.inc" );
 $wgTitle = Title::newFromText( "Rebuild text index script" );
-set_time_limit(0);
 
 $wgDBuser			= $wgDBadminuser;
 $wgDBpassword		= $wgDBadminpassword;
diff --git a/maintenance/refreshLinks.inc b/maintenance/refreshLinks.inc
new file mode 100644
index 000000000000..42e1138e0e86
--- /dev/null
+++ b/maintenance/refreshLinks.inc
@@ -0,0 +1,46 @@
+<?php
+
+define( "REPORTING_INTERVAL", 50 );
+define( "PAUSE_INTERVAL", 50 );
+
+function refreshLinks( $start ) {
+	global $wgUser, $wgTitle, $wgArticle, $wgEnablePersistentLC, $wgLinkCache, $wgOut;
+
+	$res = wfQuery("SELECT max(cur_id) as m FROM cur", DB_READ);
+	$row = wfFetchObject( $res );
+	$end = $row->m;
+
+	print("Refreshing link table. Starting from cur_id $start of $end.\n");
+
+	# Don't generate TeX PNGs (lack of a sensible current directory causes errors anyway)
+	$wgUser->setOption("math", 3);
+
+	for ($id = $start; $id <= $end; $id++) {
+		if ( !($id % REPORTING_INTERVAL) ) {
+			print "$id\n";
+		}
+
+		if ( !($id % PAUSE_INTERVAL) ) {
+			sleep(1);
+		}
+		
+		$wgTitle = Title::newFromID( $id );
+		if ( is_null( $wgTitle ) ) {
+			continue;
+		}
+		
+		$wgArticle = new Article( $wgTitle );
+		$text = $wgArticle->getContent( true );
+		$wgLinkCache = new LinkCache;
+		$wgOut->addWikiText( $text );
+
+		if ( $wgEnablePersistentLC ) {
+			$wgLinkCache->saveToLinkscc( $id, wfStrencode( $wgTitle->getPrefixedDBkey() ) );
+		}
+
+		$linksUpdate = new LinksUpdate( $id, $wgTitle->getPrefixedDBkey() );
+		$linksUpdate->doDumbUpdate();
+		$linksUpdate->fixBrokenLinks();
+	}
+}
+?>
diff --git a/maintenance/refreshLinks.php b/maintenance/refreshLinks.php
index 089e6a70b5b0..6d6ddc3f0494 100644
--- a/maintenance/refreshLinks.php
+++ b/maintenance/refreshLinks.php
@@ -1,8 +1,8 @@
 <?php
-define( "REPORTING_INTERVAL", 50 );
-define( "PAUSE_INTERVAL", 50 );
 
 require_once( "commandLine.inc" );
+require_once( "refreshLinks.inc" );
+
 error_reporting( E_ALL & (~E_NOTICE) );
 
 
@@ -12,42 +12,7 @@ if ($argv[2]) {
 	$start = 1;
 }
 
-$res = wfQuery("SELECT max(cur_id) as m FROM cur", DB_READ);
-$row = wfFetchObject( $res );
-$end = $row->m;
-
-print("Refreshing link table. Starting from cur_id $start of $end.\n");
-
-# Don't generate TeX PNGs (lack of a sensible current directory causes errors anyway)
-$wgUser->setOption("math", 3);
-
-for ($id = $start; $id <= $end; $id++) {
-	if ( !($id % REPORTING_INTERVAL) ) {
-		print "$id\n";
-	}
-
-	if ( !($id % PAUSE_INTERVAL) ) {
-		sleep(1);
-	}
-	
-	$wgTitle = Title::newFromID( $id );
-	if ( is_null( $wgTitle ) ) {
-		continue;
-	}
-	
-	$wgArticle = new Article( $wgTitle );
-	$text = $wgArticle->getContent( true );
-	$wgLinkCache = new LinkCache;
-	@$wgOut->addWikiText( $text );
-
-	if ( $wgEnablePersistentLC ) {
-		$wgLinkCache->saveToLinkscc( $id, wfStrencode( $wgTitle->getPrefixedDBkey() ) );
-	}
-
-	$linksUpdate = new LinksUpdate( $id, $wgTitle->getPrefixedDBkey() );
-	$linksUpdate->doDumbUpdate();
-	$linksUpdate->fixBrokenLinks();
-}
+refreshLinks( $start );
 
 exit();
 
diff --git a/maintenance/remove-brokenlinks.php b/maintenance/remove-brokenlinks.php
index 89a30033990e..7faecef568dd 100644
--- a/maintenance/remove-brokenlinks.php
+++ b/maintenance/remove-brokenlinks.php
@@ -1,25 +1,9 @@
 <?php
 
 # Remove spurious brokenlinks
-
-if ( ! is_readable( "../LocalSettings.php" ) ) {
-	print "A copy of your installation's LocalSettings.php\n" .
-	  "must exist in the source directory.\n";
-	exit();
-}
-
-$wgCommandLineMode = true;
-$DP = "../includes";
-require_once( "../LocalSettings.php" );
-require_once( "../AdminSettings.php" );
-
-$sep = strchr( $include_path = ini_get( "include_path" ), ";" ) ? ";" : ":";
-ini_set( "include_path", "$IP$sep$include_path" );
-
-require_once( "Setup.php" );
+require_once( "commandLine.inc" );
 require_once( "./rebuildrecentchanges.inc" );
 $wgTitle = Title::newFromText( "Rebuild brokenlinks script" );
-set_time_limit(0);
 
 $wgDBuser			= $wgDBadminuser;
 $wgDBpassword		= $wgDBadminpassword;
diff --git a/maintenance/tables.sql b/maintenance/tables.sql
index fb8e57868783..f3fefe876835 100644
--- a/maintenance/tables.sql
+++ b/maintenance/tables.sql
@@ -243,3 +243,10 @@ CREATE TABLE objectcache (
   unique key (keyname),
   key (exptime)
 );
+
+-- For storing revision text
+CREATE TABLE blobs (
+  blob_index char(255) binary NOT NULL default '',
+  blob_data longblob NOT NULL default '',
+  UNIQUE key blob_index (blob_index),
+);
diff --git a/maintenance/trivialCmdLine.php b/maintenance/trivialCmdLine.php
index 51628bc98370..4a0bb39cfa4b 100644
--- a/maintenance/trivialCmdLine.php
+++ b/maintenance/trivialCmdLine.php
@@ -1,5 +1,5 @@
 <?php
-require_once( "liveCmdLine.inc" );
+require_once( "commandLine.inc" );
 print "DB name: $wgDBname\n";
 print "DB user: $wgDBuser\n";
 print "DB password: $wgDBpassword\n";
diff --git a/maintenance/update2.php b/maintenance/update2.php
index 7d4a19929c31..b8712091cde6 100644
--- a/maintenance/update2.php
+++ b/maintenance/update2.php
@@ -1,4 +1,7 @@
 <?php
+
+# This script was used to convert the live Wikimedia wikis from 1.2 to 1.3
+
 $maintenance = "/home/wikipedia/common/php-new/maintenance";
 require_once( "$maintenance/liveCmdLine.inc" );
 require_once( "$maintenance/InitialiseMessages.inc" );