External storage handling

3 files changed, 125 insertions, 33 deletions

diff --git a/maintenance/storage/blobs.sql b/maintenance/storage/blobs.sql
index 9bf4d148705a..3e72f77f3339 100644
--- a/maintenance/storage/blobs.sql
+++ b/maintenance/storage/blobs.sql
@@ -1,7 +1,7 @@
 -- Blobs table for external storage
 
 CREATE TABLE /*$wgDBprefix*/blobs (
-	blob_id int(8) NOT NULL default '0',
+	blob_id int(8) NOT NULL default '0' AUTO_INCREMENT,
 	blob_text mediumtext,
 	PRIMARY KEY  (blob_id)
 ) TYPE=InnoDB;
diff --git a/maintenance/storage/moveToExternal.php b/maintenance/storage/moveToExternal.php
new file mode 100644
index 000000000000..8b441d2023bd
--- /dev/null
+++ b/maintenance/storage/moveToExternal.php
@@ -0,0 +1,80 @@
+<?php
+
+define( 'REPORTING_INTERVAL', 100 );
+define( 'STUB_HEADER', 'O:15:"historyblobstub"' );
+
+if ( !defined( 'MEDIAWIKI' ) ) {
+	$optionsWithArgs = array( 'm' );
+
+	require_once( '../commandLine.inc' );
+	require_once( 'ExternalStoreDB.php' );
+	require_once( 'resolveStubs.php' );
+
+	$fname = 'moveToExternal';
+
+	if ( !isset( $args[0] ) ) {
+		print "Usage: php moveToExternal.php [-m <maxid>] <cluster>\n";
+		exit;
+	}
+
+	$cluster = $args[0];
+	$dbw =& wfGetDB( DB_MASTER );
+
+	if ( isset( $options['m'] ) ) {
+		$maxID = $options['m'];
+	} else {
+		$maxID = $dbw->selectField( 'text', 'MAX(old_id)', false, $fname );
+	}
+
+	moveToExternal( $cluster, $maxID );
+}
+
+
+
+function moveToExternal( $cluster, $maxID ) {
+	$fname = 'moveToExternal';
+	$dbw =& wfGetDB( DB_MASTER );
+
+	print "Moving $maxID text rows to external storage\n";
+	$ext = new ExternalStoreDB;
+	for ( $id = 1; $id <= $maxID; $id++ ) {
+		if ( !($id % REPORTING_INTERVAL) ) {
+			print "$id\n";
+			wfWaitForSlaves( 5 );
+		}
+		$row = $dbw->selectRow( 'text', array( 'old_flags', 'old_text' ),
+			array( 
+				'old_id' => $id,
+				"old_flags NOT LIKE '%external%'",
+			), $fname );
+		if ( !$row ) {
+			# Non-existent or already done
+			continue;
+		}
+
+		# Resolve stubs
+		$flags = explode( ',', $row->old_flags );
+		if ( in_array( 'object', $flags ) 
+			&& substr( $row->old_text, 0, strlen( STUB_HEADER ) ) === STUB_HEADER ) 
+		{
+			resolveStub( $id, $row->old_text, $row->old_flags );
+			continue;
+		}
+
+		$url = $ext->store( $cluster, $row->old_text );
+		if ( !$url ) {
+			print "Error writing to external storage\n";
+			exit;
+		}
+		if ( $row->old_flags === '' ) {
+			$flags = 'external';
+		} else {
+			$flags = "{$row->old_flags},external";
+		}
+		$dbw->update( 'text', 
+			array( 'old_flags' => $flags, 'old_text' => $url ),
+			array( 'old_id' => $id ), $fname );
+	}
+}
+
+?>
diff --git a/maintenance/storage/resolveStubs.php b/maintenance/storage/resolveStubs.php
index 840ae3f78758..d22976c5f75d 100644
--- a/maintenance/storage/resolveStubs.php
+++ b/maintenance/storage/resolveStubs.php
@@ -55,42 +55,54 @@ function resolveStubs() {
 			wfWaitForSlaves( 5 );
 		}
 
-		$stub = unserialize( $stub );
-		if ( get_class( $stub ) !== 'historyblobstub' ) {
-			print "Error, invalid stub object\n";
-			return;
-		}
+		resolveStub( $id, $stub, $flagsArray[$id] );
+	}
+}
 
-		# Get the (maybe) external row
-		$externalRow = $dbr->selectRow( 'text', array( 'old_text' ), 
-			array( 'old_id' => $stub->mOldId, "old_flags LIKE '%external%'" ),
-			$fname 
-		);
+/**
+ * Resolve a history stub
+ */
+function resolveStub( $id, $stubText, $flags ) {
+	$fname = 'resolveStub';
 
-		if ( !$externalRow ) {
-			# Object wasn't external
-			continue;
-		}
+	$stub = unserialize( $stubText );
+	$flags = explode( ',', $flags );
 
-		# Preserve the legacy encoding flag, but switch from object to external
-		$flags = explode( ',', $flagsArray[$id] );
-		if ( in_array( 'utf-8', $flags ) ) {
-			$newFlags = 'external,utf-8';
-		} else {
-			$newFlags = 'external';
-		}
+	$dbr =& wfGetDB( DB_SLAVE );
+	$dbw =& wfGetDB( DB_MASTER );
+	
+	if ( get_class( $stub ) !== 'historyblobstub' ) {
+		print "Error, invalid stub object\n";
+		return;
+	}
 
-		# Update the row
-		$dbw->update( 'text',
-			array( /* SET */
-				'old_flags' => $newFlags, 
-				'old_text' => $externalRow->old_text . '/' . $stub->mHash
-			), 
-			array( /* WHERE */ 
-				'old_id' => $id 
-			), $fname 
-		);
+	# Get the (maybe) external row
+	$externalRow = $dbr->selectRow( 'text', array( 'old_text' ), 
+		array( 'old_id' => $stub->mOldId, "old_flags LIKE '%external%'" ),
+		$fname 
+	);
+
+	if ( !$externalRow ) {
+		# Object wasn't external
+		continue;
 	}
-}
 
+	# Preserve the legacy encoding flag, but switch from object to external
+	if ( in_array( 'utf-8', $flags ) ) {
+		$newFlags = 'external,utf-8';
+	} else {
+		$newFlags = 'external';
+	}
+
+	# Update the row
+	$dbw->update( 'text',
+		array( /* SET */
+			'old_flags' => $newFlags, 
+			'old_text' => $externalRow->old_text . '/' . $stub->mHash
+		), 
+		array( /* WHERE */ 
+			'old_id' => $id 
+		), $fname 
+	);
+}
 ?>