Add `phpunit:prepare-parallel:extensions` command

In T361190 and Quibble 1.9.0, we introduced parallel execution of
PHPUnit tests to speed up the CI jobs. The existing implementation
is purely Python/Quibble, and cannot directly be used by developers
locally. With this patch, we re-implement the test splitting logic
already implemented in CI as a composer task so that the parallel
tests can be run locally.

There are a couple of different approaches to running PHPUnit tests
in parallel. The different approaches have been discussed at length
in T50217. Ideally, we would just install the `paratest` extension
and use that to parallelise the execution. Unfortunately we have
complex test suites (specifically Parser tests and the Scribunto
test suite) that dynamically create tests as they run, which makes
it hard for `paratest` to work out which tests will run.

To overcome this limitation, we use the `phpunit --list-tests`
function to create a list of test classes that would be included in
the execution of the test suite, then scan the filesystem for
classes named in the `tests-list.xml` output. The classes we find
are then collected into smaller groups (`split_group_X`) which we
can run in parallel in separate processes.

We split into 7-8 groups here, as that experimentally leads to an
even spread of the tests and consumes 100% of all cores on a 4-core
processor.

Because `ParserIntegrationTest.php` is a single test class that
generates thousands of integration tests, we put that in its own
bucket rather than allocating it round-robin to one of the split
buckets. This again helps to keep the buckets roughly the same size.

The current implementation only supports splitting the `extensions`
test suite. We need to do some more development and testing to
support splitting other suites.

The new composer command `phpunit:prepare-parallel:extensions` will
generate a `phpunit.xml` file with the same contents as
`phpunit.xml.dist`, but with the split-group suites added. The
result of running all of the split groups should be the same as the
result of running the whole test suite.

Bug: T365976
Change-Id: I2d841ab236c5367961603bb526319053551bec2e

6 files changed, 275 insertions, 0 deletions

diff --git a/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitTestFileScannerTest.php b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitTestFileScannerTest.php
new file mode 100644
index 000000000000..b694f0a0b4d5
--- /dev/null
+++ b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitTestFileScannerTest.php
@@ -0,0 +1,26 @@
+<?php
+
+declare( strict_types = 1 );
+
+namespace MediaWiki\Tests\Unit\composer\PhpUnitSplitter;
+
+use MediaWiki\Composer\PhpUnitSplitter\PhpUnitTestFileScanner;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @license GPL-2.0-or-later
+ * @covers \MediaWiki\Composer\PhpUnitSplitter\PhpUnitTestFileScanner
+ */
+class PhpUnitTestFileScannerTest extends TestCase {
+
+	public function testScanForTestFiles() {
+		$scanner = new PhpUnitTestFileScanner( __DIR__ );
+		$files = $scanner->scanForFiles();
+		$expected = [];
+		foreach ( glob( __DIR__ . DIRECTORY_SEPARATOR . "*Test.php" ) as $testFile ) {
+			$expected[ basename( $testFile ) ] = [ $testFile ];
+		}
+		$this->assertEquals( $expected, $files, "Expected PhpUnitSplitter test files to be found" );
+	}
+
+}
diff --git a/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitTestListProcessorTest.php b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitTestListProcessorTest.php
new file mode 100644
index 000000000000..04526c1b7e8a
--- /dev/null
+++ b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitTestListProcessorTest.php
@@ -0,0 +1,29 @@
+<?php
+
+declare( strict_types = 1 );
+
+namespace MediaWiki\Tests\Unit\composer\PhpUnitSplitter;
+
+use MediaWiki\Composer\PhpUnitSplitter\PhpUnitTestListProcessor;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @license GPL-2.0-or-later
+ * @covers \MediaWiki\Composer\PhpUnitSplitter\PhpUnitTestListProcessor
+ */
+class PhpUnitTestListProcessorTest extends TestCase {
+
+	private const FIXTURE_FILE = __DIR__ . "/fixtures/tests-list.xml";
+
+	public function testGetTestClasses() {
+		$testList = new PhpUnitTestListProcessor( self::FIXTURE_FILE );
+		$this->assertCount(
+			5, $testList->getTestClasses(), "Expected classes to be loaded"
+		);
+		$test5 = $testList->getTestClasses()[4];
+		$this->assertEquals(
+			[ [ 'MediaWiki', 'Tests', 'Unit', 'composer', 'PhpUnitSplitter' ], 'TestSuiteBuilderTest' ],
+			[ $test5->getNamespace(), $test5->getClassName() ]
+		);
+	}
+}
diff --git a/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitXmlManagerTest.php b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitXmlManagerTest.php
new file mode 100644
index 000000000000..af4c0d1a2d9d
--- /dev/null
+++ b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitXmlManagerTest.php
@@ -0,0 +1,96 @@
+<?php
+
+declare( strict_types = 1 );
+
+namespace MediaWiki\Tests\Unit\composer\PhpUnitSplitter;
+
+use MediaWiki\Composer\PhpUnitSplitter\PhpUnitXmlManager;
+use MediaWiki\Composer\PhpUnitSplitter\TestListMissingException;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @license GPL-2.0-or-later
+ * @covers \MediaWiki\Composer\PhpUnitSplitter\PhpUnitXmlManager
+ */
+class PhpUnitXmlManagerTest extends TestCase {
+
+	private string $testDir;
+	private PhpUnitXmlManager $manager;
+
+	public function setUp(): void {
+		parent::setUp();
+		$this->testDir = implode( DIRECTORY_SEPARATOR, [ sys_get_temp_dir(), uniqid( 'PhpUnitTest' ) ] );
+		mkdir( $this->testDir );
+		$this->manager = new PhpUnitXmlManager( $this->testDir );
+		$this->setupTestFolder();
+	}
+
+	private static function getSourcePhpUnitDistXml(): string {
+		return __DIR__ . DIRECTORY_SEPARATOR . implode(
+		DIRECTORY_SEPARATOR, [ '..', '..', '..', '..', '..', '..', 'phpunit.xml.dist' ]
+		);
+	}
+
+	private function setupTestFolder() {
+		copy(
+			self::getSourcePhpUnitDistXml(),
+			$this->testDir . DIRECTORY_SEPARATOR . 'phpunit.xml.dist'
+		);
+		mkdir( $this->testDir . DIRECTORY_SEPARATOR . "tests" );
+		foreach ( glob( __DIR__ . DIRECTORY_SEPARATOR . "*Test.php" ) as $file ) {
+			copy(
+				$file,
+				implode( DIRECTORY_SEPARATOR, [ $this->testDir, "tests", basename( $file ) ] )
+			);
+		}
+	}
+
+	private function tearDownTestFolder() {
+		foreach ( [ 'phpunit.xml', 'phpunit.xml.dist', 'tests-list.xml' ] as $file ) {
+			$path = $this->testDir . DIRECTORY_SEPARATOR . $file;
+			if ( file_exists( $path ) ) {
+				unlink( $path );
+			}
+		}
+		$testsFolder = $this->testDir . DIRECTORY_SEPARATOR . "tests";
+		foreach ( glob( $testsFolder . DIRECTORY_SEPARATOR . "*.php" ) as $file ) {
+			unlink( $file );
+		}
+		rmdir( $testsFolder );
+		rmdir( $this->testDir );
+	}
+
+	private function copyTestListIntoPlace() {
+		copy(
+			__DIR__ . DIRECTORY_SEPARATOR . 'fixtures' . DIRECTORY_SEPARATOR . 'tests-list.xml',
+			$this->testDir . DIRECTORY_SEPARATOR . 'tests-list.xml'
+		);
+	}
+
+	public function tearDown(): void {
+		parent::tearDown();
+		$this->tearDownTestFolder();
+	}
+
+	public function testIsPrepared() {
+		$this->copyTestListIntoPlace();
+		$this->assertFalse( $this->manager->isPhpUnitXmlPrepared(), "Expected no PHPUnit Xml to be present" );
+		$this->manager->createPhpUnitXml( 4 );
+		$this->assertTrue( $this->manager->isPhpUnitXmlPrepared(), "Expected PHPUnit Xml to have been prepared" );
+	}
+
+	public function testFailsIfNoListIsPresent() {
+		$this->assertFalse( $this->manager->isPhpUnitXmlPrepared(), "Expected no PHPUnit Xml to be present" );
+		$this->expectException( TestListMissingException::class );
+		$this->manager->createPhpUnitXml( 4 );
+	}
+
+	public function testPhpUnitXmlDistNotPrepared() {
+		$this->assertFalse( $this->manager->isPhpUnitXmlPrepared(), "Expected no PHPUnit Xml to be present" );
+		copy( self::getSourcePhpUnitDistXml(), implode( DIRECTORY_SEPARATOR, [ $this->testDir, "phpunit.xml" ] ) );
+		$this->copyTestListIntoPlace();
+		$this->manager->createPhpUnitXml( 4 );
+		copy( self::getSourcePhpUnitDistXml(), implode( DIRECTORY_SEPARATOR, [ $this->testDir, "phpunit.xml" ] ) );
+		$this->assertFalse( $this->manager->isPhpUnitXmlPrepared(), "Expected phpunit.dist.xml to be treated as unprepared" );
+	}
+}
diff --git a/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitXmlTest.php b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitXmlTest.php
new file mode 100644
index 000000000000..6b30596e90bf
--- /dev/null
+++ b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/PhpUnitXmlTest.php
@@ -0,0 +1,52 @@
+<?php
+
+declare( strict_types = 1 );
+
+namespace MediaWiki\Tests\Unit\composer\PhpUnitSplitter;
+
+use MediaWiki\Composer\PhpUnitSplitter\PhpUnitXml;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @license GPL-2.0-or-later
+ * @covers \MediaWiki\Composer\PhpUnitSplitter\PhpUnitXml
+ */
+class PhpUnitXmlTest extends TestCase {
+
+	private const BASIC_XML = '<?xml version="1.0" encoding="UTF-8"?>
+<phpunit bootstrap="tests/phpunit/bootstrap.php">
+<testsuites>
+	<testsuite name="core:unit">
+		<directory>tests/phpunit/unit</directory>
+	</testsuite>
+</testsuites>
+</phpunit>';
+
+	public function createFixtureFile( string $data ): string {
+		$filename = tempnam( sys_get_temp_dir(), "phpunit-test" );
+		file_put_contents( $filename, $data );
+		return $filename;
+	}
+
+	public function testFixtureContainsNoSplitGroups() {
+		$phpUnitXmlFile = $this->createFixtureFile( self::BASIC_XML );
+		$phpUnitXml = new PhpUnitXml( $phpUnitXmlFile );
+		$this->assertFalse( $phpUnitXml->containsSplitGroups(), "No split groups expected in fixture" );
+		unlink( $phpUnitXmlFile );
+	}
+
+	public function testAddSplitGroups() {
+		$phpUnitXmlFile = $this->createFixtureFile( self::BASIC_XML );
+		$phpUnitXml = new PhpUnitXml( $phpUnitXmlFile );
+		$phpUnitXml->addSplitGroups( [
+			[ "file1.php", "file2.php" ],
+			[ "file3.php", "file4.php" ],
+			[ "file7.php", "file6.php" ],
+			[ "file9.php", "file8.php" ],
+			[ "file11.php", "file10.php" ],
+			[ "file13.php", "file12.php" ],
+		] );
+		$this->assertTrue( $phpUnitXml->containsSplitGroups(), "Expected groups to be added" );
+		unlink( $phpUnitXmlFile );
+	}
+}
diff --git a/tests/phpunit/unit/includes/composer/PhpUnitSplitter/TestSuiteBuilderTest.php b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/TestSuiteBuilderTest.php
new file mode 100644
index 000000000000..ce8a99504a6e
--- /dev/null
+++ b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/TestSuiteBuilderTest.php
@@ -0,0 +1,50 @@
+<?php
+
+declare( strict_types = 1 );
+
+namespace MediaWiki\Tests\Unit\composer\PhpUnitSplitter;
+
+use MediaWiki\Composer\PhpUnitSplitter\TestDescriptor;
+use MediaWiki\Composer\PhpUnitSplitter\TestSuiteBuilder;
+use PHPUnit\Framework\TestCase;
+
+/**
+ * @license GPL-2.0-or-later
+ * @covers \MediaWiki\Composer\PhpUnitSplitter\TestSuiteBuilder
+ */
+class TestSuiteBuilderTest extends TestCase {
+
+	public function testBuildSuites() {
+		$testList = [
+			new TestDescriptor( "ATest", [ "MediaWiki" ], "MediaWiki/ATest.php" ),
+			new TestDescriptor( "BTest", [ "MediaWiki" ], "MediaWiki/BTest.php" ),
+			new TestDescriptor( "CTest", [ "MediaWiki" ], "MediaWiki/CTest.php" ),
+			new TestDescriptor( "DTest", [ "MediaWiki" ], "MediaWiki/DTest.php" ),
+			new TestDescriptor( "ETest", [ "MediaWiki" ], "MediaWiki/ETest.php" ),
+		];
+		$suites = ( new TestSuiteBuilder() )->buildSuites( $testList, 3 );
+		$expected = [
+			[
+				"list" => [
+					"MediaWiki/ATest.php",
+					"MediaWiki/DTest.php",
+				],
+				"time" => 0
+			],
+			[
+				"list" => [
+					"MediaWiki/BTest.php",
+					"MediaWiki/ETest.php",
+				],
+				"time" => 0
+			],
+			[
+				"list" => [
+					"MediaWiki/CTest.php",
+				],
+				"time" => 0
+			]
+		];
+		$this->assertEquals( $expected, $suites, "Expected suites to be built correctly" );
+	}
+}
diff --git a/tests/phpunit/unit/includes/composer/PhpUnitSplitter/fixtures/tests-list.xml b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/fixtures/tests-list.xml
new file mode 100644
index 000000000000..942fefe72e48
--- /dev/null
+++ b/tests/phpunit/unit/includes/composer/PhpUnitSplitter/fixtures/tests-list.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<tests>
+	<testCaseClass name="MediaWiki\Tests\Unit\composer\PhpUnitSplitter\PhpUnitTestFileScannerTest">
+		<testCaseMethod name="testSchemaChangesPassValidation" groups="default" dataSet="&quot;patch-change_tag-rename-indexes.json&quot;"/>
+	</testCaseClass>
+	<testCaseClass name="MediaWiki\Tests\Unit\composer\PhpUnitSplitter\PhpUnitTestListProcessorTest">
+		<testCaseMethod name="testPrefixes" groups="API,Database"/>
+		<testCaseMethod name="testValidCovers" groups="API,Database"/>
+	</testCaseClass>
+	<testCaseClass name="MediaWiki\Tests\Unit\composer\PhpUnitSplitter\PhpUnitXmlManagerTest">
+		<testCaseMethod name="testConfigSchemaIsLoadable" groups="default"/>
+		<testCaseMethod name="testConfigSchemaDefaultsValidate" groups="default"/>
+		<testCaseMethod name="testCurrentSettingsValidate" groups="default"/>
+		<testCaseMethod name="testCurrentSettingsNotDeprecated" groups="default"/>
+	</testCaseClass>
+	<testCaseClass name="MediaWiki\Tests\Unit\composer\PhpUnitSplitter\PhpUnitXmlTest">
+		<testCaseMethod name="testOnBeforePageDisplay" groups="Marius Hoch &lt; hoo@online.de &gt;,WikimediaBadges,__phpunit_covers_wikimediabadges\beforepagedisplayhookhandler"/>
+	</testCaseClass>
+	<testCaseClass name="MediaWiki\Tests\Unit\composer\PhpUnitSplitter\TestSuiteBuilderTest">
+		<testCaseMethod name="testOnBeforePageDisplay" groups="Marius Hoch &lt; hoo@online.de &gt;,WikimediaBadges,__phpunit_covers_wikimediabadges\beforepagedisplayhookhandler"/>
+	</testCaseClass>
+</tests>