Run tests in parallel by default

.gitignore

@@ -253,7 +253,7 @@ php
 php_test_results_*.txt
 
 # Temporary test information generated by `./run-tests.php`
-/run-test-info.php
+/run-test-info-*.php
 
 # Temporary POST data placeholder files generated by `./run-tests.php`
 phpt.*

README.md

@@ -97,15 +97,15 @@ can be determined using `nproc`.
 PHP ships with an extensive test suite, the command `make test` is used after
 successful compilation of the sources to run this test suite.
 
-It is possible to run tests using multiple cores by setting `-jN` in
-`TEST_PHP_ARGS` or `TESTS`:
+Tests run in parallel by default, using up to 10 detected logical processors.
+Set `-jN` in `TEST_PHP_ARGS` or `TESTS` to override the worker count:
 
 ```shell
 make TEST_PHP_ARGS=-j4 test
 ```
 
-Shall run `make test` with a maximum of 4 concurrent jobs: Generally the maximum
-number of jobs should not exceed the number of cores available.
+This runs `make test` with a maximum of 4 concurrent jobs. Alternatively,
+use `-j1` to run tests sequentially.
 
 Use the `TEST_PHP_ARGS` or `TESTS` variable to test only specific directories:
 

docs/source/miscellaneous/writing-tests.rst

@@ -192,6 +192,11 @@ When you are testing your test case it's really important to make sure that you
 temporary resources (eg files) that you used in the test. There is a special ``--CLEAN--`` section
 to help you do this — see `here <#clean>`_.
 
+Tests run in parallel by default. Mutable resources such as files, directories, ports, database
+objects, and IPC identifiers must therefore be unique to each test. Read-only fixtures may be
+shared. If a resource cannot be isolated, declare the narrowest applicable conflict using
+``--CONFLICTS--`` or a ``CONFLICTS`` file.
+
 Another good check is to look at what lines of code in the PHP source your test case covers. This is
 easy to do, there are some instructions on the `PHP Wiki
 <https://wiki.php.net/doc/articles/writing-tests>`_.

ext/gd/tests/createfromwbmp2.phpt

@@ -8,7 +8,7 @@ gd
 ?>
 --FILE--
 <?php
-$filename = __DIR__ . '/_tmp.wbmp';
+$filename = __DIR__ . '/_tmp_createfromwbmp2.wbmp';
 $fp = fopen($filename,"wb");
 if (!$fp) {
     exit("Failed to create <$filename>");

ext/gd/tests/createfromwbmp2_extern.phpt

@@ -4,7 +4,7 @@ imagecreatefromwbmp with invalid wbmp
 gd
 --FILE--
 <?php
-$filename = __DIR__ . '/_tmp.wbmp';
+$filename = __DIR__ . '/_tmp_createfromwbmp2_extern.wbmp';
 $fp = fopen($filename,"wb");
 if (!$fp) {
     exit("Failed to create <$filename>");
@@ -41,4 +41,4 @@ unlink($filename);
 --EXPECTF--
 Warning: imagecreatefromwbmp(): %croduct of memory allocation multiplication would exceed INT_MAX, failing operation gracefully%win %s on line %d
 
-Warning: imagecreatefromwbmp(): "%s_tmp.wbmp" is not a valid WBMP file in %s on line %d
+Warning: imagecreatefromwbmp(): "%s_tmp_createfromwbmp2_extern.wbmp" is not a valid WBMP file in %s on line %d

ext/zip/tests/oo_addglob_leak.phpt

@@ -12,7 +12,7 @@ if(!defined("GLOB_BRACE")) die ('skip requires GLOB_BRACE');
 $dirname = __DIR__ . '/';
 include $dirname . 'utils.inc';
 
-$dirname = __DIR__ . '/__tmp_oo_addglob2/';
+$dirname = __DIR__ . '/__tmp_oo_addglob_leak/';
 $file = $dirname . 'test.zip';
 
 @mkdir($dirname);
@@ -38,7 +38,7 @@ var_dump($zip->addGlob($dirname . 'bar.*', GLOB_BRACE, $options));
 <?php
 $dirname = __DIR__ . '/';
 include $dirname . 'utils.inc';
-rmdir_rf(__DIR__ . '/__tmp_oo_addglob2/');
+rmdir_rf(__DIR__ . '/__tmp_oo_addglob_leak/');
 ?>
 --EXPECTF--
 array(1) {

run-tests.php

@@ -34,9 +34,9 @@ function show_usage(): void
     php run-tests.php [options] [files] [directories]
 
 Options:
-    -j<workers> Run up to <workers> simultaneous testing processes in parallel for
-                quicker testing on systems with multiple logical processors.
-                Note that this is experimental feature.
+    -j<workers> Run up to <workers> simultaneous testing processes. By default,
+                the worker count is detected automatically. Use -j1 to run
+                tests sequentially.
 
     -l <file>   Read the testfiles to be executed from <file>. After the test
                 has finished all failed tests are written to the same <file>.
@@ -351,6 +351,7 @@ function main(): void
     $shuffle = false;
     $bless = false;
     $workers = null;
+    $workersExplicit = false;
     $context_line_count = 3;
     $num_repeats = 1;
     $show_progress = true;
@@ -412,6 +413,7 @@ function main(): void
 
             switch ($switch) {
                 case 'j':
+                    $workersExplicit = true;
                     $workers = substr($argv[$i], 2);
                     if ($workers == 0 || !preg_match('/^\d+$/', $workers)) {
                         error("'$workers' is not a valid number of workers, try e.g. -j16 for 16 workers");
@@ -641,6 +643,19 @@ function main(): void
         }
     }
 
+    if (!$workersExplicit && (!$selected_tests || count($test_files) > 1)) {
+        $workers = get_default_worker_count();
+        if (
+            $workers !== null
+            && ($valgrind !== null || isset($environment['SKIP_ASAN']))
+        ) {
+            $workers = min($workers, 2);
+        }
+        if ($workers !== null && !can_create_parallel_worker_socket()) {
+            $workers = null;
+        }
+    }
+
     if ($online === null && !isset($environment['SKIP_ONLINE_TESTS'])) {
         $online = false;
     }
@@ -800,6 +815,55 @@ function main(): void
     }
 }
 
+function get_default_worker_count(): ?int
+{
+    if (IS_WINDOWS) {
+        $workerCount = getenv('NUMBER_OF_PROCESSORS');
+        return is_string($workerCount) ? parse_default_worker_count($workerCount) : null;
+    }
+
+    $commands = [
+        'nproc 2>/dev/null',
+        'getconf _NPROCESSORS_ONLN 2>/dev/null',
+        'getconf NPROCESSORS_ONLN 2>/dev/null',
+        'sysctl -n hw.logicalcpu 2>/dev/null',
+        'sysctl -n hw.ncpu 2>/dev/null',
+    ];
+    foreach ($commands as $command) {
+        $workerCount = shell_exec($command);
+        if (
+            is_string($workerCount)
+            && ($workerCount = parse_default_worker_count($workerCount)) !== null
+        ) {
+            return $workerCount;
+        }
+    }
+
+    return null;
+}
+
+function parse_default_worker_count(string $workerCount): ?int
+{
+    $workerCount = trim($workerCount);
+    if (preg_match('/^[0-9]+$/D', $workerCount) !== 1) {
+        return null;
+    }
+
+    $workerCount = (int) $workerCount;
+    return $workerCount >= 2 ? min($workerCount, 10) : null;
+}
+
+function can_create_parallel_worker_socket(): bool
+{
+    $socket = @stream_socket_server('tcp://127.0.0.1:0');
+    if ($socket === false) {
+        return false;
+    }
+
+    fclose($socket);
+    return true;
+}
+
 function verify_config(string $php): void
 {
     if (empty($php) || !file_exists($php)) {
@@ -820,7 +884,7 @@ function write_information(array $user_tests, $phpdbg): void
     $php_escaped = escapeshellarg($php);
 
     // Get info from php
-    $info_file = __DIR__ . '/run-test-info.php';
+    $info_file = __DIR__ . '/run-test-info-' . getmypid() . '.php';
     @unlink($info_file);
     $php_info = '<?php echo "
 PHP_SAPI    : " , PHP_SAPI , "
@@ -1571,10 +1635,10 @@ function run_all_tests_parallel(array $test_files, array $env, ?string $redir_te
                             // - If this is running a small enough number of tests,
                             //   reduce the batch size to give batches to more workers.
                             $files = [];
-                            $maxBatchSize = $valgrind ? 1 : ($shuffle ? 4 : 32);
+                            $maxBatchSize = $valgrind ? 1 : 4;
                             $averageFilesPerWorker = max(1, (int) ceil($totalFileCount / count($workerProcs)));
                             $batchSize = min($maxBatchSize, $averageFilesPerWorker);
-                            while (count($files) <= $batchSize && $file = array_pop($test_files)) {
+                            while (count($files) < $batchSize && $file = array_pop($test_files)) {
                                 foreach ($fileConflictsWith[$file] as $conflictKey) {
                                     if (isset($activeConflicts[$conflictKey])) {
                                         $waitingTests[$conflictKey][] = $file;

sapi/cli/tests/010-2.phpt

@@ -12,7 +12,7 @@ if (substr(PHP_OS, 0, 3) == 'WIN') {
 
 $php = getenv('TEST_PHP_EXECUTABLE_ESCAPED');
 
-$filename_txt = __DIR__."/010.test.txt";
+$filename_txt = __DIR__."/010-R.test.txt";
 $filename_txt_escaped = escapeshellarg($filename_txt);
 
 $txt = '

sapi/cli/tests/010.phpt

@@ -14,7 +14,7 @@ $php = getenv('TEST_PHP_EXECUTABLE_ESCAPED');
 
 $filename = __DIR__."/010.test.php";
 $filename_escaped = escapeshellarg($filename);
-$filename_txt = __DIR__."/010.test.txt";
+$filename_txt = __DIR__."/010-F.test.txt";
 $filename_txt_escaped = escapeshellarg($filename_txt);
 
 $code = '
@@ -37,7 +37,7 @@ var_dump(shell_exec("cat $filename_txt_escaped | $php -n -F $filename_escaped"))
 --CLEAN--
 <?php
 @unlink(__DIR__."/010.test.php");
-@unlink(__DIR__."/010.test.txt");
+@unlink(__DIR__."/010-F.test.txt");
 ?>
 --EXPECT--
 string(25) "

tests/run-test/automatic_worker_limit.phpt

@@ -0,0 +1,107 @@
+--TEST--
+Automatic worker detection is capped for regular and instrumented runs
+--SKIPIF--
+<?php
+if (PHP_OS_FAMILY === 'Windows') {
+    die('skip requires a POSIX shell');
+}
+?>
+--ENV--
+TEST_PHP_FORK_SERVER=0
+--FILE--
+<?php
+$root = __DIR__ . '/automatic_worker_limit_' . getmypid();
+$bin = $root . '/bin';
+$tests = $root . '/tests';
+mkdir($bin, recursive: true);
+mkdir($tests);
+
+$nproc = $bin . '/nproc';
+file_put_contents($nproc, "#!/bin/sh\nprintf '64\\n'\n");
+chmod($nproc, 0755);
+
+$testFiles = [];
+for ($i = 0; $i < 11; $i++) {
+    $testFiles[] = $file = $tests . "/$i.phpt";
+    file_put_contents($file, <<<PHPT
+        --TEST--
+        worker cap $i
+        --FILE--
+        <?php echo "ok\\n"; ?>
+        --EXPECT--
+        ok
+        PHPT);
+}
+
+$environment = [
+    'PATH' => $bin . PATH_SEPARATOR . getenv('PATH'),
+    'TEST_PHP_EXECUTABLE' => getenv('TEST_PHP_EXECUTABLE'),
+    'TEST_PHP_FORK_SERVER' => '0',
+];
+foreach (['TEMP', 'TMPDIR'] as $name) {
+    if (($value = getenv($name)) !== false) {
+        $environment[$name] = $value;
+    }
+}
+
+$runTests = static function (array $arguments) use ($environment, $testFiles): array {
+    $process = proc_open(
+        [
+            getenv('TEST_PHP_EXECUTABLE'),
+            dirname(__DIR__, 2) . '/run-tests.php',
+            '-q',
+            '--no-progress',
+            ...$arguments,
+            ...$testFiles,
+        ],
+        [
+            0 => ['pipe', 'r'],
+            1 => ['pipe', 'w'],
+            2 => ['redirect', 1],
+        ],
+        $pipes,
+        null,
+        $environment,
+    );
+    fclose($pipes[0]);
+    $output = stream_get_contents($pipes[1]);
+    fclose($pipes[1]);
+
+    return [proc_close($process), $output];
+};
+
+[$exitCode, $output] = $runTests([]);
+var_dump($exitCode);
+var_dump(str_contains($output, 'Spawning 10 workers...'));
+var_dump(str_contains($output, 'Spawning 11 workers...'));
+
+[$exitCode, $output] = $runTests(['--asan']);
+var_dump($exitCode);
+var_dump(str_contains($output, 'Spawning 2 workers...'));
+var_dump(str_contains($output, 'Spawning 10 workers...'));
+
+[$exitCode, $output] = $runTests(['--asan', '-j3']);
+var_dump($exitCode);
+var_dump(str_contains($output, 'Spawning 3 workers...'));
+?>
+--CLEAN--
+<?php
+foreach (glob(__DIR__ . '/automatic_worker_limit_*') ?: [] as $root) {
+    foreach (glob($root . '/tests/*.phpt') ?: [] as $file) {
+        unlink($file);
+    }
+    @unlink($root . '/bin/nproc');
+    @rmdir($root . '/tests');
+    @rmdir($root . '/bin');
+    @rmdir($root);
+}
+?>
+--EXPECT--
+int(0)
+bool(true)
+bool(false)
+int(0)
+bool(true)
+bool(false)
+int(0)
+bool(true)