Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit c868be4

Browse filesBrowse files
nicolas-grekasnicolas-grekas
committed
feature #50877 [Finder] Add early directory prunning filter support (mvorisek)
This PR was squashed before being merged into the 6.4 branch. Discussion ---------- [Finder] Add early directory prunning filter support | Q | A | ------------- | --- | Branch? | 6.4 | Bug fix? | no | New feature? | yes | Deprecations? | no | Tickets | - | License | MIT | Doc PR | symfony/symfony-docs#18499 Filtering terminology first: - "exclude" - whole filesystem subtree is traversed, directories (and files) are excluded after, slow - "prune" - pruned directories are not traversed, fast Currently there is early directory prunning support possible, but with string patterns only, not even full regex is supported as the input is always quoted - https://github.com/symfony/symfony/blob/v6.2.12/src/Symfony/Component/Finder/Iterator/ExcludeDirectoryFilterIterator.php#L45. This PR adds early directory prunning with full callback support. Tested with VFS to assert the pruned directories are really not traversed and the count of IO syscalls is minimal. Commits ------- 840cb28 [Finder] Add early directory prunning filter support
2 parents f3a09b9 + 840cb28 commit c868be4
Copy full SHA for c868be4

File tree

5 files changed

+285
-3
lines changed
Filter options

5 files changed

+285
-3
lines changed

‎src/Symfony/Component/Finder/CHANGELOG.md

Copy file name to clipboardExpand all lines: src/Symfony/Component/Finder/CHANGELOG.md
+5Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,11 @@
11
CHANGELOG
22
=========
33

4+
6.4
5+
---
6+
7+
* Add early directory prunning to `Finder::filter()`
8+
49
6.2
510
---
611

‎src/Symfony/Component/Finder/Finder.php

Copy file name to clipboardExpand all lines: src/Symfony/Component/Finder/Finder.php
+14-1Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ class Finder implements \IteratorAggregate, \Countable
5050
private array $notNames = [];
5151
private array $exclude = [];
5252
private array $filters = [];
53+
private array $pruneFilters = [];
5354
private array $depths = [];
5455
private array $sizes = [];
5556
private bool $followLinks = false;
@@ -580,14 +581,22 @@ public function sortByModifiedTime(): static
580581
* The anonymous function receives a \SplFileInfo and must return false
581582
* to remove files.
582583
*
584+
* @param \Closure(SplFileInfo): bool $closure
585+
* @param bool $prune Whether to skip traversing directories further
586+
*
583587
* @return $this
584588
*
585589
* @see CustomFilterIterator
586590
*/
587-
public function filter(\Closure $closure): static
591+
public function filter(\Closure $closure /* , bool $prune = false */): static
588592
{
593+
$prune = 1 < \func_num_args() ? func_get_arg(1) : false;
589594
$this->filters[] = $closure;
590595

596+
if ($prune) {
597+
$this->pruneFilters[] = $closure;
598+
}
599+
591600
return $this;
592601
}
593602

@@ -741,6 +750,10 @@ private function searchInDirectory(string $dir): \Iterator
741750
$exclude = $this->exclude;
742751
$notPaths = $this->notPaths;
743752

753+
if ($this->pruneFilters) {
754+
$exclude = array_merge($exclude, $this->pruneFilters);
755+
}
756+
744757
if (static::IGNORE_VCS_FILES === (static::IGNORE_VCS_FILES & $this->ignore)) {
745758
$exclude = array_merge($exclude, self::$vcsPatterns);
746759
}

‎src/Symfony/Component/Finder/Iterator/ExcludeDirectoryFilterIterator.php

Copy file name to clipboardExpand all lines: src/Symfony/Component/Finder/Iterator/ExcludeDirectoryFilterIterator.php
+23-2Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,19 +27,32 @@ class ExcludeDirectoryFilterIterator extends \FilterIterator implements \Recursi
2727
/** @var \Iterator<string, SplFileInfo> */
2828
private \Iterator $iterator;
2929
private bool $isRecursive;
30+
/** @var array<string, true> */
3031
private array $excludedDirs = [];
3132
private ?string $excludedPattern = null;
33+
/** @var list<callable(SplFileInfo):bool> */
34+
private array $pruneFilters = [];
3235

3336
/**
34-
* @param \Iterator<string, SplFileInfo> $iterator The Iterator to filter
35-
* @param string[] $directories An array of directories to exclude
37+
* @param \Iterator<string, SplFileInfo> $iterator The Iterator to filter
38+
* @param list<string|callable(SplFileInfo):bool> $directories An array of directories to exclude
3639
*/
3740
public function __construct(\Iterator $iterator, array $directories)
3841
{
3942
$this->iterator = $iterator;
4043
$this->isRecursive = $iterator instanceof \RecursiveIterator;
4144
$patterns = [];
4245
foreach ($directories as $directory) {
46+
if (!\is_string($directory)) {
47+
if (!\is_callable($directory)) {
48+
throw new \InvalidArgumentException('Invalid PHP callback.');
49+
}
50+
51+
$this->pruneFilters[] = $directory;
52+
53+
continue;
54+
}
55+
4356
$directory = rtrim($directory, '/');
4457
if (!$this->isRecursive || str_contains($directory, '/')) {
4558
$patterns[] = preg_quote($directory, '#');
@@ -70,6 +83,14 @@ public function accept(): bool
7083
return !preg_match($this->excludedPattern, $path);
7184
}
7285

86+
if ($this->pruneFilters && $this->hasChildren()) {
87+
foreach ($this->pruneFilters as $pruneFilter) {
88+
if (!$pruneFilter($this->current())) {
89+
return false;
90+
}
91+
}
92+
}
93+
7394
return true;
7495
}
7596

‎src/Symfony/Component/Finder/Tests/FinderTest.php

Copy file name to clipboardExpand all lines: src/Symfony/Component/Finder/Tests/FinderTest.php
+68Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
class FinderTest extends Iterator\RealIteratorTestCase
1818
{
19+
use Iterator\VfsIteratorTestTrait;
20+
1921
public function testCreate()
2022
{
2123
$this->assertInstanceOf(Finder::class, Finder::create());
@@ -989,6 +991,72 @@ public function testFilter()
989991
$this->assertIterator($this->toAbsolute(['test.php', 'test.py']), $finder->in(self::$tmpDir)->getIterator());
990992
}
991993

994+
public function testFilterPrune()
995+
{
996+
$this->setupVfsProvider([
997+
'x' => [
998+
'a.php' => '',
999+
'b.php' => '',
1000+
'd' => [
1001+
'u.php' => '',
1002+
],
1003+
'x' => [
1004+
'd' => [
1005+
'u2.php' => '',
1006+
],
1007+
],
1008+
],
1009+
'y' => [
1010+
'c.php' => '',
1011+
],
1012+
]);
1013+
1014+
$finder = $this->buildFinder();
1015+
$finder
1016+
->in($this->vfsScheme.'://x')
1017+
->filter(fn (): bool => true, true) // does nothing
1018+
->filter(function (\SplFileInfo $file): bool {
1019+
$path = $this->stripSchemeFromVfsPath($file->getPathname());
1020+
1021+
$res = 'x/d' !== $path;
1022+
1023+
$this->vfsLog[] = [$path, 'exclude_filter', $res];
1024+
1025+
return $res;
1026+
}, true)
1027+
->filter(fn (): bool => true, true); // does nothing
1028+
1029+
$this->assertSameVfsIterator([
1030+
'x/a.php',
1031+
'x/b.php',
1032+
'x/x',
1033+
'x/x/d',
1034+
'x/x/d/u2.php',
1035+
], $finder->getIterator());
1036+
1037+
// "x/d" directory must be pruned early
1038+
// "x/x/d" directory must not be pruned
1039+
$this->assertSame([
1040+
['x', 'is_dir', true],
1041+
['x', 'list_dir_open', ['a.php', 'b.php', 'd', 'x']],
1042+
['x/a.php', 'is_dir', false],
1043+
['x/a.php', 'exclude_filter', true],
1044+
['x/b.php', 'is_dir', false],
1045+
['x/b.php', 'exclude_filter', true],
1046+
['x/d', 'is_dir', true],
1047+
['x/d', 'exclude_filter', false],
1048+
['x/x', 'is_dir', true],
1049+
['x/x', 'exclude_filter', true], // from ExcludeDirectoryFilterIterator::accept() (prune directory filter)
1050+
['x/x', 'exclude_filter', true], // from CustomFilterIterator::accept() (regular filter)
1051+
['x/x', 'list_dir_open', ['d']],
1052+
['x/x/d', 'is_dir', true],
1053+
['x/x/d', 'exclude_filter', true],
1054+
['x/x/d', 'list_dir_open', ['u2.php']],
1055+
['x/x/d/u2.php', 'is_dir', false],
1056+
['x/x/d/u2.php', 'exclude_filter', true],
1057+
], $this->vfsLog);
1058+
}
1059+
9921060
public function testFollowLinks()
9931061
{
9941062
if ('\\' == \DIRECTORY_SEPARATOR) {

‎src/Symfony/Component/Finder/Tests/Iterator/VfsIteratorTestTrait.php

Copy file name to clipboard
+175Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
<?php
2+
3+
/*
4+
* This file is part of the Symfony package.
5+
*
6+
* (c) Fabien Potencier <fabien@symfony.com>
7+
*
8+
* For the full copyright and license information, please view the LICENSE
9+
* file that was distributed with this source code.
10+
*/
11+
12+
namespace Symfony\Component\Finder\Tests\Iterator;
13+
14+
trait VfsIteratorTestTrait
15+
{
16+
private static int $vfsNextSchemeIndex = 0;
17+
18+
/** @var array<string, \Closure(string, 'list_dir_open'|'list_dir_rewind'|'is_dir'): (list<string>|bool)> */
19+
public static array $vfsProviders;
20+
21+
protected string $vfsScheme;
22+
23+
/** @var list<array{string, string, mixed}> */
24+
protected array $vfsLog = [];
25+
26+
protected function setUp(): void
27+
{
28+
parent::setUp();
29+
30+
$this->vfsScheme = 'symfony-finder-vfs-test-'.++self::$vfsNextSchemeIndex;
31+
32+
$vfsWrapperClass = \get_class(new class() {
33+
/** @var array<string, \Closure(string, 'list_dir_open'|'list_dir_rewind'|'is_dir'): (list<string>|bool)> */
34+
public static array $vfsProviders = [];
35+
36+
/** @var resource */
37+
public $context;
38+
39+
private string $scheme;
40+
41+
private string $dirPath;
42+
43+
/** @var list<string> */
44+
private array $dirData;
45+
46+
private function parsePathAndSetScheme(string $url): string
47+
{
48+
$urlArr = parse_url($url);
49+
\assert(\is_array($urlArr));
50+
\assert(isset($urlArr['scheme']));
51+
\assert(isset($urlArr['host']));
52+
53+
$this->scheme = $urlArr['scheme'];
54+
55+
return str_replace(\DIRECTORY_SEPARATOR, '/', $urlArr['host'].($urlArr['path'] ?? ''));
56+
}
57+
58+
public function processListDir(bool $fromRewind): bool
59+
{
60+
$providerFx = self::$vfsProviders[$this->scheme];
61+
$data = $providerFx($this->dirPath, 'list_dir'.($fromRewind ? '_rewind' : '_open'));
62+
\assert(\is_array($data));
63+
$this->dirData = $data;
64+
65+
return true;
66+
}
67+
68+
public function dir_opendir(string $url): bool
69+
{
70+
$this->dirPath = $this->parsePathAndSetScheme($url);
71+
72+
return $this->processListDir(false);
73+
}
74+
75+
public function dir_readdir(): string|false
76+
{
77+
return array_shift($this->dirData) ?? false;
78+
}
79+
80+
public function dir_closedir(): bool
81+
{
82+
unset($this->dirPath);
83+
unset($this->dirData);
84+
85+
return true;
86+
}
87+
88+
public function dir_rewinddir(): bool
89+
{
90+
return $this->processListDir(true);
91+
}
92+
93+
/**
94+
* @return array<string, mixed>
95+
*/
96+
public function stream_stat(): array
97+
{
98+
return [];
99+
}
100+
101+
/**
102+
* @return array<string, mixed>
103+
*/
104+
public function url_stat(string $url): array
105+
{
106+
$path = $this->parsePathAndSetScheme($url);
107+
$providerFx = self::$vfsProviders[$this->scheme];
108+
$isDir = $providerFx($path, 'is_dir');
109+
\assert(\is_bool($isDir));
110+
111+
return ['mode' => $isDir ? 0040755 : 0100644];
112+
}
113+
});
114+
self::$vfsProviders = &$vfsWrapperClass::$vfsProviders;
115+
116+
stream_wrapper_register($this->vfsScheme, $vfsWrapperClass);
117+
}
118+
119+
protected function tearDown(): void
120+
{
121+
stream_wrapper_unregister($this->vfsScheme);
122+
123+
parent::tearDown();
124+
}
125+
126+
/**
127+
* @param array<string, mixed> $data
128+
*/
129+
protected function setupVfsProvider(array $data): void
130+
{
131+
self::$vfsProviders[$this->vfsScheme] = function (string $path, string $op) use ($data) {
132+
$pathArr = explode('/', $path);
133+
$fileEntry = $data;
134+
while (($name = array_shift($pathArr)) !== null) {
135+
if (!isset($fileEntry[$name])) {
136+
$fileEntry = false;
137+
138+
break;
139+
}
140+
141+
$fileEntry = $fileEntry[$name];
142+
}
143+
144+
if ('list_dir_open' === $op || 'list_dir_rewind' === $op) {
145+
/** @var list<string> $res */
146+
$res = array_keys($fileEntry);
147+
} elseif ('is_dir' === $op) {
148+
$res = \is_array($fileEntry);
149+
} else {
150+
throw new \Exception('Unexpected operation type');
151+
}
152+
153+
$this->vfsLog[] = [$path, $op, $res];
154+
155+
return $res;
156+
};
157+
}
158+
159+
protected function stripSchemeFromVfsPath(string $url): string
160+
{
161+
$urlArr = parse_url($url);
162+
\assert(\is_array($urlArr));
163+
\assert($urlArr['scheme'] === $this->vfsScheme);
164+
\assert(isset($urlArr['host']));
165+
166+
return str_replace(\DIRECTORY_SEPARATOR, '/', $urlArr['host'].($urlArr['path'] ?? ''));
167+
}
168+
169+
protected function assertSameVfsIterator(array $expected, \Traversable $iterator)
170+
{
171+
$values = array_map(fn (\SplFileInfo $fileinfo) => $this->stripSchemeFromVfsPath($fileinfo->getPathname()), iterator_to_array($iterator));
172+
173+
$this->assertEquals($expected, array_values($values));
174+
}
175+
}

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.