Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit fbf55c2

Browse filesBrowse files
nicolas-grekasnicolas-grekas
committed
feature #33461 [Cache] Improve RedisTagAwareAdapter invalidation logic & requirements (andrerom)
This PR was merged into the 4.4 branch. Discussion ---------- [Cache] Improve RedisTagAwareAdapter invalidation logic & requirements | Q | A | ------------- | --- | Branch? | 4.4 | Bug fix? | yes, _and improvment_ | New feature? | no | BC breaks? | no | Deprecations? | no | Tests pass? | yes | License | MIT | Doc PR | Changes logic of invalidation in RedisTagAwareAdapter in order to: - Delete the tag key on invalidation => _avoiding possible left behind empty tag keys that Redis is not allowed to evict, gradually consuming more and more memory_ Positive side effects of no longer using sPOP: - Lowered requirements to Redis 2.8, and no specific version constraint for phpredis - Lift limitation of 2 billion keys per tag _(Now only limited by Redis Set datatype: 4 billion)_ Commits ------- 3d38c58 [Cache] Improve RedisTagAwareAdapter invalidation logic & requirements
2 parents e3b513b + 3d38c58 commit fbf55c2
Copy full SHA for fbf55c2

File tree

4 files changed

+92
-97
lines changed
Filter options

4 files changed

+92
-97
lines changed

‎src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php

Copy file name to clipboardExpand all lines: src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php
+67-50Lines changed: 67 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -11,33 +11,32 @@
1111

1212
namespace Symfony\Component\Cache\Adapter;
1313

14-
use Predis;
1514
use Predis\Connection\Aggregate\ClusterInterface;
15+
use Predis\Connection\Aggregate\PredisCluster;
1616
use Predis\Response\Status;
17-
use Symfony\Component\Cache\CacheItem;
18-
use Symfony\Component\Cache\Exception\LogicException;
17+
use Symfony\Component\Cache\Exception\InvalidArgumentException;
1918
use Symfony\Component\Cache\Marshaller\MarshallerInterface;
2019
use Symfony\Component\Cache\Traits\RedisTrait;
2120

2221
/**
23-
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation using sPOP.
22+
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation using RENAME+SMEMBERS.
2423
*
2524
* Set (tag relation info) is stored without expiry (non-volatile), while cache always gets an expiry (volatile) even
2625
* if not set by caller. Thus if you configure redis with the right eviction policy you can be safe this tag <> cache
2726
* relationship survives eviction (cache cleanup when Redis runs out of memory).
2827
*
2928
* Requirements:
30-
* - Server: Redis 3.2+
31-
* - Client: PHP Redis 3.1.3+ OR Predis
32-
* - Redis Server(s) configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
29+
* - Client: PHP Redis or Predis
30+
* Note: Due to lack of RENAME support it is NOT recommended to use Cluster on Predis, instead use phpredis.
31+
* - Server: Redis 2.8+
32+
* Configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
3333
*
3434
* Design limitations:
35-
* - Max 2 billion cache keys per cache tag
36-
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to 2 billion cache items as well
35+
* - Max 4 billion cache keys per cache tag as limited by Redis Set datatype.
36+
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to 4 billion cache items also.
3737
*
3838
* @see https://redis.io/topics/lru-cache#eviction-policies Documentation for Redis eviction policies.
3939
* @see https://redis.io/topics/data-types#sets Documentation for Redis Set datatype.
40-
* @see https://redis.io/commands/spop Documentation for sPOP operation, capable of retriving AND emptying a Set at once.
4140
*
4241
* @author Nicolas Grekas <p@tchwork.com>
4342
* @author André Rømcke <andre.romcke+symfony@gmail.com>
@@ -46,11 +45,6 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
4645
{
4746
use RedisTrait;
4847

49-
/**
50-
* Redis "Set" can hold more than 4 billion members, here we limit ourselves to PHP's > 2 billion max int (32Bit).
51-
*/
52-
private const POP_MAX_LIMIT = 2147483647 - 1;
53-
5448
/**
5549
* Limits for how many keys are deleted in batch.
5650
*/
@@ -62,26 +56,18 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
6256
*/
6357
private const DEFAULT_CACHE_TTL = 8640000;
6458

65-
/**
66-
* @var bool|null
67-
*/
68-
private $redisServerSupportSPOP = null;
69-
7059
/**
7160
* @param \Redis|\RedisArray|\RedisCluster|\Predis\ClientInterface $redisClient The redis client
7261
* @param string $namespace The default namespace
7362
* @param int $defaultLifetime The default lifetime
74-
*
75-
* @throws \Symfony\Component\Cache\Exception\LogicException If phpredis with version lower than 3.1.3.
7663
*/
7764
public function __construct($redisClient, string $namespace = '', int $defaultLifetime = 0, MarshallerInterface $marshaller = null)
7865
{
79-
$this->init($redisClient, $namespace, $defaultLifetime, $marshaller);
80-
81-
// Make sure php-redis is 3.1.3 or higher configured for Redis classes
82-
if (!$this->redis instanceof \Predis\ClientInterface && version_compare(phpversion('redis'), '3.1.3', '<')) {
83-
throw new LogicException('RedisTagAwareAdapter requires php-redis 3.1.3 or higher, alternatively use predis/predis');
66+
if ($redisClient instanceof \Predis\ClientInterface && $redisClient->getConnection() instanceof ClusterInterface && !$redisClient->getConnection() instanceof PredisCluster) {
67+
throw new InvalidArgumentException(sprintf('Unsupported Predis cluster connection: only "%s" is, "%s" given.', PredisCluster::class, \get_class($redisClient->getConnection())));
8468
}
69+
70+
$this->init($redisClient, $namespace, $defaultLifetime, $marshaller);
8571
}
8672

8773
/**
@@ -121,7 +107,7 @@ protected function doSave(array $values, ?int $lifetime, array $addTagData = [],
121107
continue;
122108
}
123109
// setEx results
124-
if (true !== $result && (!$result instanceof Status || $result !== Status::get('OK'))) {
110+
if (true !== $result && (!$result instanceof Status || Status::get('OK') !== $result)) {
125111
$failed[] = $id;
126112
}
127113
}
@@ -138,9 +124,10 @@ protected function doDelete(array $ids, array $tagData = []): bool
138124
return true;
139125
}
140126

141-
$predisCluster = $this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof ClusterInterface;
127+
$predisCluster = $this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof PredisCluster;
142128
$this->pipeline(static function () use ($ids, $tagData, $predisCluster) {
143129
if ($predisCluster) {
130+
// Unlike phpredis, Predis does not handle bulk calls for us against cluster
144131
foreach ($ids as $id) {
145132
yield 'del' => [$id];
146133
}
@@ -161,46 +148,76 @@ protected function doDelete(array $ids, array $tagData = []): bool
161148
*/
162149
protected function doInvalidate(array $tagIds): bool
163150
{
164-
if (!$this->redisServerSupportSPOP()) {
151+
if (!$this->redis instanceof \Predis\ClientInterface || !$this->redis->getConnection() instanceof PredisCluster) {
152+
$movedTagSetIds = $this->renameKeys($this->redis, $tagIds);
153+
} else {
154+
$clusterConnection = $this->redis->getConnection();
155+
$tagIdsByConnection = new \SplObjectStorage();
156+
$movedTagSetIds = [];
157+
158+
foreach ($tagIds as $id) {
159+
$connection = $clusterConnection->getConnectionByKey($id);
160+
$slot = $tagIdsByConnection[$connection] ?? $tagIdsByConnection[$connection] = new \ArrayObject();
161+
$slot[] = $id;
162+
}
163+
164+
foreach ($tagIdsByConnection as $connection) {
165+
$slot = $tagIdsByConnection[$connection];
166+
$movedTagSetIds = array_merge($movedTagSetIds, $this->renameKeys(new $this->redis($connection, $this->redis->getOptions()), $slot->getArrayCopy()));
167+
}
168+
}
169+
170+
// No Sets found
171+
if (!$movedTagSetIds) {
165172
return false;
166173
}
167174

168-
// Pop all tag info at once to avoid race conditions
169-
$tagIdSets = $this->pipeline(static function () use ($tagIds) {
170-
foreach ($tagIds as $tagId) {
171-
// Client: Predis or PHP Redis 3.1.3+ (https://github.com/phpredis/phpredis/commit/d2e203a6)
172-
// Server: Redis 3.2 or higher (https://redis.io/commands/spop)
173-
yield 'sPop' => [$tagId, self::POP_MAX_LIMIT];
175+
// Now safely take the time to read the keys in each set and collect ids we need to delete
176+
$tagIdSets = $this->pipeline(static function () use ($movedTagSetIds) {
177+
foreach ($movedTagSetIds as $movedTagId) {
178+
yield 'sMembers' => [$movedTagId];
174179
}
175180
});
176181

177-
// Flatten generator result from pipeline, ignore keys (tag ids)
178-
$ids = array_unique(array_merge(...iterator_to_array($tagIdSets, false)));
182+
// Return combination of the temporary Tag Set ids and their values (cache ids)
183+
$ids = array_merge($movedTagSetIds, ...iterator_to_array($tagIdSets, false));
179184

180185
// Delete cache in chunks to avoid overloading the connection
181-
foreach (array_chunk($ids, self::BULK_DELETE_LIMIT) as $chunkIds) {
186+
foreach (array_chunk(array_unique($ids), self::BULK_DELETE_LIMIT) as $chunkIds) {
182187
$this->doDelete($chunkIds);
183188
}
184189

185190
return true;
186191
}
187192

188-
private function redisServerSupportSPOP(): bool
193+
/**
194+
* Renames several keys in order to be able to operate on them without risk of race conditions.
195+
*
196+
* Filters out keys that do not exist before returning new keys.
197+
*
198+
* @see https://redis.io/commands/rename
199+
* @see https://redis.io/topics/cluster-spec#keys-hash-tags
200+
*
201+
* @return array Filtered list of the valid moved keys (only those that existed)
202+
*/
203+
private function renameKeys($redis, array $ids): array
189204
{
190-
if (null !== $this->redisServerSupportSPOP) {
191-
return $this->redisServerSupportSPOP;
192-
}
205+
$newIds = [];
206+
$uniqueToken = bin2hex(random_bytes(10));
193207

194-
foreach ($this->getHosts() as $host) {
195-
$info = $host->info('Server');
196-
$info = isset($info['Server']) ? $info['Server'] : $info;
197-
if (version_compare($info['redis_version'], '3.2', '<')) {
198-
CacheItem::log($this->logger, 'Redis server needs to be version 3.2 or higher, your Redis server was detected as '.$info['redis_version']);
208+
$results = $this->pipeline(static function () use ($ids, $uniqueToken) {
209+
foreach ($ids as $id) {
210+
yield 'rename' => [$id, '{'.$id.'}'.$uniqueToken];
211+
}
212+
}, $redis);
199213

200-
return $this->redisServerSupportSPOP = false;
214+
foreach ($results as $id => $result) {
215+
if (true === $result || ($result instanceof Status && Status::get('OK') === $result)) {
216+
// Only take into account if ok (key existed), will be false on phpredis if it did not exist
217+
$newIds[] = '{'.$id.'}'.$uniqueToken;
201218
}
202219
}
203220

204-
return $this->redisServerSupportSPOP = true;
221+
return $newIds;
205222
}
206223
}

‎src/Symfony/Component/Cache/CHANGELOG.md

Copy file name to clipboardExpand all lines: src/Symfony/Component/Cache/CHANGELOG.md
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ CHANGELOG
66

77
* added support for connecting to Redis Sentinel clusters
88
* added argument `$prefix` to `AdapterInterface::clear()`
9+
* improved `RedisTagAwareAdapter` to support Redis server >= 2.8 and up to 4B items per tag
10+
* [BC BREAK] `RedisTagAwareAdapter` is not compatible with `RedisCluster` from `Predis` anymore, use `phpredis` instead
911

1012
4.3.0
1113
-----

‎src/Symfony/Component/Cache/Tests/Adapter/PredisTagAwareRedisClusterAdapterTest.php

Copy file name to clipboardExpand all lines: src/Symfony/Component/Cache/Tests/Adapter/PredisTagAwareRedisClusterAdapterTest.php
-35Lines changed: 0 additions & 35 deletions
This file was deleted.

‎src/Symfony/Component/Cache/Traits/RedisTrait.php

Copy file name to clipboardExpand all lines: src/Symfony/Component/Cache/Traits/RedisTrait.php
+23-12Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -55,9 +55,17 @@ private function init($redisClient, string $namespace, int $defaultLifetime, ?Ma
5555
if (preg_match('#[^-+_.A-Za-z0-9]#', $namespace, $match)) {
5656
throw new InvalidArgumentException(sprintf('RedisAdapter namespace contains "%s" but only characters in [-+_.A-Za-z0-9] are allowed.', $match[0]));
5757
}
58+
5859
if (!$redisClient instanceof \Redis && !$redisClient instanceof \RedisArray && !$redisClient instanceof \RedisCluster && !$redisClient instanceof \Predis\ClientInterface && !$redisClient instanceof RedisProxy && !$redisClient instanceof RedisClusterProxy) {
5960
throw new InvalidArgumentException(sprintf('%s() expects parameter 1 to be Redis, RedisArray, RedisCluster or Predis\ClientInterface, %s given.', __METHOD__, \is_object($redisClient) ? \get_class($redisClient) : \gettype($redisClient)));
6061
}
62+
63+
if ($redisClient instanceof \Predis\ClientInterface && $redisClient->getOptions()->exceptions) {
64+
$options = clone $redisClient->getOptions();
65+
\Closure::bind(function () { $this->options['exceptions'] = false; }, $options, $options)();
66+
$redisClient = new $redisClient($redisClient->getConnection(), $options);
67+
}
68+
6169
$this->redis = $redisClient;
6270
$this->marshaller = $marshaller ?? new DefaultMarshaller();
6371
}
@@ -277,6 +285,7 @@ public static function createConnection($dsn, array $options = [])
277285
$params['replication'] = true;
278286
$hosts[0] += ['alias' => 'master'];
279287
}
288+
$params['exceptions'] = false;
280289

281290
$redis = new $class($hosts, array_diff_key($params, self::$defaultConnectionOptions));
282291
if (isset($params['redis_sentinel'])) {
@@ -414,40 +423,42 @@ protected function doSave(array $values, $lifetime)
414423
}
415424
}
416425
});
426+
417427
foreach ($results as $id => $result) {
418-
if (true !== $result && (!$result instanceof Status || $result !== Status::get('OK'))) {
428+
if (true !== $result && (!$result instanceof Status || Status::get('OK') !== $result)) {
419429
$failed[] = $id;
420430
}
421431
}
422432

423433
return $failed;
424434
}
425435

426-
private function pipeline(\Closure $generator): \Generator
436+
private function pipeline(\Closure $generator, $redis = null): \Generator
427437
{
428438
$ids = [];
439+
$redis = $redis ?? $this->redis;
429440

430-
if ($this->redis instanceof RedisClusterProxy || $this->redis instanceof \RedisCluster || ($this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof RedisCluster)) {
441+
if ($redis instanceof RedisClusterProxy || $redis instanceof \RedisCluster || ($redis instanceof \Predis\ClientInterface && $redis->getConnection() instanceof RedisCluster)) {
431442
// phpredis & predis don't support pipelining with RedisCluster
432443
// see https://github.com/phpredis/phpredis/blob/develop/cluster.markdown#pipelining
433444
// see https://github.com/nrk/predis/issues/267#issuecomment-123781423
434445
$results = [];
435446
foreach ($generator() as $command => $args) {
436-
$results[] = $this->redis->{$command}(...$args);
447+
$results[] = $redis->{$command}(...$args);
437448
$ids[] = $args[0];
438449
}
439-
} elseif ($this->redis instanceof \Predis\ClientInterface) {
440-
$results = $this->redis->pipeline(function ($redis) use ($generator, &$ids) {
450+
} elseif ($redis instanceof \Predis\ClientInterface) {
451+
$results = $redis->pipeline(static function ($redis) use ($generator, &$ids) {
441452
foreach ($generator() as $command => $args) {
442453
$redis->{$command}(...$args);
443454
$ids[] = $args[0];
444455
}
445456
});
446-
} elseif ($this->redis instanceof \RedisArray) {
457+
} elseif ($redis instanceof \RedisArray) {
447458
$connections = $results = $ids = [];
448459
foreach ($generator() as $command => $args) {
449-
if (!isset($connections[$h = $this->redis->_target($args[0])])) {
450-
$connections[$h] = [$this->redis->_instance($h), -1];
460+
if (!isset($connections[$h = $redis->_target($args[0])])) {
461+
$connections[$h] = [$redis->_instance($h), -1];
451462
$connections[$h][0]->multi(\Redis::PIPELINE);
452463
}
453464
$connections[$h][0]->{$command}(...$args);
@@ -461,12 +472,12 @@ private function pipeline(\Closure $generator): \Generator
461472
$results[$k] = $connections[$h][$c];
462473
}
463474
} else {
464-
$this->redis->multi(\Redis::PIPELINE);
475+
$redis->multi(\Redis::PIPELINE);
465476
foreach ($generator() as $command => $args) {
466-
$this->redis->{$command}(...$args);
477+
$redis->{$command}(...$args);
467478
$ids[] = $args[0];
468479
}
469-
$results = $this->redis->exec();
480+
$results = $redis->exec();
470481
}
471482

472483
foreach ($ids as $k => $id) {

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.