Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 28948ee

Browse filesBrowse files
andreromnicolas-grekas
andrerom
authored and
nicolas-grekas
committed
[Cache] Improve RedisTagAwareAdapter invalidation logic & requirements
1 parent a0bbae7 commit 28948ee
Copy full SHA for 28948ee

File tree

3 files changed

+74
-86
lines changed
Filter options

3 files changed

+74
-86
lines changed

‎src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php

Copy file name to clipboardExpand all lines: src/Symfony/Component/Cache/Adapter/RedisTagAwareAdapter.php
+72-51Lines changed: 72 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -11,33 +11,32 @@
1111

1212
namespace Symfony\Component\Cache\Adapter;
1313

14-
use Predis;
1514
use Predis\Connection\Aggregate\ClusterInterface;
15+
use Predis\Connection\Aggregate\PredisCluster;
1616
use Predis\Response\Status;
17-
use Symfony\Component\Cache\CacheItem;
18-
use Symfony\Component\Cache\Exception\LogicException;
17+
use Symfony\Component\Cache\Exception\InvalidArgumentException;
1918
use Symfony\Component\Cache\Marshaller\MarshallerInterface;
2019
use Symfony\Component\Cache\Traits\RedisTrait;
2120

2221
/**
23-
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation using sPOP.
22+
* Stores tag id <> cache id relationship as a Redis Set, lookup on invalidation using RENAME+SMEMBERS.
2423
*
2524
* Set (tag relation info) is stored without expiry (non-volatile), while cache always gets an expiry (volatile) even
2625
* if not set by caller. Thus if you configure redis with the right eviction policy you can be safe this tag <> cache
2726
* relationship survives eviction (cache cleanup when Redis runs out of memory).
2827
*
2928
* Requirements:
30-
* - Server: Redis 3.2+
31-
* - Client: PHP Redis 3.1.3+ OR Predis
32-
* - Redis Server(s) configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
29+
* - Client: PHP Redis or Predis
30+
* Note: Due to lack of RENAME support it is NOT recommended to use Cluster on Predis, instead use phpredis.
31+
* - Server: Redis 2.8+
32+
* Configured with any `volatile-*` eviction policy, OR `noeviction` if it will NEVER fill up memory
3333
*
3434
* Design limitations:
35-
* - Max 2 billion cache keys per cache tag
36-
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to 2 billion cache items as well
35+
* - Max 4 billion cache keys per cache tag as limited by Redis Set datatype.
36+
* E.g. If you use a "all" items tag for expiry instead of clear(), that limits you to 4 billion cache items also.
3737
*
3838
* @see https://redis.io/topics/lru-cache#eviction-policies Documentation for Redis eviction policies.
3939
* @see https://redis.io/topics/data-types#sets Documentation for Redis Set datatype.
40-
* @see https://redis.io/commands/spop Documentation for sPOP operation, capable of retriving AND emptying a Set at once.
4140
*
4241
* @author Nicolas Grekas <p@tchwork.com>
4342
* @author André Rømcke <andre.romcke+symfony@gmail.com>
@@ -46,11 +45,6 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
4645
{
4746
use RedisTrait;
4847

49-
/**
50-
* Redis "Set" can hold more than 4 billion members, here we limit ourselves to PHP's > 2 billion max int (32Bit).
51-
*/
52-
private const POP_MAX_LIMIT = 2147483647 - 1;
53-
5448
/**
5549
* Limits for how many keys are deleted in batch.
5650
*/
@@ -62,26 +56,18 @@ class RedisTagAwareAdapter extends AbstractTagAwareAdapter
6256
*/
6357
private const DEFAULT_CACHE_TTL = 8640000;
6458

65-
/**
66-
* @var bool|null
67-
*/
68-
private $redisServerSupportSPOP = null;
69-
7059
/**
7160
* @param \Redis|\RedisArray|\RedisCluster|\Predis\ClientInterface $redisClient The redis client
7261
* @param string $namespace The default namespace
7362
* @param int $defaultLifetime The default lifetime
74-
*
75-
* @throws \Symfony\Component\Cache\Exception\LogicException If phpredis with version lower than 3.1.3.
7663
*/
7764
public function __construct($redisClient, string $namespace = '', int $defaultLifetime = 0, MarshallerInterface $marshaller = null)
7865
{
79-
$this->init($redisClient, $namespace, $defaultLifetime, $marshaller);
80-
81-
// Make sure php-redis is 3.1.3 or higher configured for Redis classes
82-
if (!$this->redis instanceof \Predis\ClientInterface && version_compare(phpversion('redis'), '3.1.3', '<')) {
83-
throw new LogicException('RedisTagAwareAdapter requires php-redis 3.1.3 or higher, alternatively use predis/predis');
66+
if ($redisClient instanceof \Predis\ClientInterface && $redisClient->getConnection() instanceof ClusterInterface && !$redisClient->getConnection() instanceof PredisCluster) {
67+
throw new InvalidArgumentException(sprintf('Unsupported Predis cluster connection: only "%s" is, "%s" given.', PredisCluster::class, \get_class($redisClient->getConnection())));
8468
}
69+
70+
$this->init($redisClient, $namespace, $defaultLifetime, $marshaller);
8571
}
8672

8773
/**
@@ -138,9 +124,10 @@ protected function doDelete(array $ids, array $tagData = []): bool
138124
return true;
139125
}
140126

141-
$predisCluster = $this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof ClusterInterface;
127+
$predisCluster = $this->redis instanceof \Predis\ClientInterface && $this->redis->getConnection() instanceof PredisCluster;
142128
$this->pipeline(static function () use ($ids, $tagData, $predisCluster) {
143129
if ($predisCluster) {
130+
// Unlike phpredis, Predis does not handle bulk calls for us against cluster
144131
foreach ($ids as $id) {
145132
yield 'del' => [$id];
146133
}
@@ -161,46 +148,80 @@ protected function doDelete(array $ids, array $tagData = []): bool
161148
*/
162149
protected function doInvalidate(array $tagIds): bool
163150
{
164-
if (!$this->redisServerSupportSPOP()) {
151+
if (!$this->redis instanceof \Predis\ClientInterface || !$this->redis->getConnection() instanceof PredisCluster) {
152+
$movedTagSetIds = $this->renameKeys($this->redis, $tagIds);
153+
} else {
154+
$clusterConnection = $this->redis->getConnection();
155+
$tagIdsByConnection = new \SplObjectStorage();
156+
$movedTagSetIds = [];
157+
158+
foreach ($tagIds as $id) {
159+
$connection = $clusterConnection->getConnectionByKey($id);
160+
$slot = $tagIdsByConnection[$connection] ?? $tagIdsByConnection[$connection] = new \ArrayObject();
161+
$slot[] = $id;
162+
}
163+
164+
foreach ($tagIdsByConnection as $connection) {
165+
$slot = $tagIdsByConnection[$connection];
166+
$redis = new \Predis\Client($connection, $this->redis->getOptions());
167+
$movedTagSetIds = array_merge($movedTagSetIds, $this->renameKeys($redis, $slot->getArrayCopy()));
168+
}
169+
}
170+
171+
// No Sets found
172+
if (!$movedTagSetIds) {
165173
return false;
166174
}
167175

168-
// Pop all tag info at once to avoid race conditions
169-
$tagIdSets = $this->pipeline(static function () use ($tagIds) {
170-
foreach ($tagIds as $tagId) {
171-
// Client: Predis or PHP Redis 3.1.3+ (https://github.com/phpredis/phpredis/commit/d2e203a6)
172-
// Server: Redis 3.2 or higher (https://redis.io/commands/spop)
173-
yield 'sPop' => [$tagId, self::POP_MAX_LIMIT];
176+
// Now safely take the time to read the keys in each set and collect ids we need to delete
177+
$tagIdSets = $this->pipeline(static function () use ($movedTagSetIds) {
178+
foreach ($movedTagSetIds as $movedTagId) {
179+
yield 'sMembers' => [$movedTagId];
174180
}
175181
});
176182

177-
// Flatten generator result from pipeline, ignore keys (tag ids)
178-
$ids = array_unique(array_merge(...iterator_to_array($tagIdSets, false)));
183+
// Return combination of the temporary Tag Set ids and their values (cache ids)
184+
$ids = array_merge($movedTagSetIds, ...iterator_to_array($tagIdSets, false));
179185

180186
// Delete cache in chunks to avoid overloading the connection
181-
foreach (array_chunk($ids, self::BULK_DELETE_LIMIT) as $chunkIds) {
187+
foreach (array_chunk(array_unique($ids), self::BULK_DELETE_LIMIT) as $chunkIds) {
182188
$this->doDelete($chunkIds);
183189
}
184190

185191
return true;
186192
}
187193

188-
private function redisServerSupportSPOP(): bool
194+
/**
195+
* Renames several keys in order to be able to operate on them without risk of race conditions.
196+
*
197+
* Filters out keys that do not exist before returning new keys.
198+
*
199+
* @see https://redis.io/commands/rename
200+
*
201+
* @return array Filtered list of the valid moved keys (only those that existed)
202+
*/
203+
private function renameKeys($redis, array $ids): array
189204
{
190-
if (null !== $this->redisServerSupportSPOP) {
191-
return $this->redisServerSupportSPOP;
192-
}
193-
194-
foreach ($this->getHosts() as $host) {
195-
$info = $host->info('Server');
196-
$info = isset($info['Server']) ? $info['Server'] : $info;
197-
if (version_compare($info['redis_version'], '3.2', '<')) {
198-
CacheItem::log($this->logger, 'Redis server needs to be version 3.2 or higher, your Redis server was detected as '.$info['redis_version']);
199-
200-
return $this->redisServerSupportSPOP = false;
205+
// 1. Due to Predis exception we don't do this in pipeline
206+
// 2. https://redis.io/topics/cluster-spec#keys-hash-tags is used to place in same hash slot on cluster
207+
$newIds = [];
208+
$uniqueToken = bin2hex(random_bytes(10));
209+
foreach ($ids as $id) {
210+
$newId = '{'.$id.'}'.$uniqueToken;
211+
try {
212+
$ok = $redis->rename($id, $newId);
213+
if (true === $ok || ($ok instanceof Status && $ok === Status::get('OK'))) {
214+
// Only take into account if ok (key existed), will be false on phpredis if it did not exist
215+
$newIds[] = $newId;
216+
}
217+
} catch (\Predis\Response\ServerException $e) {
218+
// Silence errors when key does not exists on Predis. Otherwise re-throw exception
219+
if ('ERR no such key' !== $e->getMessage()) {
220+
throw $e;
221+
}
201222
}
202223
}
203224

204-
return $this->redisServerSupportSPOP = true;
225+
return $newIds;
205226
}
206227
}

‎src/Symfony/Component/Cache/CHANGELOG.md

Copy file name to clipboardExpand all lines: src/Symfony/Component/Cache/CHANGELOG.md
+2Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ CHANGELOG
66

77
* added support for connecting to Redis Sentinel clusters
88
* added argument `$prefix` to `AdapterInterface::clear()`
9+
* improved `RedisTagAwareAdapter` to support Redis server >= 2.8 and up to 4B items per tag
10+
* [BC BREAK] `RedisTagAwareAdapter` is not compatible with `RedisCluster` from `Predis` anymore, use `phpredis` instead
911

1012
4.3.0
1113
-----

‎src/Symfony/Component/Cache/Tests/Adapter/PredisTagAwareRedisClusterAdapterTest.php

Copy file name to clipboardExpand all lines: src/Symfony/Component/Cache/Tests/Adapter/PredisTagAwareRedisClusterAdapterTest.php
-35Lines changed: 0 additions & 35 deletions
This file was deleted.

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.