From cbfdaec2f9390d1144846fde365d41c180573893 Mon Sep 17 00:00:00 2001 From: michael-grunder Date: Sat, 19 Apr 2025 18:02:19 -0700 Subject: [PATCH 1/6] Rework how and when update slot cache and remap topology. When a replica node is taken offline, it will briefly respond to commands with some form of `CLUSTERDOWN` error. Presently while this did cause PhpRedis to invalidate the slot cache, we did not immediately remap the topology. What this meant, is that for an already connected cluster, we might keep sending readonly commands to this now decomissioned cluster replica for the duration of the request. Additionally, if the request was long lived enough we may still have it mapped if the replica was brought back online at which point the client might see dozens of `LOADING` exceptions. This commit just changes our behavior when we detect any `CLUSTERDOWN` error such that we flush the slot cache, disconnect the particular node, and also attempt a remap of the topology. --- cluster_library.c | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/cluster_library.c b/cluster_library.c index 38e1a6505b..954754cadb 100644 --- a/cluster_library.c +++ b/cluster_library.c @@ -1597,29 +1597,27 @@ PHP_REDIS_API short cluster_send_command(redisCluster *c, short slot, const char timedout = c->waitms ? mstime() - msstart >= c->waitms : 0; } while (!c->clusterdown && !timedout); - // If we've detected the cluster is down, throw an exception - if (c->clusterdown) { - cluster_cache_clear(c); - CLUSTER_THROW_EXCEPTION("The Redis Cluster is down (CLUSTERDOWN)", 0); - return -1; - } else if (timedout || resp == -1) { - // Make sure the socket is reconnected, it such that it is in a clean state + if (c->clusterdown || (timedout || resp == -1)) { + /* Flush slot cache and ensure a reconnection to reread the topology */ redis_sock_disconnect(c->cmd_sock, 1, 1); cluster_cache_clear(c); - if (timedout) { - CLUSTER_THROW_EXCEPTION("Timed out attempting to find data in the correct node!", 0); + if (c->clusterdown) { + cluster_map_keyspace(c); + CLUSTER_THROW_EXCEPTION("The Redis Cluster is down (CLUSTERDOWN)", 0); + } else if (timedout) { + CLUSTER_THROW_EXCEPTION( + "Timed out attempting to find data in the correct node!", 0); } else { - CLUSTER_THROW_EXCEPTION("Error processing response from Redis node!", 0); + CLUSTER_THROW_EXCEPTION( + "Error processing response from Redis node!", 0); } return -1; } - /* Clear redirection flag */ + /* Clear redirection flag and return success */ c->redir_type = REDIR_NONE; - - // Success, return the slot where data exists. return 0; } From bc8cd2c4bc42992cec05545349fcf06f3f17c607 Mon Sep 17 00:00:00 2001 From: michael-grunder Date: Sun, 20 Apr 2025 06:53:23 -0700 Subject: [PATCH 2/6] Add a mechanism for the user to flush the slot cache. --- cluster_library.c | 6 ++++-- cluster_library.h | 2 +- redis_cluster.c | 8 ++++++++ redis_cluster.stub.php | 6 ++++++ redis_cluster_arginfo.h | 16 ++++++++++------ redis_cluster_legacy_arginfo.h | 6 +++++- 6 files changed, 34 insertions(+), 10 deletions(-) diff --git a/cluster_library.c b/cluster_library.c index 954754cadb..f8b4f724ac 100644 --- a/cluster_library.c +++ b/cluster_library.c @@ -3128,11 +3128,13 @@ PHP_REDIS_API void cluster_cache_store(zend_string *hash, HashTable *nodes) { redis_register_persistent_resource(cc->hash, cc, le_cluster_slot_cache); } -void cluster_cache_clear(redisCluster *c) +zend_bool cluster_cache_clear(redisCluster *c) { if (c->cache_key) { - zend_hash_del(&EG(persistent_list), c->cache_key); + return zend_hash_del(&EG(persistent_list), c->cache_key) == SUCCESS; } + + return false; } diff --git a/cluster_library.h b/cluster_library.h index aa5152cb67..f75170426d 100644 --- a/cluster_library.h +++ b/cluster_library.h @@ -392,7 +392,7 @@ PHP_REDIS_API char **cluster_sock_read_multibulk_reply(RedisSock *redis_sock, in PHP_REDIS_API void cluster_cache_store(zend_string *hash, HashTable *nodes); PHP_REDIS_API redisCachedCluster *cluster_cache_load(zend_string *hash); -void cluster_cache_clear(redisCluster *c); +zend_bool cluster_cache_clear(redisCluster *c); /* * Redis Cluster response handlers. Our response handlers generally take the diff --git a/redis_cluster.c b/redis_cluster.c index 1cbd825925..c1b6b88d68 100644 --- a/redis_cluster.c +++ b/redis_cluster.c @@ -1768,6 +1768,14 @@ static void redisClearNodeBytes(redisClusterNode *node) { } } +PHP_METHOD(RedisCluster, flushSlotCache) { + redisCluster *c = GET_CONTEXT(); + + ZEND_PARSE_PARAMETERS_NONE(); + + RETURN_BOOL(cluster_cache_clear(c)); +} + PHP_METHOD(RedisCluster, gettransferredbytes) { redisCluster *c = GET_CONTEXT(); zend_long rx = 0, tx = 0; diff --git a/redis_cluster.stub.php b/redis_cluster.stub.php index 58cced5777..1a123e037b 100644 --- a/redis_cluster.stub.php +++ b/redis_cluster.stub.php @@ -88,6 +88,12 @@ public function _masters(): array; public function _redir(): string|null; + /** + * Flush the persistent slot cache, if one exists. + * @return bool Whether the slot cache was flushed. + */ + public function flushSlotCache(): bool; + /** * @see Redis::acl */ diff --git a/redis_cluster_arginfo.h b/redis_cluster_arginfo.h index b3fb58475a..bbc3447838 100644 --- a/redis_cluster_arginfo.h +++ b/redis_cluster_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 43a43fa735ced4b48a361078ac8a10fb62cb1244 */ + * Stub hash: db05ec768efb80715ac68fe0a1ab65d37de1d390 */ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_RedisCluster___construct, 0, 0, 1) ZEND_ARG_TYPE_INFO(0, name, IS_STRING, 1) @@ -41,6 +41,9 @@ ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_RedisCluster__redir, 0, 0, IS_STRING, 1) ZEND_END_ARG_INFO() +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_RedisCluster_flushSlotCache, 0, 0, _IS_BOOL, 0) +ZEND_END_ARG_INFO() + ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_RedisCluster_acl, 0, 2, IS_MIXED, 0) ZEND_ARG_TYPE_MASK(0, key_or_address, MAY_BE_STRING|MAY_BE_ARRAY, NULL) ZEND_ARG_TYPE_INFO(0, subcmd, IS_STRING, 0) @@ -141,8 +144,7 @@ ZEND_END_ARG_INFO() #define arginfo_class_RedisCluster_lmpop arginfo_class_RedisCluster_zmpop -ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_RedisCluster_clearlasterror, 0, 0, _IS_BOOL, 0) -ZEND_END_ARG_INFO() +#define arginfo_class_RedisCluster_clearlasterror arginfo_class_RedisCluster_flushSlotCache ZEND_BEGIN_ARG_WITH_RETURN_TYPE_MASK_EX(arginfo_class_RedisCluster_client, 0, 2, MAY_BE_ARRAY|MAY_BE_STRING|MAY_BE_BOOL) ZEND_ARG_TYPE_MASK(0, key_or_address, MAY_BE_STRING|MAY_BE_ARRAY, NULL) @@ -150,7 +152,7 @@ ZEND_BEGIN_ARG_WITH_RETURN_TYPE_MASK_EX(arginfo_class_RedisCluster_client, 0, 2, ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, arg, IS_STRING, 1, "null") ZEND_END_ARG_INFO() -#define arginfo_class_RedisCluster_close arginfo_class_RedisCluster_clearlasterror +#define arginfo_class_RedisCluster_close arginfo_class_RedisCluster_flushSlotCache ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_RedisCluster_cluster, 0, 2, IS_MIXED, 0) ZEND_ARG_TYPE_MASK(0, key_or_address, MAY_BE_STRING|MAY_BE_ARRAY, NULL) @@ -198,7 +200,7 @@ ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_RedisCluster_del, 0, 1 ZEND_ARG_VARIADIC_TYPE_INFO(0, other_keys, IS_STRING, 0) ZEND_END_ARG_INFO() -#define arginfo_class_RedisCluster_discard arginfo_class_RedisCluster_clearlasterror +#define arginfo_class_RedisCluster_discard arginfo_class_RedisCluster_flushSlotCache ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_RedisCluster_dump, 0, 1, RedisCluster, MAY_BE_STRING|MAY_BE_FALSE) ZEND_ARG_TYPE_INFO(0, key, IS_STRING, 0) @@ -817,7 +819,7 @@ ZEND_END_ARG_INFO() #define arginfo_class_RedisCluster_unlink arginfo_class_RedisCluster_del -#define arginfo_class_RedisCluster_unwatch arginfo_class_RedisCluster_clearlasterror +#define arginfo_class_RedisCluster_unwatch arginfo_class_RedisCluster_flushSlotCache ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_RedisCluster_watch, 0, 1, RedisCluster, MAY_BE_BOOL) ZEND_ARG_TYPE_INFO(0, key, IS_STRING, 0) @@ -1073,6 +1075,7 @@ ZEND_METHOD(RedisCluster, _unpack); ZEND_METHOD(RedisCluster, _prefix); ZEND_METHOD(RedisCluster, _masters); ZEND_METHOD(RedisCluster, _redir); +ZEND_METHOD(RedisCluster, flushSlotCache); ZEND_METHOD(RedisCluster, acl); ZEND_METHOD(RedisCluster, append); ZEND_METHOD(RedisCluster, bgrewriteaof); @@ -1304,6 +1307,7 @@ static const zend_function_entry class_RedisCluster_methods[] = { ZEND_ME(RedisCluster, _prefix, arginfo_class_RedisCluster__prefix, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, _masters, arginfo_class_RedisCluster__masters, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, _redir, arginfo_class_RedisCluster__redir, ZEND_ACC_PUBLIC) + ZEND_ME(RedisCluster, flushSlotCache, arginfo_class_RedisCluster_flushSlotCache, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, acl, arginfo_class_RedisCluster_acl, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, append, arginfo_class_RedisCluster_append, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, bgrewriteaof, arginfo_class_RedisCluster_bgrewriteaof, ZEND_ACC_PUBLIC) diff --git a/redis_cluster_legacy_arginfo.h b/redis_cluster_legacy_arginfo.h index d117db522a..85a0630071 100644 --- a/redis_cluster_legacy_arginfo.h +++ b/redis_cluster_legacy_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: 43a43fa735ced4b48a361078ac8a10fb62cb1244 */ + * Stub hash: db05ec768efb80715ac68fe0a1ab65d37de1d390 */ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_RedisCluster___construct, 0, 0, 1) ZEND_ARG_INFO(0, name) @@ -34,6 +34,8 @@ ZEND_END_ARG_INFO() #define arginfo_class_RedisCluster__redir arginfo_class_RedisCluster__masters +#define arginfo_class_RedisCluster_flushSlotCache arginfo_class_RedisCluster__masters + ZEND_BEGIN_ARG_INFO_EX(arginfo_class_RedisCluster_acl, 0, 0, 2) ZEND_ARG_INFO(0, key_or_address) ZEND_ARG_INFO(0, subcmd) @@ -915,6 +917,7 @@ ZEND_METHOD(RedisCluster, _unpack); ZEND_METHOD(RedisCluster, _prefix); ZEND_METHOD(RedisCluster, _masters); ZEND_METHOD(RedisCluster, _redir); +ZEND_METHOD(RedisCluster, flushSlotCache); ZEND_METHOD(RedisCluster, acl); ZEND_METHOD(RedisCluster, append); ZEND_METHOD(RedisCluster, bgrewriteaof); @@ -1146,6 +1149,7 @@ static const zend_function_entry class_RedisCluster_methods[] = { ZEND_ME(RedisCluster, _prefix, arginfo_class_RedisCluster__prefix, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, _masters, arginfo_class_RedisCluster__masters, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, _redir, arginfo_class_RedisCluster__redir, ZEND_ACC_PUBLIC) + ZEND_ME(RedisCluster, flushSlotCache, arginfo_class_RedisCluster_flushSlotCache, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, acl, arginfo_class_RedisCluster_acl, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, append, arginfo_class_RedisCluster_append, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, bgrewriteaof, arginfo_class_RedisCluster_bgrewriteaof, ZEND_ACC_PUBLIC) From 47d3239fe2a6cd08e92393aca2d8fff26bfab365 Mon Sep 17 00:00:00 2001 From: michael-grunder Date: Sun, 20 Apr 2025 07:44:22 -0700 Subject: [PATCH 3/6] Add a slot cache expiration mechanism This commit adds a new `ini` setting `redis.clusters.slot_cache_expiry` which specifies in seconds how long a cached slot map should live before being refreshed. The setting is disabled by default to avoid breaking backward compatibility. --- cluster.md | 5 ++++- cluster_library.c | 40 ++++++++++++++++++++++++++++------------ cluster_library.h | 3 ++- redis.c | 1 + redis_cluster.c | 2 +- 5 files changed, 36 insertions(+), 15 deletions(-) diff --git a/cluster.md b/cluster.md index 3949f88fae..158174cf25 100644 --- a/cluster.md +++ b/cluster.md @@ -51,7 +51,10 @@ $obj_cluster = new RedisCluster('mycluster'); On construction, the RedisCluster class will iterate over the provided seed nodes until it can attain a connection to the cluster and run CLUSTER SLOTS to map every node in the cluster locally. Once the keyspace is mapped, RedisCluster will only connect to nodes when it needs to (e.g. you're getting a key that we believe is on that node.) ## Slot caching -Each time the `RedisCluster` class is constructed from scratch, phpredis needs to execute a `CLUSTER SLOTS` command to map the keyspace. Although this isn't an expensive command, it does require a round trip for each newly created object, which is inefficient. Starting from PhpRedis 5.0.0 these slots can be cached by setting `redis.clusters.cache_slots = 1` in `php.ini`. +Each time the `RedisCluster` class is constructed from scratch, phpredis needs to execute a `CLUSTER SLOTS` command to map the keyspace. Although this isn't an expensive command, it does require a round trip for each newly created object, which is inefficient. Starting from PhpRedis 5.0.0 these slots can be cached by setting `redis.clusters.cache_slots = 1` in `php.ini`. + +### Slot cache expiration +You can also configure the cached slot maps to expire after a certain number of seconds. To do this set a positive value in `redis.clusters.slot_cache_expiry`. Expiring the cache could be beneficial in situations where new replica(s) are added to a cluster when PhpRedis has the topology cached. A non-destructiv change like this will not result in `MOVED` or `ASKING` responses from Redis so PhpRedis won't know to refresh the slot topology. ## Timeouts Because Redis cluster is intended to provide high availability, timeouts do not work in the same way they do in normal socket communication. It's fully possible to have a timeout or even exception on a given socket (say in the case that a master node has failed), and continue to serve the request if and when a slave can be promoted as the new master. diff --git a/cluster_library.c b/cluster_library.c index f8b4f724ac..bbcd54c682 100644 --- a/cluster_library.c +++ b/cluster_library.c @@ -883,6 +883,16 @@ cluster_free(redisCluster *c, int free_ctx) if (free_ctx) efree(c); } +static zend_long cluster_slot_cache_expiry(void) { + zend_long expiry; + + expiry = INI_INT("redis.clusters.slot_cache_expiry"); + if (expiry <= 0) + return 0; + + return time(NULL) + expiry; +} + /* Create a cluster slot cache structure */ PHP_REDIS_API redisCachedCluster *cluster_cache_create(zend_string *hash, HashTable *nodes) { @@ -892,6 +902,7 @@ redisCachedCluster *cluster_cache_create(zend_string *hash, HashTable *nodes) { cc = pecalloc(1, sizeof(*cc), 1); cc->hash = zend_string_dup(hash, 1); + cc->expiry = cluster_slot_cache_expiry(); /* Copy nodes */ cc->master = pecalloc(zend_hash_num_elements(nodes), sizeof(*cc->master), 1); @@ -3103,22 +3114,26 @@ zend_string *cluster_hash_seeds(zend_string **seeds, uint32_t count) { } PHP_REDIS_API redisCachedCluster *cluster_cache_load(zend_string *hash) { + redisCachedCluster *cc; zend_resource *le; /* Look for cached slot information */ le = zend_hash_find_ptr(&EG(persistent_list), hash); - if (le != NULL) { - /* Sanity check on our list type */ - if (le->type == le_cluster_slot_cache) { - /* Success, return the cached entry */ - return le->ptr; - } + if (le == NULL) + return NULL; + + if (le->type != le_cluster_slot_cache) { php_error_docref(0, E_WARNING, "Invalid slot cache resource"); + return NULL; } - /* Not found */ - return NULL; + cc = le->ptr; + if (cc->expiry != 0 && cc->expiry <= time(NULL)) { + return NULL; + } + + return cc; } /* Cache a cluster's slot information in persistent_list if it's enabled */ @@ -3128,13 +3143,14 @@ PHP_REDIS_API void cluster_cache_store(zend_string *hash, HashTable *nodes) { redis_register_persistent_resource(cc->hash, cc, le_cluster_slot_cache); } -zend_bool cluster_cache_clear(redisCluster *c) -{ +/* Flush the slot cache for the provided cluster, if one exists. Success and + * failure in this context just means "did we remove it" */ +int cluster_cache_clear(redisCluster *c) { if (c->cache_key) { - return zend_hash_del(&EG(persistent_list), c->cache_key) == SUCCESS; + return zend_hash_del(&EG(persistent_list), c->cache_key); } - return false; + return FAILURE; } diff --git a/cluster_library.h b/cluster_library.h index f75170426d..965d28ef60 100644 --- a/cluster_library.h +++ b/cluster_library.h @@ -161,6 +161,7 @@ typedef struct redisCachedCluster { zend_string *hash; /* What we're cached by */ redisCachedMaster *master; /* Array of masters */ size_t count; /* Number of masters */ + uint64_t expiry; /* Expiry time (if any) */ } redisCachedCluster; /* A Redis Cluster master node */ @@ -392,7 +393,7 @@ PHP_REDIS_API char **cluster_sock_read_multibulk_reply(RedisSock *redis_sock, in PHP_REDIS_API void cluster_cache_store(zend_string *hash, HashTable *nodes); PHP_REDIS_API redisCachedCluster *cluster_cache_load(zend_string *hash); -zend_bool cluster_cache_clear(redisCluster *c); +int cluster_cache_clear(redisCluster *c); /* * Redis Cluster response handlers. Our response handlers generally take the diff --git a/redis.c b/redis.c index 3f13a59888..b380ed012c 100644 --- a/redis.c +++ b/redis.c @@ -95,6 +95,7 @@ PHP_INI_BEGIN() /* redis cluster */ PHP_INI_ENTRY("redis.clusters.cache_slots", "0", PHP_INI_ALL, NULL) + PHP_INI_ENTRY("redis.clusters.slot_cache_expiry", "0", PHP_INI_ALL, NULL) PHP_INI_ENTRY("redis.clusters.auth", "", PHP_INI_ALL, NULL) PHP_INI_ENTRY("redis.clusters.persistent", "0", PHP_INI_ALL, NULL) PHP_INI_ENTRY("redis.clusters.read_timeout", "0", PHP_INI_ALL, NULL) diff --git a/redis_cluster.c b/redis_cluster.c index c1b6b88d68..9a6e7dff17 100644 --- a/redis_cluster.c +++ b/redis_cluster.c @@ -1773,7 +1773,7 @@ PHP_METHOD(RedisCluster, flushSlotCache) { ZEND_PARSE_PARAMETERS_NONE(); - RETURN_BOOL(cluster_cache_clear(c)); + RETURN_BOOL(cluster_cache_clear(c) == SUCCESS); } PHP_METHOD(RedisCluster, gettransferredbytes) { From fead6c7f27ab2426bbf72951fa2155e829d3af3a Mon Sep 17 00:00:00 2001 From: michael-grunder Date: Wed, 23 Apr 2025 18:56:13 -0700 Subject: [PATCH 4/6] Add `invalidateSlotCaches()` function to invalidate any slot caches. This commit adds a mechanism to invalidate every `RedisCluster` slot cache across all forked workers with one call to any of the workers. The invalidation itslef works by using a `uint64_t` generation counter which is allocated in shared memory on `MINIT` which is stored in every slot cache when it is created. When a user wants to invalidate any of the slot caches they can call `RedisCluster::invalidateSlotCaches()` which simply does an atomic increment on this global genertion value. When a slot cache is retreived from the persistent list, the internal generation is compared to the global shared memory generation and if they don't match, the cache is not used, which is the same as invalidting it as RedisCluster will write a new slot cache on object destruction. The feature is only compiled if `config.m4` determines that using `mmap` to allocate shared memory is available and works. Additionally, the feature must be explicitly enabled by setting the `redis.clusters.shared_slot_cache_invalidation`. --- cluster_library.c | 58 +++++++++++++++++++++++++++++++--- cluster_library.h | 15 ++++++++- config.m4 | 19 +++++++++++ redis.c | 20 +++++++++++- redis_cluster.c | 15 ++++++++- redis_cluster.stub.php | 10 ++++++ redis_cluster_arginfo.h | 32 ++++++++++++------- redis_cluster_legacy_arginfo.h | 30 ++++++++++++------ 8 files changed, 171 insertions(+), 28 deletions(-) diff --git a/cluster_library.c b/cluster_library.c index bbcd54c682..a6b8e9f366 100644 --- a/cluster_library.c +++ b/cluster_library.c @@ -6,6 +6,14 @@ #include "crc16.h" #include +#ifdef HAVE_REDIS_ATOMICS_MMAP +#include +#include + +static _Atomic uint64_t *g_cluster_cache_gen; +static pid_t g_cluster_cache_pid; +#endif + extern zend_class_entry *redis_cluster_exception_ce; int le_cluster_slot_cache; @@ -883,7 +891,7 @@ cluster_free(redisCluster *c, int free_ctx) if (free_ctx) efree(c); } -static zend_long cluster_slot_cache_expiry(void) { +static zend_long cluster_cache_expiry(void) { zend_long expiry; expiry = INI_INT("redis.clusters.slot_cache_expiry"); @@ -893,6 +901,38 @@ static zend_long cluster_slot_cache_expiry(void) { return time(NULL) + expiry; } +#ifdef HAVE_REDIS_ATOMICS_MMAP +void cluster_cache_gen_init(void) { + g_cluster_cache_pid = getpid(); + g_cluster_cache_gen = mmap(NULL, sizeof(uint64_t), PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); +} + +void cluster_cache_gen_free(void) { + if (g_cluster_cache_gen && g_cluster_cache_pid == getpid()) { + munmap(g_cluster_cache_gen, sizeof(uint64_t)); + g_cluster_cache_gen = NULL; + } +} + +int cluster_cache_gen_invalidate(void) { + if (g_cluster_cache_gen == NULL) + return FAILURE; + + atomic_fetch_add_explicit(g_cluster_cache_gen, 1, memory_order_relaxed); + + return SUCCESS; +} + +static uint64_t cluster_cache_gen(void) { + if (g_cluster_cache_gen == NULL) + return 0; + + return atomic_load(g_cluster_cache_gen); +} + +#endif + /* Create a cluster slot cache structure */ PHP_REDIS_API redisCachedCluster *cluster_cache_create(zend_string *hash, HashTable *nodes) { @@ -902,7 +942,10 @@ redisCachedCluster *cluster_cache_create(zend_string *hash, HashTable *nodes) { cc = pecalloc(1, sizeof(*cc), 1); cc->hash = zend_string_dup(hash, 1); - cc->expiry = cluster_slot_cache_expiry(); + cc->expiry = cluster_cache_expiry(); +#ifdef HAVE_REDIS_ATOMICS_MMAP + cc->generation = cluster_cache_gen(); +#endif /* Copy nodes */ cc->master = pecalloc(zend_hash_num_elements(nodes), sizeof(*cc->master), 1); @@ -3119,7 +3162,6 @@ PHP_REDIS_API redisCachedCluster *cluster_cache_load(zend_string *hash) { /* Look for cached slot information */ le = zend_hash_find_ptr(&EG(persistent_list), hash); - if (le == NULL) return NULL; @@ -3129,9 +3171,15 @@ PHP_REDIS_API redisCachedCluster *cluster_cache_load(zend_string *hash) { } cc = le->ptr; - if (cc->expiry != 0 && cc->expiry <= time(NULL)) { + /* Short circuit if it should be expired */ + if (cc->expiry != 0 && cc->expiry <= time(NULL)) return NULL; - } + +#ifdef HAVE_REDIS_ATOMICS_MMAP + /* Short circuit if it has been globally invalidated */ + if (cluster_cache_gen() != cc->generation) + return NULL; +#endif return cc; } diff --git a/cluster_library.h b/cluster_library.h index 965d28ef60..646d7846aa 100644 --- a/cluster_library.h +++ b/cluster_library.h @@ -7,6 +7,10 @@ #include "TSRM.h" #endif +#ifdef HAVE_CONFIG_H +#include "config.h" +#endif + /* Redis cluster hash slots and N-1 which we'll use to find it */ #define REDIS_CLUSTER_SLOTS 16384 #define REDIS_CLUSTER_MOD (REDIS_CLUSTER_SLOTS-1) @@ -157,11 +161,13 @@ typedef struct redisCachedMaster { } redisCachedMaster; typedef struct redisCachedCluster { - // int rsrc_id; /* Zend resource ID */ zend_string *hash; /* What we're cached by */ redisCachedMaster *master; /* Array of masters */ size_t count; /* Number of masters */ uint64_t expiry; /* Expiry time (if any) */ +#ifdef HAVE_REDIS_ATOMICS_MMAP + uint64_t generation; /* Shared invalidation generation */ +#endif } redisCachedCluster; /* A Redis Cluster master node */ @@ -387,6 +393,13 @@ PHP_REDIS_API redisCachedCluster *cluster_cache_create(zend_string *hash, HashTa PHP_REDIS_API void cluster_cache_free(redisCachedCluster *rcc); PHP_REDIS_API void cluster_init_cache(redisCluster *c, redisCachedCluster *rcc); +/* Conditionally compiled shared slot cache invalidation functions */ +#ifdef HAVE_REDIS_ATOMICS_MMAP +void cluster_cache_gen_init(void); +void cluster_cache_gen_free(void); +int cluster_cache_gen_invalidate(void); +#endif + /* Functions to facilitate cluster slot caching */ PHP_REDIS_API char **cluster_sock_read_multibulk_reply(RedisSock *redis_sock, int *len); diff --git a/config.m4 b/config.m4 index c84ce1e99f..d2dc5da429 100644 --- a/config.m4 +++ b/config.m4 @@ -319,6 +319,25 @@ if test "$PHP_REDIS" != "no"; then fi fi + dnl Check if we can use C11 atomics and anonymous shared mmap + AC_LINK_IFELSE( + [AC_LANG_PROGRAM([[ + #include + #include + #include + ]], [[ + static _Atomic int test = 0; + void *ptr = mmap(NULL, 8, PROT_READ | PROT_WRITE, + MAP_SHARED | MAP_ANONYMOUS, -1, 0); + if (ptr == (void *)-1) return 1; + atomic_fetch_add(&test, 1); + return 0; + ]])], + [AC_DEFINE([HAVE_REDIS_ATOMICS_MMAP], [1], + [Define if C11 atomics and MAP_SHARED|MAP_ANONYMOUS mmap are usable])], + [] + ) + AC_CHECK_PROG([GIT], [git], [yes], [no]) if test "$GIT" = "yes" && test -d "$srcdir/.git"; then AC_DEFINE_UNQUOTED(GIT_REVISION, ["$(git log -1 --format=%H)"], [ ]) diff --git a/redis.c b/redis.c index b380ed012c..cf711e1870 100644 --- a/redis.c +++ b/redis.c @@ -96,6 +96,11 @@ PHP_INI_BEGIN() /* redis cluster */ PHP_INI_ENTRY("redis.clusters.cache_slots", "0", PHP_INI_ALL, NULL) PHP_INI_ENTRY("redis.clusters.slot_cache_expiry", "0", PHP_INI_ALL, NULL) + +#ifdef HAVE_REDIS_ATOMICS_MMAP + PHP_INI_ENTRY("redis.clusters.shared_slot_cache_invalidation", "0", + PHP_INI_ALL, NULL) +#endif PHP_INI_ENTRY("redis.clusters.auth", "", PHP_INI_ALL, NULL) PHP_INI_ENTRY("redis.clusters.persistent", "0", PHP_INI_ALL, NULL) PHP_INI_ENTRY("redis.clusters.read_timeout", "0", PHP_INI_ALL, NULL) @@ -145,7 +150,7 @@ zend_module_entry redis_module_entry = { "redis", NULL, PHP_MINIT(redis), - NULL, + PHP_MSHUTDOWN(redis), NULL, NULL, PHP_MINFO(redis), @@ -380,6 +385,11 @@ PHP_MINIT_FUNCTION(redis) "Redis cluster slot cache", module_number); +#ifdef HAVE_REDIS_ATOMICS_MMAP + /* Initialize shared slot cache invalidation */ + cluster_cache_gen_init(); +#endif + /* RedisException class */ redis_exception_ce = register_class_RedisException(spl_ce_RuntimeException); @@ -395,6 +405,14 @@ PHP_MINIT_FUNCTION(redis) return SUCCESS; } +PHP_MSHUTDOWN_FUNCTION(redis) { +#ifdef HAVE_REDIS_ATOMICS_MMAP + cluster_cache_gen_free(); +#endif + + return SUCCESS; +} + static const char * get_available_serializers(void) { diff --git a/redis_cluster.c b/redis_cluster.c index 9a6e7dff17..32904c0bbd 100644 --- a/redis_cluster.c +++ b/redis_cluster.c @@ -1770,12 +1770,25 @@ static void redisClearNodeBytes(redisClusterNode *node) { PHP_METHOD(RedisCluster, flushSlotCache) { redisCluster *c = GET_CONTEXT(); - + ZEND_PARSE_PARAMETERS_NONE(); RETURN_BOOL(cluster_cache_clear(c) == SUCCESS); } +#ifdef HAVE_REDIS_ATOMICS_MMAP +PHP_METHOD(RedisCluster, invalidateSlotCaches) { + ZEND_PARSE_PARAMETERS_NONE(); + + if (INI_INT("redis.clusters.shared_slot_cache_invalidation") == 0) { + php_error_docref(NULL, E_WARNING, "Shared slot cache invalidation disabled"); + RETURN_FALSE; + } + + RETURN_BOOL(cluster_cache_gen_invalidate() == SUCCESS); +} +#endif + PHP_METHOD(RedisCluster, gettransferredbytes) { redisCluster *c = GET_CONTEXT(); zend_long rx = 0, tx = 0; diff --git a/redis_cluster.stub.php b/redis_cluster.stub.php index 1a123e037b..4c5b9d5fa4 100644 --- a/redis_cluster.stub.php +++ b/redis_cluster.stub.php @@ -94,6 +94,16 @@ public function _redir(): string|null; */ public function flushSlotCache(): bool; +#ifdef HAVE_REDIS_ATOMICS_MMAP + /** + * Invaalidate all slot caches for across all workers. Only available on + * linux like systems with c11 atomics and shared memory allocation + * + * @return bool Whether we could invalidate any cache(es) + */ + public static function invalidateSlotCaches(): bool; +#endif + /** * @see Redis::acl */ diff --git a/redis_cluster_arginfo.h b/redis_cluster_arginfo.h index bbc3447838..81deed6116 100644 --- a/redis_cluster_arginfo.h +++ b/redis_cluster_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: db05ec768efb80715ac68fe0a1ab65d37de1d390 */ + * Stub hash: b1d3eb09f86ccffaa4cf0cc6885550e7d4b6da04 */ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_RedisCluster___construct, 0, 0, 1) ZEND_ARG_TYPE_INFO(0, name, IS_STRING, 1) @@ -44,6 +44,11 @@ ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_RedisCluster_flushSlotCache, 0, 0, _IS_BOOL, 0) ZEND_END_ARG_INFO() +#if defined(HAVE_REDIS_ATOMICS_MMAP) +ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_RedisCluster_invalidateSlotCaches, 0, 0, _IS_BOOL, 0) +ZEND_END_ARG_INFO() +#endif + ZEND_BEGIN_ARG_WITH_RETURN_TYPE_INFO_EX(arginfo_class_RedisCluster_acl, 0, 2, IS_MIXED, 0) ZEND_ARG_TYPE_MASK(0, key_or_address, MAY_BE_STRING|MAY_BE_ARRAY, NULL) ZEND_ARG_TYPE_INFO(0, subcmd, IS_STRING, 0) @@ -1064,7 +1069,6 @@ ZEND_BEGIN_ARG_WITH_RETURN_OBJ_TYPE_MASK_EX(arginfo_class_RedisCluster_zdiff, 0, ZEND_ARG_TYPE_INFO_WITH_DEFAULT_VALUE(0, options, IS_ARRAY, 1, "null") ZEND_END_ARG_INFO() - ZEND_METHOD(RedisCluster, __construct); ZEND_METHOD(RedisCluster, _compress); ZEND_METHOD(RedisCluster, _uncompress); @@ -1076,6 +1080,9 @@ ZEND_METHOD(RedisCluster, _prefix); ZEND_METHOD(RedisCluster, _masters); ZEND_METHOD(RedisCluster, _redir); ZEND_METHOD(RedisCluster, flushSlotCache); +#if defined(HAVE_REDIS_ATOMICS_MMAP) +ZEND_METHOD(RedisCluster, invalidateSlotCaches); +#endif ZEND_METHOD(RedisCluster, acl); ZEND_METHOD(RedisCluster, append); ZEND_METHOD(RedisCluster, bgrewriteaof); @@ -1295,7 +1302,6 @@ ZEND_METHOD(RedisCluster, zdiffstore); ZEND_METHOD(RedisCluster, zunion); ZEND_METHOD(RedisCluster, zdiff); - static const zend_function_entry class_RedisCluster_methods[] = { ZEND_ME(RedisCluster, __construct, arginfo_class_RedisCluster___construct, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, _compress, arginfo_class_RedisCluster__compress, ZEND_ACC_PUBLIC) @@ -1308,6 +1314,9 @@ static const zend_function_entry class_RedisCluster_methods[] = { ZEND_ME(RedisCluster, _masters, arginfo_class_RedisCluster__masters, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, _redir, arginfo_class_RedisCluster__redir, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, flushSlotCache, arginfo_class_RedisCluster_flushSlotCache, ZEND_ACC_PUBLIC) +#if defined(HAVE_REDIS_ATOMICS_MMAP) + ZEND_ME(RedisCluster, invalidateSlotCaches, arginfo_class_RedisCluster_invalidateSlotCaches, ZEND_ACC_PUBLIC|ZEND_ACC_STATIC) +#endif ZEND_ME(RedisCluster, acl, arginfo_class_RedisCluster_acl, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, append, arginfo_class_RedisCluster_append, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, bgrewriteaof, arginfo_class_RedisCluster_bgrewriteaof, ZEND_ACC_PUBLIC) @@ -1529,17 +1538,16 @@ static const zend_function_entry class_RedisCluster_methods[] = { ZEND_FE_END }; - -static const zend_function_entry class_RedisClusterException_methods[] = { - ZEND_FE_END -}; - static zend_class_entry *register_class_RedisCluster(void) { zend_class_entry ce, *class_entry; INIT_CLASS_ENTRY(ce, "RedisCluster", class_RedisCluster_methods); +#if (PHP_VERSION_ID >= 80400) + class_entry = zend_register_internal_class_with_flags(&ce, NULL, 0); +#else class_entry = zend_register_internal_class_ex(&ce, NULL); +#endif zval const_OPT_SLAVE_FAILOVER_value; ZVAL_LONG(&const_OPT_SLAVE_FAILOVER_value, REDIS_OPT_FAILOVER); @@ -1570,13 +1578,11 @@ static zend_class_entry *register_class_RedisCluster(void) zend_string *const_FAILOVER_DISTRIBUTE_SLAVES_name = zend_string_init_interned("FAILOVER_DISTRIBUTE_SLAVES", sizeof("FAILOVER_DISTRIBUTE_SLAVES") - 1, 1); zend_declare_class_constant_ex(class_entry, const_FAILOVER_DISTRIBUTE_SLAVES_name, &const_FAILOVER_DISTRIBUTE_SLAVES_value, ZEND_ACC_PUBLIC, NULL); zend_string_release(const_FAILOVER_DISTRIBUTE_SLAVES_name); -#if (PHP_VERSION_ID >= 80000) zend_string *attribute_name_SensitiveParameter_func___construct_arg5_0 = zend_string_init_interned("SensitiveParameter", sizeof("SensitiveParameter") - 1, 1); zend_add_parameter_attribute(zend_hash_str_find_ptr(&class_entry->function_table, "__construct", sizeof("__construct") - 1), 5, attribute_name_SensitiveParameter_func___construct_arg5_0, 0); zend_string_release(attribute_name_SensitiveParameter_func___construct_arg5_0); -#endif return class_entry; } @@ -1585,8 +1591,12 @@ static zend_class_entry *register_class_RedisClusterException(zend_class_entry * { zend_class_entry ce, *class_entry; - INIT_CLASS_ENTRY(ce, "RedisClusterException", class_RedisClusterException_methods); + INIT_CLASS_ENTRY(ce, "RedisClusterException", NULL); +#if (PHP_VERSION_ID >= 80400) + class_entry = zend_register_internal_class_with_flags(&ce, class_entry_RuntimeException, 0); +#else class_entry = zend_register_internal_class_ex(&ce, class_entry_RuntimeException); +#endif return class_entry; } diff --git a/redis_cluster_legacy_arginfo.h b/redis_cluster_legacy_arginfo.h index 85a0630071..f588a69787 100644 --- a/redis_cluster_legacy_arginfo.h +++ b/redis_cluster_legacy_arginfo.h @@ -1,5 +1,5 @@ /* This is a generated file, edit the .stub.php file instead. - * Stub hash: db05ec768efb80715ac68fe0a1ab65d37de1d390 */ + * Stub hash: b1d3eb09f86ccffaa4cf0cc6885550e7d4b6da04 */ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_RedisCluster___construct, 0, 0, 1) ZEND_ARG_INFO(0, name) @@ -36,6 +36,11 @@ ZEND_END_ARG_INFO() #define arginfo_class_RedisCluster_flushSlotCache arginfo_class_RedisCluster__masters +#if defined(HAVE_REDIS_ATOMICS_MMAP) +ZEND_BEGIN_ARG_INFO_EX(arginfo_class_RedisCluster_invalidateSlotCaches, 0, 0, 0) +ZEND_END_ARG_INFO() +#endif + ZEND_BEGIN_ARG_INFO_EX(arginfo_class_RedisCluster_acl, 0, 0, 2) ZEND_ARG_INFO(0, key_or_address) ZEND_ARG_INFO(0, subcmd) @@ -906,7 +911,6 @@ ZEND_BEGIN_ARG_INFO_EX(arginfo_class_RedisCluster_zdiff, 0, 0, 1) ZEND_ARG_INFO(0, options) ZEND_END_ARG_INFO() - ZEND_METHOD(RedisCluster, __construct); ZEND_METHOD(RedisCluster, _compress); ZEND_METHOD(RedisCluster, _uncompress); @@ -918,6 +922,9 @@ ZEND_METHOD(RedisCluster, _prefix); ZEND_METHOD(RedisCluster, _masters); ZEND_METHOD(RedisCluster, _redir); ZEND_METHOD(RedisCluster, flushSlotCache); +#if defined(HAVE_REDIS_ATOMICS_MMAP) +ZEND_METHOD(RedisCluster, invalidateSlotCaches); +#endif ZEND_METHOD(RedisCluster, acl); ZEND_METHOD(RedisCluster, append); ZEND_METHOD(RedisCluster, bgrewriteaof); @@ -1137,7 +1144,6 @@ ZEND_METHOD(RedisCluster, zdiffstore); ZEND_METHOD(RedisCluster, zunion); ZEND_METHOD(RedisCluster, zdiff); - static const zend_function_entry class_RedisCluster_methods[] = { ZEND_ME(RedisCluster, __construct, arginfo_class_RedisCluster___construct, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, _compress, arginfo_class_RedisCluster__compress, ZEND_ACC_PUBLIC) @@ -1150,6 +1156,9 @@ static const zend_function_entry class_RedisCluster_methods[] = { ZEND_ME(RedisCluster, _masters, arginfo_class_RedisCluster__masters, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, _redir, arginfo_class_RedisCluster__redir, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, flushSlotCache, arginfo_class_RedisCluster_flushSlotCache, ZEND_ACC_PUBLIC) +#if defined(HAVE_REDIS_ATOMICS_MMAP) + ZEND_ME(RedisCluster, invalidateSlotCaches, arginfo_class_RedisCluster_invalidateSlotCaches, ZEND_ACC_PUBLIC|ZEND_ACC_STATIC) +#endif ZEND_ME(RedisCluster, acl, arginfo_class_RedisCluster_acl, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, append, arginfo_class_RedisCluster_append, ZEND_ACC_PUBLIC) ZEND_ME(RedisCluster, bgrewriteaof, arginfo_class_RedisCluster_bgrewriteaof, ZEND_ACC_PUBLIC) @@ -1371,17 +1380,16 @@ static const zend_function_entry class_RedisCluster_methods[] = { ZEND_FE_END }; - -static const zend_function_entry class_RedisClusterException_methods[] = { - ZEND_FE_END -}; - static zend_class_entry *register_class_RedisCluster(void) { zend_class_entry ce, *class_entry; INIT_CLASS_ENTRY(ce, "RedisCluster", class_RedisCluster_methods); +#if (PHP_VERSION_ID >= 80400) + class_entry = zend_register_internal_class_with_flags(&ce, NULL, 0); +#else class_entry = zend_register_internal_class_ex(&ce, NULL); +#endif zval const_OPT_SLAVE_FAILOVER_value; ZVAL_LONG(&const_OPT_SLAVE_FAILOVER_value, REDIS_OPT_FAILOVER); @@ -1420,8 +1428,12 @@ static zend_class_entry *register_class_RedisClusterException(zend_class_entry * { zend_class_entry ce, *class_entry; - INIT_CLASS_ENTRY(ce, "RedisClusterException", class_RedisClusterException_methods); + INIT_CLASS_ENTRY(ce, "RedisClusterException", NULL); +#if (PHP_VERSION_ID >= 80400) + class_entry = zend_register_internal_class_with_flags(&ce, class_entry_RuntimeException, 0); +#else class_entry = zend_register_internal_class_ex(&ce, class_entry_RuntimeException); +#endif return class_entry; } From f01f80589552c8e5037e72103574e9b0cb7c03b1 Mon Sep 17 00:00:00 2001 From: michael-grunder Date: Thu, 1 May 2025 10:58:07 -0700 Subject: [PATCH 5/6] Use zend_hrtime instead of time(NULL) since it's monotonic `time(NULL)` isn't monotonic and can run backwards so switch to `zend_hrtime` which uses `clock_gettime(CLOCK_MONOTONIC)` on Linux and `QueryPerformanceCounter`on Windows. --- cluster_library.c | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/cluster_library.c b/cluster_library.c index a6b8e9f366..a097c58857 100644 --- a/cluster_library.c +++ b/cluster_library.c @@ -6,6 +6,12 @@ #include "crc16.h" #include +#if PHP_VERSION_ID < 80300 +#include "ext/standard/hrtime.h" +#else +#include "Zend/zend_hrtime.h" +#endif + #ifdef HAVE_REDIS_ATOMICS_MMAP #include #include @@ -891,6 +897,18 @@ cluster_free(redisCluster *c, int free_ctx) if (free_ctx) efree(c); } +static inline uint64_t redis_time(void) { + #define REDIS_NANO_IN_SEC ((uint64_t)1000000000) + +#if PHP_VERSION_ID < 80300 + return php_hrtime_current() / REDIS_NANO_IN_SEC; +#else + return zend_hrtime() / REDIS_NANO_IN_SEC; +#endif + + #undef REDIS_NANO_IN_SEC +} + static zend_long cluster_cache_expiry(void) { zend_long expiry; @@ -898,7 +916,7 @@ static zend_long cluster_cache_expiry(void) { if (expiry <= 0) return 0; - return time(NULL) + expiry; + return redis_time() + expiry; } #ifdef HAVE_REDIS_ATOMICS_MMAP @@ -3172,7 +3190,7 @@ PHP_REDIS_API redisCachedCluster *cluster_cache_load(zend_string *hash) { cc = le->ptr; /* Short circuit if it should be expired */ - if (cc->expiry != 0 && cc->expiry <= time(NULL)) + if (cc->expiry != 0 && cc->expiry <= redis_time()) return NULL; #ifdef HAVE_REDIS_ATOMICS_MMAP From cceb3c5767b296bd758e6ec9304b27d0a6591106 Mon Sep 17 00:00:00 2001 From: michael-grunder Date: Thu, 1 May 2025 14:47:17 -0700 Subject: [PATCH 6/6] Clean up invalidated slot cache --- cluster_library.c | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cluster_library.c b/cluster_library.c index a097c58857..d990458e99 100644 --- a/cluster_library.c +++ b/cluster_library.c @@ -3191,15 +3191,19 @@ PHP_REDIS_API redisCachedCluster *cluster_cache_load(zend_string *hash) { cc = le->ptr; /* Short circuit if it should be expired */ if (cc->expiry != 0 && cc->expiry <= redis_time()) - return NULL; + goto invalidated; #ifdef HAVE_REDIS_ATOMICS_MMAP /* Short circuit if it has been globally invalidated */ if (cluster_cache_gen() != cc->generation) - return NULL; + goto invalidated; #endif return cc; + +invalidated: + zend_hash_del(&EG(persistent_list), hash); + return NULL; } /* Cache a cluster's slot information in persistent_list if it's enabled */