triggerdotdev · ericallam · Jun 15, 2026 · Jun 15, 2026 · Jun 15, 2026
diff --git a/.server-changes/harden-realtime-change-publishing.md b/.server-changes/harden-realtime-change-publishing.md
@@ -0,0 +1,6 @@
+---
+area: webapp
+type: improvement
+---
+
+Harden the native realtime backend's run-change publishing so a publish can never throw into a run lifecycle operation and never buffers commands in memory during a pub/sub Redis outage.
diff --git a/apps/webapp/app/redis.server.ts b/apps/webapp/app/redis.server.ts
@@ -11,6 +11,8 @@ export type RedisWithClusterOptions = {
  clusterMode?: boolean;
  clusterOptions?: Omit<ClusterOptions, "redisOptions">;
  keyPrefix?: string;
+  /** Cap retries for a command before it rejects; `null` means unlimited (default: ioredis's default of 20). */
+  maxRetriesPerRequest?: number | null;
 };

 export type RedisClient = Redis | Cluster;
@@ -44,6 +46,9 @@ export function createRedisClient(
        password: options.password,
        enableAutoPipelining: true,
        reconnectOnError: defaultReconnectOnError,
+        ...(options.maxRetriesPerRequest !== undefined
+          ? { maxRetriesPerRequest: options.maxRetriesPerRequest }
+          : {}),
        ...(options.tlsDisabled
          ? {
              checkServerIdentity: () => {
@@ -72,6 +77,9 @@ export function createRedisClient(
      enableAutoPipelining: true,
      keyPrefix: options.keyPrefix,
      reconnectOnError: defaultReconnectOnError,
+      ...(options.maxRetriesPerRequest !== undefined
+        ? { maxRetriesPerRequest: options.maxRetriesPerRequest }
+        : {}),
      ...(options.tlsDisabled ? {} : { tls: {} }),
    });
  }

diff --git a/apps/webapp/app/services/realtime/runChangeNotifier.server.ts b/apps/webapp/app/services/realtime/runChangeNotifier.server.ts
@@ -225,7 +225,13 @@ export class RunChangeNotifier {

  #ensurePublisher(): RedisClient {
    if (!this.#publisher) {
-      this.#publisher = createRedisClient(`${this.#connectionName}:pub`, this.options.redis);
+      // Publishes are fire-and-forget with a consumer-side backstop, so a dropped publish is
+      // latency-only. Cap retries (vs ioredis's default 20) so a pub/sub outage rejects publishes
+      // after ~1 reconnect cycle instead of buffering them in memory across the fleet.
+      this.#publisher = createRedisClient(`${this.#connectionName}:pub`, {
+        ...this.options.redis,
+        maxRetriesPerRequest: 1,
+      });
    }
    return this.#publisher;
  }

diff --git a/apps/webapp/app/services/realtime/runChangeNotifierInstance.server.ts b/apps/webapp/app/services/realtime/runChangeNotifierInstance.server.ts
@@ -1,6 +1,7 @@
 import { getMeter } from "@internal/tracing";
 import { env } from "~/env.server";
 import { singleton } from "~/utils/singleton";
+import { logger } from "../logger.server";
 import { RunChangeNotifier, type ChangeRecordInput } from "./runChangeNotifier.server";

 /**
@@ -74,12 +75,24 @@ export function publishChangeRecord(input: ChangeRecordInput): void {
  if (!nativeBackendEnabled) {
    return;
  }
-  getRunChangeNotifier().publish(input);
+  // Publish runs on the run-engine event bus / metadata flush loop; lazy init + encoding happen
+  // before the notifier's own try/catch, so guard the whole call — it must never throw at its caller.
+  try {
+    getRunChangeNotifier().publish(input);
+  } catch (error) {
+    logger.error("[runChangeNotifier] publishChangeRecord threw; dropping notification", { error });
+  }
 }

 export function publishManyChangeRecords(inputs: ChangeRecordInput[]): void {
  if (!nativeBackendEnabled) {
    return;
  }
-  getRunChangeNotifier().publishMany(inputs);
+  try {
+    getRunChangeNotifier().publishMany(inputs);
+  } catch (error) {
+    logger.error("[runChangeNotifier] publishManyChangeRecords threw; dropping notifications", {
+      error,
+    });
+  }
 }