Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

Appearance settings
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions 5 .changeset/chatty-snakes-hope.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@trigger.dev/sdk": patch
---

fix: Logging large objects is now much more performant and uses less memory
4 changes: 2 additions & 2 deletions 4 apps/webapp/app/env.server.ts
Original file line number Diff line number Diff line change
Expand Up @@ -283,8 +283,8 @@ const EnvironmentSchema = z.object({
PROD_OTEL_LOG_EXPORT_TIMEOUT_MILLIS: z.string().default("30000"),
PROD_OTEL_LOG_MAX_QUEUE_SIZE: z.string().default("512"),

TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"),
TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("256"),
TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"),
TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT: z.string().default("1024"),
Comment thread
nicktrn marked this conversation as resolved.
TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"),
TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT: z.string().default("131072"),
TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT: z.string().default("10"),
Expand Down
4 changes: 2 additions & 2 deletions 4 docs/self-hosting/env/webapp.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -105,8 +105,8 @@ mode: "wide"
| `MAXIMUM_DEV_QUEUE_SIZE` | No | — | Max dev queue size. |
| `MAXIMUM_DEPLOYED_QUEUE_SIZE` | No | — | Max deployed queue size. |
| **OTel limits** | | | |
| `TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT` | No | 256 | OTel span attribute count limit. |
| `TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT` | No | 256 | OTel log attribute count limit. |
| `TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT` | No | 1024 | OTel span attribute count limit. |
| `TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT` | No | 1024 | OTel log attribute count limit. |
| `TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT` | No | 131072 | OTel span attribute value length limit. |
| `TRIGGER_OTEL_LOG_ATTRIBUTE_VALUE_LENGTH_LIMIT` | No | 131072 | OTel log attribute value length limit. |
| `TRIGGER_OTEL_SPAN_EVENT_COUNT_LIMIT` | No | 10 | OTel span event count limit. |
Expand Down
5 changes: 4 additions & 1 deletion 5 packages/cli-v3/src/entryPoints/dev-run-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
localsAPI,
logger,
LogLevel,
OTEL_LOG_ATTRIBUTE_COUNT_LIMIT,
resourceCatalog,
runMetadata,
runtime,
Expand Down Expand Up @@ -191,7 +192,8 @@ async function doBootstrap() {
typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true,
typeof config.disableConsoleInterceptor === "boolean"
? config.disableConsoleInterceptor
: false
: false,
OTEL_LOG_ATTRIBUTE_COUNT_LIMIT
);

const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info";
Expand All @@ -200,6 +202,7 @@ async function doBootstrap() {
logger: otelLogger,
tracer: tracer,
level: logLevels.includes(configLogLevel as any) ? (configLogLevel as LogLevel) : "info",
maxAttributeCount: OTEL_LOG_ATTRIBUTE_COUNT_LIMIT,
});

logger.setGlobalTaskLogger(otelTaskLogger);
Expand Down
5 changes: 4 additions & 1 deletion 5 packages/cli-v3/src/entryPoints/managed-run-worker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import {
localsAPI,
logger,
LogLevel,
OTEL_LOG_ATTRIBUTE_COUNT_LIMIT,
resourceCatalog,
runMetadata,
runtime,
Expand Down Expand Up @@ -182,7 +183,8 @@ async function doBootstrap() {
typeof config.enableConsoleLogging === "boolean" ? config.enableConsoleLogging : true,
typeof config.disableConsoleInterceptor === "boolean"
? config.disableConsoleInterceptor
: false
: false,
OTEL_LOG_ATTRIBUTE_COUNT_LIMIT
);

const configLogLevel = triggerLogLevel ?? config.logLevel ?? "info";
Expand All @@ -191,6 +193,7 @@ async function doBootstrap() {
logger: otelLogger,
tracer: tracer,
level: logLevels.includes(configLogLevel as any) ? (configLogLevel as LogLevel) : "info",
maxAttributeCount: OTEL_LOG_ATTRIBUTE_COUNT_LIMIT,
});

logger.setGlobalTaskLogger(otelTaskLogger);
Expand Down
8 changes: 6 additions & 2 deletions 8 packages/core/src/v3/consoleInterceptor.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@ export class ConsoleInterceptor {
constructor(
private readonly logger: logsAPI.Logger,
private readonly sendToStdIO: boolean,
private readonly interceptingDisabled: boolean
private readonly interceptingDisabled: boolean,
private readonly maxAttributeCount?: number
) {}

// Intercept the console and send logs to the OpenTelemetry logger
Expand Down Expand Up @@ -92,7 +93,10 @@ export class ConsoleInterceptor {
severityNumber,
severityText,
body: getLogMessage(parsed.value, severityText),
attributes: { ...this.#getAttributes(severityNumber), ...flattenAttributes(parsed.value) },
attributes: {
...this.#getAttributes(severityNumber),
...flattenAttributes(parsed.value, undefined, this.maxAttributeCount),
},
timestamp,
});

Expand Down
8 changes: 2 additions & 6 deletions 8 packages/core/src/v3/limits.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,11 +13,11 @@ function getOtelEnvVarLimit(key: string, defaultValue: number) {

export const OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit(
"TRIGGER_OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT",
256
1024
);
export const OTEL_LOG_ATTRIBUTE_COUNT_LIMIT = getOtelEnvVarLimit(
"TRIGGER_OTEL_LOG_ATTRIBUTE_COUNT_LIMIT",
256
1024
);
export const OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT = getOtelEnvVarLimit(
"TRIGGER_OTEL_SPAN_ATTRIBUTE_VALUE_LENGTH_LIMIT",
Expand Down Expand Up @@ -51,10 +51,6 @@ export function imposeAttributeLimits(attributes: Attributes): Attributes {
continue;
}

if (Object.keys(newAttributes).length >= OTEL_SPAN_ATTRIBUTE_COUNT_LIMIT) {
break;
}

newAttributes[key] = value;
}

Expand Down
28 changes: 7 additions & 21 deletions 28 packages/core/src/v3/logger/taskLogger.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ export type TaskLoggerConfig = {
logger: Logger;
tracer: TriggerTracer;
level: LogLevel;
maxAttributeCount?: number;
};

export type TraceOptions = Prettify<
Expand Down Expand Up @@ -78,7 +79,12 @@ export class OtelTaskLogger implements TaskLogger {
severityNumber: SeverityNumber,
properties?: Record<string, unknown>
) {
let attributes: Attributes = { ...flattenAttributes(safeJsonProcess(properties)) };
let attributes: Attributes = {};

if (properties) {
// Use flattenAttributes directly - it now handles all non-JSON friendly values efficiently
attributes = flattenAttributes(properties, undefined, this._config.maxAttributeCount);
}

const icon = iconStringForSeverity(severityNumber);
if (icon !== undefined) {
Expand Down Expand Up @@ -136,23 +142,3 @@ export class NoopTaskLogger implements TaskLogger {
return {} as Span;
}
}

function safeJsonProcess(value?: Record<string, unknown>): Record<string, unknown> | undefined {
try {
return JSON.parse(JSON.stringify(value, jsonErrorReplacer));
} catch {
return value;
}
}

function jsonErrorReplacer(key: string, value: unknown) {
if (value instanceof Error) {
return {
name: value.name,
message: value.message,
stack: value.stack,
};
}

return value;
}
Loading
Morty Proxy This is a proxified and sanitized view of the page, visit original site.