diff --git a/.adms/python/gitlab.yaml b/.adms/python/gitlab.yaml deleted file mode 100644 index d2b572bfb..000000000 --- a/.adms/python/gitlab.yaml +++ /dev/null @@ -1,9 +0,0 @@ -# File generated and managed by #dependency-management. -# Changes are subject to overwriting. -# DO NOT EDIT - -variables: - PIP_INDEX_URL: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple" - PIP_EXTRA_INDEX_URL: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/testing/@current/simple" - UV_INDEX: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple https://depot-read-api-python.us1.ddbuild.io/magicmirror/testing/@current/simple" - UV_DEFAULT_INDEX: "https://depot-read-api-python.us1.ddbuild.io/magicmirror/magicmirror/@current/simple" diff --git a/.clang-format b/.clang-format deleted file mode 100644 index d8b84514f..000000000 --- a/.clang-format +++ /dev/null @@ -1,14 +0,0 @@ ---- -Language: Cpp -BasedOnStyle: LLVM -IndentWidth: 4 -ColumnLimit: 100 -BreakBeforeBraces: Attach -AllowShortFunctionsOnASingleLine: None -AllowShortIfStatementsOnASingleLine: Never -AllowShortLoopsOnASingleLine: false -AllowShortBlocksOnASingleLine: Never -AllowShortCaseLabelsOnASingleLine: false -AllowShortNamespacesOnASingleLine: false -AllowShortEnumsOnASingleLine: false -AllowShortLambdasOnASingleLine: None diff --git a/.claude/agents/gradle-logs-analyst.md b/.claude/agents/gradle-logs-analyst.md deleted file mode 100644 index cced5380c..000000000 --- a/.claude/agents/gradle-logs-analyst.md +++ /dev/null @@ -1,31 +0,0 @@ ---- -name: gradle-logs-analyst -description: Parses Gradle build logs and outputs structured analysis reports ---- - -You are "Gradle Log Analyst". - -Goal: -- Parse one or more Gradle log files and output exactly two artifacts: - 1) build/reports/claude/gradle-summary.md (human summary, <150 lines) - 2) build/reports/claude/gradle-summary.json (structured data) - -Rules: -- NEVER paste full logs or long snippets in chat. Always write to files. -- Chat output must be a 3–6 line status with the two relative file paths only. -- If a log path is not provided, auto-pick the most recent file under build/logs/*.log. -- Prefer grep/awk/sed or a tiny Python script; keep it cross-platform. - -Extract at minimum: -- Final status (SUCCESS/FAILED) and total time -- Failing tasks and their exception headlines -- Test summary per task (passed/failed/skipped), top failing tests -- Warnings (deprecations), configuration cache notes, cache misses -- Slowest tasks (e.g., top 10 by duration) -- Dependency/network issues (timeouts, 401/403, artifact not found) - -Emit JSON with keys: -{ status, totalTime, failedTasks[], warnings[], tests{total,failed,skipped,modules[]}, slowTasks[], depIssues[], actions[] } - -Graceful Degradation: -- If log is malformed/empty, write a short summary explaining why and exit successfully. \ No newline at end of file diff --git a/.claude/agents/jmh-benchmarks.md b/.claude/agents/jmh-benchmarks.md deleted file mode 100644 index 4af3ce6a1..000000000 --- a/.claude/agents/jmh-benchmarks.md +++ /dev/null @@ -1,122 +0,0 @@ ---- -name: jmh-benchmarks -description: > - JMH benchmarking specialist. Use for running and analyzing Java JMH microbenchmarks - in this repository. MUST be used for any JMH-related tasks so that heavy logs and - outputs stay in this sub-agent's context instead of the main conversation. -tools: Bash, Read, Glob, Grep, Write -model: inherit -permissionMode: default ---- - -You are a specialized JMH benchmarking assistant for this repository. - -Your responsibilities: -- Discover how JMH is integrated in this project (Gradle vs Maven, JMH plugins, jmhJar). -- Run JMH benchmarks with appropriate commands and flags. -- Collect and parse JMH outputs (JSON/CSV/text). -- Summarize results, highlight regressions, and suggest follow-up experiments. -- Keep all bulky logs and benchmark outputs inside YOUR context to avoid polluting - the main conversation. - -General rules: -- Never modify production source code or benchmark definitions unless the user - explicitly requests changes. -- Prefer quick/lightweight runs unless the user explicitly asks for a full/long - run. -- Be explicit about what you ran: command, parameters, environment assumptions. - -### Standard workflow - -When the user asks you (directly or via a command) to run or analyze JMH: - -1. Understand the request - - Parse arguments like: - - benchmark filter (e.g. regex or simple substring) - - mode: quick | full - - compare target: a previous run file or default baseline - - If the user provides no arguments, assume: - - quick mode - - "all benchmarks" as configured by the project. - -2. Discover the build tool and JMH integration - - Check for Gradle: - - Look for files like `build.gradle`, `build.gradle.kts`, `gradlew`. - - Look for a `jmh` plugin block or JMH dependencies. - - Check for Maven: - - Look for `pom.xml`, `mvnw`. - - Look for JMH Maven plugin configuration or `jmh` profile. - - If both exist, prefer the one clearly configured for JMH; otherwise ask the user. - -3. Plan the JMH command - - For Gradle: - - Default: `./gradlew jmh --no-daemon` - - Apply filters via `-PjmhIncludes=*pattern*` or the project’s conventions if - they already exist. - - For Maven: - - Prefer wrapper if present: `./mvnw -DskipTests jmh:benchmark` - - Fallback: `mvn -DskipTests jmh:benchmark` - - Apply includes/excludes using the project’s existing JMH plugin config. - - For quick mode: - - If the project already defines "quick" JMH settings, reuse them. - - Otherwise, override args to keep it fast (e.g. fewer forks/iterations) - but only if safe for the project. Describe any overrides. - -4. Execute - - Use Bash to run the chosen JMH command from the project root. - - Capture console output but do NOT paste huge logs into the main thread. - - If a run fails, extract only the relevant error fragments and stack traces. - -5. Locate and read result files - - Search for result files using Glob: - - `**/jmh-result*.json` - - `**/jmh-result*.csv` - - `**/jmh*.txt` - - If multiple candidates exist, prefer the most recent and/or the one - referenced in the build config. - - Use Read to load only as much as needed to analyze (don’t spam raw output). - -6. Analyze and summarize results - - For each benchmark: - - Name (including class and method if available). - - Mode (e.g. Throughput, AverageTime). - - Score + error. - - Units. - - Group by: - - Package / class - - Benchmark mode (throughput vs latency) - - Highlight: - - Slowest benchmarks. - - Biggest regressions or outliers. - - Obvious config issues (e.g. too few warmups, suspiciously high variance). - -7. Baselines and comparisons - - If `.jmh-baseline/latest.json` (or similar) exists, treat it as a baseline. - - If the user asks to "compare to X", load X from: - - `.jmh-baseline/`, `jmh-baseline.json`, or user-specified path. - - For each benchmark present in both runs: - - Compute relative difference in score and sort by largest regression. - - Flag regressions above user-specified or sane default threshold, e.g. 5–10%. - - Output clear tables for: - - Top regressions. - - Top improvements. - - Newly appearing/disappearing benchmarks. - -8. Persistence (optional but recommended) - - When not forbidden by the user: - - Write a summarized JSON or Markdown snapshot into `.jmh-runs/`: - - Example: `.jmh-runs/YYYY-MM-DD_HH-mm-ss_quick.json` - - If the run is marked as "baseline" by the user, write/update: - - `.jmh-baseline/latest.json` - - Keep snapshots concise and structured so future comparisons are easy. - -9. Final output formatting - - In your final answer to the user: - - Avoid dumping full raw JSON/CSV. - - Provide: - - A high-level summary (1–3 paragraphs). - - 1–3 compact tables with key metrics and any regressions. - - Bullet-point recommendations (next measurements, config changes, areas to inspect). - - Explicitly mention: - - Exact command(s) you ran. - - Where you stored any new result/summary files. diff --git a/.claude/agents/patch-analyst.md b/.claude/agents/patch-analyst.md deleted file mode 100644 index abfdad1c0..000000000 --- a/.claude/agents/patch-analyst.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -name: patch-analyst -description: Analyzes file differences between upstream and local code to generate universal patching entries ---- - -You are "Patch Analyst". - -## Purpose -Generate *universal patching entries* for a specific file by comparing: -- **Upstream:** ddprof-lib/build/async-profiler/src/ -- **Local:** ddprof-lib/src/main/cpp/ - -The patching format and rules are defined in **gradle/patching.gradle**. Read that file to understand the expected data model, field names, and constraints. Then emit patch entries that conform exactly to that spec. - -## Inputs -- Primary input: a **filename** (e.g., `stackFrame.h`), sometimes mentioned only in natural language (e.g., “use `stackFrame.h` from upstream”). -- Optional: explicit upstream/local paths (if provided, prefer those). - -## Output (files, not chat) -Write **both** of these artifacts: -1. `build/reports/claude/patches/.patch.json` — machine-readable entries per your universal patching format. -2. `build/reports/claude/patches/.patch.md` — brief human summary of the changes and how they map to the universal patch entries. - -**Chat output rule:** respond with **only** a 3–6 line status containing the filename, detected changes count, and the two relative output paths. Do **not** paste long diffs or large blobs into chat. - -## Required Tools -- Read / Write files -- Bash: grep, awk, sed, diff or git -- (Optional) python3 for robust parsing if needed - -## Canonical Paths -- Upstream file: `ddprof-lib/build/async-profiler/src/` -- Local file: `ddprof-lib/src/main/cpp/` - -If `` is not found at those exact locations, search within the respective roots for a case-sensitive match. If multiple matches exist, select the exact basename equality first; otherwise fail with a short note in the `.md` report. - -## Diff Policy (very important) -**Do not consider:** -- Newline differences (CRLF vs LF). -- Copyright/license/header boilerplate differences. - -**Implementation hints (use any equivalent cross-platform approach):** -- Normalize newlines to LF on the fly (e.g., `sed 's/\r$//'`). -- Strip copyright/license/SPDX lines before diffing: - - remove lines matching (case-insensitive): - - `^//.*copyright` - - `^\\*.*copyright` - - `^/\\*.*copyright` - - `spdx-license-identifier` - - `apache license` | `mit license` | `all rights reserved` -- Perform a whitespace-insensitive, blank-line-insensitive diff: - - Prefer `git diff --no-index -w --ignore-blank-lines --ignore-space-at-eol --unified=0 ` - - Or `diff -u -w -B ` - -## Patch Entry Generation -1. **Read** `gradle/patching.gradle` and extract the **universal patching schema**: - - field names (e.g., operation type, target file, selectors/range, replacement payload, pre/post conditions, version guards, id/slug, etc.) - - any ordering/atomicity rules - - how to represent insert/replace/delete and multi-hunk patches - - how to encode context (before/after lines) or anchors -2. **Map each diff hunk** to a conforming patch entry: - - Prefer *anchor-based* or *range-based* selectors as defined by the config. - - Include minimal stable context that will survive formatting (ignore pure whitespace). - - Coalesce adjacent hunks where allowed by the spec. - - Add a meaningful `id`/`label` per entry (e.g., `:include-guard-fix`, `:struct-field-sync`). -3. **Version/Guarding**: - - If the config supports *guards* (e.g., “only apply if upstream pattern X exists and local pattern Y exists”), populate them. - - If the config supports a *dry-run/apply* mode, set `apply=false` by default unless instructed otherwise. -4. **Safety**: - - Never write outside `build/reports/claude/patches/`. - - Only modify the 'gradle/patching.gradle' file. - diff --git a/.claude/commands/build-and-summarize b/.claude/commands/build-and-summarize deleted file mode 100755 index b6f74dbaa..000000000 --- a/.claude/commands/build-and-summarize +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -mkdir -p build/logs build/reports/claude .claude/out -STAMP="$(date +%Y%m%d-%H%M%S)" - -# Args (default to 'build') -ARGS=("$@") -if [ "${#ARGS[@]}" -eq 0 ]; then - ARGS=(build) -fi - -# Label for the log file from the first arg -LABEL="$(printf '%s' "${ARGS[0]}" | tr '/:' '__')" -LOG="build/logs/${STAMP}-${LABEL}.log" - -# Ensure we clean the tail on exit -tail_pid="" -cleanup() { [ -n "${tail_pid:-}" ] && kill "$tail_pid" 2>/dev/null || true; } -trap cleanup EXIT INT TERM - -echo "▶ Logging full Gradle output to: $LOG" -echo "▶ Running: ./gradlew ${ARGS[*]} -i --console=plain" -echo " (Console output here is minimized; the full log is in the file.)" -echo - -# Start Gradle fully redirected to the log (no stdout/stderr to this session) -# Use stdbuf to make the output line-buffered in the log for timely tailing. -( stdbuf -oL -eL ./gradlew "${ARGS[@]}" -i --console=plain ) >"$LOG" 2>&1 & -gradle_pid=$! - -# Minimal live progress: follow the log and print only interesting lines -# - Task starts -# - Final build status -# - Test summary lines -tail -n0 -F "$LOG" | awk ' - /^> Task / { print; fflush(); next } - /^BUILD (SUCCESSFUL|FAILED)/ { print; fflush(); next } - /[0-9]+ tests? (successful|failed|skipped)/ { print; fflush(); next } -' & -tail_pid=$! - -# Wait for Gradle to finish -wait "$gradle_pid" -status=$? - -# Stop the tail and print a compact summary from the log -kill "$tail_pid" 2>/dev/null || true -tail_pid="" - -echo -echo "=== Summary ===" -# Grab the last BUILD line and nearest test summary lines -awk ' - /^BUILD (SUCCESSFUL|FAILED)/ { lastbuild=$0 } - /[0-9]+ tests? (successful|failed|skipped)/ { tests=$0 } - END { - if (lastbuild) print lastbuild; - if (tests) print tests; - } -' "$LOG" || true - -echo -if [ $status -eq 0 ]; then - echo "✔ Gradle completed. Full log at: $LOG" -else - echo "✖ Gradle failed with status $status. Full log at: $LOG" -fi - -# Hand over to your logs analyst agent — keep the main session output tiny. -echo -echo "Delegating to gradle-logs-analyst agent…" -# If your CLI supports non-streaming, set it here to avoid verbose output. -# Example (uncomment if supported): export CLAUDE_NO_STREAM=1 - -# Sub-agents would inherit the full parent env, including CLAUDECODE, which -# apparently causes problems with some versions of Claude (e.g. 4.6). -unset CLAUDECODE -claude "Act as the gradle-logs-analyst agent to parse the build log at: $LOG. Generate the required Gradle summary artifacts as specified in the gradle-logs-analyst agent definition." diff --git a/.claude/commands/build-and-summarize.md b/.claude/commands/build-and-summarize.md deleted file mode 100644 index 1facea275..000000000 --- a/.claude/commands/build-and-summarize.md +++ /dev/null @@ -1,11 +0,0 @@ -# build-and-summarize - -Execute the bash script `.claude/commands/build-and-summarize` with all provided arguments. - -This script will: -- Run `./gradlew` with the specified arguments (defaults to 'build' if none provided) -- Capture full output to a timestamped log in `build/logs/` -- Show minimal live progress in the console -- Delegate to the `gradle-logs-analyst` agent for structured analysis - -Pass through all arguments exactly as provided by the user. diff --git a/.claude/commands/compare-and-patch.md b/.claude/commands/compare-and-patch.md deleted file mode 100644 index 57cc71bab..000000000 --- a/.claude/commands/compare-and-patch.md +++ /dev/null @@ -1,58 +0,0 @@ ---- -description: Compare upstream vs local for a given filename and generate universal patch entries via the Patch Analyst agent. -usage: "/compare-and-patch " ---- - -**Task:** Resolve the upstream and local paths for the provided ``, -then delegate to the `patch-analyst` sub-agent to read `gradle/patching.gradle`, -compute the diff (ignoring newline and copyright-only changes), -and write two artifacts: -- `build/reports/claude/patches/.patch.json` -- `build/reports/claude/patches/.patch.md` - -```bash -set -euo pipefail - -if [ $# -lt 1 ]; then - echo "Usage: /compare-and-patch " - exit 2 -fi - -FILE="$1" -UP_ROOT="ddprof-lib/build/async-profiler/src" -LOCAL_ROOT="ddprof-lib/src/main/cpp" - -mkdir -p build/reports/claude/patches - -# Resolve canonical upstream and local paths -UP_CANON="$UP_ROOT/$FILE" -LOCAL_CANON="$LOCAL_ROOT/$FILE" - -# If direct paths don't exist, try to find by basename match inside each root -if [ ! -f "$UP_CANON" ]; then - FOUND_UP=$(find "$UP_ROOT" -type f -name "$(basename "$FILE")" -maxdepth 1 2>/dev/null | head -n1 || true) - if [ -n "$FOUND_UP" ]; then UP_CANON="$FOUND_UP"; fi -fi - -if [ ! -f "$LOCAL_CANON" ]; then - FOUND_LOCAL=$(find "$LOCAL_ROOT" -type f -name "$(basename "$FILE")" -maxdepth 1 2>/dev/null | head -n1 || true) - if [ -n "$FOUND_LOCAL" ]; then LOCAL_CANON="$FOUND_LOCAL"; fi -fi - -# Minimal existence check—agent will handle edge cases and write a status -if [ ! -f "$UP_CANON" ]; then - echo "Upstream file not found under $UP_ROOT for $FILE" -fi -if [ ! -f "$LOCAL_CANON" ]; then - echo "Local file not found under $LOCAL_ROOT for $FILE" -fi - -echo "Resolved:" -echo " upstream: $UP_CANON" -echo " local: $LOCAL_CANON" - -# NOTE: Do not compute the diff here; let the agent do normalization and policy (ignore EOL/copyright). -# Delegate to the patch-analyst agent with resolved paths - -echo "Delegating to patch-analyst agent..." -claude "Act as the patch-analyst agent to analyze $FILE. Use upstream file: $UP_CANON and local file: $LOCAL_CANON. Generate the required patch analysis artifacts as specified in the patch-analyst agent definition." \ No newline at end of file diff --git a/.claude/commands/contribute-upstream.md b/.claude/commands/contribute-upstream.md deleted file mode 100644 index f8d0273a7..000000000 --- a/.claude/commands/contribute-upstream.md +++ /dev/null @@ -1,169 +0,0 @@ ---- -description: Analyze divergences from upstream async-profiler, propose grouped PRs, and open draft PRs. ---- - -# Contribute Upstream Workflow - -You are orchestrating the contribution of java-profiler divergences back to the upstream async-profiler project. These are all changes in our repo relative to upstream — not just uncommitted local modifications. Follow these steps precisely. - -## Configuration - -- **Fork repo**: `git@github.com:DataDog/async-profiler.git` -- **Upstream repo**: `async-profiler/async-profiler` (for PR target) -- **Upstream branch**: `master` -- **Analysis script**: `utils/check_contribution_candidates.sh` -- **Report dir**: `build/contribution-reports/` - -## Step 1: Run Analysis - -Execute the analysis script to generate reports: - -```bash -./utils/check_contribution_candidates.sh -``` - -If it fails, diagnose and report the error to the user. Do not proceed. - -## Step 2: Parse Results - -Find the most recent JSON report in `build/contribution-reports/` (highest timestamp). Read it to get the list of files with contributable hunks. - -Also read the corresponding markdown report to understand the actual diff hunks for each file. - -If there are zero candidates, tell the user and stop. - -## Step 3: Filter Out Existing PRs - -Before proposing new PRs, check what's already open from the DataDog fork against upstream. Note: `--author DataDog` does not work because fork PRs are authored by the pushing user, not the org. Instead, query the API and filter by head repo: - -```bash -gh api 'repos/async-profiler/async-profiler/pulls?state=open&per_page=100' \ - --jq '.[] | select(.head.repo.full_name == "DataDog/async-profiler") | {number, title}' -``` - -Then for each matching PR, fetch the files it touches: - -```bash -gh api 'repos/async-profiler/async-profiler/pulls//files' --jq '.[].filename' -``` - -For each open PR, extract the list of files it touches. Then cross-reference with the candidate files from Step 2: - -- If a candidate file is already fully covered by an open PR (i.e., the PR touches that file and addresses the same type of change), **exclude it** from proposals -- If a candidate file is only partially covered (the open PR addresses some hunks but not others), keep the uncovered hunks as candidates -- When presenting proposals in Step 4, mention any skipped files and the existing PR that covers them, so the user has full visibility - -Analyze the remaining contributable hunks across candidate files and group them into logical PR proposals. Guidelines: - -- **Related changes go together**: e.g., a bug fix touching `stackWalker.cpp` and `vmStructs.cpp` for the same issue = one PR -- **Independent changes are separate**: unrelated fixes in different files = separate PRs -- **Each PR should be self-contained**: it should make sense on its own, compile on its own, and have a clear rationale -- **Keep PRs small**: prefer multiple small PRs over one large one - -For each proposed PR, prepare: -- A descriptive title (e.g., "Fix null pointer check in stackWalker", "Add bounds validation in VMStruct") -- The list of files and hunks it covers -- A brief rationale explaining why this change benefits upstream - -## Step 5: Present Proposals to User - -Show the user a numbered list of proposed PRs with: -- Title -- Files involved -- Brief description of the change -- Number of hunks - -Use `AskUserQuestion` with `multiSelect: true` to let the user pick which PRs to create. Offer all proposals as options. - -If the user selects none, stop. - -## Step 6: Create Selected PRs - -For each selected PR, perform the following: - -### 6a. Clone the Fork (once) - -Clone `git@github.com:DataDog/async-profiler.git` to a temp directory. Reuse this clone for all PRs. - -```bash -FORK_DIR=$(mktemp -d "${TMPDIR:-/tmp}/async-profiler-fork.XXXXXX") -git clone git@github.com:DataDog/async-profiler.git "$FORK_DIR" -cd "$FORK_DIR" -git remote add upstream https://github.com/async-profiler/async-profiler.git -git fetch upstream -git checkout -b temp upstream/master -git branch -D master 2>/dev/null || true -git checkout -b master -git branch -D temp -``` - -### 6b. Create Feature Branch - -For each PR, create a branch from upstream master: - -```bash -cd "$FORK_DIR" -git checkout master -BRANCH_NAME="contribute/-$(date +%Y%m%d)" -git checkout -b "$BRANCH_NAME" -``` - -Where `` is a short kebab-case description derived from the PR title. - -### 6c. Port Changes - -Apply only the relevant contributable hunks for this PR to the upstream files: - -1. For each file in the PR, find the corresponding upstream file in `src/` of the fork -2. Apply the contributable hunks using careful manual editing (use the Edit tool) -3. **Critical**: Ensure NO Datadog-specific references leak through (DD_, ddprof, Datadog, datadog, DDPROF, context.h, counters.h, tagger, QueueItem) -4. If a hunk cannot be cleanly applied because the upstream file diverged, skip it and note it for the user - -### 6d. Verify Build - -Attempt a basic build check: - -```bash -cd "$FORK_DIR" -make -``` - -If the build fails, analyze the error. If it's a simple fix (e.g., missing include), fix it. If it's complex, note it for the user and proceed anyway (the PR is draft). - -### 6e. Commit and Push - -```bash -cd "$FORK_DIR" -git add -A -git commit -m "" -git push origin "$BRANCH_NAME" -``` - -### 6f. Open Draft PR - -Use `gh` to create a draft PR against upstream. **Important**: Before creating the first PR, fetch the target project's PR template from the upstream repo (`gh api repos/async-profiler/async-profiler/contents/.github/PULL_REQUEST_TEMPLATE.md` and decode the base64 content). The PR body **must** follow that template exactly — use all its sections, checkboxes, and footer verbatim. Fill in each section with the relevant content for this change. - -```bash -gh pr create \ - --repo async-profiler/async-profiler \ - --base master \ - --head "DataDog:$BRANCH_NAME" \ - --draft \ - --title "" \ - --body "" -``` - -## Step 7: Report - -After all selected PRs are created, show the user: -- A summary table of created PRs with their URLs -- Any hunks that could not be applied -- Any build issues encountered -- The temp directory path in case manual follow-up is needed - -## Error Handling - -- If `gh` is not authenticated, tell the user to run `gh auth login` and stop -- If the fork clone fails, check SSH key setup and report -- If a branch already exists on the fork, append a counter suffix (e.g., `-2`) -- Always clean up on fatal errors (remove temp directory) diff --git a/.claude/commands/jmh.md b/.claude/commands/jmh.md deleted file mode 100644 index 484408b06..000000000 --- a/.claude/commands/jmh.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -description: Run and analyze JMH benchmarks via the jmh-benchmarks subagent. -argument-hint: "[filter or 'all'] [mode: quick|full] [baseline-path(optional)]" -allowed-tools: Bash(./gradlew:*), Bash(gradlew:*), Bash(./mvnw:*), Bash(mvn:*), Bash(java:*), Read, Glob, Grep, Write ---- - -You MUST route this request through the `jmh-benchmarks` subagent for all heavy lifting, -so that JMH logs and result files stay in its separate context and do not pollute the -main conversation. - -User arguments (raw): `$ARGUMENTS` - -Interpretation: -- $1 → benchmark filter or "all" (optional) -- $2 → mode: "quick" (default) or "full" (optional) -- $3 → optional baseline path to compare against - -If $1 is empty: -- Treat it as "all" benchmarks in quick mode. - -If $1 is provided and not "all": -- Treat it as a JMH benchmark include pattern (e.g. regex or substring) - and pass it to JMH according to the project's conventions. - -If $2 is "full": -- Ask the `jmh-benchmarks` subagent to use the project's full/fat JMH configuration. - Otherwise: -- Use "quick" mode settings as defined in the subagent prompt. - -If $3 is provided: -- Treat it as a file path to a previous JMH results file and request a comparison - against the new run. - -Your task: - -1. Summarize what will be done based on the provided arguments (filter, mode, baseline). -2. Explicitly instruct the `jmh-benchmarks` subagent to: - - Detect the build tool (Gradle vs Maven) and JMH integration. - - Choose an appropriate JMH command and run it, respecting the filter/mode. - - Locate JMH result files (JSON/CSV/text). - - Analyze and summarize the results. - - If a baseline is available (either from $3 or default baseline locations), - perform a regression comparison. -3. Ask the subagent to keep all large logs and raw results in its own context, and - only return a concise summary to this main thread. -4. Present the final answer as: - - A short description of what was run (command, mode, filter). - - A table of key benchmarks with score, error, units. - - If baseline available: a table of regressions/improvements with percentage change. - - Bullet points with recommendations for further investigation. - -Do not run JMH directly yourself in this command body. Always delegate to the -`jmh-benchmarks` subagent and act as a thin orchestrator and summarizer. diff --git a/.claude/settings.json b/.claude/settings.json deleted file mode 100644 index 39f7edcee..000000000 --- a/.claude/settings.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "permissions": { - "allow": [ - "Read", - "Write", - "Bash(grep:*)", - "Bash(awk:*)", - "Bash(sed:*)", - "Bash(python3:*)", - "Bash(./gradlew:*)", - "Bash(sh:*)", - "Bash(ls:*)", - "Bash(date:*)", - "Bash(mkdir:*)", - "Bash(tee:*)" - ], - "ask": [], - "deny": [ - "Bash(sudo:*)", - "Bash(rm:*)" - ] - }, - "hooks": { - "SubagentStop": [ - { - "hooks": [ - { - "type": "command", - "command": "echo '[gradle-log-analyst] Wrote build/reports/claude/gradle-summary.{md,json}' >&2" - } - ] - } - ] - } -} \ No newline at end of file diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index 50ca329f2..000000000 --- a/.gitattributes +++ /dev/null @@ -1 +0,0 @@ -*.sh eol=lf diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index a95104170..000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2026, Datadog, Inc - -# Default owners for all files in the repository -* @DataDog/profiling-java diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index 3c6c3e186..000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,26 +0,0 @@ -**What does this PR do?**: - - -**Motivation**: - - -**Additional Notes**: - - -**How to test the change?**: - - -**For Datadog employees**: -- [ ] If this PR touches code that signs or publishes builds or packages, or handles - credentials of any kind, I've requested a security review (run the `dd:platform-security-review` - skill, or file a request via the [PSEC review form](https://datadoghq.atlassian.net/jira/software/c/projects/PSEC/forms/form/direct/7861446195161534/37715)). - `bewaire` also runs automatically on every PR. -- [ ] This PR doesn't touch any of that. -- [ ] JIRA: [JIRA-XXXX] - -Unsure? Have a question? Request a review! diff --git a/.github/actions/extract_versions/action.yml b/.github/actions/extract_versions/action.yml deleted file mode 100644 index d2bb62823..000000000 --- a/.github/actions/extract_versions/action.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: Extract Java and Gradle Versions -description: Versions are reported in JAVA_VERSION and GRADLE_VERSION environment variables - -runs: - using: "composite" - steps: - - name: Extract Versions - id: extract_versions - shell: bash - run: | - set +e - - # Extract Java version - ${{ env.JAVA_TEST_HOME }}/bin/java -version - JAVA_VERSION=$(${{ env.JAVA_TEST_HOME }}/bin/java -version 2>&1 | awk -F '"' '/version/ { - split($2, v, "[._]"); - if (v[2] == "") { - # Version is like "24": assume it is major only and add .0.0 - printf "%s.0.0\n", v[1] - } else if (v[1] == "1") { - # Java 8 or older: Format is "1.major.minor_update" - printf "%s.%s.%s\n", v[2], v[3], v[4] - } else { - # Java 9 or newer: Format is "major.minor.patch" - printf "%s.%s.%s\n", v[1], v[2], v[3] - } - }') - echo "JAVA_VERSION=${JAVA_VERSION}" - echo "JAVA_VERSION=${JAVA_VERSION}" >> $GITHUB_ENV - - # Extract Gradle version from gradle-wrapper.properties - gradle_version=$(grep 'distributionUrl' gradle/wrapper/gradle-wrapper.properties | cut -d'=' -f2) - gradle_version=${gradle_version#*gradle-} - gradle_version=${gradle_version%-bin.zip} - echo "GRADLE_VERSION=${gradle_version}" >> $GITHUB_ENV \ No newline at end of file diff --git a/.github/actions/setup_cached_java/action.yml b/.github/actions/setup_cached_java/action.yml deleted file mode 100644 index d5b683840..000000000 --- a/.github/actions/setup_cached_java/action.yml +++ /dev/null @@ -1,75 +0,0 @@ -name: "Setup test Java environment" -description: "Setup Java environment for testing" - -inputs: - version: - description: "The test JDK version to install" - required: true - default: "11" - arch: - description: "The architecture" - required: true - default: "amd64" - -runs: - using: composite - steps: - - name: Infer Build JDK - shell: bash - id: infer_build_jdk - run: | - # Gradle 9 requires JDK 17+ to run; using JDK 21 (LTS) - echo "Inferring JDK 21 [${{ inputs.arch }}]" - if [[ ${{ inputs.arch }} =~ "-musl" ]]; then - echo "build_jdk=jdk21-librca" >> $GITHUB_OUTPUT - else - echo "build_jdk=jdk21" >> $GITHUB_OUTPUT - fi - - name: Cache Build JDK [${{ inputs.arch }}] - id: cache_build_jdk - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: | - jdks/${{ steps.infer_build_jdk.outputs.build_jdk }} - key: ${{ steps.infer_build_jdk.outputs.build_jdk }}-${{ inputs.arch }}--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - restore-keys: | - ${{ steps.infer_build_jdk.outputs.build_jdk }}-${{ inputs.arch }}-- - enableCrossOsArchive: true - - name: Cache JDK ${{ inputs.version }} [${{ inputs.arch }}] - id: cache_jdk - uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: | - jdks/jdk${{ inputs.version }} - key: jdk${{ inputs.version }}-${{ inputs.arch }}--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - restore-keys: | - jdk${{ inputs.version }}-${{ inputs.arch }}-- - enableCrossOsArchive: true - - name: JDK cache miss - if: steps.cache_jdk.outputs.cache-hit != 'true' || steps.cache_build_jdk.outputs.cache-hit != 'true' - shell: bash - run: | - # well, the cache-hit is not alway set to 'true', even when cache is hit (but it is not freshly recreated, whatever that means) - if [ ! -d "jdks/jdk${{ inputs.version }}" ]; then - OWNER=${{ github.repository_owner }} - REPO=${{ github.event.repository.name }} - BRANCH=${{ github.ref_name }} - WORKFLOW="cache_java.yml" - - URL="https://github.com/$OWNER/$REPO/actions/workflows/$WORKFLOW" - - echo "### ⚠️ JDK Cache Miss Detected" >> $GITHUB_STEP_SUMMARY - echo "🛠️ [Click here and select ${BRANCH} branch to manually refresh the cache](<$URL>)" >> $GITHUB_STEP_SUMMARY - exit 1 - fi - - name: Setup Environment - shell: bash - run: | - chmod a+rx -R jdks - echo "Setting up JDK ${{ inputs.version }} [${{ inputs.arch }}]" - JAVA_HOME=$(pwd)/jdks/${{ steps.infer_build_jdk.outputs.build_jdk }} - JAVA_TEST_HOME=$(pwd)/jdks/jdk${{ inputs.version }} - PATH=$JAVA_HOME/bin:$PATH - echo "JAVA_HOME=$JAVA_HOME" >> $GITHUB_ENV - echo "JAVA_TEST_HOME=$JAVA_TEST_HOME" >> $GITHUB_ENV - echo "PATH=$JAVA_HOME/bin:$PATH" >> $GITHUB_ENV diff --git a/.github/actions/upsert-pr-comment/action.yml b/.github/actions/upsert-pr-comment/action.yml deleted file mode 100644 index bbefdd5ea..000000000 --- a/.github/actions/upsert-pr-comment/action.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: "Upsert PR Comment with Octo-STS" -description: > - Exchanges OIDC for an Octo-STS GitHub-App token and - creates or updates a single comment on the PR. - -inputs: - body-file: - description: "Path to file whose contents become the comment body" - required: true - comment-id: - description: "Unique identifier for this comment type (used to find/update existing comment)" - required: false - default: "upsert-pr-comment" - repo: # optional; defaults to triggering repo - description: "Repository (owner/repo)." - required: false - pr-number: # optional; defaults to triggering PR - description: "Pull-request number." - required: false - -runs: - using: "composite" - steps: - # 1. Get installation token from DD-Octo-STS - - name: Obtain Octo-STS token - id: octo-sts - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4 - with: - audience: dd-octo-sts - scope: DataDog/java-profiler - policy: self.pr-comment - - # 2. Upsert the comment - - name: Upsert PR comment - env: - GH_TOKEN: ${{ steps.octo-sts.outputs.token }} - BODY_FILE: ${{ inputs['body-file'] }} - COMMENT_ID: ${{ inputs['comment-id'] }} - REPO: ${{ inputs.repo || github.repository }} - PR: ${{ inputs['pr-number'] || github.event.pull_request.number }} - shell: bash - run: | - if [[ -s "$BODY_FILE" ]]; then - set -e - # Create marker to identify this comment type - MARKER="" - - # Find existing comment with this marker - cid=$(gh api "repos/$REPO/issues/$PR/comments?per_page=100" \ - --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | head -n1) - - # Prepend marker to body - BODY="${MARKER}"$'\n'"$(< "$BODY_FILE")" - - if [[ -n "$cid" ]]; then - gh api --method PATCH "repos/$REPO/issues/comments/$cid" \ - --raw-field body="$BODY" - echo "✏️ Updated comment $cid" - else - gh api --method POST "repos/$REPO/issues/$PR/comments" \ - --raw-field body="$BODY" - echo "💬 Created new comment" - fi - else - echo "⚠️ Skipping empty comment" - fi diff --git a/.github/chainguard/async-profiler-build.ci.sts.yaml b/.github/chainguard/async-profiler-build.ci.sts.yaml deleted file mode 100644 index e651df88d..000000000 --- a/.github/chainguard/async-profiler-build.ci.sts.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Allow java-profiler GitLab CI to post reports -issuer: https://gitlab.ddbuild.io - -subject_pattern: "project_path:DataDog/java-profiler:ref_type:branch:ref:.*" - -permissions: - contents: write - issues: write - # write (not read) is required to post comments on pull requests via the - # issues/comments endpoint — GitLab CI back-reports benchmark & reliability - # results to the PR (see .gitlab/scripts/upsert-github-pr-comment.sh). - pull_requests: write diff --git a/.github/chainguard/benchmarking-platform-reports.ci.sts.yaml b/.github/chainguard/benchmarking-platform-reports.ci.sts.yaml deleted file mode 100644 index 8c52fe3e6..000000000 --- a/.github/chainguard/benchmarking-platform-reports.ci.sts.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Allow the benchmarking-platform GitLab CI to back-report results to PRs only. -# -# Scoped deliberately narrower than async-profiler-build.ci: the BP project lives -# in a separate repository, so it is granted *only* the permissions needed to -# upsert a result comment on a java-profiler PR — never contents: write. -issuer: https://gitlab.ddbuild.io - -subject_pattern: "project_path:DataDog/apm-reliability/benchmarking-platform:ref_type:branch:ref:.*" - -permissions: - # Posting a comment on a pull request via the issues/comments endpoint requires - # pull_requests: write for a GitHub App (PRs are issues, but comment-writes on a - # PR are governed by the Pull requests permission — issues: write alone yields a - # 403 "Resource not accessible by integration"). No contents access is granted. - issues: write - pull_requests: write diff --git a/.github/chainguard/gh-pages.sts.yaml b/.github/chainguard/gh-pages.sts.yaml deleted file mode 100644 index fe170dd12..000000000 --- a/.github/chainguard/gh-pages.sts.yaml +++ /dev/null @@ -1,26 +0,0 @@ -# Octo-STS Trust Policy for GitHub Pages Publishing -# This policy allows GitLab CI to push integration test reports to gh-pages branch -# -# Trust Policy Location: .github/chainguard/gh-pages.sts.yaml -# Referenced by: scripts/get-github-token-via-octo-sts.sh (OCTO_STS_POLICY=gh-pages) -# -# How it works: -# 1. GitLab CI generates OIDC token with issuer: https://gitlab.ddbuild.io -# 2. Token includes claims: project_path, ref, namespace_path, etc. -# 3. Octo-STS validates token against this policy -# 4. If valid, Octo-STS returns short-lived GitHub token with specified permissions - -# GitLab OIDC issuer -issuer: https://gitlab.ddbuild.io - -# Match GitLab CI jobs from any branch (needed for PR comments) -# GitLab token includes: project_path=DataDog/java-profiler, ref= -subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.* - -# GitHub API permissions for the returned token -# contents:write - Required to push to gh-pages branch -permissions: - contents: write - -# Token lifetime (default: 1 hour) -# Short-lived tokens reduce security risk diff --git a/.github/chainguard/self.approve-trivial.approve-pr.sts.yaml b/.github/chainguard/self.approve-trivial.approve-pr.sts.yaml deleted file mode 100644 index eb52fc892..000000000 --- a/.github/chainguard/self.approve-trivial.approve-pr.sts.yaml +++ /dev/null @@ -1,14 +0,0 @@ -issuer: https://token.actions.githubusercontent.com - -subject: repo:DataDog/java-profiler:pull_request - -claim_pattern: - event_name: pull_request_target - ref: refs/heads/main - ref_protected: "true" - job_workflow_ref: DataDog/java-profiler/\.github/workflows/approve-trivial\.yml@refs/heads/main - -permissions: - contents: read - pull_requests: write - diff --git a/.github/chainguard/self.dependabot-automerge.schedule.sts.yaml b/.github/chainguard/self.dependabot-automerge.schedule.sts.yaml deleted file mode 100644 index 20d7b2b85..000000000 --- a/.github/chainguard/self.dependabot-automerge.schedule.sts.yaml +++ /dev/null @@ -1,12 +0,0 @@ -issuer: https://token.actions.githubusercontent.com - -subject: repo:DataDog/java-profiler:ref:refs/heads/main - -claim_pattern: - event_name: schedule - ref: refs/heads/main - ref_protected: "true" - job_workflow_ref: DataDog/java-profiler/\.github/workflows/dependabot-automerge\.yml@refs/heads/main - -permissions: - contents: write diff --git a/.github/chainguard/self.pr-comment.sts.yaml b/.github/chainguard/self.pr-comment.sts.yaml deleted file mode 100644 index ac1797eb5..000000000 --- a/.github/chainguard/self.pr-comment.sts.yaml +++ /dev/null @@ -1,13 +0,0 @@ -issuer: https://token.actions.githubusercontent.com - -subject: repo:DataDog/java-profiler:pull_request -claim_pattern: - event_name: pull_request - # Allow codecheck.yml (scan-build) and ci.yml (test summary) to post PR comments - job_workflow_ref: DataDog/java-profiler/.github/workflows/(codecheck|ci)\.yml@.* - -permissions: - issues: write - contents: read - metadata: read - pull_requests: write diff --git a/.github/chainguard/update-images.sts.yaml b/.github/chainguard/update-images.sts.yaml deleted file mode 100644 index c09e441b2..000000000 --- a/.github/chainguard/update-images.sts.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# Octo-STS Trust Policy for Image Update PRs -# -# Allows the GitLab CI check-image-updates and rebuild-images-pr jobs to push -# branches and create pull requests for CI image reference updates. -# -# Referenced by: scripts/create-image-update-pr.sh (OCTO_STS_POLICY=update-images) - -# GitLab OIDC issuer -issuer: https://gitlab.ddbuild.io - -# Match GitLab CI jobs from the async-profiler-build project on any branch -subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.* - -# GitHub API permissions -permissions: - contents: write - pull_requests: write diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index 799e64238..000000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,64 +0,0 @@ -version: 2 -updates: - # Gradle dependencies (root project) - - package-ecosystem: "gradle" - directory: "/" - schedule: - interval: "weekly" - day: "monday" - open-pull-requests-limit: 5 - labels: - - "dependencies" - - "no-release-notes" - - "no-review" - ignore: - # JUnit 5.10+ dropped Java 8 support; 5.12+ dropped Java 11; 6.x requires Java 17. - # CI targets on Java 8 and 11 (HotSpot and J9) run the Gradle test worker on the - # test JDK itself (the profiler attaches to its own process), so the JUnit Platform - # classes must be loadable by Java 8/11. Keep the entire JUnit stack at 5.9.x / 1.9.x. - - dependency-name: "org.junit.jupiter:*" - versions: [">=5.10.0"] - - dependency-name: "org.junit.platform:*" - versions: [">=1.10.0"] - - dependency-name: "org.junit-pioneer:junit-pioneer" - versions: [">=2.0.0"] - groups: - gradle-minor: - update-types: - - "minor" - - "patch" - - # Gradle dependencies (build-logic composite build) - cooldown: - default-days: 2 - - package-ecosystem: "gradle" - directory: "/build-logic" - schedule: - interval: "weekly" - day: "monday" - open-pull-requests-limit: 5 - labels: - - "dependencies" - - "no-release-notes" - - "no-review" - groups: - gradle-minor: - update-types: - - "minor" - - "patch" - - # GitHub Actions - cooldown: - default-days: 2 - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "weekly" - day: "monday" - open-pull-requests-limit: 5 - labels: - - "dependencies" - - "no-release-notes" - - "no-review" - cooldown: - default-days: 2 diff --git a/.github/release.yml b/.github/release.yml deleted file mode 100644 index 8f207cb31..000000000 --- a/.github/release.yml +++ /dev/null @@ -1,7 +0,0 @@ -changelog: - exclude: - labels: - - "no-release-notes" - authors: - - "dependabot" - - "dependabot[bot]" diff --git a/.github/scripts/cppcheck-gh.xslt b/.github/scripts/cppcheck-gh.xslt deleted file mode 100644 index 10e4354c2..000000000 --- a/.github/scripts/cppcheck-gh.xslt +++ /dev/null @@ -1,109 +0,0 @@ - - - - - - - - - - - - - Results - - - -

CppCheck Report

- - - -
- -

Errors ()

-
- -
-
- -
- -

Warnings ()

-
- -
-
- -
- -

Style Violations ()

-
- -
-
-
- -

No issues found

-
-
- - -
- - - - - - - - -
- - - - - -
- -
diff --git a/.github/scripts/cppcheck-html.xslt b/.github/scripts/cppcheck-html.xslt deleted file mode 100644 index 67981a1ac..000000000 --- a/.github/scripts/cppcheck-html.xslt +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - - - - - - - - Results - - - -

CppCheck Report

-

Errors

- -
-

Warnings

- -
-

Style Violations

- - - -
- - - - - - - - -
- - - - - -
- -
diff --git a/.github/scripts/cppcheck-suppressions.txt b/.github/scripts/cppcheck-suppressions.txt deleted file mode 100644 index 357b4b604..000000000 --- a/.github/scripts/cppcheck-suppressions.txt +++ /dev/null @@ -1,4 +0,0 @@ -cstyleCast -constParameter -obsoleteFunctions:flightRecorder.cpp -obsoleteFunctionsalloca:flightRecorder.cpp \ No newline at end of file diff --git a/.github/scripts/filter_gradle_log.py b/.github/scripts/filter_gradle_log.py deleted file mode 100755 index 369a0433e..000000000 --- a/.github/scripts/filter_gradle_log.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 -""" -Streaming filter for Gradle test output. - -Compresses verbose test logs: - - PASSED tests: single summary line; all buffered output (including [TEST::INFO]) discarded - - FAILED tests: full context emitted: STARTED line + buffered stdout/stderr + FAILED line - followed by exception/stack trace that comes after the FAILED marker - - SKIPPED tests: single summary line - - CRASHED tests: if the stream ends mid-test (JVM kill, OOM, sanitizer abort), the full - buffer is emitted with a warning header - -Designed for inline use with `tee` so the unfiltered raw log is preserved: - - ./gradlew ... 2>&1 \\ - | tee -a "${RAW_LOG}" \\ - | python3 -u .github/scripts/filter_gradle_log.py - -Exit code and PIPESTATUS: - The filter always exits 0 regardless of test outcomes; use ${PIPESTATUS[0]} in bash - to capture the Gradle exit code: - - ./gradlew ... 2>&1 | tee -a raw.log | python3 -u filter_gradle_log.py - GRADLE_EXIT=${PIPESTATUS[0]} - -Limitations: - - [TEST::INFO] lines emitted from class-level lifecycle methods (@BeforeAll, static - initializers) appear before any STARTED marker and are suppressed in OUTSIDE state. - They remain visible in the raw log preserved by tee. -""" - -import re -import sys - -# Matches Gradle per-test event lines emitted by the Test task: -# -# com.example.FooTest > testBar STARTED -# com.example.FooTest > testBar[1] PASSED (0.456s) -# com.example.FooTest > testBar(int) FAILED -# com.example.FooTest > testBar SKIPPED -# -# The class name starts with a word character (not '>'), which prevents matching -# "> Task :project:taskName FAILED" build-level lines. -_TEST_EVENT = re.compile( - r'^([\w.$][\w.$ ]* > \S.*?) (STARTED|PASSED|FAILED|SKIPPED)(\s+\([^)]+\))?\s*$' -) - - -def emit(line: str) -> None: - print(line, flush=True) - - -def main() -> None: - # --- States --- - OUTSIDE = 0 # between tests: pass lines through directly - BUFFERING = 1 # inside a running test: accumulate output - FAILING = 2 # after FAILED marker: pass lines through until next test - - state = OUTSIDE - buf: list = [] - - for raw in sys.stdin: - line = raw.rstrip('\n') - m = _TEST_EVENT.match(line) - - if m: - event = m.group(2) - - if event == 'STARTED': - if state == BUFFERING: - # Previous test had no outcome line (shouldn't normally happen). - # Emit the buffer so we don't silently discard output. - for buffered_line in buf: - emit(buffered_line) - elif state == FAILING: - emit('') # blank line to visually separate failure blocks - - # Include the STARTED line in the buffer so it appears in failure output. - buf = [line] - state = BUFFERING - - elif event == 'PASSED': - buf = [] - emit(line) - state = OUTSIDE - - elif event == 'FAILED': - # Emit everything collected since STARTED (includes [TEST::INFO] lines). - for buffered_line in buf: - emit(buffered_line) - buf = [] - emit(line) - state = FAILING - - elif event == 'SKIPPED': - buf = [] - emit(line) - state = OUTSIDE - - elif state == BUFFERING: - buf.append(line) - - else: - # OUTSIDE or FAILING: pass through directly. - # In FAILING state this captures exception lines, stack traces, etc. - # In OUTSIDE state, suppress [TEST::INFO] lines: they originate from - # class-level init (@BeforeAll, static blocks) and are noise when no - # test has failed; the raw log still contains them for reference. - if state == FAILING or not line.startswith('[TEST::INFO]'): - emit(line) - - # EOF handling: if still inside a test the JVM likely crashed (SIGABRT from sanitizer, - # OOM kill, etc.). Emit everything so the failure is visible in the filtered log. - if state == BUFFERING and buf: - emit('# WARNING: stream ended inside a test (crash / OOM / sanitizer abort?)') - for buffered_line in buf: - emit(buffered_line) - - -if __name__ == '__main__': - main() diff --git a/.github/scripts/generate-test-summary.sh b/.github/scripts/generate-test-summary.sh deleted file mode 100755 index a6cbcfcc5..000000000 --- a/.github/scripts/generate-test-summary.sh +++ /dev/null @@ -1,335 +0,0 @@ -#!/usr/bin/env bash -# Generate CI test results summary for PR comments -# -# Usage: generate-test-summary.sh -# -# Fetches job data from GitHub API, parses test results, and generates -# a markdown summary suitable for posting as a PR comment. - -set -euo pipefail - -RUN_ID="$1" -OUTPUT_FILE="$2" - -# --- Configuration --- -REPO="${GITHUB_REPOSITORY:-DataDog/java-profiler}" -SERVER_URL="${GITHUB_SERVER_URL:-https://github.com}" -RUN_URL="${SERVER_URL}/${REPO}/actions/runs/${RUN_ID}" -COMMIT_SHA="${GITHUB_SHA:-unknown}" -SHORT_SHA="${COMMIT_SHA:0:7}" - -# --- Helper functions --- -log() { - echo "[generate-test-summary] $*" >&2 -} - -# Convert seconds to human-readable duration -format_duration() { - local seconds=$1 - local hours=$((seconds / 3600)) - local minutes=$(((seconds % 3600) / 60)) - local secs=$((seconds % 60)) - - if ((hours > 0)); then - printf "%dh %dm %ds" "$hours" "$minutes" "$secs" - elif ((minutes > 0)); then - printf "%dm %ds" "$minutes" "$secs" - else - printf "%ds" "$secs" - fi -} - -# Parse ISO timestamp to epoch seconds -parse_timestamp() { - local ts="$1" - if [[ -n "$ts" && "$ts" != "null" ]]; then - date -j -f "%Y-%m-%dT%H:%M:%SZ" "$ts" "+%s" 2>/dev/null || \ - date -d "$ts" "+%s" 2>/dev/null || \ - echo "0" - else - echo "0" - fi -} - -# Get status emoji for job conclusion -status_emoji() { - case "$1" in - success) echo ":white_check_mark:" ;; - failure) echo ":x:" ;; - skipped) echo ":white_circle:" ;; - cancelled) echo ":no_entry_sign:" ;; - *) echo ":grey_question:" ;; - esac -} - -# --- Fetch job data --- -log "Fetching job data for run $RUN_ID..." -jobs_json=$(gh api "/repos/$REPO/actions/runs/$RUN_ID/jobs" --paginate -q '.jobs') - -# --- Parse test jobs --- -log "Parsing test job statuses..." - -# Arrays to store parsed data -# Note: Associative arrays need dummy init+unset to work with 'set -u' when empty -declare -A job_status=() -job_status["__init__"]=1; unset 'job_status[__init__]' -declare -A job_url=() -job_url["__init__"]=1; unset 'job_url[__init__]' -declare -A job_duration=() -job_duration["__init__"]=1; unset 'job_duration[__init__]' -declare -a failed_jobs=() -declare -a all_platforms=() -declare -a all_java_versions=() - -# Parse each job -while IFS= read -r job; do - name=$(echo "$job" | jq -r '.name') - conclusion=$(echo "$job" | jq -r '.conclusion // "pending"') - html_url=$(echo "$job" | jq -r '.html_url') - started_at=$(echo "$job" | jq -r '.started_at') - completed_at=$(echo "$job" | jq -r '.completed_at') - - # Only process test jobs (match pattern: test-linux-{libc}-{arch} ({java}, {config})) - # Note: regex stored in variable to avoid bash parsing issues with ) character - # Note: No ^ anchor because reusable workflow jobs are prefixed with caller job name - # e.g., "test-matrix / test-linux-glibc-amd64 (8, debug)" - test_job_pattern='test-linux-([a-z]+)-([a-z0-9]+) \(([^,]+), ([^)]+)\)$' - if [[ "$name" =~ $test_job_pattern ]]; then - libc="${BASH_REMATCH[1]}" - arch="${BASH_REMATCH[2]}" - java_version="${BASH_REMATCH[3]}" - config="${BASH_REMATCH[4]}" - - platform="${libc}-${arch}/${config}" - - # Calculate duration - if [[ -n "$started_at" && "$started_at" != "null" && -n "$completed_at" && "$completed_at" != "null" ]]; then - start_epoch=$(parse_timestamp "$started_at") - end_epoch=$(parse_timestamp "$completed_at") - duration=$((end_epoch - start_epoch)) - else - duration=0 - fi - - # Store in associative arrays - key="${platform}|${java_version}" - job_status["$key"]="$conclusion" - job_url["$key"]="$html_url" - job_duration["$key"]="$duration" - - # Track failed jobs - if [[ "$conclusion" == "failure" ]]; then - failed_jobs+=("$key") - fi - - # Collect unique platforms and java versions - # shellcheck disable=SC2076 # Intentional literal match for array membership - if [[ ! " ${all_platforms[*]} " =~ " ${platform} " ]]; then - all_platforms+=("$platform") - fi - # shellcheck disable=SC2076 # Intentional literal match for array membership - if [[ ! " ${all_java_versions[*]} " =~ " ${java_version} " ]]; then - all_java_versions+=("$java_version") - fi - fi -done < <(echo "$jobs_json" | jq -c '.[]') - -# Sort java versions (natural sort for versions like 8, 11, 17, 21, 25) -sorted_java=() -if ((${#all_java_versions[@]} > 0)); then - mapfile -t sorted_java < <(printf '%s\n' "${all_java_versions[@]}" | sort -t'-' -k1,1n -k2,2) -fi - -# Sort platforms -sorted_platforms=() -if ((${#all_platforms[@]} > 0)); then - mapfile -t sorted_platforms < <(printf '%s\n' "${all_platforms[@]}" | sort) -fi - -# --- Calculate statistics --- -total_jobs=${#job_status[@]} -passed_jobs=0 -failed_count=${#failed_jobs[@]} -skipped_jobs=0 -cancelled_jobs=0 -total_duration=0 -max_duration=0 - -for key in "${!job_status[@]}"; do - status="${job_status[$key]}" - dur="${job_duration[$key]:-0}" - - case "$status" in - success) ((++passed_jobs)) ;; - failure) ;; # already counted - skipped) ((++skipped_jobs)) ;; - cancelled) ((++cancelled_jobs)) ;; - esac - - ((total_duration += dur)) || true - if ((dur > max_duration)); then - max_duration=$dur - fi -done - -# --- Download failure artifacts (if any failures) --- -declare -A failure_details=() -failure_details["__init__"]=1; unset 'failure_details[__init__]' - -if ((failed_count > 0)); then - log "Downloading failure artifacts..." - mkdir -p ./failure-artifacts - - # Try to download test reports - gh run download "$RUN_ID" --pattern '(test-reports)*' --dir ./failure-artifacts 2>/dev/null || true - - # Parse JUnit XML for failure details - for key in "${failed_jobs[@]}"; do - IFS='|' read -r platform java_version <<< "$key" - - # Find matching test report directory - # Pattern: (test-reports) test-linux-{libc}-{arch} ({java}, {config}) - IFS='/' read -r libc_arch config <<< "$platform" - report_pattern="./failure-artifacts/*${libc_arch}*${java_version}*${config}*" - - failures="" - for report_dir in $report_pattern; do - if [[ -d "$report_dir" ]]; then - # Parse JUnit XML files - for xml_file in "$report_dir"/**/TEST-*.xml; do - if [[ -f "$xml_file" ]]; then - # Extract failed test cases - while IFS= read -r testcase; do - classname=$(echo "$testcase" | grep -oP 'classname="\K[^"]+' || echo "") - testname=$(echo "$testcase" | grep -oP 'name="\K[^"]+' || echo "") - # Get failure message (first line only, truncated) - failure_msg=$(echo "$testcase" | grep -oP ']*message="\K[^"]*' | head -c 100 || echo "") - - if [[ -n "$classname" && -n "$testname" ]]; then - short_class="${classname##*.}" - failures+="| \`${short_class}.${testname}\` | ${failure_msg:-Test failed} |"$'\n' - fi - done < <(grep -Pzo '(?s)]*>.*?' "$xml_file" 2>/dev/null | tr '\0' '\n' | grep -E '<(failure|error)' || true) - fi - done - fi - done - - failure_details["$key"]="$failures" - done - - # Cleanup - rm -rf ./failure-artifacts -fi - -# --- Generate markdown --- -log "Generating markdown summary..." - -{ - echo "## CI Test Results" - echo "" - echo "**Run:** [#${RUN_ID}]($RUN_URL) | **Commit:** \`$SHORT_SHA\` | **Duration:** $(format_duration "$max_duration") (longest job)" - echo "" - - # Overall status - if ((failed_count == 0)); then - echo "> :white_check_mark: **All $total_jobs test jobs passed**" - else - echo "> :x: **$failed_count of $total_jobs test jobs failed**" - fi - echo "" - - # Status matrix table (JDK versions as rows, platforms as columns) - if ((${#sorted_platforms[@]} > 0 && ${#sorted_java[@]} > 0)); then - echo "### Status Overview" - echo "" - - # Header row - platforms as columns - printf "| JDK |" - for platform in "${sorted_platforms[@]}"; do - printf " %s |" "$platform" - done - echo "" - - # Separator row - printf "%s" "|-----|" - for _ in "${sorted_platforms[@]}"; do - printf "%s" "--------|" - done - echo "" - - # Data rows - JDK versions as rows - for java in "${sorted_java[@]}"; do - printf "| %s |" "$java" - for platform in "${sorted_platforms[@]}"; do - key="${platform}|${java}" - status="${job_status[$key]:-}" - url="${job_url[$key]:-}" - - if [[ -n "$status" ]]; then - emoji=$(status_emoji "$status") - if [[ -n "$url" ]]; then - printf " [%s](%s) |" "$emoji" "$url" - else - printf " %s |" "$emoji" - fi - else - printf " - |" - fi - done - echo "" - done - echo "" - echo "**Legend:** :white_check_mark: passed | :x: failed | :white_circle: skipped | :no_entry_sign: cancelled" - echo "" - fi - - # Failed tests details - if ((failed_count > 0)); then - echo "### Failed Tests" - echo "" - - for key in "${failed_jobs[@]}"; do - IFS='|' read -r platform java_version <<< "$key" - url="${job_url[$key]:-}" - details="${failure_details[$key]:-}" - - echo "
" - echo "${platform} / ${java_version}" - echo "" - if [[ -n "$url" ]]; then - echo "**Job:** [View logs]($url)" - echo "" - fi - - if [[ -n "$details" ]]; then - echo "| Test | Error |" - echo "|------|-------|" - echo -n "$details" - else - echo "_No detailed failure information available. Check the job logs._" - fi - echo "" - echo "
" - echo "" - done - fi - - # Summary statistics (single line) - printf "**Summary:** Total: %d | Passed: %d | Failed: %d" "$total_jobs" "$passed_jobs" "$failed_count" - if ((skipped_jobs > 0)); then - printf " | Skipped: %d" "$skipped_jobs" - fi - if ((cancelled_jobs > 0)); then - printf " | Cancelled: %d" "$cancelled_jobs" - fi - echo "" - echo "" - - echo "---" - echo "*Updated: $(date -u '+%Y-%m-%d %H:%M:%S UTC')*" - -} > "$OUTPUT_FILE" - -log "Summary written to $OUTPUT_FILE" -log "Total jobs: $total_jobs, Passed: $passed_jobs, Failed: $failed_count" diff --git a/.github/scripts/java_setup.sh b/.github/scripts/java_setup.sh deleted file mode 100644 index c17a43c6a..000000000 --- a/.github/scripts/java_setup.sh +++ /dev/null @@ -1,92 +0,0 @@ -#!/usr/bin/env bash - -function prepareJdk() { - local variant=$1 - local arch=$2 - local version=${variant%%-*} - local qualifier=${variant#*-} - - local target_path="${GITHUB_WORKSPACE}/${JDKS_DIR}/jdk${variant}" - - mkdir -p ${target_path} - - if [[ ${qualifier} == "librca" ]] && [[ "${arch}" =~ "-musl" ]]; then - local osarch="${arch%-musl}" - local suffix= - if [[ "${osarch}" == "aarch64" ]]; then - suffix="AARCH64_" - fi - URL_VAR="JAVA_${version}_MUSL_${suffix}URL" - URL="${!URL_VAR}" - if [[ -z "${URL}" ]]; then - echo "Musl/Liberica JDK URL not found for ${arch}/${variant}" - exit 1 - fi - curl -L --fail "${URL}" | tar -xvzf - -C ${target_path} --strip-components 1 - return - fi - - if [[ ${qualifier} == "orcl" ]]; then - if [[ ${version} == "8" ]]; then - mkdir -p "${target_path}" - curl -L --fail "${JAVA_8_ORACLE_URL}" | sudo tar -xvzf - -C ${target_path} --strip-components 1 - return - else - echo "Oracle JDK 8 only!" - exit 1 - fi - fi - - if [[ ${qualifier} == "ibm" ]]; then - if [[ ${version} == "8" ]]; then - mkdir -p "${target_path}" - curl -L --fail "${JAVA_8_IBM_URL}" | sudo tar -xvzf - -C ${target_path} --strip-components 2 - return - else - echo "IBM JDK 8 only!" - exit 1 - fi - fi - - if [[ ${qualifier} == "zing" ]]; then - URL_VAR="JAVA_${version}_ZING_URL" - if [[ "${arch}" == "aarch64" ]]; then - URL_VAR="JAVA_${version}_ZING_AARCH64_URL" - fi - - URL="${!URL_VAR}" - if [[ -z "${URL}" ]]; then - echo "Zing JDK URL not found for ${variant}" - exit 1 - fi - curl -L --fail "${URL}" | sudo tar -xvzf - -C ${target_path} --strip-components 1 - if [[ "${arch}" != "aarch64" ]]; then - # rename the bundled libstdc++.so to avoid conflicts with the system one - sudo mv ${target_path}/etc/libc++/libstdc++.so.6 ${target_path}/etc/libc++/libstdc++.so.6.bak - fi - return - fi - - # below the installation of the SDKMAN-managed JDK - source ~/.sdkman/bin/sdkman-init.sh - - local suffix="tem" - local versionVar="JAVA_${version}_VERSION" - if [[ "${qualifier}" == "j9" ]]; then - suffix="sem" - versionVar="JAVA_${version}_J9_VERSION" - elif [[ "${qualifier}" == "graal" ]]; then - suffix="graal" - versionVar="JAVA_${version}_GRAAL_VERSION" - fi - - local distro_base - distro_base="${!versionVar}" - local jdk_distro="${distro_base}-${suffix}" - - echo 'n' | sdk install java ${jdk_distro} > /dev/null - - rm -rf ${target_path} - mkdir -p "$(dirname ${target_path})" - mv ${SDKMAN_DIR}/candidates/java/${jdk_distro} ${target_path} -} diff --git a/.github/scripts/prepare_reports.sh b/.github/scripts/prepare_reports.sh deleted file mode 100755 index 4ff852450..000000000 --- a/.github/scripts/prepare_reports.sh +++ /dev/null @@ -1,20 +0,0 @@ - #!/usr/bin/env bash - -set -e -mkdir -p test-reports -mkdir -p unwinding-reports -cp build/test-raw.log test-reports/ || true -cp /tmp/hs_err* test-reports/ || true -cp /tmp/asan_*.log test-reports/ || true -cp /tmp/ubsan_*.log test-reports/ || true -cp /tmp/tsan_*.log test-reports/ || true -cp ddprof-test/javacore*.txt test-reports/ || true -cp ddprof-test/build/hs_err* test-reports/ || true -cp -r ddprof-lib/build/tmp test-reports/native_build || true -cp -r ddprof-test/build/reports/tests test-reports/tests || true -cp build/logs/gdb-watchdog.log test-reports/ || true -cp -r /tmp/recordings test-reports/recordings || true -find ddprof-lib/build -name 'libjavaProfiler.*' -exec cp {} test-reports/ \; || true - -cp -r ddprof-test/build/reports/unwinding-summary.md unwinding-reports/ || true -cp -r /tmp/unwinding-recordings/* unwinding-reports/ || true diff --git a/.github/scripts/python_utils.py b/.github/scripts/python_utils.py deleted file mode 100644 index d15ad56f1..000000000 --- a/.github/scripts/python_utils.py +++ /dev/null @@ -1,59 +0,0 @@ -import sys -from bs4 import BeautifulSoup - -def remove_tags(soup, tags_to_remove): - for tag in tags_to_remove: - for element in soup.find_all(tag): - element.decompose() - -def create_scanbuild_code_links(soup, target_branch): - target = None - for element in soup.find_all("td"): - clz = element.get('class') - if clz is None: - src = element.text - target = element - elif 'Q' in clz and target is not None and target.text != 'Function/Method': - line = element.text - link = soup.new_tag('a', href=f'https://github.com/DataDog/java-profiler/blob/{target_branch}/ddprof-lib/src/main/cpp/{src}#L{line}') - link.string = src - target.clear() - target.append(link) - target = None -def parse_table(table): - markdown_table = [] - for row in table.find_all('tr'): - cells = row.find_all(['th', 'td']) - markdown_cells = [cell.get_text(strip=True) for cell in cells] - markdown_table.append('| ' + ' | '.join(markdown_cells) + ' |') - return '\n'.join(markdown_table) - -def scanbuild_cleanup(soup, args): - target_branch = args[0] - remove_tags(soup, ["title", "script", "a"]) - create_scanbuild_code_links(soup, target_branch) - title = soup.find('h1') - title.string = 'Scan-Build Report' - return str(soup) - -def cppcheck_cleanup(soup, args): - remove_tags(soup, ["title", "style", "head"]) - return str(soup) - -def usage(soup, args): - return "Usage" - -if __name__ == "__main__": - actions = { - "scanbuild_cleanup": scanbuild_cleanup, - "cppcheck_cleanup": cppcheck_cleanup, - } - action = actions.get(sys.argv[1], usage) - input_file = sys.argv[2] - args = sys.argv[3:] - - with open(input_file, "r") as file: - html_content = file.read() - - soup = BeautifulSoup(html_content, "html.parser") - print(action(soup, args)) diff --git a/.github/scripts/release.sh b/.github/scripts/release.sh deleted file mode 100755 index e0984dde8..000000000 --- a/.github/scripts/release.sh +++ /dev/null @@ -1,159 +0,0 @@ -#!/usr/bin/env bash - -set -x -set -e - -TYPE=$1 -DRYRUN=$2 - -BRANCH=$(git branch --show-current) -RELEASE_BRANCH= -SKIP_RELEASE_CREATION=false - -BASE=$(./gradlew printVersion -Psnapshot=false | grep 'Version:' | cut -f2 -d' ') -# BASE == 0.0.1 - -create_annotated_tag() { - local version=$1 - local type=$2 - local branch=$3 - - local tag_name="v_${version}" - local tag_message="Release v_${version} (${type,,}) from ${branch}" - - # Check if tag exists - if git rev-parse "$tag_name" >/dev/null 2>&1; then - if [ -z "$DRYRUN" ]; then - echo "::error::Tag $tag_name already exists" - exit 1 - else - echo "[DRY-RUN] Tag $tag_name exists (would fail)" - return - fi - fi - - if [ -z "$DRYRUN" ]; then - git tag -a "$tag_name" -m "$tag_message" - echo "✓ Created annotated tag: $tag_name" - else - echo "[DRY-RUN] Would create tag: $tag_name" - echo "[DRY-RUN] Message: $tag_message" - fi -} - -if [ "$TYPE" == "MINOR" ] || [ "$TYPE" == "MAJOR" ]; then - if [ "$BRANCH" != "main" ] && [ -z "$DRYRUN" ]; then - echo "Major or minor release can be performed only from 'main' branch." - exit 1 - fi - if [ "$TYPE" == "MAJOR" ]; then - # 0.1.0 -> 1.0.0 - ./gradlew incrementVersion --versionIncrementType=MAJOR - BASE=$(./gradlew printVersion -Psnapshot=false | grep 'Version:' | cut -f2 -d' ') - # BASE == 1.0.0 - fi - RELEASE_BRANCH="release/${BASE%.*}._" - if [ -z "$DRYRUN" ] && git rev-parse "v_${BASE}" >/dev/null 2>&1; then - echo "Tag v_${BASE} already exists; skipping tag and release branch creation — will only produce a version-bump PR" - SKIP_RELEASE_CREATION=true - else - create_annotated_tag "$BASE" "$TYPE" "$BRANCH" - fi -fi - -if [ "$TYPE" == "PATCH" ]; then - if [[ ! $BRANCH =~ ^release\/[0-9]+\.[0-9]+\._$ ]] && [ -z "$DRYRUN" ]; then - echo "Patch release can be created only for 'release/*' branch." - exit 1 - fi - RELEASE_BRANCH="release/${BASE%.*}._" - create_annotated_tag "$BASE" "$TYPE" "$BRANCH" -fi - -# RETAG: re-point an existing tag at the current HEAD of a release branch. -# Use when a partial release (tag + branch created, but no Maven artifacts and -# no final GitHub release yet) needs additional commits (e.g. a cherry-picked fix). -if [ "$TYPE" == "RETAG" ]; then - if [[ ! $BRANCH =~ ^release\/[0-9]+\.[0-9]+\._$ ]] && [ -z "$DRYRUN" ]; then - echo "Retag can only be performed from a 'release/*' branch." - exit 1 - fi - TAG_NAME="v_${BASE}" - if ! git rev-parse "$TAG_NAME" >/dev/null 2>&1; then - echo "::error::Tag $TAG_NAME does not exist. Use a normal release to create a new tag." - exit 1 - fi - - # Refuse to retag if the GitHub release is already public - if command -v gh >/dev/null 2>&1; then - IS_DRAFT=$(gh release view "$TAG_NAME" --json isDraft --jq '.isDraft' 2>/dev/null || echo "not-found") - if [ "$IS_DRAFT" == "false" ]; then - echo "::error::GitHub release $TAG_NAME is already public. Retagging is not allowed." - exit 1 - fi - fi - - if [ -z "$DRYRUN" ]; then - git tag -f -a "$TAG_NAME" -m "Release v_${BASE} (retag) from ${BRANCH}" - git push --force-with-lease origin "$BRANCH" - git push origin :"$TAG_NAME" - git push origin "$TAG_NAME" - else - echo "[DRY-RUN] Would force-move tag $TAG_NAME to $(git rev-parse HEAD)" - echo "[DRY-RUN] Would push $BRANCH with --force-with-lease" - echo "[DRY-RUN] Would delete and re-push remote tag $TAG_NAME" - fi - - echo "==================== RETAG SUMMARY ====================" - echo "Release Branch: $BRANCH" - echo "Retagged Version: $BASE" - echo "Tag: $TAG_NAME -> $(git rev-parse HEAD)" - echo "========================================================" - exit 0 -fi - -if [ "$SKIP_RELEASE_CREATION" == "false" ] && [ "$BRANCH" != "$RELEASE_BRANCH" ]; then - git checkout -b $RELEASE_BRANCH - if ! git diff --quiet; then - git add build.gradle.kts - git commit -m "[Automated] Release ${BASE}" - fi - git push $DRYRUN --atomic --set-upstream origin $RELEASE_BRANCH - git checkout $BRANCH -fi - -if [ "$TYPE" == "MAJOR" ] || [ "$TYPE" == "MINOR" ]; then - ./gradlew incrementVersion --versionIncrementType=MINOR -else - ./gradlew incrementVersion --versionIncrementType=PATCH -fi - -CANDIDATE=$(./gradlew printVersion -Psnapshot=false | grep 'Version:' | cut -f2 -d' ') - -git add build.gradle.kts -git commit -m "[Automated] Bump dev version to ${CANDIDATE}" - -if [ -z "$DRYRUN" ]; then - BUMP_BRANCH="automated/bump-${CANDIDATE//./-}" - git checkout -b "$BUMP_BRANCH" - git push --force-with-lease --set-upstream origin "$BUMP_BRANCH" - REPO="${GITHUB_REPOSITORY:-$(git remote get-url origin | sed 's|.*github.com[:/]\(.*\)\.git|\1|')}" - BUMP_PR_URL="https://github.com/${REPO}/compare/${BRANCH}...${BUMP_BRANCH}?quick_pull=1&title=%5BAutomated%5D+Bump+dev+version+to+${CANDIDATE}" - echo "BUMP_PR_URL=$BUMP_PR_URL" >> "${GITHUB_OUTPUT:-/dev/null}" - echo "⚠ Open this URL to create the version bump PR: $BUMP_PR_URL" -else - git push $DRYRUN --atomic --set-upstream origin $BRANCH -fi - -git push $DRYRUN --atomic --tags - -echo "==================== RELEASE SUMMARY ====================" -echo "Release Type: $TYPE" -echo "Released Version: $BASE" -echo "Next Dev Version: $CANDIDATE" -echo "Release Branch: $RELEASE_BRANCH" -echo "Tag: v_$BASE" -if [ -z "$DRYRUN" ]; then - echo "Tag Message: $(git tag -l v_$BASE -n1 --format='%(contents:subject)')" -fi -echo "==========================================================" diff --git a/.github/scripts/test_alpine_aarch64.sh b/.github/scripts/test_alpine_aarch64.sh deleted file mode 100755 index c81600482..000000000 --- a/.github/scripts/test_alpine_aarch64.sh +++ /dev/null @@ -1,36 +0,0 @@ -#! /bin/sh - -set -e -set +x - -export KEEP_JFRS=true -export TEST_COMMIT="${1}" -export TEST_CONFIGURATION="${2}" -export LIBRARY="musl" -export CONFIG="${3}" -export JAVA_HOME="${4}" -export JAVA_TEST_HOME="${5}" - -export PATH="${JAVA_HOME}/bin":${PATH} - -# due to env hell in GHA containers, we need to re-do the logic from Extract Versions here -JAVA_VERSION=$("${JAVA_TEST_HOME}/bin/java" -version 2>&1 | awk -F '"' '/version/ { - split($2, v, "[._]"); - if (v[2] == "") { - # Version is like "24": assume it is major only and add .0.0 - printf "%s.0.0\n", v[1] - } else if (v[1] == "1") { - # Java 8 or older: Format is "1.major.minor_update" - printf "%s.%s.%s\n", v[2], v[3], v[4] - } else { - # Java 9 or newer: Format is "major.minor.patch" - printf "%s.%s.%s\n", v[1], v[2], v[3] - } -}') -export JAVA_VERSION - -apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock tar binutils >/dev/null -# Install debug symbols for musl libc -apk add musl-dbg - -./gradlew -PCI -PkeepJFRs :ddprof-test:test${CONFIG} --no-daemon --parallel --build-cache --no-watch-fs \ No newline at end of file diff --git a/.github/scripts/unwinding_report_alpine_aarch64.sh b/.github/scripts/unwinding_report_alpine_aarch64.sh deleted file mode 100755 index e6144d2c8..000000000 --- a/.github/scripts/unwinding_report_alpine_aarch64.sh +++ /dev/null @@ -1,36 +0,0 @@ -#! /bin/sh - -set -e -set +x - -export KEEP_JFRS=true -export TEST_COMMIT="${1}" -export TEST_CONFIGURATION="${2}" -export LIBRARY="musl" -export CONFIG="${3}" -export JAVA_HOME="${4}" -export JAVA_TEST_HOME="${5}" - -export PATH="${JAVA_HOME}/bin":${PATH} - -# due to env hell in GHA containers, we need to re-do the logic from Extract Versions here -JAVA_VERSION=$("${JAVA_TEST_HOME}/bin/java" -version 2>&1 | awk -F '"' '/version/ { - split($2, v, "[._]"); - if (v[2] == "") { - # Version is like "24": assume it is major only and add .0.0 - printf "%s.0.0\n", v[1] - } else if (v[1] == "1") { - # Java 8 or older: Format is "1.major.minor_update" - printf "%s.%s.%s\n", v[2], v[3], v[4] - } else { - # Java 9 or newer: Format is "major.minor.patch" - printf "%s.%s.%s\n", v[1], v[2], v[3] - } -}') -export JAVA_VERSION - -apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock tar binutils >/dev/null -# Install debug symbols for musl libc -apk add musl-dbg - -./gradlew -PCI :ddprof-test:unwindingReport --no-daemon --parallel --build-cache --no-watch-fs diff --git a/.github/workflows/add-milestone-to-pull-requests.yaml b/.github/workflows/add-milestone-to-pull-requests.yaml deleted file mode 100644 index 0981e9311..000000000 --- a/.github/workflows/add-milestone-to-pull-requests.yaml +++ /dev/null @@ -1,47 +0,0 @@ -name: Add milestone to pull requests -on: - pull_request: - types: [closed] - branches: - - main - -permissions: - contents: read - pull-requests: write - issues: write - -jobs: - add_milestone_to_merged: - if: github.event.pull_request.merged && github.event.pull_request.milestone == null - name: Add milestone to merged pull requests - runs-on: ubuntu-latest - steps: - - name: Get project milestones - id: milestones - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # 9.0.0 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const list = await github.rest.issues.listMilestones({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'open' - }) - // Need to manually sort because "sort by number" isn't part of the api - // highest number first - const milestones = list.data.sort((a,b) => (b.number - a.number)) - - return milestones.length == 0 ? null : milestones[0].number - - name: Update Pull Request - if: steps.milestones.outputs.result != null - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # 9.0.0 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - // Confusingly, the issues api is used because pull requests are issues - await github.rest.issues.update({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: ${{ github.event.pull_request.number }}, - milestone: ${{ steps.milestones.outputs.result }}, - }); diff --git a/.github/workflows/approve-trivial.yml b/.github/workflows/approve-trivial.yml deleted file mode 100644 index f737036d4..000000000 --- a/.github/workflows/approve-trivial.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Auto-Approve Trivial PRs - -on: - pull_request_target: - types: [labeled] - -jobs: - auto-approve: - if: contains(github.event.pull_request.labels.*.name, 'trivial') || contains(github.event.pull_request.labels.*.name, 'no-review') - runs-on: ubuntu-latest - permissions: - id-token: write # Needed to federate tokens - steps: - - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4 - id: octo-sts - with: - scope: DataDog/java-profiler - policy: self.approve-trivial.approve-pr - - name: Auto-approve PR - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # 9.0.0 - with: - github-token: ${{ steps.octo-sts.outputs.token }} - script: | - await github.rest.pulls.createReview({ - owner: context.repo.owner, - repo: context.repo.repo, - pull_number: context.payload.pull_request.number, - event: 'APPROVE' - }) diff --git a/.github/workflows/cache_java.yml b/.github/workflows/cache_java.yml deleted file mode 100644 index 031f23c80..000000000 --- a/.github/workflows/cache_java.yml +++ /dev/null @@ -1,360 +0,0 @@ -name: Cache Java Distributions - -on: - push: - branches: - - 'main' - paths: - - '.github/workflows/cache_java.yml' - - '.github/scripts/java_setup.sh' - schedule: - # Runs every day at 03:00 UTC every 4 days - # This should keep the caches fresh and not expiring after 7 days - - cron: '0 3 */4 * *' - workflow_dispatch: - workflow_call: - -env: - JDKS_DIR: jdks - JAVA_8_VERSION: 8.0.462 - JAVA_8_J9_VERSION: 8.0.462 - JAVA_11_VERSION: 11.0.28 - JAVA_11_J9_VERSION: 11.0.28 - JAVA_17_VERSION: 17.0.16 - JAVA_17_J9_VERSION: 17.0.16 - JAVA_21_VERSION: 21.0.8 - JAVA_21_J9_VERSION: 21.0.8 - JAVA_25_VERSION: 25 - - JAVA_17_GRAAL_VERSION: 17.0.12 - JAVA_21_GRAAL_VERSION: 21.0.8 - JAVA_25_GRAAL_VERSION: 25 - - # https://gist.github.com/wavezhang/ba8425f24a968ec9b2a8619d7c2d86a6?permalink_comment_id=4444663#gistcomment-4444663 - # jdk1.8.0_361 - JAVA_8_ORACLE_URL: "https://javadl.oracle.com/webapps/download/AutoDL?BundleId=247926_0ae14417abb444ebb02b9815e2103550" - - JAVA_8_IBM_URL: "https://public.dhe.ibm.com/ibmdl/export/pub/systems/cloud/runtimes/java/8.0.8.60/linux/x86_64/ibm-java-jre-8.0-8.60-linux-x86_64.tgz" - - # FIXME: Azul pulled public CDN access to Zing/Prime downloads - all URLs return 404 - # JAVA_8_ZING_URL : "https://cdn.azul.com/zing-zvm/ZVM23.05.0.0/zing23.05.0.0-2-jdk8.0.372-linux_x64.tar.gz" - # JAVA_8_ZING_AARCH64_URL : "https://cdn.azul.com/zing-zvm/ZVM24.10.0.0/zing24.10.0.0-4-jdk8.0.431-linux_aarch64.tar.gz" - # JAVA_11_ZING_URL : "https://cdn.azul.com/zing-zvm/ZVM23.05.0.0/zing23.05.0.0-2-jdk11.0.19-linux_x64.tar.gz" - # JAVA_11_ZING_AARCH64_URL: "https://cdn.azul.com/zing-zvm/ZVM24.10.0.0/zing24.10.0.0-4-jdk11.0.24.0.101-linux_aarch64.tar.gz" - # JAVA_17_ZING_URL : "https://cdn.azul.com/zing-zvm/ZVM23.05.0.0/zing23.05.0.0-2-jdk17.0.7-linux_x64.tar.gz" - # JAVA_17_ZING_AARCH64_URL: "https://cdn.azul.com/zing-zvm/ZVM24.10.0.0/zing24.10.0.0-4-jdk17.0.12.0.101-linux_aarch64.tar.gz" - # JAVA_21_ZING_URL : "https://cdn.azul.com/zing-zvm/ZVM23.10.0.0/zing23.10.0.0-3-jdk21.0.1-linux_x64.tar.gz" - # JAVA_21_ZING_AARCH64_URL: "https://cdn.azul.com/zing-zvm/ZVM24.10.0.0/zing24.10.0.0-4-jdk21.0.4.0.101-linux_aarch64.tar.gz" - - JAVA_8_MUSL_URL : "https://download.bell-sw.com/java/8u462+11/bellsoft-jdk8u462+11-linux-x64-musl-lite.tar.gz" - JAVA_8_MUSL_AARCH64_URL: "https://download.bell-sw.com/java/8u462+11/bellsoft-jdk8u462+11-linux-aarch64-musl-lite.tar.gz" - JAVA_11_MUSL_URL: "https://download.bell-sw.com/java/11.0.28+12/bellsoft-jdk11.0.28+12-linux-x64-musl-lite.tar.gz" - JAVA_11_MUSL_AARCH64_URL: "https://download.bell-sw.com/java/11.0.28+12/bellsoft-jdk11.0.28+12-linux-aarch64-musl-lite.tar.gz" - JAVA_17_MUSL_URL: "https://download.bell-sw.com/java/17.0.16+12/bellsoft-jdk17.0.16+12-linux-x64-musl-lite.tar.gz" - JAVA_17_MUSL_AARCH64_URL: "https://download.bell-sw.com/java/17.0.16+12/bellsoft-jdk17.0.16+12-linux-aarch64-musl-lite.tar.gz" - JAVA_21_MUSL_URL: "https://download.bell-sw.com/java/21.0.8+12/bellsoft-jdk21.0.8+12-linux-x64-musl-lite.tar.gz" - JAVA_21_MUSL_AARCH64_URL: "https://download.bell-sw.com/java/21.0.8+12/bellsoft-jdk21.0.8+12-linux-aarch64-musl-lite.tar.gz" - JAVA_25_MUSL_URL: "https://download.bell-sw.com/java/25.0.2+12/bellsoft-jdk25.0.2+12-linux-x64-musl-lite.tar.gz" - JAVA_25_MUSL_AARCH64_URL: "https://download.bell-sw.com/java/25.0.2+12/bellsoft-jdk25.0.2+12-linux-aarch64-musl-lite.tar.gz" - -permissions: - contents: read - actions: read - -jobs: - setup-sdkman-amd64: - runs-on: ubuntu-latest - outputs: - sdkman_path: ${{ steps.export-path.outputs.sdkman_path }} - steps: - - name: Cache SDKMan! AMD64 - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: sdkman - key: sdkman-amd64-${{ github.run_id }} - restore-keys: | - sdkman-amd64- - enableCrossOsArchive: true - - name: Check if SDKMAN! is Already Installed - id: check-sdkman - run: | - if [ -e "${GITHUB_WORKSPACE}/sdkman/bin/sdkman-init.sh" ]; then - echo "SDKMAN! already installed at ${GITHUB_WORKSPACE}/sdkman." - echo "skip_install=true" >> $GITHUB_ENV - else - echo "SDKMAN! not found, proceeding with installation." - echo "skip_install=false" >> $GITHUB_ENV - fi - echo "SDKMAN_DIR=${GITHUB_WORKSPACE}/sdkman" >> $GITHUB_ENV - - name: Setup OS - if: env.skip_install == 'false' - run: | - sudo apt-get update -y - sudo apt-get install -y curl zip unzip - - name: Install SDKMAN! - if: env.skip_install == 'false' - run: | - curl -s "https://get.sdkman.io" | bash - - name: Upload SDKMAN! as Artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: sdkman-installation-amd64 - path: ${{ env.SDKMAN_DIR }} - - setup-sdkman-aarch64: - runs-on: - group: ARM LINUX SHARED - labels: arm-4core-linux - outputs: - sdkman_path: ${{ steps.export-path.outputs.sdkman_path }} - steps: - - name: Cache SDKMan! AARCH64 - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: sdkman - key: sdkman-aarch64-${{ github.run_id }} - restore-keys: | - sdkman-aarch64- - - name: Check if SDKMAN! is Already Installed - id: check-sdkman - run: | - if [ -e "${GITHUB_WORKSPACE}/sdkman/bin/sdkman-init.sh" ]; then - echo "SDKMAN! already installed at ${GITHUB_WORKSPACE}/sdkman." - echo "skip_install=true" >> $GITHUB_ENV - else - echo "SDKMAN! not found, proceeding with installation." - echo "skip_install=false" >> $GITHUB_ENV - fi - echo "SDKMAN_DIR=${GITHUB_WORKSPACE}/sdkman" >> $GITHUB_ENV - - name: Setup OS - if: env.skip_install == 'false' - run: | - sudo apt-get update -y - sudo apt-get install -y curl zip unzip - - name: Install SDKMAN! - if: env.skip_install == 'false' - run: | - curl -s "https://get.sdkman.io" | bash - - name: Upload SDKMAN! as Artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: sdkman-installation-aarch64 - path: ${{ env.SDKMAN_DIR }} - - cache-amd64: - needs: setup-sdkman-amd64 - runs-on: ubuntu-latest - strategy: - fail-fast: true - matrix: - # java_variant: [ "8", "8-orcl", "8-zing", "8-j9", "8-ibm", "11", "11-zing", "11-j9", "17", "17-zing", "17-j9", "17-graal", "21", "21-j9", "21-zing", "21-graal", "25", "25-graal" ] - # FIXME: Zing disabled - Azul pulled public CDN access - java_variant: [ "8", "8-orcl", "8-j9", "8-ibm", "11", "11-j9", "17", "17-j9", "17-graal", "21", "21-j9", "21-graal", "25", "25-graal" ] - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Try restore cache JDK ${{ matrix.java_variant }} - id: cache-jdk - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ${{ env.JDKS_DIR }}/jdk${{ matrix.java_variant }} - key: jdk${{matrix.java_variant }}-amd64--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - enableCrossOsArchive: true - - name: Is JDK cached? - id: check-cache - run: | - if [ -d "jdks" ]; then - echo "cache-hit=true" >> $GITHUB_OUTPUT - else - echo "cache-hit=false" >> $GITHUB_OUTPUT - fi - - name: Setup OS - if: steps.check-cache.outputs.cache-hit != 'true' - run: | - sudo apt-get update -y - sudo apt-get install -y curl zip unzip - - name: Download SDKMAN! from Artifact - if: steps.check-cache.outputs.cache-hit != 'true' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: sdkman-installation-amd64 - path: sdkman - - - name: Install JDK ${{ matrix.java_variant }} - if: steps.check-cache.outputs.cache-hit != 'true' - run: | - mv $GITHUB_WORKSPACE/sdkman ~/.sdkman - mkdir -p ~/.sdkman/ext # Create ext directory; it is empty and not uploaded - mkdir -p ~/.sdkman/tmp # Create tmp directory; it is empty and not uploaded - - source .github/scripts/java_setup.sh - - prepareJdk ${{ matrix.java_variant }} amd64 - - - name: Save JDK ${{ matrix.java_variant }} cache - if: steps.check-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: | - ${{ env.JDKS_DIR }}/jdk${{ matrix.java_variant }} - key: jdk${{matrix.java_variant }}-amd64--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - enableCrossOsArchive: true - - cache-amd64-musl: - runs-on: ubuntu-latest - container: - image: "alpine:3.23" - options: --cpus 2 - strategy: - fail-fast: true - matrix: - java_variant: [ "8-librca", "11-librca", "17-librca", "21-librca", "25-librca" ] - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Setup OS - run: | - # This needs to be done early because alpine does not have bash and tar is also iffy - apk update && apk add curl zip unzip bash tar - - name: Cache JDK ${{ matrix.java_variant }} - id: cache-jdk - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ${{ env.JDKS_DIR }}/jdk${{ matrix.java_variant }} - key: jdk${{matrix.java_variant }}-amd64-musl--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - enableCrossOsArchive: true - - - name: Is JDK cached? - id: check-cache - run: | - if [ -d "jdks" ]; then - echo "cache-hit=true" >> $GITHUB_OUTPUT - else - echo "cache-hit=false" >> $GITHUB_OUTPUT - fi - - - name: Install JDK ${{ matrix.java_variant }} - if: steps.check-cache.outputs.cache-hit != 'true' - shell: bash - run: | - source .github/scripts/java_setup.sh - - prepareJdk ${{ matrix.java_variant }} amd64-musl - - - name: Save JDK ${{ matrix.java_variant }} cache - if: steps.check-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: | - ${{ env.JDKS_DIR }}/jdk${{ matrix.java_variant }} - key: jdk${{matrix.java_variant }}-amd64-musl--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - enableCrossOsArchive: true - - cache-aarch64: - needs: setup-sdkman-aarch64 - runs-on: - group: ARM LINUX SHARED - labels: arm-4core-linux - strategy: - fail-fast: true - matrix: - # java_variant: [ "8", "8-zing", "8-j9", "11", "11-zing", "11-j9", "17", "17-zing", "17-j9", "17-graal", "21", "21-j9", "21-zing", "21-graal", "25", "25-graal" ] - # FIXME: Zing disabled - Azul pulled public CDN access - java_variant: [ "8", "8-j9", "11", "11-j9", "17", "17-j9", "17-graal", "21", "21-j9", "21-graal", "25", "25-graal" ] - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Cache JDK ${{ matrix.java_variant }} - id: cache-jdk - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ${{ env.JDKS_DIR }}/jdk${{ matrix.java_variant }} - key: jdk${{matrix.java_variant }}-aarch64--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - enableCrossOsArchive: true - - name: Is JDK cached? - id: check-cache - run: | - if [ -d "jdks" ]; then - echo "cache-hit=true" >> $GITHUB_OUTPUT - else - echo "cache-hit=false" >> $GITHUB_OUTPUT - fi - - name: Setup OS - if: steps.check-cache.outputs.cache-hit != 'true' - run: | - sudo apt-get update -y - sudo apt-get install -y curl zip unzip - - name: Download SDKMAN! from Artifact - if: steps.check-cache.outputs.cache-hit != 'true' - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - name: sdkman-installation-aarch64 - path: sdkman - - name: Install JDK ${{ matrix.java_variant }} - if: steps.check-cache.outputs.cache-hit != 'true' - run: | - mv $GITHUB_WORKSPACE/sdkman ~/.sdkman - mkdir -p ~/.sdkman/ext # Create ext directory; it is empty and not uploaded - mkdir -p ~/.sdkman/tmp # Create tmp directory; it is empty and not uploaded - - source .github/scripts/java_setup.sh - - prepareJdk ${{ matrix.java_variant }} aarch64 - - - name: Save JDK ${{ matrix.java_variant }} cache - if: steps.check-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: | - ${{ env.JDKS_DIR }}/jdk${{ matrix.java_variant }} - key: jdk${{matrix.java_variant }}-aarch64--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - enableCrossOsArchive: true - - cache-aarch64-musl: - runs-on: - group: ARM LINUX SHARED - labels: arm-4core-linux-ubuntu24.04 - strategy: - fail-fast: true - matrix: - java_variant: [ "8-librca", "11-librca", "17-librca", "21-librca", "25-librca" ] - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Setup OS - run: | - # This needs to be done early because alpine does not have bash and tar is also iffy - sudo apt update && sudo apt install -y curl zip unzip bash tar - - name: Cache JDK ${{ matrix.java_variant }} - id: cache-jdk - uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: | - ${{ env.JDKS_DIR }}/jdk${{ matrix.java_variant }} - key: jdk${{matrix.java_variant }}-aarch64-musl--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - enableCrossOsArchive: true - - - name: Is JDK cached? - id: check-cache - run: | - if [ -d "jdks" ]; then - echo "cache-hit=true" >> $GITHUB_OUTPUT - else - echo "cache-hit=false" >> $GITHUB_OUTPUT - fi - - - name: Install JDK ${{ matrix.java_variant }} - if: steps.check-cache.outputs.cache-hit != 'true' - shell: bash - run: | - source .github/scripts/java_setup.sh - - prepareJdk ${{ matrix.java_variant }} aarch64-musl - - - name: Save JDK ${{ matrix.java_variant }} cache - if: steps.check-cache.outputs.cache-hit != 'true' - uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 - with: - path: | - ${{ env.JDKS_DIR }}/jdk${{ matrix.java_variant }} - key: jdk${{matrix.java_variant }}-aarch64-musl--${{ hashFiles('.github/workflows/cache_java.yml', '.github/scripts/java_setup.sh') }} - enableCrossOsArchive: true diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index c2c3cd72c..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,238 +0,0 @@ -name: CI Run - -concurrency: - group: pr-ci_${{ github.event.pull_request.number }} - cancel-in-progress: true - -on: - push: - branches: - - '*' - tags-ignore: - - v* - pull_request: - types: [opened, synchronize, reopened, labeled] - workflow_dispatch: - -permissions: - contents: read - pull-requests: read - actions: read - -jobs: - check-for-pr: - runs-on: ubuntu-latest - outputs: - skip: ${{ steps.check.outputs.skip }} - steps: - - name: Check if PR exists for this branch (skip push if PR exists) - id: check - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - HEAD_REF: ${{ github.ref_name }} - run: | - # On push events, base_ref is empty; skip if a PR already exists for this branch - if [ -z "${{ github.base_ref }}" ]; then - prs=$(gh pr list \ - --repo "$GITHUB_REPOSITORY" \ - --json headRefName \ - --jq "[.[] | select(.headRefName == env.HEAD_REF)] | length") - if ((prs > 0)); then - echo "skip=true" >> "$GITHUB_OUTPUT" - fi - fi - check-formatting: - runs-on: ubuntu-22.04 - needs: check-for-pr - if: needs.check-for-pr.outputs.skip != 'true' - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Setup Java - uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 - with: - distribution: 'zulu' - java-version: '21' - - - name: Setup OS - run: | - sudo apt-get update - sudo apt-get install -y clang-format-11 - # we need this to make sure we are actually using clang-format v. 11 - sudo mv /usr/bin/clang-format /usr/bin/clang-format-14 - sudo mv /usr/bin/clang-format-11 /usr/bin/clang-format - - - name: Cache Gradle Wrapper Binaries - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/wrapper/dists - key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - gradle-wrapper-${{ runner.os }}- - - - name: Cache Gradle User Home - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/caches - key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - gradle-caches-${{ runner.os }}- - - - name: Check - run: | - ./gradlew spotlessCheck --no-daemon --parallel --build-cache --no-watch-fs - - check-javadoc: - runs-on: ubuntu-22.04 - needs: check-for-pr - if: needs.check-for-pr.outputs.skip != 'true' - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Setup Java - uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 - with: - distribution: 'zulu' - java-version: '21' - - - name: Setup OS - run: | - sudo apt-get update - sudo apt-get install -y curl zip unzip binutils - - - name: Cache Gradle Wrapper Binaries - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/wrapper/dists - key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - gradle-wrapper-${{ runner.os }}- - - - name: Cache Gradle User Home - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/caches - key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - gradle-caches-${{ runner.os }}- - - - name: Validate Javadoc - run: | - # Note: javadoc task depends on copyReleaseLibs which requires building native libraries - # This ensures javadoc validation matches the exact conditions used during publishing - ./gradlew :ddprof-lib:javadoc --no-daemon --parallel --build-cache --no-watch-fs - - compute-configurations: - runs-on: ubuntu-latest - needs: check-for-pr - if: needs.check-for-pr.outputs.skip != 'true' - outputs: - configurations: ${{ steps.compute.outputs.configurations }} - run_fuzz: ${{ steps.compute.outputs.run_fuzz }} - steps: - - name: Debounce label events - if: github.event.action == 'labeled' - run: | - echo "Waiting 30s to allow adding multiple labels..." - sleep 30 - - name: Compute test configurations from PR labels - id: compute - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Always include debug - configs='["debug"' - - # For PRs, check labels; for push/workflow_dispatch, use debug only - if [ "${{ github.event_name }}" = "pull_request" ]; then - labels=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }} --jq '.labels[].name' 2>/dev/null) || labels="" - - if echo "$labels" | grep -Fq "test:release"; then - configs="$configs"',"release"' - fi - if echo "$labels" | grep -Fq "test:asan"; then - configs="$configs"',"asan"' - fi - if echo "$labels" | grep -Fq "test:tsan"; then - configs="$configs"',"tsan"' - fi - if echo "$labels" | grep -Fq "test:fuzz"; then - echo "run_fuzz=true" >> $GITHUB_OUTPUT - else - echo "run_fuzz=false" >> $GITHUB_OUTPUT - fi - else - echo "run_fuzz=false" >> $GITHUB_OUTPUT - fi - - configs="$configs]" - echo "configurations=$configs" >> $GITHUB_OUTPUT - echo "Test configurations: $configs" - - test-matrix: - needs: [check-for-pr, check-formatting, compute-configurations] - if: needs.check-for-pr.outputs.skip != 'true' - uses: ./.github/workflows/test_workflow.yml - with: - configuration: ${{ needs.compute-configurations.outputs.configurations }} - - summarize-tests: - needs: [test-matrix] - if: always() && !cancelled() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork == false - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - actions: read - id-token: write - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Generate test summary - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - .github/scripts/generate-test-summary.sh \ - "${{ github.run_id }}" \ - "test-summary.md" - - - name: Post PR comment - uses: ./.github/actions/upsert-pr-comment - with: - body-file: test-summary.md - comment-id: ci-test-results - - fuzz: - needs: [check-for-pr, compute-configurations] - if: needs.check-for-pr.outputs.skip != 'true' && needs.compute-configurations.outputs.run_fuzz == 'true' - runs-on: ubuntu-latest - continue-on-error: true - timeout-minutes: 30 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Cache Gradle Wrapper Binaries - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/wrapper/dists - key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - gradle-wrapper-${{ runner.os }}- - - name: Cache Gradle User Home - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/caches - key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - gradle-caches-${{ runner.os }}- - - name: Setup OS - run: | - sudo apt-get update - sudo apt-get install -y clang - - name: Fuzz - run: ./gradlew :ddprof-lib:fuzz:fuzz -Pfuzz-duration=120 --no-daemon - - name: Upload crash artifacts - if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fuzz-crashes - path: ddprof-lib/fuzz/build/fuzz-crashes/ diff --git a/.github/workflows/codecheck.yml b/.github/workflows/codecheck.yml deleted file mode 100644 index 09a94041b..000000000 --- a/.github/workflows/codecheck.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Code Quality Checks - -concurrency: - group: pr-code_quality_${{ github.event.pull_request.number }} - cancel-in-progress: true - -on: - pull_request: - types: [opened, synchronize, reopened] - -permissions: - contents: read - pull-requests: write - actions: read - id-token: write - -jobs: - scan-build: - if: needs.check-for-pr.outputs.skip != 'true' - runs-on: ubuntu-latest - env: - HEAD_REF: ${{ github.head_ref }} - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: System setup - run: | - sudo apt-get update - sudo apt install -y clang clang-tools openjdk-11-jdk - - name: Set up Python - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 - with: - python-version: 3.14 - - name: Install Python dependencies - run: | - python -m pip install --upgrade pip - pip install beautifulsoup4 - - name: Scan Build - run: | - ./gradlew scanBuild --no-daemon - - name: Upload logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: always() - with: - name: scan-build-reports - path: | - ddprof-lib/build/reports/scan-build - - name: Read Report - id: read-report - run: | - find ddprof-lib/build/reports/scan-build -name 'index.html' | xargs -I {} python .github/scripts/python_utils.py scanbuild_cleanup {} ${HEAD_REF} > comment.html - - name: Comment on PR - if: github.event.pull_request.head.repo.fork == false - uses: ./.github/actions/upsert-pr-comment - with: - body-file: comment.html - comment-id: scan-build-report - - codeql: - if: needs.check-for-pr.outputs.skip != 'true' - runs-on: ubuntu-latest - permissions: - actions: read - contents: read - security-events: write - - strategy: - fail-fast: false - matrix: - language: [ 'cpp', 'java' ] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] - # Learn more about CodeQL language support at https://git.io/codeql-language-support - - steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - # Initializes the CodeQL tools for scanning. - - name: Initialize CodeQL - uses: github/codeql-action/init@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 - with: - languages: ${{ matrix.language }} - # If you wish to specify custom queries, you can do so here or in a config file. - # By default, queries listed here will override any specified in a config file. - # Prefix the list here with "+" to use these queries and those in the config file. - # queries: ./path/to/local/query, your-org/your-repo/queries@main - - run: ./gradlew -x test assembleReleaseJar - - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2 diff --git a/.github/workflows/create-next-milestone.yaml b/.github/workflows/create-next-milestone.yaml deleted file mode 100644 index 44ed27c62..000000000 --- a/.github/workflows/create-next-milestone.yaml +++ /dev/null @@ -1,31 +0,0 @@ -name: Create next milestone -on: - milestone: - types: [closed] - -permissions: - contents: read - issues: write - -jobs: - create_next_milestone: - runs-on: ubuntu-latest - steps: - - name: Get next minor version - id: semvers - env: - MILESTONE_TITLE: ${{ github.event.milestone.title }} - run: | - MAJOR=$(echo "$MILESTONE_TITLE" | cut -d. -f1) - MINOR=$(echo "$MILESTONE_TITLE" | cut -d. -f2) - echo "minor=${MAJOR}.$((MINOR + 1)).0" >> "$GITHUB_OUTPUT" - - name: Create next milestone - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # 9.0.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - await github.rest.issues.createMilestone({ - owner: context.repo.owner, - repo: context.repo.repo, - title: '${{ steps.semvers.outputs.minor }}' - }) diff --git a/.github/workflows/dependabot-automerge.yml b/.github/workflows/dependabot-automerge.yml deleted file mode 100644 index 821c7d3eb..000000000 --- a/.github/workflows/dependabot-automerge.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Dependabot Auto-Merge - -on: - schedule: - - cron: '0 */4 * * *' # Every 4 hours - -jobs: - scheduled-automerge: - name: Merge approved Dependabot PRs - runs-on: ubuntu-latest - permissions: - id-token: write # Needed to federate tokens - steps: - - uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4 - id: octo-sts - with: - scope: DataDog/java-profiler - policy: self.dependabot-automerge.schedule - - name: Merge approved Dependabot PRs - env: - GH_TOKEN: ${{ steps.octo-sts.outputs.token }} - run: | - gh pr list \ - --repo "$GITHUB_REPOSITORY" \ - --author "dependabot[bot]" \ - --json number \ - --jq '.[].number' \ - | while read pr_number; do - echo "Processing PR #$pr_number" - gh pr merge "$pr_number" --squash --repo "$GITHUB_REPOSITORY" \ - || echo "Could not merge PR #$pr_number" - done diff --git a/.github/workflows/gh_release.yml b/.github/workflows/gh_release.yml deleted file mode 100644 index e1a5e93c5..000000000 --- a/.github/workflows/gh_release.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Github Release -run-name: Release ${{ inputs.release_tag }} ${{ github.event.ref_name }} -on: - workflow_dispatch: - inputs: - release_tag: - type: string - description: "Release tag" - required: true - workflow_call: - inputs: - release_tag: - type: string - description: "Release tag" - required: false - push: - tags: - - v_*.*.* - -permissions: - contents: write - actions: read - -jobs: - gh-release: - if: (startsWith(github.event.ref, 'refs/tags/v_') || inputs.release_tag != '') && !endsWith(github.event.ref, '-SNAPSHOT') - runs-on: ubuntu-latest - steps: - - name: Create Release [automatic] - id: create_release_auto - if: ${{ startsWith(github.ref, 'refs/tags/') }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - run: | - if gh release view "${GITHUB_REF_NAME}" >/dev/null 2>&1; then - gh release edit "${GITHUB_REF_NAME}" --draft - else - gh release create "${GITHUB_REF_NAME}" \ - --title "${GITHUB_REF_NAME}" \ - --generate-notes \ - --draft - fi - - name: Create Release [manual] - id: create_release_manual - if: ${{ !startsWith(github.ref, 'refs/tags/') }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - run: | - TAG="${{ inputs.release_tag }}" - if gh release view "${TAG}" >/dev/null 2>&1; then - gh release edit "${TAG}" --draft - else - gh release create "${TAG}" \ - --title "${TAG}" \ - --generate-notes \ - --draft - fi diff --git a/.github/workflows/increment-milestones-on-tag.yaml b/.github/workflows/increment-milestones-on-tag.yaml deleted file mode 100644 index 8fb0e3d6c..000000000 --- a/.github/workflows/increment-milestones-on-tag.yaml +++ /dev/null @@ -1,72 +0,0 @@ -name: Increment milestones on tag -on: - create - -permissions: - contents: read - issues: write - -jobs: - increment_milestone: - if: github.event.ref_type == 'tag' && github.event.master_branch == 'main' - runs-on: ubuntu-latest - steps: - - name: Get milestone title - id: milestoneTitle - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # 9.0.0 - with: - result-encoding: string - script: | - // Our tags are of the form v_X.X.X and milestones don't have the "v" - return '${{github.event.ref}}'.startsWith('v_') ? '${{github.event.ref}}'.substring(2) : '${{github.event.ref}}'; - - name: Get milestone for tag - id: milestone - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # 9.0.0 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const milestones = await github.paginate(github.rest.issues.listMilestones, { - owner: context.repo.owner, - repo: context.repo.repo, - state: 'all' - }) - - const milestone = milestones.find(milestone => milestone.title == '${{steps.milestoneTitle.outputs.result}}') - - if (milestone) { - return milestone.number - } else { - return null - } - - name: Close milestone - if: fromJSON(steps.milestone.outputs.result) - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # 9.0.0 - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - await github.rest.issues.updateMilestone({ - owner: context.repo.owner, - repo: context.repo.repo, - state: 'closed', - milestone_number: ${{steps.milestone.outputs.result}} - }) - - name: Get next minor version - if: fromJSON(steps.milestone.outputs.result) - id: semvers - env: - MILESTONE_TITLE: ${{ steps.milestoneTitle.outputs.result }} - run: | - MAJOR=$(echo "$MILESTONE_TITLE" | cut -d. -f1) - MINOR=$(echo "$MILESTONE_TITLE" | cut -d. -f2) - echo "minor=${MAJOR}.$((MINOR + 1)).0" >> "$GITHUB_OUTPUT" - - name: Create next milestone - if: fromJSON(steps.milestone.outputs.result) - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # 9.0.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - await github.rest.issues.createMilestone({ - owner: context.repo.owner, - repo: context.repo.repo, - title: '${{ steps.semvers.outputs.minor }}' - }) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml deleted file mode 100644 index c17455771..000000000 --- a/.github/workflows/nightly.yml +++ /dev/null @@ -1,84 +0,0 @@ -name: Nightly Sanitized Run - -on: - schedule: - # Runs every day at 03:00 UTC - - cron: '0 3 * * *' - workflow_dispatch: - -permissions: - contents: read - actions: read - -jobs: - run-test: - uses: ./.github/workflows/test_workflow.yml - with: - configuration: '["asan"]' - # C++ gtests (ASan + TSan) run on every PR via native-sanitizer-tests in ci.yml. - # Skip them here so the nightly focuses on Java functional tests under ASan. - skip_gtest: true - fuzz: - runs-on: ubuntu-latest - continue-on-error: true - timeout-minutes: 30 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Cache Gradle Wrapper Binaries - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/wrapper/dists - key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - gradle-wrapper-${{ runner.os }}- - - name: Cache Gradle User Home - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/caches - key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - gradle-caches-${{ runner.os }}- - - name: Setup OS - run: | - sudo apt-get update - sudo apt-get install -y clang - - name: Fuzz - run: ./gradlew :ddprof-lib:fuzz:fuzz -Pfuzz-duration=120 --no-daemon - - name: Upload crash artifacts - if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: fuzz-crashes - path: ddprof-lib/fuzz/build/fuzz-crashes/ - report-failures: - runs-on: ubuntu-latest - needs: run-test - if: failure() - steps: - - name: Download all failure artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - pattern: failures-* - path: ./artifacts - merge-multiple: true - - name: Report failures - run: | - # Check if any failure files exist - if ! ls ./artifacts/failures_*.txt 1> /dev/null 2>&1; then - echo "No failure artifacts found" - exit 0 - fi - - # Combine all failure files - find ./artifacts -name 'failures_*.txt' -exec cat {} \; > ./artifacts/all_failures.txt - scenarios=$(cat ./artifacts/all_failures.txt | tr '\n' ',' | sed 's/,$//') - - echo "Failed scenarios: $scenarios" - - if [ -n "${{ secrets.SLACK_WEBHOOK }}" ]; then - curl -X POST "${{ secrets.SLACK_WEBHOOK }}" \ - -H 'Content-Type: application/json' \ - -d "{\"scenarios\": \"${scenarios}\", \"failed_run_url\": \"${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}\"}" - else - echo "SLACK_WEBHOOK not configured, skipping notification" - fi diff --git a/.github/workflows/release-validated.yml b/.github/workflows/release-validated.yml deleted file mode 100644 index 92b0ebc08..000000000 --- a/.github/workflows/release-validated.yml +++ /dev/null @@ -1,248 +0,0 @@ -name: Validated Release -run-name: "${{ inputs.dry_run && 'Dry-run for ' || 'Perform ' }}${{ inputs.release_type }} release of ${{ github.ref_name }} branch" - -on: - workflow_dispatch: - inputs: - release_type: - type: choice - description: The release type - options: - - "major" - - "minor" - - "patch" - - "retag" - default: "minor" - dry_run: - description: Perform the release dry-run - required: true - type: boolean - default: true - skip_tests: - description: Skip pre-release tests (emergency releases only) - required: false - type: boolean - default: false - -permissions: - contents: write - actions: read - -jobs: - validate-inputs: - runs-on: ubuntu-latest - outputs: - release_version: ${{ steps.compute-version.outputs.release_version }} - next_version: ${{ steps.compute-version.outputs.next_version }} - release_branch: ${{ steps.compute-version.outputs.release_branch }} - steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - with: - fetch-depth: 0 - - - name: Setup Java - uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 - with: - distribution: 'zulu' - java-version: '21' - - - name: Validate branch and compute versions - id: compute-version - run: | - BRANCH="${GITHUB_REF_NAME}" - TYPE="${{ inputs.release_type }}" - - echo "Current branch: $BRANCH" - echo "Release type: $TYPE" - - # Branch validation - if [ "$TYPE" == "patch" ] || [ "$TYPE" == "retag" ]; then - if [[ ! $BRANCH =~ ^release/[0-9]+\.[0-9]+\._$ ]]; then - echo "::error::${TYPE^} can only be performed from 'release/*' branches (format: release/X.Y._)" - exit 1 - fi - else - if [ "$BRANCH" != "main" ]; then - echo "::error::Major or minor releases can only be performed from 'main' branch" - exit 1 - fi - fi - - # Get current version - BASE=$(./gradlew printVersion -Psnapshot=false | grep 'Version:' | cut -f2 -d' ') - echo "Current version: $BASE" - - # Parse version components - if [[ $BASE =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)$ ]]; then - MAJOR="${BASH_REMATCH[1]}" - MINOR="${BASH_REMATCH[2]}" - PATCH="${BASH_REMATCH[3]}" - else - echo "::error::Invalid version format: $BASE (expected X.Y.Z)" - exit 1 - fi - - # Check if this version is already released - if git rev-parse "v_${BASE}" >/dev/null 2>&1; then - ALREADY_RELEASED=true - echo "Version $BASE is already released" - else - ALREADY_RELEASED=false - echo "Version $BASE is not yet released" - fi - - # Compute release version based on type - if [ "$TYPE" == "MAJOR" ] || [ "$TYPE" == "major" ]; then - if [ "$ALREADY_RELEASED" == "true" ]; then - RELEASE_VERSION="$((MAJOR + 1)).0.0" - else - RELEASE_VERSION="$BASE" - fi - elif [ "$TYPE" == "MINOR" ] || [ "$TYPE" == "minor" ]; then - if [ "$ALREADY_RELEASED" == "true" ]; then - RELEASE_VERSION="$MAJOR.$((MINOR + 1)).0" - else - RELEASE_VERSION="$BASE" - fi - elif [ "$TYPE" == "retag" ]; then - # Retag reuses the current version; tag must already exist - RELEASE_VERSION="$BASE" - if ! git rev-parse "v_${RELEASE_VERSION}" >/dev/null 2>&1; then - echo "::error::Tag v_${RELEASE_VERSION} does not exist. Use a normal release to create a new tag." - exit 1 - fi - # Refuse if the GitHub release is already public - IS_DRAFT=$(gh release view "v_${RELEASE_VERSION}" --json isDraft --jq '.isDraft' 2>/dev/null || echo "not-found") - if [ "$IS_DRAFT" == "false" ]; then - echo "::error::GitHub release v_${RELEASE_VERSION} is already public. Retagging is not allowed." - exit 1 - fi - else - # PATCH always increments - RELEASE_VERSION="$MAJOR.$MINOR.$((PATCH + 1))" - fi - - # Compute release branch - RELEASE_BRANCH="release/${RELEASE_VERSION%.*}._" - - # Check if tag already exists (skip for retag, which requires it to exist) - if [ "$TYPE" != "retag" ] && git rev-parse "v_${RELEASE_VERSION}" >/dev/null 2>&1; then - echo "::error::Tag v_${RELEASE_VERSION} already exists" - exit 1 - fi - - echo "Release version: $RELEASE_VERSION" - echo "Release branch: $RELEASE_BRANCH" - - # Set outputs - echo "release_version=$RELEASE_VERSION" >> $GITHUB_OUTPUT - echo "release_branch=$RELEASE_BRANCH" >> $GITHUB_OUTPUT - - # Output summary - echo "## Release Validation" >> $GITHUB_STEP_SUMMARY - echo "- **Release Type**: $TYPE" >> $GITHUB_STEP_SUMMARY - echo "- **Branch**: $BRANCH" >> $GITHUB_STEP_SUMMARY - echo "- **Release Version**: $RELEASE_VERSION" >> $GITHUB_STEP_SUMMARY - echo "- **Release Branch**: $RELEASE_BRANCH" >> $GITHUB_STEP_SUMMARY - echo "- **Tag**: v_$RELEASE_VERSION" >> $GITHUB_STEP_SUMMARY - echo "- **Dry Run**: ${{ inputs.dry_run }}" >> $GITHUB_STEP_SUMMARY - echo "- **Skip Tests**: ${{ inputs.skip_tests }}" >> $GITHUB_STEP_SUMMARY - - pre-release-tests: - needs: validate-inputs - if: ${{ inputs.dry_run != true && inputs.skip_tests != true && inputs.release_type != 'retag' }} - uses: ./.github/workflows/test_workflow.yml - with: - configuration: '["debug", "asan"]' - - create-release: - needs: [validate-inputs, pre-release-tests] - if: always() && needs.validate-inputs.result == 'success' && (needs.pre-release-tests.result == 'success' || needs.pre-release-tests.result == 'skipped') - runs-on: ubuntu-latest - steps: - - name: Check test results - if: ${{ inputs.dry_run != true && inputs.skip_tests != true && inputs.release_type != 'retag' && needs.pre-release-tests.result != 'success' }} - run: | - echo "::error::Pre-release tests failed. Cannot proceed with release." - exit 1 - - - name: Setup SSH agent - run: | - eval $(ssh-agent -s) - echo "${{ secrets.SSH_PUSH_SECRET }}" | ssh-add - - mkdir -p ~/.ssh - ssh-keyscan github.com >> ~/.ssh/known_hosts - echo "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" >> $GITHUB_ENV - echo "SSH_AGENT_PID=$SSH_AGENT_PID" >> $GITHUB_ENV - - - name: Checkout repository - run: git clone git@github.com:$GITHUB_REPOSITORY.git java-profiler - - - name: Configure git - run: | - cd java-profiler - git config --global user.email "java-profiler@datadoghq.com" - git config --global user.name "Datadog Java Profiler" - git checkout $GITHUB_REF_NAME - - - name: Setup Java - uses: actions/setup-java@1bcf9fb12cf4aa7d266a90ae39939e61372fe520 # v5.4.0 - with: - distribution: 'zulu' - java-version: '21' - - - name: Create release - id: create-release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - cd java-profiler - - if [ "${{ inputs.dry_run }}" == "true" ]; then - DRY_RUN="--dry-run" - else - DRY_RUN="" - fi - - TYPE="${{ inputs.release_type }}" - ./.github/scripts/release.sh ${TYPE^^} $DRY_RUN - - - name: Output Release Summary - if: ${{ inputs.dry_run != true }} - env: - BUMP_PR_URL: ${{ steps.create-release.outputs.BUMP_PR_URL }} - run: | - echo "## ✅ Release Created Successfully" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Release Details" >> $GITHUB_STEP_SUMMARY - echo "- **Tag**: v_${{ needs.validate-inputs.outputs.release_version }}" >> $GITHUB_STEP_SUMMARY - echo "- **Branch**: ${{ needs.validate-inputs.outputs.release_branch }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - if [ -n "$BUMP_PR_URL" ]; then - echo "### ⚠ Action Required" >> $GITHUB_STEP_SUMMARY - echo "A version bump PR needs review and merge before the next snapshot builds carry the correct version:" >> $GITHUB_STEP_SUMMARY - echo "- $BUMP_PR_URL" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - fi - echo "### Next Steps (Automatic)" >> $GITHUB_STEP_SUMMARY - echo "1. 🔨 GitLab pipeline will build multi-platform artifacts" >> $GITHUB_STEP_SUMMARY - echo "2. 📦 GitLab will publish to Maven Central" >> $GITHUB_STEP_SUMMARY - echo "3. 🚀 GitHub workflows will create release when Maven artifacts are available" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Monitoring" >> $GITHUB_STEP_SUMMARY - echo "- GitHub Releases: ${{ github.server_url }}/${{ github.repository }}/releases" >> $GITHUB_STEP_SUMMARY - echo "- Maven Central: https://repo1.maven.org/maven2/com/datadoghq/ddprof/${{ needs.validate-inputs.outputs.release_version }}/" >> $GITHUB_STEP_SUMMARY - - - name: Output Dry-Run Summary - if: ${{ inputs.dry_run == true }} - run: | - echo "## 🔍 Dry-Run Complete" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "This was a dry-run. No changes were made." >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "### Would have created:" >> $GITHUB_STEP_SUMMARY - echo "- **Tag**: v_${{ needs.validate-inputs.outputs.release_version }}" >> $GITHUB_STEP_SUMMARY - echo "- **Branch**: ${{ needs.validate-inputs.outputs.release_branch }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "To perform the actual release, run this workflow again with **dry_run = false**" >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 69a295eee..000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: Automated Release -run-name: "${{ inputs.dry_run && 'Dry-run for ' || 'Preform ' }} ${{ inputs.release_type }} release of ${{ github.ref }} branch" - -on: - workflow_dispatch: - inputs: - release_type: - type: choice - description: The release type - options: - - "major" - - "minor" - - "patch" - default: "minor" - dry_run: - description: Perform the release dry-run - required: true - type: boolean - default: true - -permissions: - contents: write - actions: read - -jobs: - deprecation-warning: - runs-on: ubuntu-latest - steps: - - name: Show Deprecation Notice - run: | - echo "::error::==========================================" - echo "::error:: This workflow is DEPRECATED" - echo "::error::==========================================" - echo "::error::" - echo "::error::Please use the new workflow:" - echo "::error:: 'Validated Release'" - echo "::error::" - echo "::error::Location:" - echo "::error:: .github/workflows/release-validated.yml" - echo "::error::" - echo "::error::The new workflow includes:" - echo "::error:: - Pre-release test validation (testDebug + testAsan)" - echo "::error:: - Annotated git tags (fixes GitLab trigger)" - echo "::error:: - Improved error handling and rollback procedures" - echo "::error::" - echo "::error::==========================================" - exit 1 \ No newline at end of file diff --git a/.github/workflows/test_workflow.yml b/.github/workflows/test_workflow.yml deleted file mode 100644 index 9c2a32cb4..000000000 --- a/.github/workflows/test_workflow.yml +++ /dev/null @@ -1,639 +0,0 @@ -name: Shared Test Workflow - -on: - workflow_call: - inputs: - configuration: - required: true - type: string - skip_gtest: - description: "Skip C++ gtest execution (use when gtests run in a separate job)" - required: false - type: boolean - default: false - -permissions: - contents: read - actions: read - -jobs: - cache-jdks: - # This job is used to cache the JDKs for the test jobs - uses: ./.github/workflows/cache_java.yml - filter-musl-configs: - # Sanitizers (asan/tsan) are not supported on musl - filter them out - runs-on: ubuntu-latest - outputs: - configs: ${{ steps.filter.outputs.configs }} - has_configs: ${{ steps.filter.outputs.has_configs }} - steps: - - id: filter - run: | - configs=$(echo '${{ inputs.configuration }}' | jq -c '[.[] | select(. != "asan" and . != "tsan")]') - if [ "$configs" = "[]" ]; then - echo "has_configs=false" >> $GITHUB_OUTPUT - else - echo "has_configs=true" >> $GITHUB_OUTPUT - fi - echo "configs=$configs" >> $GITHUB_OUTPUT - test-linux-glibc-amd64: - needs: cache-jdks - strategy: - fail-fast: false - matrix: - # java_version: [ "8", "8-orcl", "8-j9", "8-zing", "8-ibm", "11", "11-j9", "11-zing", "17", "17-j9", "17-zing", "17-graal", "21", "21-zing", "21-graal", "25", "25-graal" ] - # FIXME: Zing disabled - Azul pulled public CDN access - java_version: [ "8", "8-orcl", "8-j9", "8-ibm", "11", "11-j9", "17", "17-j9", "17-graal", "21", "21-graal", "25", "25-graal" ] - config: ${{ fromJson(inputs.configuration) }} - runs-on: ubuntu-latest - timeout-minutes: 180 - steps: - - name: Set enabled flag - id: set_enabled - run: | - echo "enabled=true" >> $GITHUB_OUTPUT - if [[ "${{ matrix.java_version }}" =~ -zing ]]; then - if [[ "${{ matrix.config }}" != "release" ]] && [[ "${{ matrix.config }}" != "debug" ]]; then - echo "enabled=false" >> $GITHUB_OUTPUT - fi - fi - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - if: steps.set_enabled.outputs.enabled == 'true' - - name: Cache Gradle Wrapper Binaries - if: steps.set_enabled.outputs.enabled == 'true' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/wrapper/dists - key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - gradle-wrapper-${{ runner.os }}- - - name: Cache Gradle User Home - if: steps.set_enabled.outputs.enabled == 'true' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/caches - key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - gradle-caches-${{ runner.os }}- - - name: Setup cached JDK - id: cache-jdk - if: steps.set_enabled.outputs.enabled == 'true' - uses: ./.github/actions/setup_cached_java - with: - version: ${{ matrix.java_version }} - arch: 'amd64' - - name: Setup OS - if: steps.set_enabled.outputs.enabled == 'true' - run: | - sudo apt-get update - sudo apt-get install -y curl zip unzip libgtest-dev libgmock-dev binutils - # Install debug symbols for system libraries - sudo apt-get install -y libc6-dbg - if [[ ${{ matrix.java_version }} =~ "-zing" ]]; then - sudo apt-get install -y g++-9 gcc-9 - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9 - sudo update-alternatives --set gcc /usr/bin/gcc-9 - fi - - name: Extract Versions - if: steps.set_enabled.outputs.enabled == 'true' - uses: ./.github/actions/extract_versions - - name: Test - if: steps.set_enabled.outputs.enabled == 'true' - run: | - set +e - export KEEP_JFRS=true - export TEST_COMMIT=${{ github.sha }} - export TEST_CONFIGURATION=glibc/${{ matrix.java_version }}-${{ matrix.config }}-amd64 - export LIBC=glibc - export SANITIZER=${{ matrix.config }} - - GRADLEW_PREFIX="" - if [[ "${{ matrix.config }}" == "asan" ]]; then - # Reduce ASLR entropy so the JVM doesn't land in ASan's shadow range - # [0x00007fff7000-0x10007fff7fff]; 28 (the default) causes random aborts. - if ! sudo sysctl -w vm.mmap_rnd_bits=8 2>/dev/null; then - echo "::warning::Cannot set vm.mmap_rnd_bits; falling back to setarch --addr-no-randomize" - GRADLEW_PREFIX="setarch $(uname -m) --addr-no-randomize" - fi - fi - - # GraalVM pre-allocates fixed memory that conflicts with ASan's shadow range - # [0x00007fff7000-0x10007fff7fff]; vm.mmap_rnd_bits tuning does not help (google/sanitizers#856). - if [[ "${{ matrix.config }}" == "asan" && "${{ matrix.java_version }}" =~ graal ]]; then - echo "::notice::Skipping ASan for GraalVM (incompatible shadow memory ranges — google/sanitizers#856)" - exit 0 - fi - - MAX_ATTEMPTS=1 - if [[ "${{ matrix.config }}" == "asan" && "${{ matrix.java_version }}" =~ (j9|ibm) ]]; then - MAX_ATTEMPTS=2 - fi - - for attempt in $(seq 1 $MAX_ATTEMPTS); do - mkdir -p build/logs - ${GRADLEW_PREFIX} ./gradlew -PCI -PkeepJFRs ${{ inputs.skip_gtest == true && '-Pskip-gtest' || '' }} :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs 2>&1 \ - | tee -a build/test-raw.log \ - | python3 -u .github/scripts/filter_gradle_log.py - EXIT_CODE=${PIPESTATUS[0]} - - if [ $EXIT_CODE -eq 0 ]; then break; fi - if [ $attempt -lt $MAX_ATTEMPTS ]; then - echo "::warning::Attempt $attempt failed (exit $EXIT_CODE), retrying..." - ./gradlew --stop 2>/dev/null || true - fi - done - - if [ $EXIT_CODE -ne 0 ]; then - echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64" >> failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt - exit 1 - fi - - name: Generate Unwinding Report - if: success() && matrix.config == 'debug' - run: | - ./gradlew -PCI :ddprof-test:unwindingReport --no-daemon - - name: Add Unwinding Report to Job Summary - if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != '' - run: | - echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (amd64)" >> $GITHUB_STEP_SUMMARY - cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY - - name: Upload build artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: success() - with: - name: (build) test-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: build/ - - name: Upload failures - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: failures-glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64 - path: failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt - - name: Prepare reports - if: always() && steps.set_enabled.outputs.enabled == 'true' - run: | - .github/scripts/prepare_reports.sh - - name: Upload unwinding reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: success() && matrix.config == 'debug' - with: - name: (unwinding-reports) unwinding-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: unwinding-reports - - name: Upload test reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: (test-reports) test-linux-glibc-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: test-reports - - name: Upload signal-safety violation log - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: signal-safety-violation-glibc-${{ matrix.java_version }}-${{ matrix.config }}-amd64 - path: /tmp/signal-safety-violation.txt - if-no-files-found: ignore - - name: Upload ASan logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() && matrix.config == 'asan' - with: - name: asan-logs-glibc-${{ matrix.java_version }}-amd64 - path: /tmp/asan.log.* - if-no-files-found: ignore - - test-linux-musl-amd64: - needs: [cache-jdks, filter-musl-configs] - if: needs.filter-musl-configs.outputs.has_configs == 'true' - strategy: - fail-fast: false - matrix: - java_version: [ "8-librca", "11-librca", "17-librca", "21-librca", "25-librca" ] - config: ${{ fromJson(needs.filter-musl-configs.outputs.configs) }} - runs-on: ubuntu-latest - container: - image: "alpine:3.23" - options: --cpus 4 --workdir /github/workspace -v /home/runner/work/_temp:/home/runner/work/_temp - timeout-minutes: 180 - steps: - - name: Setup OS - run: | - apk update && apk add curl moreutils wget hexdump linux-headers bash make g++ clang git cppcheck jq cmake gtest-dev gmock tar binutils python3 >/dev/null - # Install debug symbols for musl libc - apk add musl-dbg - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Cache Gradle Wrapper Binaries - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/wrapper/dists - key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - gradle-wrapper-${{ runner.os }}- - - name: Cache Gradle User Home - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/caches - key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - gradle-caches-${{ runner.os }}- - - name: Setup cached JDK - id: cache-jdk - uses: ./.github/actions/setup_cached_java - with: - version: ${{ matrix.java_version }} - arch: 'amd64-musl' - - name: Extract Versions - uses: ./.github/actions/extract_versions - - name: Test - shell: bash - run: | - set +e - - export KEEP_JFRS=true - export TEST_COMMIT=${{ github.sha }} - export TEST_CONFIGURATION=musl/${{ matrix.java_version }}-${{ matrix.config }}-amd64 - # make sure the job knows it is running on musl - export LIBC=musl - export SANITIZER=${{ matrix.config }} - - # due to env hell in GHA containers, we need to re-do the logic from Extract Versions here - JAVA_VERSION=$(${{ env.JAVA_TEST_HOME }}/bin/java -version 2>&1 | awk -F '"' '/version/ { - split($2, v, "[._]"); - if (v[2] == "") { - # Version is like "24": assume it is major only and add .0.0 - printf "%s.0.0\n", v[1] - } else if (v[1] == "1") { - # Java 8 or older: Format is "1.major.minor_update" - printf "%s.%s.%s\n", v[2], v[3], v[4] - } else { - # Java 9 or newer: Format is "major.minor.patch" - printf "%s.%s.%s\n", v[1], v[2], v[3] - } - }') - export JAVA_VERSION - echo "JAVA_VERSION=${JAVA_VERSION}" - - mkdir -p build/logs - ./gradlew -PCI -PkeepJFRs :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs 2>&1 \ - | tee -a build/test-raw.log \ - | python3 -u .github/scripts/filter_gradle_log.py - EXIT_CODE=${PIPESTATUS[0]} - - if [ $EXIT_CODE -ne 0 ]; then - echo "musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64" >> failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt - exit 1 - fi - - name: Generate Unwinding Report - if: success() && matrix.config == 'debug' - run: | - ./gradlew -PCI :ddprof-test:unwindingReport --no-daemon - - name: Add Unwinding Report to Job Summary - if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != '' - run: | - echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (amd64-musl)" >> $GITHUB_STEP_SUMMARY - cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY - - name: Upload build artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: success() - with: - name: (build) test-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: build/ - - name: Upload failures - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: failures-musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64 - path: failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64.txt - - name: Prepare reports - if: always() - run: | - .github/scripts/prepare_reports.sh - - name: Upload unwinding reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: success() && matrix.config == 'debug' - with: - name: (unwinding-reports) unwinding-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: unwinding-reports - - name: Upload test reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: (test-reports) test-linux-musl-amd64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: test-reports - - name: Upload signal-safety violation log - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: signal-safety-violation-musl-${{ matrix.java_version }}-${{ matrix.config }}-amd64 - path: /tmp/signal-safety-violation.txt - if-no-files-found: ignore - - name: Upload ASan logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() && matrix.config == 'asan' - with: - name: asan-logs-musl-${{ matrix.java_version }}-amd64 - path: /tmp/asan.log.* - if-no-files-found: ignore - - test-linux-glibc-aarch64: - needs: cache-jdks - strategy: - fail-fast: false - matrix: - # java_version: [ "8", "8-j9", "8-zing", "11", "11-j9", "11-zing", "17", "17-j9", "17-zing", "17-graal", "21", "21-zing", "21-graal", "23", "23-graal" ] - # FIXME: Hotspot 8 and 11 versions are rather crashy in ASGCT on aarch64, so we are skipping them for now - # java_version: [ "8-j9", "8-zing", "11-j9", "11-zing", "17", "17-j9", "17-zing", "17-graal", "21", "21-zing", "21-graal", "25", "25-graal" ] - # FIXME: Zing disabled - Azul pulled public CDN access - java_version: [ "8-j9", "11-j9", "17", "17-j9", "17-graal", "21", "21-graal", "25", "25-graal" ] - config: ${{ fromJson(inputs.configuration) }} - runs-on: - group: ARM LINUX SHARED - labels: arm-4core-linux - timeout-minutes: 180 - steps: - - name: Set enabled flag - id: set_enabled - run: | - echo "enabled=true" >> $GITHUB_OUTPUT - if [[ "${{ matrix.java_version }}" =~ -zing ]]; then - if [[ "${{ matrix.config }}" != "release" ]] && [[ "${{ matrix.config }}" != "debug" ]]; then - echo "enabled=false" >> $GITHUB_OUTPUT - fi - fi - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - if: steps.set_enabled.outputs.enabled == 'true' - - name: Cache Gradle Wrapper Binaries - if: steps.set_enabled.outputs.enabled == 'true' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/wrapper/dists - key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - gradle-wrapper-${{ runner.os }}- - - name: Cache Gradle User Home - if: steps.set_enabled.outputs.enabled == 'true' - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/caches - key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - gradle-caches-${{ runner.os }}- - - name: Setup cached JDK - id: cache-jdk - if: steps.set_enabled.outputs.enabled == 'true' - uses: ./.github/actions/setup_cached_java - with: - version: ${{ matrix.java_version }} - arch: 'aarch64' - - name: Setup OS - if: steps.set_enabled.outputs.enabled == 'true' - run: | - sudo apt update -y - sudo apt remove -y g++ - sudo apt autoremove -y - sudo apt install -y curl zip unzip clang make build-essential binutils gdb - # Install debug symbols for system libraries - sudo apt install -y libc6-dbg - if [[ ${{ matrix.java_version }} =~ "-zing" ]]; then - sudo apt -y install g++-9 gcc-9 - sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-9 100 --slave /usr/bin/g++ g++ /usr/bin/g++-9 - sudo update-alternatives --set gcc /usr/bin/gcc-9 - fi - - name: Extract Versions - if: steps.set_enabled.outputs.enabled == 'true' - uses: ./.github/actions/extract_versions - - name: Test - if: steps.set_enabled.outputs.enabled == 'true' - run: | - set +e - export KEEP_JFRS=true - export TEST_COMMIT=${{ github.sha }} - export TEST_CONFIGURATION=glibc/${{ matrix.java_version }}-${{ matrix.config }}-aarch64 - export LIBC=glibc - export SANITIZER=${{ matrix.config }} - - GRADLEW_PREFIX="" - # For ASAN: launch a gdb watchdog that dumps all native threads before the - # 30-minute Gradle timeout kills the JVM, so we can diagnose hangs in native code. - if [[ "${{ matrix.config }}" == "asan" ]]; then - # Reduce ASLR entropy so the JVM doesn't land in ASan's shadow range; 28 causes random aborts. - if ! sudo sysctl -w vm.mmap_rnd_bits=8 2>/dev/null; then - echo "::warning::Cannot set vm.mmap_rnd_bits; falling back to setarch --addr-no-randomize" - GRADLEW_PREFIX="setarch $(uname -m) --addr-no-randomize" - fi - mkdir -p build/logs - ( - sleep 1500 # 25 minutes — fires before the 30-min Gradle timeout - echo "::warning::GDB watchdog triggered after 25 minutes" - for pid in $(pgrep -f 'java.*ddprof-test'); do - echo "=== Thread dump for PID $pid ===" >> build/logs/gdb-watchdog.log - gdb -batch -ex 'thread apply all bt full' -p "$pid" >> build/logs/gdb-watchdog.log 2>&1 || true - done - ) & - GDB_WATCHDOG_PID=$! - fi - - # GraalVM pre-allocates fixed memory that conflicts with ASan's shadow range - # [0x00007fff7000-0x10007fff7fff]; vm.mmap_rnd_bits tuning does not help (google/sanitizers#856). - if [[ "${{ matrix.config }}" == "asan" && "${{ matrix.java_version }}" =~ graal ]]; then - echo "::notice::Skipping ASan for GraalVM (incompatible shadow memory ranges — google/sanitizers#856)" - exit 0 - fi - - MAX_ATTEMPTS=1 - if [[ "${{ matrix.config }}" == "asan" && "${{ matrix.java_version }}" =~ (j9|ibm) ]]; then - MAX_ATTEMPTS=2 - fi - - for attempt in $(seq 1 $MAX_ATTEMPTS); do - mkdir -p build/logs - ${GRADLEW_PREFIX} ./gradlew -PCI -PkeepJFRs ${{ inputs.skip_gtest == true && '-Pskip-gtest' || '' }} :ddprof-test:test${{ matrix.config }} --no-daemon --parallel --build-cache --no-watch-fs 2>&1 \ - | tee -a build/test-raw.log \ - | python3 -u .github/scripts/filter_gradle_log.py - EXIT_CODE=${PIPESTATUS[0]} - - if [ $EXIT_CODE -eq 0 ]; then break; fi - if [ $attempt -lt $MAX_ATTEMPTS ]; then - echo "::warning::Attempt $attempt failed (exit $EXIT_CODE), retrying..." - ./gradlew --stop 2>/dev/null || true - fi - done - - # Kill the watchdog if tests finished before it fired - if [[ -n "${GDB_WATCHDOG_PID:-}" ]]; then - kill "$GDB_WATCHDOG_PID" 2>/dev/null || true - fi - - if [ $EXIT_CODE -ne 0 ]; then - echo "glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64" >> failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt - exit 1 - fi - - name: Generate Unwinding Report - if: success() && matrix.config == 'debug' - run: | - ./gradlew -PCI :ddprof-test:unwindingReport --no-daemon - - name: Add Unwinding Report to Job Summary - if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != '' - run: | - echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (aarch64)" >> $GITHUB_STEP_SUMMARY - cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY - - name: Upload build artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: success() - with: - name: (build) test-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: build/ - - name: Upload failures - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: failures-glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64 - path: failures_glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt - - name: Prepare reports - if: always() && steps.set_enabled.outputs.enabled == 'true' - run: | - .github/scripts/prepare_reports.sh - - name: Upload unwinding reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: success() && matrix.config == 'debug' - with: - name: (unwinding-reports) unwinding-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: unwinding-reports - - name: Upload test reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: (test-reports) test-linux-glibc-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: test-reports - - name: Upload signal-safety violation log - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: signal-safety-violation-glibc-${{ matrix.java_version }}-${{ matrix.config }}-aarch64 - path: /tmp/signal-safety-violation.txt - if-no-files-found: ignore - - name: Upload ASan logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() && matrix.config == 'asan' - with: - name: asan-logs-glibc-${{ matrix.java_version }}-aarch64 - path: /tmp/asan.log.* - if-no-files-found: ignore - - test-linux-musl-aarch64: - needs: [cache-jdks, filter-musl-configs] - if: needs.filter-musl-configs.outputs.has_configs == 'true' - strategy: - fail-fast: false - matrix: - java_version: [ "8-librca", "11-librca", "17-librca", "21-librca", "25-librca" ] - config: ${{ fromJson(needs.filter-musl-configs.outputs.configs) }} - runs-on: - group: ARM LINUX SHARED - labels: arm-4core-linux-ubuntu24.04 - timeout-minutes: 180 - steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - name: Cache Gradle Wrapper Binaries - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/wrapper/dists - key: gradle-wrapper-${{ runner.os }}-${{ hashFiles('gradle/wrapper/gradle-wrapper.properties') }} - restore-keys: | - gradle-wrapper-${{ runner.os }}- - - name: Cache Gradle User Home - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - with: - path: ~/.gradle/caches - key: gradle-caches-${{ runner.os }}-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }} - restore-keys: | - gradle-caches-${{ runner.os }}- - - name: Setup cached JDK - id: cache-jdk - uses: ./.github/actions/setup_cached_java - with: - version: ${{ matrix.java_version }} - arch: 'aarch64-musl' - - name: Extract Versions - uses: ./.github/actions/extract_versions - - name: Test - run: | - set +e - # the effective JAVA_VERSION is computed in the test_alpine_aarch64.sh script - mkdir -p build/logs - docker run --cpus 4 --rm -v /tmp:/tmp -v "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" -w "${GITHUB_WORKSPACE}" alpine:3.21 /bin/sh -c " - \"$GITHUB_WORKSPACE/.github/scripts/test_alpine_aarch64.sh\" \ - \"${{ github.sha }}\" \"musl/${{ matrix.java_version }}-${{ matrix.config }}-aarch64\" \ - \"${{ matrix.config }}\" \"${{ env.JAVA_HOME }}\" \"${{ env.JAVA_TEST_HOME }}\" - " 2>&1 \ - | tee -a build/test-raw.log \ - | python3 -u .github/scripts/filter_gradle_log.py - - EXIT_CODE=${PIPESTATUS[0]} - - if [ $EXIT_CODE -ne 0 ]; then - echo "musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64" >> failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt - exit 1 - fi - - name: Fix permissions for artifact upload - if: always() - run: | - # Files created by Docker container may have restrictive permissions - # that prevent GitHub Actions from reading them during artifact creation. - # Use a+rX (umask-independent) where X adds execute only to directories. - sudo chmod -R a+rX build/ ddprof-test/build/ ddprof-lib/build/ || true - - name: Generate Unwinding Report - if: success() && matrix.config == 'debug' - run: | - docker run --cpus 4 --rm -v /tmp:/tmp -v "${GITHUB_WORKSPACE}:${GITHUB_WORKSPACE}" -w "${GITHUB_WORKSPACE}" alpine:3.21 /bin/sh -c " - \"$GITHUB_WORKSPACE/.github/scripts/unwinding_report_alpine_aarch64.sh\" \ - \"${{ github.sha }}\" \"musl/${{ matrix.java_version }}-${{ matrix.config }}-aarch64\" \ - \"${{ matrix.config }}\" \"${{ env.JAVA_HOME }}\" \"${{ env.JAVA_TEST_HOME }}\" - " - # Fix permissions after Docker container execution (both top-level and subproject builds). - # Use a+rX (umask-independent) where X adds execute only to directories. - sudo chmod -R a+rX build/ ddprof-test/build/ ddprof-lib/build/ || true - - name: Add Unwinding Report to Job Summary - if: success() && matrix.config == 'debug' && hashFiles('ddprof-test/build/reports/unwinding-summary.md') != '' - run: | - echo "## 🔧 Unwinding Quality Report - ${{ matrix.java_version }} (aarch64-musl)" >> $GITHUB_STEP_SUMMARY - cat ddprof-test/build/reports/unwinding-summary.md >> $GITHUB_STEP_SUMMARY - - name: Upload build artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: success() - with: - name: (build) test-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: build/ - - name: Upload failures - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: failures-musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64 - path: failures_musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64.txt - - name: Prepare reports - if: always() - run: | - .github/scripts/prepare_reports.sh - - name: Upload unwinding reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: success() && matrix.config == 'debug' - with: - name: (unwinding-reports) unwinding-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: unwinding-reports - - name: Upload test reports - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: (test-reports) test-linux-musl-aarch64 (${{ matrix.java_version }}, ${{ matrix.config }}) - path: test-reports - - name: Upload signal-safety violation log - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() - with: - name: signal-safety-violation-musl-${{ matrix.java_version }}-${{ matrix.config }}-aarch64 - path: /tmp/signal-safety-violation.txt - if-no-files-found: ignore - - name: Upload ASan logs - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - if: failure() && matrix.config == 'asan' - with: - name: asan-logs-musl-${{ matrix.java_version }}-aarch64 - path: /tmp/asan.log.* - if-no-files-found: ignore diff --git a/.github/workflows/update_assets.yml b/.github/workflows/update_assets.yml deleted file mode 100644 index 9c81b959d..000000000 --- a/.github/workflows/update_assets.yml +++ /dev/null @@ -1,69 +0,0 @@ -name: Update Release Assets -run-name: Update assets for ${{ inputs.release_tag }} -on: - workflow_dispatch: - inputs: - release_tag: - type: string - description: "Release tag" - required: true - push: - tags: - - v_*.*.* - -permissions: - contents: write - actions: read - -jobs: - update-assets-and-releaase: - if: (startsWith(github.event.ref, 'refs/tags/v_') || inputs.release_tag != '') && !endsWith(github.event.ref, '-SNAPSHOT') - runs-on: ubuntu-latest - steps: - - name: Setup System - id: setup-system - run: | - sudo apt update && sudo apt install -y wget unzip - - name: Download Assets - id: download-assets - timeout-minutes: 30 - run: | - # ignore errors to allow reattempted downloads - set +e - TAG=${{ inputs.release_tag }} - if [ -z "$TAG" ]; then - TAG="$GITHUB_REF_NAME" - fi - VERSION=$(echo "${TAG}" | sed -e 's/v_//g') - ASSET_URL="https://repo1.maven.org/maven2/com/datadoghq/ddprof/${VERSION}/ddprof-${VERSION}.jar" - RESULT=1 - while [ $RESULT -ne 0 ]; do - wget -q $ASSET_URL - RESULT=$? - if [ $RESULT -ne 0 ]; then - echo "Artifact not available. Retrying in 30 seconds." - sleep 30 - fi - done - echo "VERSION=${VERSION}" >> $GITHUB_ENV - - name: Prepare Assets - id: prepare-assets - run: | - LIB_BASE_DIR="META-INF/native-libs" - mkdir assets - cp ddprof-${VERSION}.jar assets/ddprof.jar - cp ddprof-${VERSION}.jar assets/ddprof-${VERSION}.jar - unzip ddprof-${VERSION}.jar - mv ${LIB_BASE_DIR}/linux-arm64/libjavaProfiler.so assets/libjavaProfiler_linux-arm64.so - mv ${LIB_BASE_DIR}/linux-x64/libjavaProfiler.so assets/libjavaProfiler_linux-x64.so - mv ${LIB_BASE_DIR}/linux-arm64-musl/libjavaProfiler.so assets/libjavaProfiler_linux-arm64-musl.so - mv ${LIB_BASE_DIR}/linux-x64-musl/libjavaProfiler.so assets/libjavaProfiler_linux-x64-musl.so - - name: Update release ${{ inputs.release_tag }} - id: update-release - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_REPO: ${{ github.repository }} - run: | - TAG="v_${VERSION}" - gh release upload "${TAG}" assets/ddprof*.jar assets/*.so --clobber - gh release edit "${TAG}" --draft=false --latest diff --git a/.github/workflows/upstream-tracker.yml b/.github/workflows/upstream-tracker.yml deleted file mode 100644 index 1fa87af82..000000000 --- a/.github/workflows/upstream-tracker.yml +++ /dev/null @@ -1,161 +0,0 @@ -name: Upstream Async-Profiler Change Tracker - -on: - schedule: - - cron: '0 3 * * *' # Daily at 3 AM UTC - workflow_dispatch: - inputs: - force_report: - description: 'Generate report even if no changes detected' - type: boolean - default: false - -permissions: - contents: write - issues: write - -jobs: - track-upstream: - runs-on: ubuntu-latest - steps: - - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 - - - name: Setup git - run: | - git config --global user.name "github-actions[bot]" - git config --global user.email "github-actions[bot]@users.noreply.github.com" - - - name: Clone upstream async-profiler - id: clone-upstream - continue-on-error: true - run: | - git clone --depth 50 https://github.com/async-profiler/async-profiler.git /tmp/async-profiler - cd /tmp/async-profiler - git fetch origin master - UPSTREAM_HEAD=$(git rev-parse origin/master) - echo "UPSTREAM_HEAD=${UPSTREAM_HEAD}" >> $GITHUB_ENV - echo "Upstream HEAD: ${UPSTREAM_HEAD}" - - - name: Handle clone failure - if: steps.clone-upstream.outcome == 'failure' - run: | - echo "::warning::Failed to clone upstream repository. Will retry on next run." - exit 0 - - - name: Get last checked commit - if: steps.clone-upstream.outcome == 'success' - id: get-last-commit - run: | - # Try to get the last commit from gh-pages branch - STATE_FILE="upstream-tracker-state.txt" - - # Fetch gh-pages branch - if git fetch origin gh-pages:gh-pages 2>/dev/null; then - # Check if state file exists in gh-pages - if git show gh-pages:"$STATE_FILE" 2>/dev/null > /tmp/state.txt; then - LAST_COMMIT=$(cat /tmp/state.txt | tr -d '[:space:]') - - # Validate that LAST_COMMIT is a valid commit SHA (40 hex characters) - if ! echo "$LAST_COMMIT" | grep -qE '^[0-9a-f]{40}$'; then - echo "Invalid commit SHA in state file: ${LAST_COMMIT}" - echo "First run detected, using current HEAD as baseline" - LAST_COMMIT="${UPSTREAM_HEAD}" - else - echo "Last checked commit from gh-pages: ${LAST_COMMIT}" - fi - else - echo "No state file in gh-pages, first run detected" - LAST_COMMIT="${UPSTREAM_HEAD}" - fi - else - echo "gh-pages branch not found, first run detected" - LAST_COMMIT="${UPSTREAM_HEAD}" - fi - - echo "last_commit=${LAST_COMMIT}" >> $GITHUB_OUTPUT - echo "Last checked commit: ${LAST_COMMIT}" - - - name: Generate tracked files list - if: steps.clone-upstream.outcome == 'success' - id: tracked-files - run: | - ./utils/generate_tracked_files.sh \ - ddprof-lib/src/main/cpp \ - /tmp/async-profiler/src \ - > /tmp/tracked_files.txt - - echo "Tracking $(wc -l < /tmp/tracked_files.txt) files" - cat /tmp/tracked_files.txt - - - name: Track upstream changes - if: steps.clone-upstream.outcome == 'success' - id: track-changes - run: | - ./utils/track_upstream_changes.sh \ - /tmp/async-profiler \ - "${{ steps.get-last-commit.outputs.last_commit }}" \ - "${{ env.UPSTREAM_HEAD }}" \ - /tmp/tracked_files.txt \ - /tmp/change_report.md \ - /tmp/change_report.json - - if [ -f /tmp/change_report.json ]; then - echo "has_changes=true" >> $GITHUB_OUTPUT - echo "Changes detected!" - else - echo "has_changes=false" >> $GITHUB_OUTPUT - echo "No changes detected" - fi - - - name: Create GitHub Issue - if: steps.track-changes.outputs.has_changes == 'true' && steps.clone-upstream.outcome == 'success' - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Verify report file exists before creating issue - if [ ! -f /tmp/change_report.md ]; then - echo "Error: Report file not found, skipping issue creation" - exit 0 - fi - - ISSUE_TITLE="Upstream async-profiler changes detected ($(date +%Y-%m-%d))" - - gh issue create \ - --title "$ISSUE_TITLE" \ - --body-file /tmp/change_report.md \ - --label "upstream-tracking,needs-review" \ - --repo ${{ github.repository }} - - - name: Update last checked commit - if: success() && steps.clone-upstream.outcome == 'success' - run: | - # Update state file in gh-pages branch - STATE_FILE="upstream-tracker-state.txt" - - git config user.name "github-actions[bot]" - git config user.email "github-actions[bot]@users.noreply.github.com" - - # Fetch or create gh-pages branch - if git fetch origin gh-pages:gh-pages 2>/dev/null; then - git checkout gh-pages - else - echo "Creating new gh-pages branch" - git checkout --orphan gh-pages - git rm -rf . 2>/dev/null || true - fi - - # Update state file - echo "${UPSTREAM_HEAD}" > "$STATE_FILE" - git add "$STATE_FILE" - - if git diff --staged --quiet; then - echo "No changes to state file" - else - git commit -m "Update upstream tracker state to ${UPSTREAM_HEAD:0:7}" - git push origin gh-pages - echo "Updated last checked commit to ${UPSTREAM_HEAD}" - fi - - # Return to main branch - git checkout main diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 152d4dfa2..000000000 --- a/.gitignore +++ /dev/null @@ -1,45 +0,0 @@ -**/build/ -**/build_*/ -**/build-*/ -!build-logic/ -!.gitlab/build-deploy/ -/nbproject/ -/out/ -/.idea/ -/target/ -**/*.class -**/*.class.h -**/*.so -**/*.o -.vscode -.classpath -.project -.settings -.gradle -.kotlin -.tmp -*.iml -/ddprof-stresstest/jmh-result.* - -**/.resources/ - -# ignore all temporary locations related to maven builds -datadog/maven/tmp -datadog/maven/repository -datadog/maven/resources - -**/harness* -**/launcher* -/gradle.properties -**/hs_err* - -# cursor AI history -.history -.claude/settings.local.json -/jmh-* - -# Temporary documentation and work state -doc/temp/ - -# CLAUDE.md is auto-generated from AGENTS.md bootstrap instructions -CLAUDE.md diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml deleted file mode 100644 index 2946b379c..000000000 --- a/.gitlab-ci.yml +++ /dev/null @@ -1,238 +0,0 @@ -image: alpine - -variables: - REGISTRY: registry.ddbuild.io - PREPARE_IMAGE: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest - # Image with dd-octo-sts for GitHub token exchange (check-image-updates, rebuild-images-pr) - DD_OCTO_STS_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - FORCE_BUILD: - value: "" - description: "Force build even if no new commits (any non-empty value)" - RUN_RELIABILITY: - value: "false" - description: "Run reliability and chaos tests. Set automatically when the test:reliability label is on the PR." - MAVEN_REPOSITORY_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" - -default: - tags: ["arch:amd64"] - interruptible: true - before_script: - - '[ "${CANCELLED:-}" != "true" ] || { echo "No PR for this branch — skipping job"; exit 0; }' - - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=${MAVEN_REPOSITORY_PROXY} - -stages: - - images - - generate-signing-key - - prepare - - sanitizer - - build - - stresstest - - deploy - - integration-test - - reliability - - benchmarks - - post-benchmarks - - fuzz - - notify - -# Detects newer images in registry and creates GitHub PR with updates -check-image-updates: - stage: images - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $CHECK_IMAGE_UPDATES == "true"' - when: always - - if: '$CI_PIPELINE_SOURCE == "web"' - when: manual - allow_failure: true - extends: .bootstrap-gh-tools - tags: ["arch:arm64"] - image: ${DD_OCTO_STS_IMAGE} - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - script: - - set -euo pipefail - - echo "Checking for image updates..." - - .gitlab/scripts/check-image-updates.sh > updates.json - - | - update_count=$(jq 'length' updates.json) - echo "Found ${update_count} update(s)" - if [ "$update_count" -gt 0 ]; then - echo "Updates available:" - jq . updates.json - .gitlab/scripts/create-image-update-pr.sh updates.json - else - echo "All images are up to date" - fi - artifacts: - when: always - paths: - - updates.json - expire_in: 7 days - -rebuild-images: - stage: images - rules: - - if: '$CI_COMMIT_TAG' - when: never - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: never - - when: manual - allow_failure: true - tags: ["arch:amd64"] - variables: - REBUILD_IMAGES: "" # comma/space-separated short names, or empty = all - image: ${DOCKER_IMAGE} - id_tokens: - DDSIGN_ID_TOKEN: - aud: image-integrity - script: - - set -euo pipefail - - .gitlab/scripts/rebuild-images.sh - artifacts: - when: always - paths: - - updates.json - expire_in: 1 day - -rebuild-images-pr: - stage: images - rules: - - if: '$CI_COMMIT_TAG' - when: never - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: never - - when: on_success - needs: - - job: rebuild-images - artifacts: true - extends: .bootstrap-gh-tools - tags: ["arch:arm64"] - image: ${DD_OCTO_STS_IMAGE} - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - script: - - set -euo pipefail - - .gitlab/scripts/create-image-update-pr.sh updates.json - -create_key: - stage: generate-signing-key - when: manual - needs: [] - tags: ["arch:amd64"] - variables: - PROJECT_NAME: "java-profiler" - EXPORT_TO_KEYSERVER: "true" - KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: java-profiler - image: $REGISTRY/ci/agent-key-management-tools/gpg:1 - script: - - /create.sh - artifacts: - expire_in: 13 mos - paths: - - pubkeys - -# Shared version detection used by benchmarks and reliability pipelines -get-versions: - extends: .get-versions - needs: - - job: prepare:start - artifacts: false - -# Triggered externally from async-profiler-build with JDK build parameters; -# kept as a child pipeline because it is mutually exclusive with the main build -jdk-integration-test: - stage: build - rules: - - if: '$JDK_VERSION == null || $DEBUG_LEVEL == null || $HASH == null || $DOWNSTREAM == null' - when: never - - if: '$CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline" || $CI_PIPELINE_SOURCE == "web"' - when: always - allow_failure: false - - when: always - trigger: - include: .gitlab/jdk-integration/.gitlab-ci.yml - strategy: depend - forward: - pipeline_variables: true - -# Generates a child pipeline YAML for reliability/chaos tests when the PR -# carries the test:reliability label (RUN_RELIABILITY=true in build.env). -generate-reliability-child-pipeline: - stage: reliability - tags: ["arch:amd64"] - image: $PREPARE_IMAGE - needs: - - job: prepare:start - artifacts: true - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: never - - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' - when: never - - when: on_success - script: - - | - if [ "${RUN_RELIABILITY:-}" = "true" ]; then - echo "Label test:reliability detected — enabling reliability child pipeline" - cp .gitlab/reliability/pr-child.gitlab-ci.yml generated-reliability.yml - else - cat > generated-reliability.yml << 'NOOP' - skip-reliability: - image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest - tags: ["arch:amd64"] - script: - - echo "Label test:reliability not set — skipping" - rules: - - when: always - NOOP - fi - artifacts: - paths: - - generated-reliability.yml - expire_in: 1 day - -run-reliability-tests: - stage: reliability - variables: - DDPROF_COMMIT_BRANCH: "$DDPROF_COMMIT_BRANCH" - DDPROF_COMMIT_SHA: "$DDPROF_COMMIT_SHA" - needs: - - job: generate-reliability-child-pipeline - artifacts: true - - job: prepare:start - artifacts: true - # Reliability/chaos tests download com.datadoghq:ddprof:-SNAPSHOT from - # Maven snapshots; that artifact is published by deploy-artifact. Without this - # gate the child pipeline can start before the snapshot exists (cold branch) or - # download a stale snapshot from a previous push. optional: true so release - # branches, where deploy-artifact never runs, stay satisfiable. - - job: deploy-artifact - artifacts: false - optional: true - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: never - - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' - when: never - - when: on_success - trigger: - include: - - artifact: generated-reliability.yml - job: generate-reliability-child-pipeline - strategy: depend - forward: - pipeline_variables: true - -include: - - local: .gitlab/common.yml - - local: .adms/python/gitlab.yaml - - local: .gitlab/benchmarks/images.yml - - local: .gitlab/build-deploy/images.yml - - local: .gitlab/build-deploy/.gitlab-ci.yml - - local: .gitlab/benchmarks/.gitlab-ci.yml - - local: .gitlab/reliability/.gitlab-ci.yml - - local: .gitlab/dd-trace-integration/.gitlab-ci.yml - - local: .gitlab/sanitizer-tests/.gitlab-ci.yml - - local: .gitlab/fuzzing/.gitlab-ci.yml diff --git a/.gitlab/Dockerfile.datadog-ci b/.gitlab/Dockerfile.datadog-ci deleted file mode 100644 index 7a1873dfe..000000000 --- a/.gitlab/Dockerfile.datadog-ci +++ /dev/null @@ -1,60 +0,0 @@ -ARG BASEIMAGE=registry.ddbuild.io/images/base/gbi-ubuntu_2404:release -FROM ${BASEIMAGE} - -USER root - -# Create non-root user for security -RUN useradd --create-home --shell /bin/bash --uid 1001 ci-user - -# Install Node.js 20 and npm -# Default seems to be 14 which does not work with datadog-ci -RUN set -x \ - && apt-get update && apt-get -y install --no-install-recommends curl xz-utils\ - && curl -fsSL https://deb.nodesource.com/setup_20.x | bash - \ - && apt-get install -y nodejs \ - pipx=1.4.3-1 \ - binutils \ - jq \ - && npm install -g @datadog/datadog-ci@3.16.0 \ - && apt-get -y clean \ - && rm -rf /var/lib/apt/lists/* - -# Install GitHub CLI -RUN set -x \ - && curl -fsSL https://cli.github.com/packages/githubcli-archive-keyring.gpg \ - -o /usr/share/keyrings/githubcli-archive-keyring.gpg \ - && chmod go+r /usr/share/keyrings/githubcli-archive-keyring.gpg \ - && echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" \ - > /etc/apt/sources.list.d/github-cli.list \ - && apt-get update \ - && apt-get install -y gh \ - && apt-get -y clean \ - && rm -rf /var/lib/apt/lists/* - -# awscli is not available in Ubuntu 2404 for some inexplicable reason so lets install in via other means -RUN PIPX_HOME=/opt/pipx PIPX_BIN_DIR=/usr/local/bin pipx install awscli - -# Install Go 1.22.3 -RUN set -x \ - && curl -LO https://golang.org/dl/go1.22.3.linux-amd64.tar.gz \ - && tar -C /usr/local -xzf go1.22.3.linux-amd64.tar.gz \ - && rm go1.22.3.linux-amd64.tar.gz - -# Set up Go environment for root and install Crane -ENV PATH="/usr/local/go/bin:${PATH}" -ENV GOPATH="/root/go" -ENV GOBIN="/usr/local/bin" - -# Install Crane version 0.19.1 directly to /usr/local/bin so it's available for all users -RUN set -x \ - && go install github.com/google/go-containerregistry/cmd/crane@v0.19.1 - -# Switch to non-root user -USER ci-user -WORKDIR /home/ci-user - -# Set PATH for the ci-user (crane is now in /usr/local/bin) -ENV PATH="/usr/local/go/bin:/usr/local/bin:${PATH}" - -# Verify installation (as non-root user) -RUN node -v && npm -v && go version && crane version && datadog-ci --help && jq --version && gh --version diff --git a/.gitlab/base/Dockerfile b/.gitlab/base/Dockerfile deleted file mode 100644 index 57bca1e80..000000000 --- a/.gitlab/base/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -ARG BASE_IMAGE=debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b -FROM ${BASE_IMAGE} as base -ARG CI_JOB_TOKEN -WORKDIR /root - -RUN mkdir -p /usr/share/man/man1 # https://github.com/debuerreotype/docker-debian-artifacts/issues/24 -RUN (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends curl git moreutils awscli amazon-ecr-credential-helper gnupg2 npm build-essential wget bsdmainutils clang jq zip unzip) || true -RUN (apk update && apk add curl git moreutils aws-cli docker-credential-ecr-login gnupg alpine-sdk build-base wget npm hexdump linux-headers clang compiler-rt bash jq gradle zip unzip) || true -# Install JDK 21 and Maven via SDKMAN (glibc image only; musl uses Dockerfile.musl). -RUN curl -s "https://get.sdkman.io" | bash && \ - bash -c "source /root/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem && sdk install maven" -ENV JAVA_HOME=/root/.sdkman/candidates/java/current -ENV PATH="/root/.sdkman/candidates/java/current/bin:/root/.sdkman/candidates/maven/current/bin:${PATH}" -RUN npm install -g --save-dev @datadog/datadog-ci -RUN rm -rf "/var/lib/apt/lists/*" \ No newline at end of file diff --git a/.gitlab/base/Dockerfile.musl b/.gitlab/base/Dockerfile.musl deleted file mode 100644 index 1bbc9ce5b..000000000 --- a/.gitlab/base/Dockerfile.musl +++ /dev/null @@ -1,7 +0,0 @@ -ARG BASE_IMAGE=eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f -FROM ${BASE_IMAGE} as base -ARG CI_JOB_TOKEN -WORKDIR /root - -RUN apk update && apk add curl git moreutils aws-cli docker-credential-ecr-login gnupg alpine-sdk build-base wget npm hexdump linux-headers clang compiler-rt bash jq zip unzip -RUN npm install -g --save-dev @datadog/datadog-ci diff --git a/.gitlab/base/centos7/Dockerfile b/.gitlab/base/centos7/Dockerfile deleted file mode 100644 index 004a00128..000000000 --- a/.gitlab/base/centos7/Dockerfile +++ /dev/null @@ -1,43 +0,0 @@ -ARG BASE_IMAGE=openjdk:11-slim-buster -FROM ${BASE_IMAGE} as base -ARG CI_JOB_TOKEN -WORKDIR /root - -# 1. Replace dead mirrorlist entries with HTTPS vault URLs -RUN set -eux; \ - sed -i -e 's/^mirrorlist/#mirrorlist/' \ - -e 's|^#baseurl=http://mirror.centos.org|baseurl=https://vault.centos.org|' \ - /etc/yum.repos.d/CentOS-*.repo - -# 2. Add a vault mirror that still contains Software Collections -RUN cat > /etc/yum.repos.d/CentOS-SCLo-Vault.repo <<'EOF' -[centos-sclo-rh] -name=CentOS-7 - SCLo rh (Rocky Vault) -baseurl=https://dl.rockylinux.org/vault/centos/7.9.2009/sclo/$basearch/rh/ -gpgcheck=0 -enabled=1 - -[centos-sclo-sclo] -name=CentOS-7 - SCLo sclo (Rocky Vault) -baseurl=https://dl.rockylinux.org/vault/centos/7.9.2009/sclo/$basearch/sclo/ -gpgcheck=0 -enabled=1 -EOF - -# 3. Expose devtoolset-11 binaries & libs by default (they are installed a bit later) -ENV PATH="/opt/rh/devtoolset-11/root/usr/bin:${PATH}" \ - LD_LIBRARY_PATH="/opt/rh/devtoolset-11/root/usr/lib64:${LD_LIBRARY_PATH}" - -RUN yum -y clean all -RUN yum -y update && yum -y install scl-utils devtoolset-11 devtoolset-11-toolchain curl zip unzip git libstdc++-static make which wget cmake binutils -RUN yum -y clean all -RUN (curl -s "https://get.sdkman.io" | bash) -RUN (source ~/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem) -RUN (curl -sL https://rpm.nodesource.com/setup_16.x | bash -) -# installing JQ requires two steps - adding the repo and then installing the tool -RUN yum install -y epel-release -RUN yum install -y jq -# now install nodejs and datadog CI support -RUN yum -y install nodejs -RUN npm install -g --save-dev @datadog/datadog-ci -RUN rm -rf "/var/lib/apt/lists/*" \ No newline at end of file diff --git a/.gitlab/benchmarks/.gitlab-ci.yml b/.gitlab/benchmarks/.gitlab-ci.yml deleted file mode 100644 index b6f7f1cef..000000000 --- a/.gitlab/benchmarks/.gitlab-ci.yml +++ /dev/null @@ -1,76 +0,0 @@ -variables: - DD_OCTO_STS_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - -# Bridge job: triggers the BP pipeline and blocks until it completes. -# Bridge jobs cannot appear in other jobs' needs: — downstream jobs use -# stage ordering (post-benchmarks stage runs after benchmarks stage). -benchmarks-trigger: - stage: benchmarks - # Bridge jobs cannot have before_script, so CANCELLED is checked via rules. - # interruptible: false prevents orphaning the BP downstream pipeline on push. - interruptible: false - needs: - - job: get-versions - artifacts: true - - job: deploy-artifact - artifacts: false - rules: - - if: '$CANCELLED == "true"' - when: never - - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' - when: never - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - - if: '$CI_PIPELINE_SOURCE == "web"' - when: manual - allow_failure: true - # Run automatically and non-blocking on any other source (push/trigger/api/etc.) - - when: on_success - allow_failure: true - variables: - CANDIDATE_VERSION: "${CURRENT_VERSION}" - BASELINE_VERSION: "${PREVIOUS_VERSION}" - BENCHMARK_ITERATIONS: "${BENCHMARK_ITERATIONS:-5}" - BENCHMARK_MODES: "${BENCHMARK_MODES:-cpu,wall,alloc,memleak}" - DDPROF_COMMIT_SHA: "${CI_COMMIT_SHA}" - DDPROF_COMMIT_BRANCH: "${CI_COMMIT_BRANCH}" - UPSTREAM_PROJECT_NAME: "java-profiler" - UPSTREAM_BRANCH: "${CI_COMMIT_BRANCH}" - UPSTREAM_PIPELINE_ID: "${CI_PIPELINE_ID}" - trigger: - project: DataDog/apm-reliability/benchmarking-platform - branch: java-profiler - strategy: depend - - -publish-benchmark-gh-pages: - stage: post-benchmarks - tags: ["arch:arm64"] - image: $DD_OCTO_STS_IMAGE - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - # Serialize concurrent GH Pages pushes. publish-gh-pages.sh uses - # 'git push --force'; two concurrent pushes race and the slower one - # silently discards the faster one's history update. - resource_group: gh-pages-publish - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: never - - if: '$CANCELLED == "true"' - when: never - - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' - when: never - - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "main"' - when: always - timeout: 15m - script: - - mkdir -p reports - - .gitlab/benchmarks/download-bp-reports.sh reports - - ./.gitlab/benchmarks/publish-gh-pages.sh reports - allow_failure: true - -include: - - local: .gitlab/common.yml diff --git a/.gitlab/benchmarks/docker/Dockerfile b/.gitlab/benchmarks/docker/Dockerfile deleted file mode 100644 index cb3833a71..000000000 --- a/.gitlab/benchmarks/docker/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -ARG BASE_IMAGE=ubuntu:22.04 -FROM ${BASE_IMAGE} - -COPY ./setup.sh /tmp/setup.sh -RUN /tmp/setup.sh \ No newline at end of file diff --git a/.gitlab/benchmarks/docker/setup.sh b/.gitlab/benchmarks/docker/setup.sh deleted file mode 100755 index 4b23f0f85..000000000 --- a/.gitlab/benchmarks/docker/setup.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash - -set -e -set -x - -apt update && apt install -y wget curl zip unzip time maven jq git libjemalloc-dev libtcmalloc-minimal4 - -pip install numpy matplotlib - -# debug output -#echo $JAVA_HOME -#mvn -version - -# retrieve the standard benchmark apps -mkdir -p /var/lib/benchmarks -wget -q -O /var/lib/benchmarks/renaissance.jar https://github.com/renaissance-benchmarks/renaissance/releases/download/v0.15.0/renaissance-mit-0.15.0.jar -wget -q -O /var/lib/dacapo.jar https://netix.dl.sourceforge.net/project/dacapobench/9.12-bach-MR1/dacapo-9.12-MR1-bach.jar - diff --git a/.gitlab/benchmarks/download-bp-reports.sh b/.gitlab/benchmarks/download-bp-reports.sh deleted file mode 100755 index 9c323dfc4..000000000 --- a/.gitlab/benchmarks/download-bp-reports.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/usr/bin/env bash -# Downloads result JSONs from the BP downstream pipeline via the GitLab CI API. -# -# Requires only curl and python3 (stdlib) — no aws CLI, pip, or boto3 needed. -# BP jobs already store artifacts in GitLab; this fetches them directly from -# the downstream pipeline triggered by benchmarks-trigger. -set -uo pipefail # intentionally no -e: we handle errors explicitly - -DEST="${1:-reports}" -mkdir -p "${DEST}" - -TMPDIR_LOCAL=$(mktemp -d) -trap 'rm -rf "${TMPDIR_LOCAL}"' EXIT - -# ── helper: curl with explicit HTTP status checking ────────────────────────── -# Usage: api_get -# Returns 0 on 2xx, prints diagnostics and returns 1 otherwise. -api_get() { - local url="$1" out="$2" - local http_code - http_code=$(curl -s -o "${out}" -w "%{http_code}" \ - --header "JOB-TOKEN: ${CI_JOB_TOKEN}" "${url}") - if [[ "${http_code}" != 2* ]]; then - echo " API ${url##*/}: HTTP ${http_code} — $(python3 -c "import sys,json; d=json.load(sys.stdin); print(d.get('message','?'))" < "${out}" 2>/dev/null || echo 'see above')" - return 1 - fi - return 0 -} - -# ── 1. find the benchmarks-trigger bridge ──────────────────────────────────── -BRIDGES_FILE="${TMPDIR_LOCAL}/bridges.json" -echo "Querying bridges for pipeline ${CI_PIPELINE_ID}…" -if ! api_get \ - "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/bridges" \ - "${BRIDGES_FILE}"; then - echo "Cannot read pipeline bridges (job token may lack Reporter access) — skipping download" - exit 0 -fi - -read -r BP_PROJECT_ID DOWNSTREAM_PIPELINE_ID < <(python3 - "${BRIDGES_FILE}" <<'PYEOF' -import json, sys -with open(sys.argv[1]) as f: - bridges = json.load(f) -for b in bridges: - if b.get("name") == "benchmarks-trigger": - dp = b.get("downstream_pipeline") or {} - if dp.get("id") and dp.get("project_id") and dp.get("status") == "success": - print(dp["project_id"], dp["id"]) - sys.exit(0) -print("", "") -PYEOF -) - -if [ -z "${DOWNSTREAM_PIPELINE_ID:-}" ]; then - echo "benchmarks-trigger bridge not found or did not run — skipping download" - exit 0 -fi -echo "BP downstream pipeline: project=${BP_PROJECT_ID} pipeline=${DOWNSTREAM_PIPELINE_ID}" - -# ── 2. list jobs in the downstream pipeline (paginated) ────────────────────── -echo "Listing BP pipeline jobs…" -JOB_IDS="" -PAGE=1 -while true; do - JOBS_FILE="${TMPDIR_LOCAL}/jobs_page${PAGE}.json" - if ! api_get \ - "${CI_API_V4_URL}/projects/${BP_PROJECT_ID}/pipelines/${DOWNSTREAM_PIPELINE_ID}/jobs?per_page=100&page=${PAGE}" \ - "${JOBS_FILE}"; then - echo "Cannot list BP pipeline jobs — skipping download" - exit 0 - fi - PAGE_IDS=$(python3 -c " -import json, sys -jobs = json.load(open(sys.argv[1])) -print(' '.join(str(j['id']) for j in jobs)) -print(len(jobs), file=sys.stderr) -" "${JOBS_FILE}" 2>"${TMPDIR_LOCAL}/page_count.txt") - JOB_IDS="${JOB_IDS} ${PAGE_IDS}" - PAGE_COUNT=$(cat "${TMPDIR_LOCAL}/page_count.txt") - if [ "${PAGE_COUNT}" -lt 100 ]; then - break - fi - PAGE=$((PAGE + 1)) -done - -# ── 3. download result_*.json from each job's artifact zip ─────────────────── -DOWNLOADED=0 -for JOB_ID in ${JOB_IDS}; do - ART_ZIP="${TMPDIR_LOCAL}/art_${JOB_ID}.zip" - ART_STATUS=$(curl -s -o "${ART_ZIP}" -w "%{http_code}" \ - --header "JOB-TOKEN: ${CI_JOB_TOKEN}" \ - "${CI_API_V4_URL}/projects/${BP_PROJECT_ID}/jobs/${JOB_ID}/artifacts" 2>/dev/null) - if [[ "${ART_STATUS}" == 2* ]]; then - # Contract with BP (DataDog/apm-reliability/benchmarking-platform#190): - # artifacts are stored under the "artifacts/" prefix with names matching - # "result_*.json". If BP renames either, this silently extracts nothing. - # -j: junk paths (strip artifacts/ prefix), -q: quiet, -o: overwrite - if unzip -q -j "${ART_ZIP}" "artifacts/result_*.json" -d "${DEST}/" 2>/dev/null; then - DOWNLOADED=$((DOWNLOADED + 1)) - fi - fi -done - -RESULT_COUNT=$(find "${DEST}" -name "result_*.json" | wc -l) -echo "result_*.json files: ${RESULT_COUNT} (from ${DOWNLOADED} BP job(s))" - -if [ "${RESULT_COUNT}" -eq 0 ]; then - echo "WARNING: no result JSONs found — BP jobs may not have run or produced artifacts yet" - exit 1 -fi diff --git a/.gitlab/benchmarks/generate-run-json.sh b/.gitlab/benchmarks/generate-run-json.sh deleted file mode 100755 index 058d56278..000000000 --- a/.gitlab/benchmarks/generate-run-json.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash - -# generate-run-json.sh - Generate run JSON for benchmark tests -# -# Usage: generate-run-json.sh [reports-dir] -# -# Parses benchmark report files and outputs a JSON object -# suitable for update-history.sh. Reads CI environment variables for metadata. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="${SCRIPT_DIR}/../.." -REPORTS_DIR="${1:-${PROJECT_ROOT}/reports}" - -# Read metadata from environment or defaults -TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -PIPELINE_ID="${CI_PIPELINE_ID:-0}" -PIPELINE_URL="${CI_PIPELINE_URL:-#}" -DDPROF_BRANCH="${DDPROF_COMMIT_BRANCH:-main}" -DDPROF_SHA="${DDPROF_COMMIT_SHA:-unknown}" - -# Read version from environment or version.txt -LIB_VERSION="${CURRENT_VERSION:-unknown}" -if [ "${LIB_VERSION}" = "unknown" ] && [ -f "${PROJECT_ROOT}/version.txt" ]; then - LIB_VERSION=$(awk -F: '{print $NF}' "${PROJECT_ROOT}/version.txt" | tr -d ' ') -fi - -# Lookup PR for branch -PR_JSON="{}" -if [ -x "${SCRIPT_DIR}/../common/lookup-pr.sh" ]; then - PR_JSON=$("${SCRIPT_DIR}/../common/lookup-pr.sh" "${DDPROF_BRANCH}" 2>/dev/null) || PR_JSON="{}" -fi - -# Parse benchmark results -python3 <= 5: - break - except Exception: - pass - - results["architectures"] = sorted(architectures) - results["modes_tested"] = sorted(modes) - - return results - -# Analyze results -summary = analyze_benchmarks(reports_dir) - -# Determine overall status -if summary["regression_detected"]: - status = "failed" -elif summary["total_benchmarks"] > 0: - status = "passed" -else: - status = "unknown" - -# Build run JSON -run = { - "id": pipeline_id, - "timestamp": timestamp, - "ddprof_branch": ddprof_branch, - "ddprof_sha": ddprof_sha, - "ddprof_pr": pr_info if pr_info.get("number") else None, - "pipeline": { - "id": pipeline_id, - "url": pipeline_url - }, - "lib_version": lib_version, - "status": status, - "summary": summary -} - -# Output JSON -print(json.dumps(run, indent=2)) -EOF diff --git a/.gitlab/benchmarks/images.yml b/.gitlab/benchmarks/images.yml deleted file mode 100644 index e367cfe7c..000000000 --- a/.gitlab/benchmarks/images.yml +++ /dev/null @@ -1,11 +0,0 @@ -stages: - - images - -variables: - BASE_BENCHMARK_IMAGE_NAME: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest - BASE_CI_IMAGE_NAME: registry.ddbuild.io/ci/async-profiler-build - - # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090 - BENCHMARK_IMAGE_AMD64: registry.ddbuild.io/ci/async-profiler-build-amd64:v107978918-amd64-benchmarks@sha256:95d4e3719717a6af63ed62437b985e36a95b4552bac04a39cd7b82bafa09271d - # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090 - BENCHMARK_IMAGE_ARM64: registry.ddbuild.io/ci/async-profiler-build-arm64:v107978918-arm64-benchmarks@sha256:b534640f415e2c1af44611c1531a0435a387c85f435876fa2e7e5011e4749221 diff --git a/.gitlab/benchmarks/publish-gh-pages.sh b/.gitlab/benchmarks/publish-gh-pages.sh deleted file mode 100755 index 390fa98ac..000000000 --- a/.gitlab/benchmarks/publish-gh-pages.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/bin/bash - -# publish-gh-pages.sh - Publish benchmark reports to GitHub Pages -# -# Usage: publish-gh-pages.sh [reports-dir] -# -# Updates benchmark history and regenerates GitHub Pages site. -# Reports are available at: https://datadog.github.io/java-profiler/benchmarks/ -# -# In CI: Uses Octo-STS for secure, short-lived GitHub tokens (no secrets needed) -# Locally: Use 'devflow gitlab auth' for Octo-STS, or set GITHUB_TOKEN env var - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="${SCRIPT_DIR}/../.." -REPORTS_DIR="${1:-${PROJECT_ROOT}/reports}" -export MAX_HISTORY=10 - -# GitHub repo for Pages -GITHUB_REPO="DataDog/java-profiler" -PAGES_URL="https://datadog.github.io/java-profiler" - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } -log_error() { echo -e "${RED}[ERROR]${NC} $*"; } - -# Obtain GitHub token -obtain_github_token() { - # Try dd-octo-sts CLI (works in CI with DDOCTOSTS_ID_TOKEN) - if command -v dd-octo-sts >/dev/null 2>&1 && [ -n "${DDOCTOSTS_ID_TOKEN:-}" ]; then - log_info "Obtaining GitHub token via dd-octo-sts CLI..." - # Policy name matches the .sts.yaml filename (without extension) - - # Run dd-octo-sts and capture only stdout (don't capture stderr to avoid error messages in token) - local TOKEN_OUTPUT - local TOKEN_EXIT_CODE - TOKEN_OUTPUT=$(dd-octo-sts token --scope DataDog/java-profiler --policy async-profiler-build.ci 2>/tmp/dd-octo-sts-error.log) - TOKEN_EXIT_CODE=$? - - if [ $TOKEN_EXIT_CODE -eq 0 ] && [ -n "${TOKEN_OUTPUT}" ]; then - # Validate token format (GitHub tokens start with ghs_, ghp_, or look like JWT) - if [[ "${TOKEN_OUTPUT}" =~ ^(ghs_|ghp_|v1\.|[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.) ]]; then - GITHUB_TOKEN="${TOKEN_OUTPUT}" - log_info "GitHub token obtained via dd-octo-sts (expires in 1 hour)" - return 0 - else - log_warn "dd-octo-sts returned invalid token format (first 50 chars): ${TOKEN_OUTPUT:0:50}" - fi - else - log_warn "dd-octo-sts token exchange failed (exit code: ${TOKEN_EXIT_CODE})" - if [ -s /tmp/dd-octo-sts-error.log ]; then - log_warn "dd-octo-sts error output:" - cat /tmp/dd-octo-sts-error.log | head -10 >&2 - fi - fi - fi - - # Fall back to GITHUB_TOKEN environment variable - if [ -n "${GITHUB_TOKEN:-}" ]; then - log_info "Using GITHUB_TOKEN from environment" - return 0 - fi - - return 1 -} - -if ! obtain_github_token; then - log_error "Failed to obtain GitHub token" - log_error "Options:" - log_error " 1. Run in GitLab CI with dd-octo-sts-ci-base image and DDOCTOSTS_ID_TOKEN" - log_error " 2. Set GITHUB_TOKEN env var (PAT with 'repo' scope)" - exit 1 -fi - -# Create temporary directory for gh-pages content -WORK_DIR=$(mktemp -d) -RUN_JSON_FILE=$(mktemp) -# shellcheck disable=SC2064 # Intentional: capture values at setup time -trap "rm -rf ${WORK_DIR} ${RUN_JSON_FILE}" EXIT - -log_info "Preparing gh-pages content in: ${WORK_DIR}" - -# Clone gh-pages branch (or create if doesn't exist) -log_info "Cloning gh-pages branch..." -cd "${WORK_DIR}" - -if git clone --depth 1 --branch gh-pages "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" pages 2>/dev/null; then - cd pages - log_info "Cloned existing gh-pages branch" -else - log_info "Creating new gh-pages branch..." - mkdir pages && cd pages - git init - git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" - git checkout -b gh-pages -fi - -# Create Jekyll config if not exists -if [ ! -f "_config.yml" ]; then - cat > "_config.yml" < "${RUN_JSON_FILE}" 2>/dev/null; then - log_info "Generated run JSON" - - # Update history (prepend new run, keep last MAX_HISTORY) - if "${SCRIPT_DIR}/../common/update-history.sh" benchmarks "${RUN_JSON_FILE}" "." 2>/dev/null; then - log_info "Updated benchmark history" - else - log_warn "Failed to update history" - fi -else - log_warn "Failed to generate run JSON" -fi - -# Generate dashboard and index pages -log_info "Generating dashboard..." -if "${SCRIPT_DIR}/../common/generate-dashboard.sh" "." 2>&1; then - log_info "Generated dashboard index.md" -else - log_warn "Failed to generate dashboard" -fi - -log_info "Generating benchmark index..." -if "${SCRIPT_DIR}/../common/generate-index.sh" benchmarks "." 2>/dev/null; then - log_info "Generated benchmarks/index.md" -else - log_warn "Failed to generate benchmark index" -fi - -# Commit and push -TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC") -log_info "Committing changes..." -git add -A -if git diff --staged --quiet; then - log_info "No changes to commit" -else - git config user.email "ci@datadoghq.com" - git config user.name "CI Bot" - git commit -m "Update benchmark reports - ${TIMESTAMP}" - - log_info "Pushing to gh-pages..." - git push origin gh-pages --force - - log_info "Reports published successfully!" - log_info "View at: ${PAGES_URL}/benchmarks/" -fi - -echo "" -echo "PAGES_URL=${PAGES_URL}/benchmarks/" diff --git a/.gitlab/benchmarks/steps/mem_watch.sh b/.gitlab/benchmarks/steps/mem_watch.sh deleted file mode 100755 index 6b4e7e66d..000000000 --- a/.gitlab/benchmarks/steps/mem_watch.sh +++ /dev/null @@ -1,14 +0,0 @@ -#! /bin/bash -set +x -ctl_file=$1 -out=$2 -while [ -f $ctl_file ]; do - pid=$(ps ax | grep 'java' | grep "${ctl_file}" | grep -v 'grep' | grep -v 'time' | sed -e 's/^[[:space:]]*//' | cut -f1 -d' ') - if [ -n "${pid}" ]; then - rss="$(cat /proc/${pid}/smaps_rollup | grep 'Rss:' | cut -f2 -d':' | sed -e 's/^[[:space:]]*//' | cut -f1 -d' ' 2>/dev/null)" - if [ -n "$rss" ]; then - echo "mem: $rss" | tee -a $out - fi - fi - sleep 5 -done \ No newline at end of file diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml deleted file mode 100644 index 7845952d4..000000000 --- a/.gitlab/build-deploy/.gitlab-ci.yml +++ /dev/null @@ -1,370 +0,0 @@ -image: alpine - -variables: - # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090 - BUILD_IMAGE_X64: registry.ddbuild.io/ci/async-profiler-build:v107978918-x64-base@sha256:4712b562eb75cf27c5d494081f459fdbd7578ef46a99162e1051ef7d1f79da4f - # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090 - BUILD_IMAGE_ARM64: registry.ddbuild.io/ci/async-profiler-build:v107978918-arm64-base@sha256:15670752fa8e67c81c1b7fd4d93ef34c668649f117584ab29327d8d7733d64d6 - # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090 - BUILD_IMAGE_X64_2_17: registry.ddbuild.io/ci/async-profiler-build:v107978918-x64-2.17-base@sha256:03fb65c5c7fb2866397ceaf18ccb5a0b91bb377a9f89c94264427b62d48cfffd - # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090 - BUILD_IMAGE_X64_MUSL: registry.ddbuild.io/ci/async-profiler-build:v107978918-x64-musl-base@sha256:0efe0c5fbd04e5647a53235d4c4942a2dd305b3d76bf5afdc052d64a7c0b9505 - # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090 - BUILD_IMAGE_ARM64_MUSL: registry.ddbuild.io/ci/async-profiler-build:v107978918-arm64-musl-base@sha256:1c8a96ded178bfa130077166db049cbea422c94c96a8c5172d66f8c3581d57cf - # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/1600939090 - DATADOG_CI_IMAGE: registry.ddbuild.io/ci/async-profiler-build:v107978918-datadog-ci@sha256:1353737e433ef58b37ae34daf6edc0d6a5fc4706bac76ac97729e981869ea680 - - LAST_COMMIT_FILE: .last.commit - CACHE_FALLBACK_KEY: async-profiler - FORCE_BUILD: $FORCE_BUILD - - REGISTRY: registry.ddbuild.io - SONATYPE_USERNAME: robot-sonatype-apm-java - -.build_job: - extends: - - .retry-config - - .cache-config - stage: build - needs: - - job: prepare:start - artifacts: true - when: on_success - tags: [ "arch:amd64" ] - image: ${BUILD_IMAGE} - script: - - .gitlab/scripts/build.sh - artifacts: - when: always - paths: - - libs/$TARGET/libjavaProfiler.so - - libs/$TARGET/libjavaProfiler.so.debug - - test/$TARGET/reports - - test/$TARGET/logs - - "**/output.txt" - - "**/options.txt" - - "ddprof-test/${TARGET}/reports/tests/test/**/*" - expire_in: 1 day - -.stresstest_job: - extends: - - .retry-config - - .cache-config-pull - stage: stresstest - needs: - - job: prepare:start - artifacts: true - when: on_success - tags: [ "arch:amd64" ] - image: ${BUILD_IMAGE} - script: - - .gitlab/scripts/stresstests.sh - artifacts: - when: always - paths: - - stresstest/$TARGET/logs - - stresstest/$TARGET/results - expire_in: 2 weeks - -prepare:start: - extends: .retry-config - stage: prepare - rules: - - if: '$DOWNSTREAM != null' - when: never - - if: '$CI_PIPELINE_SOURCE == "pipeline"' - when: always - - when: always - interruptible: true - tags: [ "arch:arm64" ] - image: ${PREPARE_IMAGE} - - script: - - .gitlab/scripts/prepare.sh - - artifacts: - paths: - - version.txt - reports: - dotenv: build.env - expire_in: 1 day - -build:x64: - extends: .build_job - variables: - # default to the libc 2.17 linked version - BUILD_IMAGE: ${BUILD_IMAGE_X64_2_17} - TARGET: linux-x64 - -build:x64-musl: - extends: .build_job - variables: - BUILD_IMAGE: ${BUILD_IMAGE_X64_MUSL} - TARGET: linux-x64-musl - -build:arm64: - extends: .build_job - tags: [ "arch:arm64" ] - variables: - BUILD_IMAGE: ${BUILD_IMAGE_ARM64} - TARGET: linux-arm64 - -build:arm64-musl: - extends: .build_job - timeout: 3h - tags: [ "arch:arm64" ] - variables: - BUILD_IMAGE: ${BUILD_IMAGE_ARM64_MUSL} - TARGET: linux-arm64-musl - -stresstest:x64: - extends: .stresstest_job - needs: - - job: prepare:start - artifacts: true - - job: build:x64 - artifacts: true - variables: - # default to the libc 2.17 linked version - BUILD_IMAGE: ${BUILD_IMAGE_X64_2_17} - TARGET: linux-x64 - -stresstest:x64-musl: - extends: .stresstest_job - needs: - - job: prepare:start - artifacts: true - - job: build:x64-musl - artifacts: true - variables: - BUILD_IMAGE: ${BUILD_IMAGE_X64_MUSL} - TARGET: linux-x64-musl - -stresstest:arm64: - extends: .stresstest_job - needs: - - job: prepare:start - artifacts: true - - job: build:arm64 - artifacts: true - tags: [ "arch:arm64" ] - variables: - BUILD_IMAGE: ${BUILD_IMAGE_ARM64} - TARGET: linux-arm64 - -stresstest:arm64-musl: - extends: .stresstest_job - needs: - - job: prepare:start - artifacts: true - - job: build:arm64-musl - artifacts: true - timeout: 3h - tags: [ "arch:arm64" ] - variables: - BUILD_IMAGE: ${BUILD_IMAGE_ARM64_MUSL} - TARGET: linux-arm64-musl - -# Builds the chaos reliability harness once per pipeline so the per-cell -# reliability jobs can pull it as an artifact instead of compiling inline. -# Schedule-only because reliability cells are schedule-only. -chaos:build: - extends: - - .retry-config - - .cache-config-pull - stage: stresstest - needs: - - job: prepare:start - artifacts: false - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: always - - when: never - tags: [ "arch:amd64" ] - image: ${BUILD_IMAGE_X64_2_17} - script: - - | - if [ -z "${JAVA_HOME}" ] || [ ! -x "${JAVA_HOME}/bin/java" ]; then - export JAVA_HOME=~/.sdkman/candidates/java/current - fi - - ./gradlew :ddprof-stresstest:chaosJar -q - artifacts: - paths: - - ddprof-stresstest/build/libs/chaos.jar - expire_in: 1 day - -build-artifact: - extends: .cache-config-pull - stage: deploy - needs: - - job: prepare:start - artifacts: true - - job: build:x64 - artifacts: true - - job: build:x64-musl - artifacts: true - - job: build:arm64 - artifacts: true - - job: build:arm64-musl - artifacts: true - - job: gtest-asan-amd64 - artifacts: false - - job: gtest-tsan-amd64 - artifacts: false - optional: true - - job: gtest-asan-arm64 - artifacts: false - - job: gtest-tsan-arm64 - artifacts: false - optional: true - rules: - - if: '$CI_COMMIT_BRANCH =~ /^release\//' - when: never - - when: on_success - tags: [ "arch:amd64" ] - image: ${BUILD_IMAGE_X64} - - script: - - .gitlab/scripts/deploy.sh assemble - - artifacts: - name: java-profiler.zip - paths: - - ddprof-lib/build/libs/ddprof-*.jar - - ddprof-lib/build/classes/java/main/META-INF/native-libs/* - - version.txt - expire_in: 28 days - -deploy-artifact: - extends: - - .cache-config-pull - - .deploy-sa - stage: deploy - needs: - - job: prepare:start - artifacts: true - - job: build-artifact - artifacts: true - - job: build:x64 - artifacts: true - - job: build:x64-musl - artifacts: true - - job: build:arm64 - artifacts: true - - job: build:arm64-musl - artifacts: true - rules: - - if: '$CI_COMMIT_BRANCH =~ /^release\//' - when: never - - when: on_success - tags: [ "arch:amd64" ] - image: ${BUILD_IMAGE_X64} - - script: - - .gitlab/scripts/deploy.sh publish - - artifacts: - name: java-profiler-published.zip - paths: - - version.txt - expire_in: 7 days - - -upload-s3: - extends: .deploy-sa - stage: deploy - needs: - - job: prepare:start - artifacts: true - optional: true - - job: build:x64 - artifacts: true - - job: build:x64-musl - artifacts: true - - job: build:arm64 - artifacts: true - - job: build:arm64-musl - artifacts: true - tags: [ "arch:amd64" ] - image: ${DATADOG_CI_IMAGE} - allow_failure: true - interruptible: true - script: - - source .gitlab/config.env - - aws --version - - datadog-ci --version - - | - if [ ! -f "version.txt" ]; then - echo "WARNING: version.txt not found. This might be a manual run without prepare:start artifacts." - export LIB_VERSION="manual-${CI_COMMIT_SHORT_SHA:-unknown}" - else - export LIB_VERSION=$(cat version.txt | awk -F ':' '{print $3}') - fi - export S3_PREFIX_RELEASE="${S3_PREFIX}/release/${LIB_VERSION}" - echo "Using version: ${LIB_VERSION}" - - echo "=== Available library files ===" - - find libs/ -name "*.so*" -type f - - | - for lib_file in libs/*/libjavaProfiler.so; do - if [ -f "$lib_file" ]; then - platform=$(basename $(dirname "$lib_file")) - echo "Uploading library: $lib_file -> libjavaProfiler-${platform}.so" - .gitlab/scripts/upload.sh -p "${S3_PREFIX_RELEASE}" -f "$lib_file" -n "libjavaProfiler-${platform}.so" - - debug_file="${lib_file}.debug" - if [ -f "$debug_file" ]; then - echo "Uploading debug file: $debug_file -> libjavaProfiler-${platform}.debug" - .gitlab/scripts/upload.sh -p "${S3_PREFIX_RELEASE}" -f "$debug_file" -n "libjavaProfiler-${platform}.debug" - fi - fi - done - - echo "=== Uploading ELF symbols to Datadog ===" - - set +x - - export DATADOG_API_KEY_PROD=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.api_key_public_symbols_prod_us1 --with-decryption --query "Parameter.Value" --out text) - - export DATADOG_API_KEY_STAGING=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.api_key_public_symbols_staging --with-decryption --query "Parameter.Value" --out text) - - | - if [ -n "${DATADOG_API_KEY_STAGING:-}" ]; then - DATADOG_API_KEY=$DATADOG_API_KEY_STAGING DATADOG_SITE=datad0g.com DD_BETA_COMMANDS_ENABLED=1 datadog-ci elf-symbols upload --disable-git ./libs - fi - if [ -n "${DATADOG_API_KEY_PROD:-}" ]; then - DATADOG_API_KEY=$DATADOG_API_KEY_PROD DATADOG_SITE=datadoghq.com DD_BETA_COMMANDS_ENABLED=1 datadog-ci elf-symbols upload --disable-git ./libs - fi - - set -x - rules: - - if: '($CI_PIPELINE_SOURCE == "trigger" || $CI_PIPELINE_SOURCE == "pipeline") && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: on_success - - if: '$CI_PIPELINE_SOURCE == "web" && $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH' - when: on_success - - when: manual - -notify-slack-on-success: - extends: .deploy-sa - stage: notify - needs: - - job: prepare:start - artifacts: true - - job: deploy-artifact - artifacts: false - when: on_success - image: registry.ddbuild.io/slack-notifier:latest - tags: ["arch:amd64"] - script: - - .gitlab/build-deploy/notify_channel.sh success $(cat version.txt) - -notify-slack-on-failure: - extends: .deploy-sa - stage: notify - needs: - - job: prepare:start - artifacts: true - - job: deploy-artifact - artifacts: true - when: on_failure - image: registry.ddbuild.io/slack-notifier:latest - tags: ["arch:amd64"] - script: - - .gitlab/build-deploy/notify_channel.sh alert $(cat version.txt) - -include: - - local: .gitlab/common.yml - - local: .gitlab/dd-trace-integration/.gitlab-ci.yml diff --git a/.gitlab/build-deploy/images.yml b/.gitlab/build-deploy/images.yml deleted file mode 100644 index 5bbf7ad76..000000000 --- a/.gitlab/build-deploy/images.yml +++ /dev/null @@ -1,15 +0,0 @@ -stages: - - images -variables: - # Base images for the build docker images - # debian:bullseye-slim = Debian 11 (glibc 2.31); pinned by digest for reproducibility. - # Multi-arch manifest covers both amd64 and arm64 — no separate arm64 tag needed. - # JDK 21 is installed at image-build time via SDKMAN (see .gitlab/base/Dockerfile). - OPENJDK_BASE_IMAGE: debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b - OPENJDK_BASE_IMAGE_ARM64: debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b - # eclipse-temurin:21-jdk-alpine = Alpine 3.x musl; arm64 manifest confirmed present. - OPENJDK_BASE_IMAGE_MUSL: eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f - OPENJDK_BASE_IMAGE_ARM64_MUSL: eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f - BASE_IMAGE_LIBC_2_17: centos:7 - - DOCKER_IMAGE: 486234852809.dkr.ecr.us-east-1.amazonaws.com/images/docker:24.0.4-gbi-focal diff --git a/.gitlab/build-deploy/notify_channel.sh b/.gitlab/build-deploy/notify_channel.sh deleted file mode 100755 index f1e8da051..000000000 --- a/.gitlab/build-deploy/notify_channel.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -if [ ! -z "${CANCELLED:-}" ]; then - exit 0 -fi - -# Source centralized configuration -HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source "${HERE}/../../.gitlab/config.env" - -COMMIT_SHA=${CI_COMMIT_SHA} -PIPELINE_URL="$CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID" -PIPELINE_LINK="<$PIPELINE_URL|pipeline #$CI_PIPELINE_ID>" - -COMMIT_URL="https://github.com/DataDog/java-profiler/commit/$COMMIT_SHA" -COMMIT_LINK="<$COMMIT_URL|${COMMIT_SHA:0:8}>" - -BRANCH="${CI_COMMIT_BRANCH:-${CI_COMMIT_REF_NAME:-unknown}}" -BRANCH_URL="https://github.com/DataDog/java-profiler/tree/$BRANCH" -BRANCH_LINK="<$BRANCH_URL|$BRANCH>" - -# get status from argument -STATUS=$1 -VERSION=$2 - -if [[ $STATUS == "success" ]]; then - if [[ ! $VERSION =~ .*?-SNAPSHOT ]]; then - MESSAGE_TEXT=":tada: Release succeeded for ${VERSION} (branch=$BRANCH_LINK, commit=$COMMIT_LINK, pipeline=$PIPELINE_LINK) - -:point_up: Now you can publish the artifacts from " - else - MESSAGE_TEXT=":done: Build succeeded for ${VERSION} (branch=$BRANCH_LINK, commit=$COMMIT_LINK, pipeline=$PIPELINE_LINK)" - fi -else - MESSAGE_TEXT=":better-siren: Build failed for ${VERSION} (branch=$BRANCH_LINK, commit=$COMMIT_LINK, pipeline=$PIPELINE_LINK)" -fi - -postmessage "$SLACK_CHANNEL" "$MESSAGE_TEXT" "$STATUS" diff --git a/.gitlab/common.yml b/.gitlab/common.yml deleted file mode 100644 index 292a1462d..000000000 --- a/.gitlab/common.yml +++ /dev/null @@ -1,85 +0,0 @@ -# Common GitLab CI templates and configurations -# Include this file in pipeline configurations to reuse common patterns - -# Retry configuration for transient failures -.retry-config: - retry: - max: 2 - when: - - unmet_prerequisites - - runner_system_failure - - data_integrity_failure - - api_failure - - scheduler_failure - - archived_failure - - stale_schedule - - unknown_failure - -# Gradle/Maven cache configuration (push+pull) -.cache-config: - cache: - key: - files: - - gradle/wrapper/gradle-wrapper.properties - prefix: build-v2-${CI_COMMIT_REF_SLUG} - fallback_keys: - - build-v2-${CI_DEFAULT_BRANCH} - - build-v2-main - paths: - - .gradle/caches/ - - .gradle/wrapper/ - - .m2/repository/ - -# Read-only variant — extends base config and overrides policy -.cache-config-pull: - extends: .cache-config - cache: - policy: pull - -# Service account for jobs that publish artifacts or read SSM secrets -.deploy-sa: - variables: - KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: java-profiler - -# Install gh and crane when not already present in the image. -# Extend this in before_script for jobs that need GitHub CLI or crane. -.bootstrap-gh-tools: - before_script: - - | - mkdir -p /tmp/bootstrap-bin - if ! command -v crane >/dev/null 2>&1; then - curl -fsSL "https://github.com/google/go-containerregistry/releases/download/v0.19.1/go-containerregistry_Linux_x86_64.tar.gz" \ - | tar -xz -C /tmp/bootstrap-bin crane - fi - if ! command -v gh >/dev/null 2>&1; then - curl -fsSL "https://github.com/cli/cli/releases/download/v2.45.0/gh_2.45.0_linux_amd64.tar.gz" \ - | tar -xz -C /tmp/bootstrap-bin --strip-components=2 'gh_2.45.0_linux_amd64/bin/gh' - fi - export PATH="/tmp/bootstrap-bin:$PATH" - -# Common job to determine versions (runs from the CI checkout) -.get-versions: - stage: prepare - rules: - - if: '$DOWNSTREAM == null' - when: always - - when: never - interruptible: true - tags: [ "arch:amd64" ] - image: ${PREPARE_IMAGE} - script: - - set -exo pipefail - - source .gitlab/scripts/includes.sh - - curl -s "https://get.sdkman.io" | bash - - source "$HOME/.sdkman/bin/sdkman-init.sh" - - sdk install java 21.0.3-tem - - CURRENT_VERSION=$(get_current_version) - - '[ -n "$CURRENT_VERSION" ] || { echo "FAIL: get_current_version returned empty"; exit 1; }' - - PREVIOUS_VERSION=$(get_previous_version) - - '[ -n "$PREVIOUS_VERSION" ] || { echo "FAIL: get_previous_version returned empty"; exit 1; }' - - echo "CURRENT_VERSION=$CURRENT_VERSION" > build.env - - echo "PREVIOUS_VERSION=$PREVIOUS_VERSION" >> build.env - artifacts: - reports: - dotenv: build.env - expire_in: 1 day diff --git a/.gitlab/common/generate-dashboard.sh b/.gitlab/common/generate-dashboard.sh deleted file mode 100755 index 1affc0af6..000000000 --- a/.gitlab/common/generate-dashboard.sh +++ /dev/null @@ -1,184 +0,0 @@ -#!/bin/bash - -# generate-dashboard.sh - Generate main index.md dashboard from history data -# -# Usage: generate-dashboard.sh [work-dir] -# -# Reads _data/{integration,benchmarks,reliability}.json and generates index.md -# with quick status table and recent runs across all test types. -# -# Pure bash/jq implementation - no Python required - -set -euo pipefail - -WORK_DIR="${1:-.}" -TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC") - -# Check if jq is available -if ! command -v jq >/dev/null 2>&1; then - echo "Error: jq is required for JSON parsing" >&2 - exit 1 -fi - -# Helper functions -status_emoji() { - case "$1" in - passed) echo "✅" ;; - failed) echo "❌" ;; - partial) echo "⚠️" ;; - *) echo "❓" ;; - esac -} - -format_pr_link() { - local pr_json="$1" - if [ "$pr_json" != "null" ] && [ -n "$pr_json" ]; then - local number=$(echo "$pr_json" | jq -r '.number // empty') - local url=$(echo "$pr_json" | jq -r '.url // empty') - if [ -n "$number" ]; then - echo "[#${number}](${url})" - return - fi - fi - echo "-" -} - -format_pipeline_link() { - local pipeline_json="$1" - if [ "$pipeline_json" != "null" ]; then - local id=$(echo "$pipeline_json" | jq -r '.id // empty') - local url=$(echo "$pipeline_json" | jq -r '.url // "#"') - if [ -n "$id" ]; then - echo "[#${id}](${url})" - return - fi - fi - echo "-" -} - -# Generate the dashboard markdown -{ -cat < **Last Updated:** ${TIMESTAMP} - -## Quick Status - -| Test Type | Latest | Status | Branch | PR | -|-----------|--------|--------|--------|-----| -EOF_HEADER - -# Quick status for each test type -for test_type in integration benchmarks reliability; do - # Capitalize first letter - display_name="$(echo "${test_type:0:1}" | tr '[:lower:]' '[:upper:]')${test_type:1}" - history_file="${WORK_DIR}/_data/${test_type}.json" - - if [ -f "$history_file" ] && [ -s "$history_file" ]; then - # Get latest run (first in array) - latest=$(jq -r '.runs[0] // empty' "$history_file" 2>/dev/null) - - if [ -n "$latest" ]; then - status=$(echo "$latest" | jq -r '.status // "unknown"') - status_symbol=$(status_emoji "$status") - branch=$(echo "$latest" | jq -r '.ddprof_branch // "unknown"') - pr_info=$(echo "$latest" | jq -c '.ddprof_pr // null') - pipeline=$(echo "$latest" | jq -c '.pipeline // null') - - pr_link=$(format_pr_link "$pr_info") - pipeline_link=$(format_pipeline_link "$pipeline") - - echo "| [$display_name]($test_type/) | $pipeline_link | $status_symbol | $branch | $pr_link |" - else - echo "| [$display_name]($test_type/) | - | - | - | - |" - fi - else - echo "| [$display_name]($test_type/) | - | - | - | - |" - fi -done - -cat <<'EOF_TEST_TYPES' - ---- - -## Test Types - -### Integration Tests -dd-trace-java compatibility tests verifying profiler works correctly with the Datadog tracer. -Tests run on every main branch build across multiple JDK versions and platforms. - -### Benchmarks -Performance regression testing using Renaissance benchmark suite. -Compares profiler overhead against baseline (no profiling). - -### Reliability Tests -Long-running stability tests checking for memory leaks and crashes. -Tests multiple allocator configurations (gmalloc, tcmalloc, jemalloc). - ---- - -## Recent Runs (All Types) - -| Date | Type | Pipeline | Branch | PR | Status | -|------|------|----------|--------|-----|--------| -EOF_TEST_TYPES - -# Collect all runs from all types (last 5 from each) into a single JSON array -tmpfile=$(mktemp) -trap "rm -f $tmpfile" EXIT - -# Build array by merging runs from all test types -all_runs="[]" -for test_type in integration benchmarks reliability; do - history_file="${WORK_DIR}/_data/${test_type}.json" - - if [ -f "$history_file" ] && [ -s "$history_file" ]; then - # Get last 5 runs, add type field, merge into all_runs - runs=$(jq --arg type "$test_type" \ - '.runs[:5] | map(. + {_type: $type})' \ - "$history_file" 2>/dev/null || echo "[]") - - all_runs=$(echo "$all_runs" "$runs" | jq -s 'add') - fi -done - -echo "$all_runs" > "$tmpfile" - -# Sort by timestamp descending, take 15 most recent -if [ "$(jq 'length' "$tmpfile" 2>/dev/null || echo 0)" -gt 0 ]; then - jq -c 'sort_by(.timestamp) | reverse | .[:15] | .[]' "$tmpfile" 2>/dev/null | \ - while IFS= read -r run; do - date=$(echo "$run" | jq -r '(.timestamp // "")[:10]') - type_val=$(echo "$run" | jq -r '._type // "unknown"') - type_name="$(echo "${type_val:0:1}" | tr '[:lower:]' '[:upper:]')${type_val:1}" - status=$(echo "$run" | jq -r '.status // "unknown"') - status_symbol=$(status_emoji "$status") - branch=$(echo "$run" | jq -r '.ddprof_branch // "unknown"') - pr_info=$(echo "$run" | jq -c '.ddprof_pr // null') - pipeline=$(echo "$run" | jq -c '.pipeline // null') - - pr_link=$(format_pr_link "$pr_info") - pipeline_link=$(format_pipeline_link "$pipeline") - - echo "| $date | $type_name | $pipeline_link | $branch | $pr_link | $status_symbol |" - done -else - echo "| - | - | - | - | - | - |" -fi - -cat <<'EOF_FOOTER' - ---- - -[Repository](https://github.com/DataDog/java-profiler) | [java-profiler](https://github.com/DataDog/java-profiler) | [View history](https://github.com/DataDog/java-profiler/commits/gh-pages) -EOF_FOOTER - -} > "${WORK_DIR}/index.md" - -echo "Generated ${WORK_DIR}/index.md" diff --git a/.gitlab/common/generate-index.sh b/.gitlab/common/generate-index.sh deleted file mode 100755 index 64e8496e8..000000000 --- a/.gitlab/common/generate-index.sh +++ /dev/null @@ -1,243 +0,0 @@ -#!/bin/bash - -# generate-index.sh - Generate test type index page with expandable details -# -# Usage: generate-index.sh [work-dir] -# -# test-type: integration|benchmarks|reliability -# Reads _data/{test-type}.json and generates {test-type}/index.md -# -# Pure bash/jq implementation - no Python required - -set -euo pipefail - -TEST_TYPE="${1:-}" -WORK_DIR="${2:-.}" - -if [ -z "${TEST_TYPE}" ]; then - echo "Usage: generate-index.sh [work-dir]" >&2 - exit 1 -fi - -# Check if jq is available -if ! command -v jq >/dev/null 2>&1; then - echo "Error: jq is required for JSON parsing" >&2 - exit 1 -fi - -# Ensure output directory exists -mkdir -p "${WORK_DIR}/${TEST_TYPE}" - -# Helper functions -status_emoji() { - case "$1" in - passed) echo "✅" ;; - failed) echo "❌" ;; - partial) echo "⚠️" ;; - *) echo "❓" ;; - esac -} - -format_pr_link() { - local pr_json="$1" - if [ "$pr_json" != "null" ] && [ -n "$pr_json" ]; then - local number=$(echo "$pr_json" | jq -r '.number // empty') - local url=$(echo "$pr_json" | jq -r '.url // empty') - if [ -n "$number" ]; then - echo "[#${number}](${url})" - return - fi - fi - echo "-" -} - -format_pipeline_link() { - local pipeline_json="$1" - if [ "$pipeline_json" != "null" ]; then - local id=$(echo "$pipeline_json" | jq -r '.id // empty') - local url=$(echo "$pipeline_json" | jq -r '.url // "#"') - if [ -n "$id" ]; then - echo "[#${id}](${url})" - return - fi - fi - echo "-" -} - -# Test type metadata -case "$TEST_TYPE" in - integration) - TITLE="DD-Trace Integration Test History" - DESCRIPTION="Tests dd-trace-java compatibility with ddprof across multiple JDK versions and platforms." - ;; - benchmarks) - TITLE="Benchmark Test History" - DESCRIPTION="Performance regression testing using Renaissance benchmark suite." - ;; - reliability) - TITLE="Reliability Test History" - DESCRIPTION="Long-running stability tests for memory leaks and crashes." - ;; - *) - # Capitalize first letter - cap_type="$(echo "${TEST_TYPE:0:1}" | tr '[:lower:]' '[:upper:]')${TEST_TYPE:1}" - TITLE="${cap_type} Test History" - DESCRIPTION="" - ;; -esac - -# Generate the index page -{ -cat </dev/null || echo "") - - if [ -z "$runs" ]; then - echo "*No test runs recorded yet.*" - else - echo "$runs" | while IFS= read -r run; do - timestamp=$(echo "$run" | jq -r '.timestamp // ""') - date_str="${timestamp:0:16}" - date_str="${date_str/T/ }" - [ -z "$date_str" ] && date_str="Unknown" - - status=$(echo "$run" | jq -r '.status // "unknown"') - status_symbol=$(status_emoji "$status") - branch=$(echo "$run" | jq -r '.ddprof_branch // "unknown"') - version=$(echo "$run" | jq -r '.lib_version // "unknown"') - sha=$(echo "$run" | jq -r '.ddprof_sha // "unknown"') - sha="${sha:0:8}" - - pr_info=$(echo "$run" | jq -c '.ddprof_pr // null') - pipeline=$(echo "$run" | jq -c '.pipeline // null') - - pr_link=$(format_pr_link "$pr_info") - pipeline_link=$(format_pipeline_link "$pipeline") - - # PR text for summary line - pr_text="" - if [ "$pr_info" != "null" ]; then - pr_num=$(echo "$pr_info" | jq -r '.number // empty') - if [ -n "$pr_num" ]; then - pr_text=" | PR $pr_link" - fi - fi - - # Print expandable details - cat < - -${date_str} | ${status_symbol} | ${branch}${pr_text} | Pipeline ${pipeline_link} - - -**Version:** ${version} -**Commit:** ${sha} - -EOF_RUN - - # Summary table based on test type - summary=$(echo "$run" | jq -c '.summary // {}') - - case "$TEST_TYPE" in - integration) - total_jobs=$(echo "$summary" | jq -r '.total_jobs // "N/A"') - passed_jobs=$(echo "$summary" | jq -r '.passed_jobs // "N/A"') - failed_jobs=$(echo "$summary" | jq -r '.failed_jobs // "N/A"') - - echo "| Metric | Value |" - echo "|--------|-------|" - echo "| Jobs | $total_jobs |" - echo "| Passed | $passed_jobs |" - echo "| Failed | $failed_jobs |" - - # Failed configs - failed_configs=$(echo "$summary" | jq -r '.failed_configs // [] | join(", ")') - if [ -n "$failed_configs" ] && [ "$failed_configs" != "" ]; then - echo "" - echo "**Failed Configs:** $failed_configs" - fi - ;; - - benchmarks) - architectures=$(echo "$summary" | jq -r '.architectures // "N/A"') - modes_tested=$(echo "$summary" | jq -r '.modes_tested // "N/A"') - regression=$(echo "$summary" | jq -r '.regression_detected // false') - regression_text=$([ "$regression" = "true" ] && echo "Yes" || echo "No") - - echo "| Metric | Value |" - echo "|--------|-------|" - echo "| Architectures | $architectures |" - echo "| Modes | $modes_tested |" - echo "| Regression | $regression_text |" - - # Regression details - regression_details=$(echo "$summary" | jq -r '.regression_details // [] | .[:5] | .[]' 2>/dev/null) - if [ -n "$regression_details" ]; then - echo "" - echo "**Regressions:**" - echo "$regression_details" | while read -r detail; do - echo "- $detail" - done - fi - ;; - - reliability) - total_configs=$(echo "$summary" | jq -r '.total_configs // "N/A"') - passed=$(echo "$summary" | jq -r '.passed // "N/A"') - failed=$(echo "$summary" | jq -r '.failed // "N/A"') - - echo "| Metric | Value |" - echo "|--------|-------|" - echo "| Configs | $total_configs |" - echo "| Passed | $passed |" - echo "| Failed | $failed |" - - # Failures - failures=$(echo "$summary" | jq -r '.failures // [] | .[:5] | .[]' 2>/dev/null) - if [ -n "$failures" ]; then - echo "" - echo "**Failures:**" - echo "$failures" | while read -r failure; do - echo "- $failure" - done - fi - ;; - esac - - echo "" - echo "" - echo "" - done - fi -else - echo "*No test runs recorded yet.*" -fi - -cat <<'EOF_FOOTER' - ---- - -[← Back to Dashboard](../) | [View git history](https://github.com/DataDog/java-profiler/commits/gh-pages) -EOF_FOOTER - -} > "${WORK_DIR}/${TEST_TYPE}/index.md" - -echo "Generated ${WORK_DIR}/${TEST_TYPE}/index.md" diff --git a/.gitlab/common/lookup-pr.sh b/.gitlab/common/lookup-pr.sh deleted file mode 100755 index 92e04a722..000000000 --- a/.gitlab/common/lookup-pr.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/bin/bash - -# lookup-pr.sh - Find GitHub PR for a java-profiler branch -# -# Usage: lookup-pr.sh -# Output: JSON with PR info {"number": N, "url": "...", "title": "..."} or {} -# -# Authentication: -# - In CI: Uses Octo-STS with java-profiler-build-read policy -# - Fallback: GITHUB_TOKEN env var or unauthenticated (public repo) -# -# Falls back gracefully if API fails or no PR found. - -set -euo pipefail - -BRANCH="${1:-}" -REPO="DataDog/java-profiler" - -# Debug logging to stderr (doesn't affect JSON output on stdout) -debug() { echo "[DEBUG] $*" >&2; } - -debug "Looking for PR in ${REPO} with head branch: ${BRANCH}" - -# Skip lookup for main/master branches (never have PRs) -if [ -z "${BRANCH}" ] || [ "${BRANCH}" = "main" ] || [ "${BRANCH}" = "master" ]; then - debug "Skipping lookup for main/master branch" - echo "{}" - exit 0 -fi - -# Obtain GitHub token via dd-octo-sts if available -GITHUB_TOKEN="" -if command -v dd-octo-sts >/dev/null 2>&1 && [ -n "${DDOCTOSTS_ID_TOKEN:-}" ]; then - debug "Attempting to get token via Octo-STS..." - TOKEN_OUTPUT=$(dd-octo-sts token --scope DataDog/java-profiler --policy async-profiler-build.ci 2>/tmp/dd-octo-sts-lookup-error.log) - TOKEN_EXIT_CODE=$? - - if [ $TOKEN_EXIT_CODE -eq 0 ] && [ -n "${TOKEN_OUTPUT}" ]; then - GITHUB_TOKEN="${TOKEN_OUTPUT}" - debug "Got GitHub token via Octo-STS" - else - debug "Failed to get token via Octo-STS (exit code: ${TOKEN_EXIT_CODE})" - if [ -s /tmp/dd-octo-sts-lookup-error.log ]; then - debug "dd-octo-sts error: $(cat /tmp/dd-octo-sts-lookup-error.log | head -5)" - fi - fi -else - debug "Octo-STS not available (dd-octo-sts: $(command -v dd-octo-sts 2>/dev/null || echo 'not found'), DDOCTOSTS_ID_TOKEN: ${DDOCTOSTS_ID_TOKEN:+set})" -fi - -# URL-encode the branch name (/ -> %2F, etc.) -# Use jq if available, otherwise use sed for common cases -url_encode() { - local string="$1" - if command -v jq >/dev/null 2>&1; then - printf '%s' "$string" | jq -sRr @uri - else - # Fallback: encode common special characters with sed - printf '%s' "$string" | sed 's|/|%2F|g; s| |%20|g; s|#|%23|g' - fi -} - -ENCODED_BRANCH=$(url_encode "${BRANCH}") -API_URL="https://api.github.com/repos/${REPO}/pulls?head=DataDog:${ENCODED_BRANCH}&state=all&per_page=1" -debug "API URL: ${API_URL}" - -if [ -n "${GITHUB_TOKEN}" ]; then - debug "Using authenticated request" - response=$(curl -s --max-time 10 \ - -H "Authorization: token ${GITHUB_TOKEN}" \ - -H "Accept: application/vnd.github+json" \ - "${API_URL}" 2>/dev/null) || { - debug "curl failed" - echo "{}" - exit 0 - } -else - # Anonymous access for public repos (60 req/hour limit) - debug "Using anonymous request (may be rate limited)" - response=$(curl -s --max-time 10 \ - -H "Accept: application/vnd.github+json" \ - "${API_URL}" 2>/dev/null) || { - debug "curl failed" - echo "{}" - exit 0 - } -fi - -debug "API response length: ${#response} chars" -debug "API response preview: ${response:0:200}" - -# Parse JSON response - use jq if available -if command -v jq >/dev/null 2>&1; then - # Check if response is a non-empty array - if ! echo "${response}" | jq -e 'if type == "array" and length > 0 then true else false end' >/dev/null 2>&1; then - debug "Response is not a valid JSON array with items (might be error response or empty)" - echo "{}" - exit 0 - fi - - debug "Found valid PR response" - - # Extract PR info - echo "${response}" | jq -c '.[0] | {number: .number, url: .html_url, title: .title}' -else - # Fallback: use grep/sed for basic JSON extraction (less reliable but works without jq) - debug "jq not available, using fallback JSON parsing" - - # Check if it looks like a non-empty array - if ! echo "${response}" | grep -q '^\[{'; then - debug "Response doesn't look like a JSON array with items" - echo "{}" - exit 0 - fi - - debug "Found valid PR response (basic check)" - - # Extract fields using grep/sed (fragile but works for simple cases) - pr_number=$(echo "${response}" | grep -o '"number":[0-9]*' | head -1 | sed 's/"number"://') - pr_url=$(echo "${response}" | grep -o '"html_url":"[^"]*"' | head -1 | sed 's/"html_url":"//; s/"$//') - pr_title=$(echo "${response}" | grep -o '"title":"[^"]*"' | head -1 | sed 's/"title":"//; s/"$//') - - if [ -n "${pr_number}" ]; then - echo "{\"number\":${pr_number},\"url\":\"${pr_url}\",\"title\":\"${pr_title}\"}" - else - echo "{}" - fi -fi diff --git a/.gitlab/common/setup-publish-env.sh b/.gitlab/common/setup-publish-env.sh deleted file mode 100755 index 256863b10..000000000 --- a/.gitlab/common/setup-publish-env.sh +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/bash - -# setup-publish-env.sh - Install dependencies for GitHub Pages publishing -# -# This script auto-detects the package manager and installs required tools - -set -euo pipefail - -echo "=== Setting up publishing environment ===" -echo "Current user: $(whoami), UID: $(id -u)" -echo "Image: ${CI_JOB_IMAGE:-unknown}" - -# Check if Python3 is already available -if command -v python3 >/dev/null 2>&1; then - echo "Python3 already available: $(python3 --version)" - echo "Skipping installation" - exit 0 -fi - -echo "Python3 not found, attempting installation..." - -# Check if we need sudo -SUDO="" -if [ "$(id -u)" -ne 0 ]; then - if command -v sudo >/dev/null 2>&1; then - echo "Running as non-root user, will use sudo" - SUDO="sudo" - else - echo "ERROR: Running as non-root but sudo not available" - echo "Current user: $(whoami), UID: $(id -u)" - exit 1 - fi -fi - -# Detect and use appropriate package manager -if command -v apt-get >/dev/null 2>&1; then - echo "Detected: apt-get (Debian/Ubuntu)" - $SUDO apt-get update -qq - $SUDO apt-get install -y python3 git curl jq -elif command -v apk >/dev/null 2>&1; then - echo "Detected: apk (Alpine)" - $SUDO apk add --no-cache python3 git curl jq -elif command -v yum >/dev/null 2>&1; then - echo "Detected: yum (RHEL/CentOS)" - $SUDO yum install -y python3 git curl jq -elif command -v dnf >/dev/null 2>&1; then - echo "Detected: dnf (Fedora/RHEL 8+)" - $SUDO dnf install -y python3 git curl jq -else - echo "ERROR: No supported package manager found (tried apt-get, apk, yum, dnf)" - echo "Available commands:" - command -v apt-get apk yum dnf 2>&1 || echo "None found" - exit 1 -fi - -echo "" -echo "=== Verifying installations ===" -echo -n "Python3: " -python3 --version || (echo "FAILED" && exit 1) - -echo -n "Git: " -git --version || echo "WARNING: git not found" - -echo -n "curl: " -curl --version | head -1 || echo "WARNING: curl not found" - -echo -n "jq: " -jq --version || echo "WARNING: jq not found" - -echo -n "dd-octo-sts: " -dd-octo-sts version || echo "WARNING: dd-octo-sts not available or failed to run" - -echo "" -echo "=== Environment ready for publishing ===" diff --git a/.gitlab/common/update-history.sh b/.gitlab/common/update-history.sh deleted file mode 100755 index 0b5b9d4ef..000000000 --- a/.gitlab/common/update-history.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -# update-history.sh - Update JSON history file with new run, keeping last N entries -# -# Usage: update-history.sh [work-dir] -# -# test-type: integration|benchmarks|reliability -# new-run-json-file: Path to file containing new run JSON -# work-dir: Directory containing _data/ (default: current directory) -# -# Updates _data/{test-type}.json, prepending new run and keeping last MAX_HISTORY entries. -# -# Pure bash/jq implementation - no Python required - -set -euo pipefail - -TEST_TYPE="${1:-}" -NEW_RUN_FILE="${2:-}" -WORK_DIR="${3:-.}" -MAX_HISTORY="${MAX_HISTORY:-10}" - -if [ -z "${TEST_TYPE}" ] || [ -z "${NEW_RUN_FILE}" ]; then - echo "Usage: update-history.sh [work-dir]" >&2 - exit 1 -fi - -if [ ! -f "${NEW_RUN_FILE}" ]; then - echo "Error: New run JSON file not found: ${NEW_RUN_FILE}" >&2 - exit 1 -fi - -HISTORY_FILE="${WORK_DIR}/_data/${TEST_TYPE}.json" - -# Ensure _data directory exists -mkdir -p "${WORK_DIR}/_data" - -# Initialize history file if it doesn't exist -if [ ! -f "${HISTORY_FILE}" ]; then - echo '{"runs":[]}' > "${HISTORY_FILE}" -fi - -# Check if jq is available -if ! command -v jq >/dev/null 2>&1; then - echo "Error: jq is required for JSON manipulation" >&2 - echo "Install with: apt-get install jq (Debian/Ubuntu) or apk add jq (Alpine)" >&2 - exit 1 -fi - -# Read new run JSON -NEW_RUN_JSON=$(cat "${NEW_RUN_FILE}") - -# Validate JSON -if ! echo "$NEW_RUN_JSON" | jq empty 2>/dev/null; then - echo "Error: Invalid JSON in ${NEW_RUN_FILE}" >&2 - exit 1 -fi - -# Update history using jq: -# 1. Read existing history (or start with empty {"runs":[]}) -# 2. Parse new run from variable -# 3. Prepend new run to .runs array -# 4. Keep only last MAX_HISTORY entries -jq --argjson newrun "$NEW_RUN_JSON" \ - --argjson maxhistory "$MAX_HISTORY" \ - '.runs = ([$newrun] + .runs) | .runs = .runs[:$maxhistory]' \ - "${HISTORY_FILE}" > "${HISTORY_FILE}.tmp" - -# Atomic move -mv "${HISTORY_FILE}.tmp" "${HISTORY_FILE}" - -# Report success -RUN_COUNT=$(jq '.runs | length' "${HISTORY_FILE}") -echo "Updated ${HISTORY_FILE}: now has ${RUN_COUNT} run(s)" diff --git a/.gitlab/config.env b/.gitlab/config.env deleted file mode 100644 index 508032d97..000000000 --- a/.gitlab/config.env +++ /dev/null @@ -1,41 +0,0 @@ -# CI Configuration - Centralized configuration for all build scripts -# This file is sourced by scripts to provide consistent configuration across the pipeline - -# Java Versions -# Build version used for compiling the library -JAVA_BUILD_VERSION=21 -# Test version used for reliability tests -JAVA_TEST_VERSION=21.0.3-tem -# Benchmark version used for performance testing -JAVA_BENCHMARK_VERSION=21 - -# Docker Base Images -# debian:bullseye-slim = Debian 11 (glibc 2.31); pinned by digest for reproducibility. -# Multi-arch manifests cover both amd64 and arm64. -# JDK 21 is installed at image-build time via SDKMAN (see .gitlab/base/Dockerfile). -OPENJDK_BASE_IMAGE_AMD64=debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b -OPENJDK_BASE_IMAGE_ARM64=debian:bullseye-slim@sha256:0083feb8da4f624e3a0245e7752af2517d4b81d8b8db50c725644672a132a31b -# eclipse-temurin:21-jdk-alpine = Alpine 3.x musl; arm64 manifest confirmed present. -OPENJDK_BASE_IMAGE_AMD64_MUSL=eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f -OPENJDK_BASE_IMAGE_ARM64_MUSL=eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f - -# AWS Configuration -AWS_REGION=us-east-1 -SSM_PREFIX=ci.java-profiler -S3_BUCKET=binaries.ddbuild.io -S3_PREFIX=async-profiler-build - -# Slack Configuration -SLACK_CHANNEL="#java-profiler-lib" - -# Build Configuration -DEFAULT_BENCHMARK_ITERATIONS=1 -DEFAULT_BENCHMARK_MODES="cpu,wall,alloc,memleak" - -# Git Configuration -# JAVA_PROFILER_REPO is no longer used; CI runs inside the repo - -# dd-trace-java Integration Test Configuration -DD_TRACE_JAVA_REPO=https://github.com/DataDog/dd-trace-java.git -DD_TRACE_JAVA_BRANCH=master -INTEGRATION_TEST_TIMEOUT=30 diff --git a/.gitlab/dd-trace-integration/.gitlab-ci.yml b/.gitlab/dd-trace-integration/.gitlab-ci.yml deleted file mode 100644 index 22aa991b8..000000000 --- a/.gitlab/dd-trace-integration/.gitlab-ci.yml +++ /dev/null @@ -1,308 +0,0 @@ -# dd-trace-java Integration Tests -# Tests the java-profiler ddprof-lib artifact with patched dd-java-agent - -variables: - DD_TRACE_VERSION: - value: "" - description: "dd-java-agent snapshot version to download (empty = auto-detect latest)" - -prepare-patched-agent: - extends: .retry-config - stage: integration-test - tags: [ "arch:amd64" ] - image: ${BUILD_IMAGE_X64} - needs: - - job: prepare:start - artifacts: true - - job: build-artifact - artifacts: true - rules: - - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - - when: on_success - interruptible: true - timeout: 10m - script: - - | - echo "=== Preparing Patched dd-java-agent ===" - - echo "Using ddprof.jar from build-artifact job" - echo "Available artifacts:" - ls -lh ddprof-lib/build/libs/ || echo "Directory not found" - - LIB_VERSION=$(awk -F ':' '{print $3}' version.txt) - DDPROF_JAR="ddprof-lib/build/libs/ddprof-lib-${LIB_VERSION}.jar" - if [ ! -f "${DDPROF_JAR}" ]; then - echo "ERROR: ddprof JAR not found at ${DDPROF_JAR} (version=${LIB_VERSION})" - ls -lh ddprof-lib/build/libs/ || true - exit 1 - fi - - echo "Found: ${DDPROF_JAR}" - cp "${DDPROF_JAR}" ddprof.jar - ls -lh ddprof.jar - - if [ -n "${DD_TRACE_VERSION}" ]; then - .gitlab/dd-trace-integration/download-snapshot-artifacts.sh \ - --dd-trace-version "${DD_TRACE_VERSION}" \ - --skip-ddprof - else - .gitlab/dd-trace-integration/download-snapshot-artifacts.sh \ - --skip-ddprof - fi - - .gitlab/dd-trace-integration/patch-dd-java-agent.sh - - DD_AGENT_JAR=dd-java-agent-original.jar \ - DDPROF_JAR=ddprof.jar \ - PATCHED_JAR=dd-java-agent-patched.jar \ - .gitlab/dd-trace-integration/verify-patch-compatibility.sh | tee compatibility-check.log - - DDPROF_SHA=$(git rev-parse HEAD) - echo "${DDPROF_SHA}" > ddprof-commit-sha.txt - echo "Captured ddprof SHA: ${DDPROF_SHA}" - - echo "" - echo "=== Patched agent prepared successfully ===" - artifacts: - name: patched-agent - paths: - - dd-java-agent-patched.jar - - dd-java-agent-original.jar - - ddprof.jar - - ddprof-commit-sha.txt - - compatibility-check.log - expire_in: 1 day - -.integration_test_base: - extends: .retry-config - stage: integration-test - needs: - - job: prepare:start - artifacts: true - - job: build-artifact - artifacts: true - - job: prepare-patched-agent - artifacts: true - rules: - - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - - when: on_success - interruptible: true - timeout: 10m - allow_failure: true - script: - - | - echo "=== Integration Test ===" - echo "Platform: ${ARCH}-${LIBC_VARIANT}" - echo "JVM: ${JVM_TYPE} JDK${JAVA_VERSION}" - echo "" - - if command -v apt-get &> /dev/null; then - apt-get update -qq && apt-get install -y -qq curl wget unzip bc ca-certificates - elif command -v apk &> /dev/null; then - apk add --no-cache curl wget unzip bc bash ca-certificates - fi - - JDK_INSTALLED=false - if command -v apt-get &> /dev/null; then - if apt-get install -y -qq openjdk-${JAVA_VERSION}-jdk 2>/dev/null || apt-get install -y -qq openjdk-${JAVA_VERSION}-jre-headless 2>/dev/null; then - JDK_INSTALLED=true - fi - elif command -v apk &> /dev/null; then - JDK_INSTALLED=false - fi - - if [ "${JDK_INSTALLED}" = false ]; then - ARCH_NAME=$(uname -m) - case "${ARCH_NAME}" in - x86_64) JDK_ARCH="amd64" ;; - aarch64) JDK_ARCH="aarch64" ;; - *) echo "ERROR: Unsupported architecture: ${ARCH_NAME}"; exit 1 ;; - esac - - if command -v apk &> /dev/null; then - JDK_LIBC="musl" - else - JDK_LIBC="glibc" - fi - - JDK_DIR="/opt/java/jdk-${JAVA_VERSION}" - mkdir -p /opt/java - - if [ "${JDK_LIBC}" = "musl" ]; then - case "${JAVA_VERSION}" in - 8) LIBERICA_VERSION="8u482+10" ;; - 11) LIBERICA_VERSION="11.0.30+9" ;; - 17) LIBERICA_VERSION="17.0.18+10" ;; - 21) LIBERICA_VERSION="21.0.10+10" ;; - 25) LIBERICA_VERSION="25.0.2+12" ;; - *) echo "ERROR: Unsupported JDK version: ${JAVA_VERSION}"; exit 1 ;; - esac - case "${JDK_ARCH}" in - amd64) LIBERICA_ARCH="x64" ;; - *) LIBERICA_ARCH="${JDK_ARCH}" ;; - esac - DOWNLOAD_URL="https://download.bell-sw.com/java/${LIBERICA_VERSION}/bellsoft-jdk${LIBERICA_VERSION}-linux-${LIBERICA_ARCH}-musl-lite.tar.gz" - else - case "${JDK_ARCH}" in - amd64) ADOPTIUM_ARCH="x64" ;; - *) ADOPTIUM_ARCH="${JDK_ARCH}" ;; - esac - DOWNLOAD_URL="https://api.adoptium.net/v3/binary/latest/${JAVA_VERSION}/ga/linux/${ADOPTIUM_ARCH}/jdk/hotspot/normal/eclipse" - fi - - MAX_RETRIES=3 - for attempt in $(seq 1 $MAX_RETRIES); do - if curl -Lk --retry 3 --retry-delay 5 "${DOWNLOAD_URL}" | tar xzf - -C /opt/java; then - break - fi - [ $attempt -lt $MAX_RETRIES ] && sleep 10 || exit 1 - done - - EXTRACTED_DIR=$(find /opt/java -maxdepth 1 -type d \( -name "jdk-*" -o -name "jdk8u*" \) ! -name "jdk-${JAVA_VERSION}" | head -1) - [ -z "${EXTRACTED_DIR}" ] && exit 1 - ln -sf "${EXTRACTED_DIR}" "${JDK_DIR}" - export JAVA_HOME="${JDK_DIR}" - else - export JAVA_HOME=$(find /usr/lib/jvm -name "java-${JAVA_VERSION}-*" -type d | head -1) - [ -z "${JAVA_HOME}" ] && export JAVA_HOME=$(find /usr/lib/jvm -name "openjdk-${JAVA_VERSION}" -type d | head -1) - [ -z "${JAVA_HOME}" ] && export JAVA_HOME=$(find /usr/lib/jvm -name "java-1.${JAVA_VERSION}*" -type d | head -1) - [ -z "${JAVA_HOME}" ] && export JAVA_HOME=$(find /usr/lib/jvm -maxdepth 1 -type d | grep -E "(jdk|java|openjdk)" | head -1) - [ -z "${JAVA_HOME}" ] && exit 1 - fi - - echo "Using JAVA_HOME: ${JAVA_HOME}" - ${JAVA_HOME}/bin/java -version - - .gitlab/dd-trace-integration/install-prerequisites.sh - .gitlab/dd-trace-integration/run-integration-test.sh - - echo "" - echo "=== All tests completed successfully ===" - artifacts: - when: always - name: "integration-test-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}" - paths: - - integration-test-results/${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}/ - expire_in: 7 days - -integration-test-x64-glibc: - extends: .integration_test_base - tags: [ "arch:amd64" ] - image: ${BUILD_IMAGE_X64} - parallel: - matrix: - - JVM_TYPE: hotspot - JAVA_VERSION: [8, 11, 17, 21, 25] - - JVM_TYPE: openj9 - JAVA_VERSION: [8, 11, 17, 21, 25] - variables: - LIBC_VARIANT: glibc - ARCH: x64 - -integration-test-x64-musl: - extends: .integration_test_base - tags: [ "arch:amd64" ] - image: ${BUILD_IMAGE_X64_MUSL} - parallel: - matrix: - - JVM_TYPE: hotspot - JAVA_VERSION: [8, 11, 17, 21, 25] - - JVM_TYPE: openj9 - JAVA_VERSION: [8, 11, 17, 21, 25] - variables: - LIBC_VARIANT: musl - ARCH: x64 - -integration-test-arm64-glibc: - extends: .integration_test_base - tags: [ "arch:arm64" ] - image: ${BUILD_IMAGE_ARM64} - parallel: - matrix: - - JVM_TYPE: hotspot - JAVA_VERSION: [8, 11, 17, 21, 25] - - JVM_TYPE: openj9 - JAVA_VERSION: [8, 11, 17, 21, 25] - variables: - LIBC_VARIANT: glibc - ARCH: arm64 - -integration-test-arm64-musl: - extends: .integration_test_base - tags: [ "arch:arm64" ] - image: ${BUILD_IMAGE_ARM64_MUSL} - parallel: - matrix: - - JVM_TYPE: hotspot - JAVA_VERSION: [8, 11, 17, 21, 25] - - JVM_TYPE: openj9 - JAVA_VERSION: [8, 11, 17, 21, 25] - variables: - LIBC_VARIANT: musl - ARCH: arm64 - -report-dd-trace-results: - extends: .retry-config - stage: integration-test - resource_group: gh-pages-publish - tags: [ "arch:arm64" ] - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - needs: - - job: prepare-patched-agent - artifacts: true - - job: integration-test-x64-glibc - artifacts: true - - job: integration-test-x64-musl - artifacts: true - - job: integration-test-arm64-glibc - artifacts: true - - job: integration-test-arm64-musl - artifacts: true - rules: - - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - - when: on_success - interruptible: true - timeout: 10m - script: - - ./.gitlab/dd-trace-integration/publish-gh-pages.sh - allow_failure: true - -post-pr-comment: - extends: .retry-config - stage: integration-test - tags: [ "arch:arm64" ] - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - needs: - - job: prepare-patched-agent - artifacts: true - - job: integration-test-x64-glibc - artifacts: true - - job: integration-test-x64-musl - artifacts: true - - job: integration-test-arm64-glibc - artifacts: true - - job: integration-test-arm64-musl - artifacts: true - rules: - - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - - when: on_success - interruptible: true - timeout: 5m - script: - - .gitlab/dd-trace-integration/post-pr-comment.sh integration-test-results - allow_failure: true diff --git a/.gitlab/dd-trace-integration/download-snapshot-artifacts.sh b/.gitlab/dd-trace-integration/download-snapshot-artifacts.sh deleted file mode 100755 index 81f2c69e1..000000000 --- a/.gitlab/dd-trace-integration/download-snapshot-artifacts.sh +++ /dev/null @@ -1,337 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Download dd-java-agent and ddprof snapshot artifacts from Maven - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -# Maven snapshot repository -SNAPSHOT_REPO="https://central.sonatype.com/repository/maven-snapshots/" - -# Default versions (can be overridden via environment) -DEFAULT_DD_TRACE_VERSION="1.50.0-SNAPSHOT" - -# Output directory for downloaded artifacts -OUTPUT_DIR="${OUTPUT_DIR:-${PROJECT_ROOT}}" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' # No Color - -function log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -function log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -function log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} - -function detect_latest_snapshot_version() { - local group_id=$1 - local artifact_id=$2 - local repo_url=$3 - - # Convert group_id to path (com.datadoghq -> com/datadoghq) - local group_path=$(echo "${group_id}" | tr '.' '/') - - # Construct metadata URL - local metadata_url="${repo_url}${group_path}/${artifact_id}/maven-metadata.xml" - - # Log to stderr so it doesn't interfere with return value - echo -e "${GREEN}[INFO]${NC} Querying for latest ${artifact_id} version from metadata" >&2 - - # Fetch metadata - local metadata=$(curl -fsSL "${metadata_url}" 2>/dev/null || echo "") - - if [ -z "${metadata}" ]; then - echo -e "${RED}[ERROR]${NC} Could not fetch metadata from ${metadata_url}" >&2 - return 1 - fi - - # Extract latest version using sed (portable across macOS and Linux) - local latest_version=$(echo "${metadata}" | sed -n 's/.*\(.*\)<\/latest>.*/\1/p') - - # Validate that latest version is vanilla (no branch name) - # Vanilla patterns: X.Y.Z-SNAPSHOT or X.Y.Z-DD-SNAPSHOT - if [ -n "${latest_version}" ]; then - if ! echo "${latest_version}" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+(-DD)?-SNAPSHOT$'; then - echo -e "${YELLOW}[WARN]${NC} Latest version '${latest_version}' contains branch name, searching for vanilla snapshot..." >&2 - latest_version="" - fi - fi - - if [ -z "${latest_version}" ]; then - # Fallback: try to find latest from versions list - # Extract all versions, filter for vanilla SNAPSHOT (no branch names) - # Vanilla patterns: X.Y.Z-SNAPSHOT or X.Y.Z-DD-SNAPSHOT - # Reject patterns with branch names: X.Y.Z-branch_name-SNAPSHOT - latest_version=$(echo "${metadata}" | \ - sed -n 's/.*\(.*\)<\/version>.*/\1/p' | \ - grep -E '^[0-9]+\.[0-9]+\.[0-9]+(-DD)?-SNAPSHOT$' | \ - sort -V | \ - tail -1 || echo "") - fi - - if [ -z "${latest_version}" ]; then - echo -e "${RED}[ERROR]${NC} Could not detect latest version for ${artifact_id}" >&2 - return 1 - fi - - echo -e "${GREEN}[INFO]${NC} Detected latest version: ${latest_version}" >&2 - echo "${latest_version}" -} - -function usage() { - cat << EOF -Usage: $0 [OPTIONS] - -Download dd-java-agent and ddprof snapshot artifacts from Maven. - -OPTIONS: - --dd-trace-version dd-java-agent version (default: auto-detect from Maven) - --ddprof-version ddprof version (auto-detected from build.env, CURRENT_VERSION, or Maven) - --output-dir Output directory (default: ${OUTPUT_DIR}) - --skip-ddprof Skip ddprof download (use when ddprof.jar already available) - --auto-detect Force auto-detection even if versions are set - --help Show this help message - -ENVIRONMENT VARIABLES: - DD_TRACE_VERSION Override dd-java-agent version - DDPROF_VERSION Override ddprof version - CURRENT_VERSION Auto-detected ddprof version from CI (build.env) - OUTPUT_DIR Output directory for artifacts - -OUTPUTS: - dd-java-agent-original.jar Downloaded dd-java-agent artifact - ddprof.jar Downloaded ddprof artifact - -NOTES: - - If no version is specified, the script will attempt to auto-detect - the latest SNAPSHOT version from the Maven repository - - Auto-detection queries maven-metadata.xml from OSSRH - - For CI/production use, explicit versions are recommended - -EXAMPLES: - # Download with auto-detected versions - $0 - - # Force auto-detection (ignore environment variables) - $0 --auto-detect - - # Download specific versions - $0 --dd-trace-version 1.49.0-SNAPSHOT --ddprof-version 1.35.0-DD-SNAPSHOT - - # Download to specific directory - $0 --output-dir /tmp/artifacts -EOF -} - -# Parse command line arguments -AUTO_DETECT=false -SKIP_DDPROF=false -DD_TRACE_VERSION_ARG="" -DDPROF_VERSION_ARG="" -while [ $# -gt 0 ]; do - case "$1" in - --dd-trace-version) - DD_TRACE_VERSION_ARG="$2" - shift 2 - ;; - --ddprof-version) - DDPROF_VERSION_ARG="$2" - shift 2 - ;; - --output-dir) - OUTPUT_DIR="$2" - shift 2 - ;; - --skip-ddprof) - SKIP_DDPROF=true - shift - ;; - --auto-detect) - AUTO_DETECT=true - shift - ;; - --help) - usage - exit 0 - ;; - *) - log_error "Unknown option: $1" - usage - exit 1 - ;; - esac -done - -# Determine dd-trace-java version -if [ "${AUTO_DETECT}" = "true" ]; then - log_info "Auto-detect flag set, detecting latest dd-java-agent version..." - if ! DD_TRACE_VERSION=$(detect_latest_snapshot_version "com.datadoghq" "dd-java-agent" "${SNAPSHOT_REPO}"); then - log_warn "Auto-detection failed, using default: ${DEFAULT_DD_TRACE_VERSION}" - DD_TRACE_VERSION="${DEFAULT_DD_TRACE_VERSION}" - fi -elif [ -n "${DD_TRACE_VERSION_ARG}" ]; then - DD_TRACE_VERSION="${DD_TRACE_VERSION_ARG}" -elif [ -n "${DD_TRACE_VERSION:-}" ]; then - DD_TRACE_VERSION="${DD_TRACE_VERSION}" -else - # Auto-detect latest - log_info "No dd-java-agent version specified, detecting latest..." - if ! DD_TRACE_VERSION=$(detect_latest_snapshot_version "com.datadoghq" "dd-java-agent" "${SNAPSHOT_REPO}"); then - log_warn "Auto-detection failed, using default: ${DEFAULT_DD_TRACE_VERSION}" - DD_TRACE_VERSION="${DEFAULT_DD_TRACE_VERSION}" - fi -fi - -# Determine ddprof version (skip if --skip-ddprof flag is set) -if [ "${SKIP_DDPROF}" = "false" ]; then - if [ "${AUTO_DETECT}" = "true" ]; then - log_info "Auto-detect flag set, detecting latest ddprof version..." - if ! DDPROF_VERSION=$(detect_latest_snapshot_version "com.datadoghq" "ddprof" "${SNAPSHOT_REPO}"); then - log_error "Could not determine ddprof version (auto-detection failed)" - exit 1 - fi - elif [ -n "${DDPROF_VERSION_ARG}" ]; then - DDPROF_VERSION="${DDPROF_VERSION_ARG}" - elif [ -n "${DDPROF_VERSION:-}" ]; then - DDPROF_VERSION="${DDPROF_VERSION}" - elif [ -n "${CURRENT_VERSION:-}" ]; then - DDPROF_VERSION="${CURRENT_VERSION}" - elif [ -f "${PROJECT_ROOT}/build.env" ]; then - # Try to source build.env to get CURRENT_VERSION - log_info "Detecting ddprof version from build.env" - # shellcheck disable=SC1091 - source "${PROJECT_ROOT}/build.env" 2>/dev/null || true - DDPROF_VERSION="${CURRENT_VERSION:-}" - fi - - if [ -z "${DDPROF_VERSION:-}" ]; then - # Auto-detect latest - log_info "No ddprof version specified, detecting latest..." - if ! DDPROF_VERSION=$(detect_latest_snapshot_version "com.datadoghq" "ddprof" "${SNAPSHOT_REPO}"); then - log_error "Could not determine ddprof version (auto-detection failed and no fallback available)" - exit 1 - fi - fi -else - log_info "Skipping ddprof download (--skip-ddprof flag set)" -fi - -# Validate Maven is available -if ! command -v mvn &> /dev/null; then - log_error "Maven (mvn) is not installed or not in PATH" - log_error "Please install Maven to download artifacts" - exit 1 -fi - -# Create output directory -mkdir -p "${OUTPUT_DIR}" - -log_info "Downloading artifacts to: ${OUTPUT_DIR}" -log_info "dd-java-agent version: ${DD_TRACE_VERSION}" -if [ "${SKIP_DDPROF}" = "false" ]; then - log_info "ddprof version: ${DDPROF_VERSION}" -else - log_info "ddprof: skipping (using existing artifact)" -fi - -# Run Maven from a temp dir to avoid picking up the Gradle project's pom.xml (or a stale empty one) -MVN_WORK_DIR=$(mktemp -d) -trap "rm -rf ${MVN_WORK_DIR}" EXIT - -# Download dd-java-agent -log_info "Downloading dd-java-agent:${DD_TRACE_VERSION}..." -if (cd "${MVN_WORK_DIR}" && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ - -DrepoUrl="${SNAPSHOT_REPO}" \ - -Dartifact="com.datadoghq:dd-java-agent:${DD_TRACE_VERSION}" \ - -q > /tmp/mvn-dd-trace.log 2>&1); then - log_info "Successfully downloaded dd-java-agent" -else - log_error "Failed to download dd-java-agent" - log_error "Maven output:" - cat /tmp/mvn-dd-trace.log - exit 1 -fi - -# Download ddprof (skip if --skip-ddprof flag is set) -if [ "${SKIP_DDPROF}" = "false" ]; then - log_info "Downloading ddprof:${DDPROF_VERSION}..." - if (cd "${MVN_WORK_DIR}" && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ - -DrepoUrl="${SNAPSHOT_REPO}" \ - -Dartifact="com.datadoghq:ddprof:${DDPROF_VERSION}" \ - -q > /tmp/mvn-ddprof.log 2>&1); then - log_info "Successfully downloaded ddprof" - else - log_error "Failed to download ddprof" - log_error "Maven output:" - cat /tmp/mvn-ddprof.log - exit 1 - fi -fi - -# Determine Maven local repository path (default: ~/.m2/repository) -MVN_REPO="${HOME}/.m2/repository" -if [ -n "${MAVEN_CONFIG:-}" ]; then - # Check if custom Maven settings specify different local repo - log_warn "Custom MAVEN_CONFIG detected, using default ~/.m2/repository" -fi - -# Copy artifacts to output directory -DD_AGENT_JAR="${MVN_REPO}/com/datadoghq/dd-java-agent/${DD_TRACE_VERSION}/dd-java-agent-${DD_TRACE_VERSION}.jar" - -if [ ! -f "${DD_AGENT_JAR}" ]; then - log_error "dd-java-agent JAR not found at: ${DD_AGENT_JAR}" - exit 1 -fi - -log_info "Copying dd-java-agent to ${OUTPUT_DIR}/dd-java-agent-original.jar" -cp "${DD_AGENT_JAR}" "${OUTPUT_DIR}/dd-java-agent-original.jar" - -if [ "${SKIP_DDPROF}" = "false" ]; then - DDPROF_JAR="${MVN_REPO}/com/datadoghq/ddprof/${DDPROF_VERSION}/ddprof-${DDPROF_VERSION}.jar" - - if [ ! -f "${DDPROF_JAR}" ]; then - log_error "ddprof JAR not found at: ${DDPROF_JAR}" - exit 1 - fi - - log_info "Copying ddprof to ${OUTPUT_DIR}/ddprof.jar" - cp "${DDPROF_JAR}" "${OUTPUT_DIR}/ddprof.jar" -fi - -# Validate JAR integrity -log_info "Validating JAR integrity..." - -if unzip -t "${OUTPUT_DIR}/dd-java-agent-original.jar" > /dev/null 2>&1; then - log_info "✓ dd-java-agent-original.jar is valid" -else - log_error "✗ dd-java-agent-original.jar is corrupted" - exit 1 -fi - -if [ "${SKIP_DDPROF}" = "false" ]; then - if unzip -t "${OUTPUT_DIR}/ddprof.jar" > /dev/null 2>&1; then - log_info "✓ ddprof.jar is valid" - else - log_error "✗ ddprof.jar is corrupted" - exit 1 - fi -fi - -# Print summary -log_info "Download complete!" -log_info "Artifacts:" -log_info " dd-java-agent: $(du -h "${OUTPUT_DIR}/dd-java-agent-original.jar" | cut -f1)" -if [ "${SKIP_DDPROF}" = "false" ]; then - log_info " ddprof: $(du -h "${OUTPUT_DIR}/ddprof.jar" | cut -f1)" -fi diff --git a/.gitlab/dd-trace-integration/generate-report.sh b/.gitlab/dd-trace-integration/generate-report.sh deleted file mode 100755 index 816f3670b..000000000 --- a/.gitlab/dd-trace-integration/generate-report.sh +++ /dev/null @@ -1,199 +0,0 @@ -#!/bin/bash - -# generate-report.sh - Generate full integration test report in Markdown -# -# Usage: generate-report.sh [output-file] -# -# Generates a comprehensive Markdown report including: -# - Test configuration -# - Both scenario results (profiler-only, tracer+profiler) -# - System diagnostics (CPU, throttling) -# - Health scores and sample rates - -set -euo pipefail - -RESULTS_DIR="${1:-}" -OUTPUT_FILE="${2:-}" - -if [ -z "${RESULTS_DIR}" ]; then - echo "Usage: $0 [output-file]" >&2 - exit 1 -fi - -if [ ! -d "${RESULTS_DIR}" ]; then - echo "Error: Results directory not found: ${RESULTS_DIR}" >&2 - exit 1 -fi - -# Extract config from directory name (e.g., glibc-x64-hotspot-jdk17) -CONFIG_NAME=$(basename "${RESULTS_DIR}") -IFS='-' read -r LIBC ARCH JVM_TYPE JDK_VERSION <<< "${CONFIG_NAME}" - -# Get timestamp -TIMESTAMP=$(date -Iseconds 2>/dev/null || date +%Y-%m-%dT%H:%M:%S%z) -DATE_HUMAN=$(date "+%Y-%m-%d %H:%M:%S %Z" 2>/dev/null || date) - -# Parse diagnostic data -DIAGNOSTICS_DIR="${RESULTS_DIR}/diagnostics" -CPU_START="N/A" -CPU_END="N/A" -THROTTLE_PCT="N/A" -CONTAINER="N/A" - -if [ -d "${DIAGNOSTICS_DIR}" ]; then - if [ -f "${DIAGNOSTICS_DIR}/system-metrics-start.json" ]; then - CPU_START=$(grep '"cpu_count"' "${DIAGNOSTICS_DIR}/system-metrics-start.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") - CONTAINER=$(grep '"container"' "${DIAGNOSTICS_DIR}/system-metrics-start.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") - fi - if [ -f "${DIAGNOSTICS_DIR}/system-metrics-end.json" ]; then - CPU_END=$(grep '"cpu_count"' "${DIAGNOSTICS_DIR}/system-metrics-end.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") - THROTTLE_PCT=$(grep '"percentage"' "${DIAGNOSTICS_DIR}/system-metrics-end.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") - elif [ -f "${DIAGNOSTICS_DIR}/system-metrics-mid.json" ]; then - CPU_END=$(grep '"cpu_count"' "${DIAGNOSTICS_DIR}/system-metrics-mid.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") - THROTTLE_PCT=$(grep '"percentage"' "${DIAGNOSTICS_DIR}/system-metrics-mid.json" | awk -F': ' '{print $2}' | tr -d ',' || echo "N/A") - fi -fi - -# Parse scenario 1 (profiler-only) results -S1_STATUS="N/A" -S1_SAMPLES="N/A" -S1_THREADS="N/A" -S1_ALLOC="N/A" -S1_LOG="${RESULTS_DIR}/profiler-only-${CONFIG_NAME}.log" - -if [ -f "${S1_LOG}" ]; then - if grep -q "SUCCESS: All validations passed" "${S1_LOG}"; then - S1_STATUS="PASS" - elif grep -q "VALIDATION_FAILED" "${S1_LOG}"; then - S1_STATUS="FAIL" - fi - S1_SAMPLES=$(grep "ExecutionSample:" "${S1_LOG}" | grep -oE '[0-9]+ events' | awk '{print $1}' | head -1 || echo "N/A") - S1_THREADS=$(grep "Thread diversity:" "${S1_LOG}" | grep -oE '[0-9]+ threads' | awk '{print $1}' | head -1 || echo "N/A") - S1_ALLOC=$(grep "Allocation samples:" "${S1_LOG}" | grep -oE '[0-9]+ events' | awk '{print $1}' | head -1 || echo "N/A") -fi - -# Parse scenario 2 (tracer+profiler) results -S2_STATUS="N/A" -S2_SAMPLES="N/A" -S2_THREADS="N/A" -S2_ALLOC="N/A" -S2_LOG="${RESULTS_DIR}/tracer-profiler-${CONFIG_NAME}.log" - -if [ -f "${S2_LOG}" ]; then - if grep -q "SUCCESS: All validations passed" "${S2_LOG}"; then - S2_STATUS="PASS" - elif grep -q "VALIDATION_FAILED" "${S2_LOG}"; then - S2_STATUS="FAIL" - fi - S2_SAMPLES=$(grep "ExecutionSample:" "${S2_LOG}" | grep -oE '[0-9]+ events' | awk '{print $1}' | head -1 || echo "N/A") - S2_THREADS=$(grep "Thread diversity:" "${S2_LOG}" | grep -oE '[0-9]+ threads' | awk '{print $1}' | head -1 || echo "N/A") - S2_ALLOC=$(grep "Allocation samples:" "${S2_LOG}" | grep -oE '[0-9]+ events' | awk '{print $1}' | head -1 || echo "N/A") -fi - -# Calculate overall status -OVERALL_STATUS="PASS" -if [ "${S1_STATUS}" = "FAIL" ] || [ "${S2_STATUS}" = "FAIL" ]; then - OVERALL_STATUS="FAIL" -fi - -# Calculate health scores (samples/sec compared to expected 1.6/sec baseline) -TEST_DURATION=60 # Default test duration (see run-integration-test.sh) -S1_RATE="N/A" -S1_HEALTH="N/A" -S2_RATE="N/A" -S2_HEALTH="N/A" - -if [ "${S1_SAMPLES}" != "N/A" ] && [ -n "${S1_SAMPLES}" ]; then - S1_RATE=$(awk "BEGIN {printf \"%.2f\", ${S1_SAMPLES} / ${TEST_DURATION}}") - S1_HEALTH=$(awk "BEGIN {printf \"%.0f\", (${S1_RATE} / 1.6) * 100}") -fi - -if [ "${S2_SAMPLES}" != "N/A" ] && [ -n "${S2_SAMPLES}" ]; then - S2_RATE=$(awk "BEGIN {printf \"%.2f\", ${S2_SAMPLES} / ${TEST_DURATION}}") - S2_HEALTH=$(awk "BEGIN {printf \"%.0f\", (${S2_RATE} / 1.6) * 100}") -fi - -# Status emoji -status_emoji() { - case "$1" in - PASS) echo "✅" ;; - FAIL) echo "❌" ;; - *) echo "⚠️" ;; - esac -} - -# Generate Markdown report -generate_report() { - cat < -CPU Timeline (${cpu_values} unique values: ${cpu_min}-${cpu_max} cores) - -\`\`\` -$(head -20 "${DIAGNOSTICS_DIR}/cpu-timeline.log") -\`\`\` - - -EOF - fi - - echo "---" - echo "" -} - -# Output report -if [ -n "${OUTPUT_FILE}" ]; then - generate_report > "${OUTPUT_FILE}" - echo "Report generated: ${OUTPUT_FILE}" -else - generate_report -fi diff --git a/.gitlab/dd-trace-integration/generate-run-json.sh b/.gitlab/dd-trace-integration/generate-run-json.sh deleted file mode 100755 index c7f7b2280..000000000 --- a/.gitlab/dd-trace-integration/generate-run-json.sh +++ /dev/null @@ -1,223 +0,0 @@ -#!/bin/bash - -# generate-run-json.sh - Generate run JSON for integration tests -# -# Usage: generate-run-json.sh [results-dir] [--verbose] -# -# Parses all test configuration directories and outputs a JSON object -# suitable for update-history.sh. Reads CI environment variables for metadata. -# -# Pure bash implementation - no Python required - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="${SCRIPT_DIR}/../.." - -# Parse arguments -VERBOSE=false -RESULTS_BASE="" -for arg in "$@"; do - case "$arg" in - --verbose) - VERBOSE=true - ;; - *) - RESULTS_BASE="$arg" - ;; - esac -done - -# Default results base if not provided -RESULTS_BASE="${RESULTS_BASE:-${PROJECT_ROOT}/integration-test-results}" - -# Read metadata from environment or defaults -TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -PIPELINE_ID="${CI_PIPELINE_ID:-0}" -PIPELINE_URL="${CI_PIPELINE_URL:-#}" -DDPROF_BRANCH="${DDPROF_COMMIT_BRANCH:-main}" -DDPROF_SHA="${DDPROF_COMMIT_SHA:-$(cat "${PROJECT_ROOT}/ddprof-commit-sha.txt" 2>/dev/null || echo unknown)}" - -# Read version from version.txt if available -LIB_VERSION="unknown" -if [ -f "${PROJECT_ROOT}/version.txt" ]; then - LIB_VERSION=$(awk -F: '{print $NF}' "${PROJECT_ROOT}/version.txt" | tr -d ' ') -fi - -# Lookup PR for branch -PR_JSON="{}" -if [ -x "${SCRIPT_DIR}/../common/lookup-pr.sh" ]; then - PR_JSON=$("${SCRIPT_DIR}/../common/lookup-pr.sh" "${DDPROF_BRANCH}" 2>/dev/null) || PR_JSON="{}" -fi - -# Parse validation log for status -parse_validation_log() { - local log_path="$1" - - if [ ! -f "$log_path" ]; then - [ "$VERBOSE" = "true" ] && echo "[DEBUG] Log not found: $log_path" >&2 - echo "missing" - return - fi - - if grep -q "SUCCESS: All validations passed" "$log_path" 2>/dev/null; then - [ "$VERBOSE" = "true" ] && echo "[DEBUG] ✓ PASSED: $log_path" >&2 - echo "passed" - elif grep -q "VALIDATION_FAILED" "$log_path" 2>/dev/null; then - [ "$VERBOSE" = "true" ] && echo "[DEBUG] ✗ FAILED: $log_path" >&2 - echo "failed" - else - # Log exists but has no status markers - this is the problem! - echo "[WARN] Log missing markers: $log_path" >&2 - if [ "$VERBOSE" = "true" ]; then - echo "[DEBUG] Log preview (first 10 lines):" >&2 - head -10 "$log_path" >&2 - echo "[DEBUG] Log preview (last 10 lines):" >&2 - tail -10 "$log_path" >&2 - fi - echo "unknown" - fi -} - -# Initialize counters -total_jobs=0 -passed_jobs=0 -failed_jobs=0 -total_scenarios=0 -passed_scenarios=0 -failed_scenarios=0 -unknown_scenarios=0 -failed_configs="" - -# Parse test results if directory exists -if [ -d "${RESULTS_BASE}" ]; then - for config_dir in "${RESULTS_BASE}"/*; do - [ -d "$config_dir" ] || continue - - config_name=$(basename "$config_dir") - [ "$VERBOSE" = "true" ] && echo "[DEBUG] Processing config: $config_name" >&2 - - # Skip history directory - [ "$config_name" = "history" ] && continue - - # Skip if no log files - [ -z "$(ls "$config_dir"/*.log 2>/dev/null)" ] && continue - - total_jobs=$((total_jobs + 1)) - - # Check profiler-only scenario - s1_log="${config_dir}/profiler-only-${config_name}.log" - s1_status=$(parse_validation_log "$s1_log") - - # Check tracer+profiler scenario - s2_log="${config_dir}/tracer-profiler-${config_name}.log" - s2_status=$(parse_validation_log "$s2_log") - - # Count scenarios - for status in "$s1_status" "$s2_status"; do - if [ "$status" != "missing" ]; then - total_scenarios=$((total_scenarios + 1)) - if [ "$status" = "passed" ]; then - passed_scenarios=$((passed_scenarios + 1)) - elif [ "$status" = "failed" ]; then - failed_scenarios=$((failed_scenarios + 1)) - elif [ "$status" = "unknown" ]; then - unknown_scenarios=$((unknown_scenarios + 1)) - fi - fi - done - - # Determine job status - if [ "$s1_status" = "passed" ] && [ "$s2_status" = "passed" ]; then - passed_jobs=$((passed_jobs + 1)) - else - failed_jobs=$((failed_jobs + 1)) - failed_configs="${failed_configs}${failed_configs:+, }\"${config_name}\"" - fi - done -fi - -# Report diagnostic summary to stderr -if [ $unknown_scenarios -gt 0 ]; then - echo "" >&2 - echo "[ERROR] ================================================" >&2 - echo "[ERROR] Found $unknown_scenarios scenario(s) with logs but NO status markers!" >&2 - echo "[ERROR] ================================================" >&2 - echo "[ERROR]" >&2 - echo "[ERROR] This means validation logs exist but don't contain:" >&2 - echo "[ERROR] - 'SUCCESS: All validations passed'" >&2 - echo "[ERROR] - 'VALIDATION_FAILED'" >&2 - echo "[ERROR]" >&2 - echo "[ERROR] Likely causes:" >&2 - echo "[ERROR] 1. Validation script crashed before completion" >&2 - echo "[ERROR] 2. jbang/jfr-shell failed to run" >&2 - echo "[ERROR] 3. Process killed/timeout before markers written" >&2 - echo "[ERROR]" >&2 - echo "[ERROR] Check the logs above for '[WARN] Log missing markers:'" >&2 - echo "[ERROR] ================================================" >&2 - echo "" >&2 -fi - -if [ "$VERBOSE" = "true" ]; then - echo "[DEBUG] Counters: total_jobs=$total_jobs, passed_jobs=$passed_jobs, failed_jobs=$failed_jobs" >&2 - echo "[DEBUG] Scenarios: total=$total_scenarios, passed=$passed_scenarios, failed=$failed_scenarios, unknown=$unknown_scenarios" >&2 - echo "[INFO] Parsing summary:" >&2 - echo "[INFO] Total configs found: $total_jobs" >&2 - echo "[INFO] Total scenarios: $total_scenarios" >&2 - echo "[INFO] Passed: $passed_scenarios" >&2 - echo "[INFO] Failed: $failed_scenarios" >&2 - echo "[INFO] Unknown: $unknown_scenarios" >&2 - echo "" >&2 -fi - -# Determine overall status -if [ $failed_jobs -eq 0 ] && [ $total_jobs -gt 0 ]; then - status="passed" -elif [ $passed_jobs -eq 0 ] && [ $total_jobs -gt 0 ]; then - status="failed" -elif [ $total_jobs -eq 0 ]; then - status="unknown" -else - status="partial" -fi - -# Extract PR number if available -pr_field="null" -if command -v jq >/dev/null 2>&1; then - pr_number=$(echo "$PR_JSON" | jq -r '.number // empty' 2>/dev/null || echo "") - if [ -n "$pr_number" ]; then - pr_field="$PR_JSON" - fi -else - # Fallback: simple grep for number field - if echo "$PR_JSON" | grep -q '"number"'; then - pr_field="$PR_JSON" - fi -fi - -# Generate JSON (without jq dependency for maximum compatibility) -cat < /dev/null; then - log_info "Installing jbang..." - - # Download and install jbang with retry logic - for attempt in $(seq 1 $MAX_RETRIES); do - log_info "jbang installation attempt $attempt of $MAX_RETRIES..." - if curl -Ls https://sh.jbang.dev | bash -s - app setup; then - break - fi - if [ "$attempt" -lt $MAX_RETRIES ]; then - log_warn "jbang installation failed, retrying in ${RETRY_DELAY}s..." - sleep $RETRY_DELAY - RETRY_DELAY=$((RETRY_DELAY * 2)) - else - log_warn "jbang installation failed after $MAX_RETRIES attempts" - fi - done - - # Add to PATH for current session - export PATH="$HOME/.jbang/bin:$PATH" - - # Verify installation - if command -v jbang &> /dev/null; then - JBANG_VERSION=$(jbang version 2>&1 | head -1) - log_info "jbang installed successfully: ${JBANG_VERSION}" - else - log_warn "jbang installation completed but not found in PATH" - log_warn "Please ensure ~/.jbang/bin is in your PATH" - fi -else - JBANG_VERSION=$(jbang version 2>&1 | head -1) - log_info "jbang already installed: ${JBANG_VERSION}" -fi - -# Add jbang trust for jfr-shell from btraceio -log_info "Adding jbang trust for btraceio catalog..." -jbang trust add https://github.com/btraceio/ 2>/dev/null || true - -# Pre-install JDK 25 for jbang (required by jfr-shell) -# First check if JDK 25 is already available (e.g., Java 25 test jobs) -log_info "Checking if JDK 25 is available for jbang..." -if jbang jdk list 2>&1 | grep -q "25"; then - log_info "JDK 25 already available for jbang" - JDK25_INSTALLED=true -else - log_info "Pre-installing JDK 25 for jbang (required by jfr-shell)..." - JDK25_INSTALLED=false - - # Try jbang's built-in install first - for attempt in $(seq 1 $MAX_RETRIES); do - log_info "JDK 25 installation attempt $attempt of $MAX_RETRIES (via jbang)..." - if jbang jdk install 25 2>&1; then - JDK25_INSTALLED=true - log_info "JDK 25 installed for jbang" - break - fi - if [ "$attempt" -lt $MAX_RETRIES ]; then - log_warn "JDK 25 installation failed, retrying in ${RETRY_DELAY}s..." - sleep $RETRY_DELAY - fi - done - - # Fallback: manually download from Adoptium if jbang failed (Foojay API down) - if [ "$JDK25_INSTALLED" = "false" ]; then - log_warn "jbang install failed (Foojay API may be down), trying direct Adoptium download..." - - # Detect architecture - ARCH=$(uname -m) - case "$ARCH" in - x86_64|amd64) ADOPTIUM_ARCH="x64" ;; - aarch64|arm64) ADOPTIUM_ARCH="aarch64" ;; - *) log_warn "Unknown arch: $ARCH"; ADOPTIUM_ARCH="x64" ;; - esac - - # Detect OS and libc - if [ -f /etc/alpine-release ]; then - ADOPTIUM_OS="alpine-linux" - else - ADOPTIUM_OS="linux" - fi - - JBANG_JDK_DIR="$HOME/.jbang/cache/jdks/25" - ADOPTIUM_URL="https://api.adoptium.net/v3/binary/latest/25/ga/${ADOPTIUM_OS}/${ADOPTIUM_ARCH}/jdk/hotspot/normal/eclipse" - - log_info "Downloading JDK 25 from Adoptium: $ADOPTIUM_URL" - if curl -fsSL -o /tmp/jdk25.tar.gz "$ADOPTIUM_URL"; then - mkdir -p "$JBANG_JDK_DIR" - tar -xzf /tmp/jdk25.tar.gz -C "$JBANG_JDK_DIR" --strip-components=1 - rm -f /tmp/jdk25.tar.gz - - if [ -x "$JBANG_JDK_DIR/bin/java" ]; then - JDK25_INSTALLED=true - log_info "JDK 25 installed manually from Adoptium" - "$JBANG_JDK_DIR/bin/java" -version 2>&1 | head -1 - else - log_warn "JDK 25 extraction failed" - fi - else - log_warn "Failed to download JDK 25 from Adoptium" - fi - fi -fi - -if [ "$JDK25_INSTALLED" = "false" ]; then - log_warn "Failed to install JDK 25 for jbang" - log_warn "JFR validation will be skipped" - echo "JDK 25 not available for jbang (Foojay API may be down)" > /tmp/skip-jfr-validation -fi - -# ======================================== -# Pre-warm jfr-shell backend -# ======================================== -# jafar-shell resolves its backend plugin (io.btrace:jfr-shell-jafar) from Maven at -# runtime. If that artifact is unavailable (network restriction, version not yet -# published), every validation run fails. Detect this early so we can skip gracefully. -if [ ! -f /tmp/skip-jfr-validation ] && command -v jbang &> /dev/null; then - log_info "Pre-warming jfr-shell backend..." - PREWARM_OUT=$(jbang --java 25 jfr-shell@btraceio script /dev/null 2>&1 || true) - if echo "$PREWARM_OUT" | grep -q "No backends found\|No JFR backends available\|Failed to resolve artifact.*jfr-shell-jafar"; then - log_warn "jfr-shell backend unavailable (io.btrace:jfr-shell-jafar not resolvable from Maven)" - log_warn "JFR validation will be skipped" - echo "jfr-shell backend unavailable (io.btrace:jfr-shell-jafar not resolvable from Maven)" > /tmp/skip-jfr-validation - else - log_info "jfr-shell backend ready" - fi -fi - -# ======================================== -# 2. Verify Java is available -# ======================================== -if [ -z "${JAVA_HOME:-}" ]; then - if command -v java &> /dev/null; then - log_info "Java found in PATH" - java -version 2>&1 | head -3 - else - echo "ERROR: Java not found. Please set JAVA_HOME or ensure java is in PATH" - exit 1 - fi -else - if [ ! -x "${JAVA_HOME}/bin/java" ]; then - echo "ERROR: Java not found at: ${JAVA_HOME}/bin/java" - exit 1 - fi - - log_info "Java found at JAVA_HOME: ${JAVA_HOME}" - "${JAVA_HOME}/bin/java" -version 2>&1 | head -3 -fi - -# ======================================== -# 3. Install basic tools (if needed) -# ======================================== -# Check for javac (needed to compile test app) -if [ -n "${JAVA_HOME:-}" ]; then - if [ ! -x "${JAVA_HOME}/bin/javac" ]; then - log_warn "javac not found at ${JAVA_HOME}/bin/javac" - log_warn "This may cause test app compilation to fail" - fi -elif ! command -v javac &> /dev/null; then - log_warn "javac not found in PATH" - log_warn "This may cause test app compilation to fail" -fi - -# ======================================== -# 4. Create output directories -# ======================================== -log_info "Creating output directories..." - -mkdir -p integration-test-results -mkdir -p /tmp/jfr-validation - -log_info "✓ Prerequisites installation complete" -log_info "" -log_info "Installed tools:" -log_info " - jbang: $(command -v jbang || echo 'not in PATH')" -log_info " - jbang JDKs: $(jbang jdk list 2>&1 | grep -v '^$' | tr '\n' ' ')" -log_info " - java: $(command -v java || echo 'not in PATH')" -log_info " - javac: $(command -v javac || echo 'not in PATH')" -log_info "" diff --git a/.gitlab/dd-trace-integration/notify_channel.sh b/.gitlab/dd-trace-integration/notify_channel.sh deleted file mode 100755 index 12c7c9926..000000000 --- a/.gitlab/dd-trace-integration/notify_channel.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -if [ ! -z "${CANCELLED:-}" ]; then - exit 0 -fi - -# Source centralized configuration -HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source "${HERE}/../../.gitlab/config.env" - -VERSION=${1:-"unknown"} -PIPELINE_URL="$CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID" -PIPELINE_LINK="<$PIPELINE_URL|pipeline #$CI_PIPELINE_ID>" - -MESSAGE_TEXT=":better-siren: dd-trace-java integration tests failed for ${VERSION} (pipeline=$PIPELINE_LINK) - -Some integration tests failed across the test matrix. -Please review the pipeline artifacts for details." - -postmessage "$SLACK_CHANNEL" "$MESSAGE_TEXT" "alert" diff --git a/.gitlab/dd-trace-integration/patch-dd-java-agent.sh b/.gitlab/dd-trace-integration/patch-dd-java-agent.sh deleted file mode 100755 index 768e319b4..000000000 --- a/.gitlab/dd-trace-integration/patch-dd-java-agent.sh +++ /dev/null @@ -1,385 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Patch dd-java-agent.jar with ddprof contents - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -# Input JARs -DD_AGENT_JAR="${DD_AGENT_JAR:-${PROJECT_ROOT}/dd-java-agent-original.jar}" -DDPROF_JAR="${DDPROF_JAR:-${PROJECT_ROOT}/ddprof.jar}" - -# Output JAR -OUTPUT_JAR="${OUTPUT_JAR:-${PROJECT_ROOT}/dd-java-agent-patched.jar}" - -# Working directory for extraction -# Use mktemp for guaranteed unique directory, fall back to PID-based if mktemp unavailable -WORK_DIR="${WORK_DIR:-$(mktemp -d 2>/dev/null || echo "/tmp/jar-patch-$$")}" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -function log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -function log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -function log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} - -function log_debug() { - if [ "${DEBUG:-false}" = "true" ]; then - echo -e "${BLUE}[DEBUG]${NC} $*" - fi -} - -function cleanup() { - if [ -d "${WORK_DIR}" ]; then - log_debug "Cleaning up work directory: ${WORK_DIR}" - rm -rf "${WORK_DIR}" - fi -} - -trap cleanup EXIT - -# Validate required tools are available -for tool in unzip zip dirname basename; do - if ! command -v "$tool" &> /dev/null; then - log_error "Required tool not found: $tool" - log_error "Please install $tool to continue" - exit 1 - fi -done - -function usage() { - cat << EOF -Usage: $0 [OPTIONS] - -Patch dd-java-agent.jar with ddprof contents. - -Mapping rules: - - Native libraries: ddprof.jar:META-INF/native-libs/** → dd-java-agent.jar:shared/META-INF/native-libs/** - - Class files: ddprof.jar:**/*.class → dd-java-agent.jar:shared/**/*.classdata (same package structure) - -OPTIONS: - --dd-agent-jar Path to dd-java-agent-original.jar (default: ${DD_AGENT_JAR}) - --ddprof-jar Path to ddprof.jar (default: ${DDPROF_JAR}) - --output-jar Path to output patched jar (default: ${OUTPUT_JAR}) - --work-dir Working directory for extraction (default: /tmp/jar-patch-\$\$) - --debug Enable debug output - --help Show this help message - -ENVIRONMENT VARIABLES: - DD_AGENT_JAR Path to dd-java-agent-original.jar - DDPROF_JAR Path to ddprof.jar - OUTPUT_JAR Path to output patched jar - WORK_DIR Working directory for extraction - DEBUG Enable debug output (true/false) - -EXAMPLES: - # Patch with default paths - $0 - - # Patch with custom paths - $0 --dd-agent-jar /path/to/agent.jar --ddprof-jar /path/to/ddprof.jar - - # Enable debug output - DEBUG=true $0 -EOF -} - -# Parse command line arguments -while [ $# -gt 0 ]; do - case "$1" in - --dd-agent-jar) - DD_AGENT_JAR="$2" - shift 2 - ;; - --ddprof-jar) - DDPROF_JAR="$2" - shift 2 - ;; - --output-jar) - OUTPUT_JAR="$2" - shift 2 - ;; - --work-dir) - WORK_DIR="$2" - shift 2 - ;; - --debug) - DEBUG=true - shift - ;; - --help) - usage - exit 0 - ;; - *) - log_error "Unknown option: $1" - usage - exit 1 - ;; - esac -done - -# Validate input JARs exist -if [ ! -f "${DD_AGENT_JAR}" ]; then - log_error "dd-java-agent JAR not found: ${DD_AGENT_JAR}" - exit 1 -fi - -if [ ! -f "${DDPROF_JAR}" ]; then - log_error "ddprof JAR not found: ${DDPROF_JAR}" - exit 1 -fi - -log_info "Starting JAR patching process" -log_info " dd-java-agent: ${DD_AGENT_JAR}" -log_info " ddprof: ${DDPROF_JAR}" -log_info " output: ${OUTPUT_JAR}" -log_info " work dir: ${WORK_DIR}" - -# Create working directories -if ! mkdir -p "${WORK_DIR}/agent" "${WORK_DIR}/ddprof"; then - log_error "Failed to create working directories in: ${WORK_DIR}" - log_error "Check disk space and permissions" - exit 1 -fi - -# Extract dd-java-agent -log_info "Extracting dd-java-agent..." -if ! unzip -q "${DD_AGENT_JAR}" -d "${WORK_DIR}/agent/"; then - log_error "Failed to extract dd-java-agent: ${DD_AGENT_JAR}" - log_error "Check if file is corrupted or disk is full" - exit 1 -fi -log_info "✓ Extracted dd-java-agent" - -# Extract ddprof -log_info "Extracting ddprof..." -if ! unzip -q "${DDPROF_JAR}" -d "${WORK_DIR}/ddprof/"; then - log_error "Failed to extract ddprof: ${DDPROF_JAR}" - log_error "Check if file is corrupted or disk is full" - exit 1 -fi -log_info "✓ Extracted ddprof" - -# Create shared directory structure in agent -mkdir -p "${WORK_DIR}/agent/shared/META-INF" - -# Copy native libraries -log_info "Copying native libraries..." -if [ -d "${WORK_DIR}/ddprof/META-INF/native-libs" ]; then - cp -r "${WORK_DIR}/ddprof/META-INF/native-libs" "${WORK_DIR}/agent/shared/META-INF/" - - # Count native libraries - NATIVE_COUNT=$(find "${WORK_DIR}/agent/shared/META-INF/native-libs" -type f -name "*.so" | wc -l | tr -d ' ') - log_info "✓ Copied ${NATIVE_COUNT} native libraries" - - # List platforms - if [ "${DEBUG:-false}" = "true" ]; then - log_debug "Native library platforms:" - find "${WORK_DIR}/agent/shared/META-INF/native-libs" -type d -mindepth 1 -maxdepth 1 -exec basename {} \; | while read -r platform; do - log_debug " - ${platform}" - done - fi -else - log_warn "No native libraries found in ddprof (META-INF/native-libs missing)" -fi - -# Copy and rename class files -log_info "Copying and renaming class files..." -CLASS_COUNT=0 -SKIPPED_COUNT=0 - -# Count total class files first for progress tracking -TOTAL_CLASSES=$(find "${WORK_DIR}/ddprof" -name "*.class" -type f | wc -l | tr -d ' ') - -# Validate TOTAL_CLASSES is a number -if ! [[ "${TOTAL_CLASSES}" =~ ^[0-9]+$ ]]; then - log_error "Failed to count class files, got: '${TOTAL_CLASSES}'" - exit 1 -fi - -log_info "Found ${TOTAL_CLASSES} class files to process" - -# Verify ddprof extraction directory exists and has content -if [ ! -d "${WORK_DIR}/ddprof" ]; then - log_error "ddprof extraction directory not found: ${WORK_DIR}/ddprof" - exit 1 -fi - -# Exit early if no files to process -if [ "${TOTAL_CLASSES}" -eq 0 ]; then - log_warn "No class files found to process" - # Skip loop but don't fail - might be expected for some builds -fi - -log_info "Starting class file copy loop..." - -# Use find to locate all .class files, excluding META-INF -while IFS= read -r -d '' classfile; do - log_debug "Processing: ${classfile}" - - # Get relative path from ddprof root - relpath="${classfile#"${WORK_DIR}"/ddprof/}" - - # Skip META-INF directory - if [[ "$relpath" == META-INF/* ]]; then - log_debug "Skipping META-INF class: ${relpath}" - SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) - continue - fi - - # Convert path: foo/bar/Baz.class → shared/foo/bar/Baz.classdata - targetpath="${WORK_DIR}/agent/shared/${relpath%.class}.classdata" - targetdir=$(dirname "$targetpath") - - # Create target directory if needed - if ! mkdir -p "$targetdir"; then - log_error "Failed to create directory: $targetdir" - log_error "Check disk space and permissions" - df -h "${WORK_DIR}" >&2 - exit 1 - fi - - # Copy file - if ! cp "$classfile" "$targetpath"; then - log_error "Failed to copy class file: $classfile" - log_error "Target: $targetpath" - log_error "Check disk space and permissions" - df -h "${WORK_DIR}" >&2 - exit 1 - fi - CLASS_COUNT=$((CLASS_COUNT + 1)) - - # Show progress every 100 files - if [ $((CLASS_COUNT % 100)) -eq 0 ]; then - log_debug "Progress: ${CLASS_COUNT}/${TOTAL_CLASSES} files copied" - fi - - log_debug "Copied: ${relpath} → shared/${relpath%.class}.classdata" -done < <(find "${WORK_DIR}/ddprof" -name "*.class" -type f -print0) - -# Verify loop completed successfully -if [ "${CLASS_COUNT}" -eq 0 ] && [ "${TOTAL_CLASSES}" -gt 0 ]; then - log_error "Class file loop failed - no files were copied despite ${TOTAL_CLASSES} files found" - exit 1 -fi - -log_info "✓ Copied and renamed ${CLASS_COUNT} class files (${SKIPPED_COUNT} skipped from META-INF)" - -# List some sample class files for verification -if [ "${DEBUG:-false}" = "true" ]; then - log_debug "Sample classdata files:" - # Use process substitution to avoid SIGPIPE from head in pipeline - count=0 - while IFS= read -r f && [ "$count" -lt 5 ]; do - relpath="${f#"${WORK_DIR}"/agent/}" - log_debug " - ${relpath}" - count=$((count + 1)) - done < <(find "${WORK_DIR}/agent/shared" -name "*.classdata" -type f) -fi - -# Repackage JAR -log_info "Repackaging JAR..." - -# Save original directory and change to agent directory -ORIG_DIR=$(pwd) -if ! cd "${WORK_DIR}/agent"; then - log_error "Failed to change directory to ${WORK_DIR}/agent" - log_error "Check if directory exists and is accessible" - exit 1 -fi - -if ! zip -r -q "${OUTPUT_JAR}" .; then - log_error "Failed to repackage JAR" - log_error "Check disk space and write permissions for: ${OUTPUT_JAR}" - cd "$ORIG_DIR" || true - exit 1 -fi - -# Restore original directory -cd "$ORIG_DIR" || log_warn "Failed to return to original directory" - -log_info "✓ Created patched JAR: ${OUTPUT_JAR}" - -# Validate patched JAR -log_info "Validating patched JAR..." - -if ! unzip -t "${OUTPUT_JAR}" > /dev/null 2>&1; then - log_error "Patched JAR is corrupted" - exit 1 -fi -log_info "✓ JAR integrity check passed" - -# Verify shared directory structure -log_info "Verifying structure..." -SHARED_NATIVE_COUNT=$(unzip -l "${OUTPUT_JAR}" | grep -c "shared/META-INF/native-libs/.*\.so$" || echo "0") -TOTAL_CLASSDATA_COUNT=$(unzip -l "${OUTPUT_JAR}" | grep -c "shared/.*\.classdata$" || echo "0") - -if [ "${SHARED_NATIVE_COUNT}" -eq 0 ] && [ "${NATIVE_COUNT}" -gt 0 ]; then - log_error "Native libraries missing in patched JAR" - exit 1 -fi - -# Diagnostic: Check for unexpected classes in ddprof-owned package namespaces -# Only check com/datadoghq/profiler - this is the actual ddprof package -log_debug "Checking for unexpected classes in com/datadoghq/profiler package..." -DDPROF_CLASSES=$(unzip -l "${DDPROF_JAR}" '*.class' 2>/dev/null | \ - awk '{print $NF}' | \ - grep "^com/datadoghq/profiler/" | \ - grep "\.class$" | \ - sed 's/\.class$//' | \ - sort || true) - -# Get classes from patched JAR in the ddprof package -PATCHED_CLASSES=$(unzip -l "${OUTPUT_JAR}" 2>/dev/null | \ - grep "shared/com/datadoghq/profiler/.*\.classdata$" | \ - awk '{print $NF}' | \ - sed 's|^shared/||' | \ - sed 's/\.classdata$//' | \ - sort || true) - -# Find classes in patched JAR that weren't in original ddprof.jar -UNEXPECTED_CLASSES=$(comm -13 <(echo "${DDPROF_CLASSES}") <(echo "${PATCHED_CLASSES}") || true) - -if [ -n "${UNEXPECTED_CLASSES}" ]; then - UNEXPECTED_COUNT=$(echo "${UNEXPECTED_CLASSES}" | grep -c . || echo "0") - log_warn "Found ${UNEXPECTED_COUNT} unexpected classes in com/datadoghq/profiler:" - log_warn "These classes exist in dd-java-agent but not in ddprof.jar:" - echo "${UNEXPECTED_CLASSES}" | while IFS= read -r cls; do - log_debug " - ${cls}" - done | head -10 - if [ "${UNEXPECTED_COUNT}" -gt 10 ]; then - log_debug " ... and $((UNEXPECTED_COUNT - 10)) more" - fi - log_warn "This may indicate:" - log_warn " - Package namespace overlap between dd-java-agent and ddprof" - log_warn " - Classes added to ddprof.jar (valid scenario)" -fi - -log_info "✓ Structure verification passed" -log_info " - ${SHARED_NATIVE_COUNT} native libraries in shared/META-INF/native-libs/" -log_info " - ${CLASS_COUNT} ddprof classes added to patched JAR" -log_info " - ${TOTAL_CLASSDATA_COUNT} total classdata files in patched JAR" - -# Print summary -JAR_SIZE=$(du -h "${OUTPUT_JAR}" | cut -f1) -log_info "" -log_info "Patching complete!" -log_info " Output: ${OUTPUT_JAR} (${JAR_SIZE})" -log_info "" -log_info "To inspect the patched JAR structure:" -log_info " unzip -l ${OUTPUT_JAR} | grep shared/" diff --git a/.gitlab/dd-trace-integration/post-pr-comment.sh b/.gitlab/dd-trace-integration/post-pr-comment.sh deleted file mode 100755 index a0d834915..000000000 --- a/.gitlab/dd-trace-integration/post-pr-comment.sh +++ /dev/null @@ -1,237 +0,0 @@ -#!/bin/bash - -# post-pr-comment.sh - Post integration test results as PR comment -# -# Usage: post-pr-comment.sh -# -# Posts a formatted comment to the java-profiler PR with: -# - Pass/fail summary with badges -# - Test matrix results -# - Link to full dashboard -# - Failure details if any -# -# Requires: -# - DDPROF_COMMIT_BRANCH: Branch name to find PR -# - CI_PIPELINE_URL: Link to pipeline -# - pr-commenter tool (available in CI images) - -set -euo pipefail - -# Colors for logging -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } -log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } - -RESULTS_DIR="${1:-integration-test-results}" -REPO="DataDog/java-profiler" - -# Dashboard URL (GitHub Pages) -DASHBOARD_URL="https://datadog.github.io/java-profiler/integration/" - -# Check required tools - try to get pr-commenter from benchmarking-platform if not available -PR_COMMENTER_AVAILABLE=false -if command -v pr-commenter >/dev/null 2>&1; then - PR_COMMENTER_AVAILABLE=true -elif [ -n "${CI_JOB_TOKEN:-}" ]; then - # In CI, clone benchmarking-platform to get pr-commenter - log_info "pr-commenter not found, cloning benchmarking-platform..." - PLATFORM_DIR=$(mktemp -d) - trap "rm -rf ${PLATFORM_DIR}" EXIT - git config --global url."https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/".insteadOf "https://github.com/DataDog/" - if git clone --depth 1 --branch dd-trace-go https://github.com/DataDog/benchmarking-platform "${PLATFORM_DIR}" 2>/dev/null; then - if [ -x "${PLATFORM_DIR}/tools/pr-commenter" ]; then - export PATH="${PLATFORM_DIR}/tools:${PATH}" - PR_COMMENTER_AVAILABLE=true - log_info "pr-commenter available from benchmarking-platform" - elif [ -f "${PLATFORM_DIR}/tools/pr-commenter.py" ]; then - # Try Python version - alias pr-commenter="python3 ${PLATFORM_DIR}/tools/pr-commenter.py" - PR_COMMENTER_AVAILABLE=true - log_info "pr-commenter.py available from benchmarking-platform" - else - log_warn "pr-commenter not found in benchmarking-platform" - ls -la "${PLATFORM_DIR}/tools/" 2>/dev/null || log_warn "No tools directory" - fi - else - log_warn "Failed to clone benchmarking-platform" - fi -else - log_warn "pr-commenter not found and not in CI - will print comment instead" -fi - -# Check required environment -if [ -z "${DDPROF_COMMIT_BRANCH:-}" ]; then - log_warn "DDPROF_COMMIT_BRANCH not set - skipping comment" - exit 0 -fi - -# Skip for main/master branches (no PR) -if [ "${DDPROF_COMMIT_BRANCH}" = "main" ] || [ "${DDPROF_COMMIT_BRANCH}" = "master" ]; then - log_info "Skipping PR comment for ${DDPROF_COMMIT_BRANCH} branch" - exit 0 -fi - -log_info "Posting comment for branch: ${DDPROF_COMMIT_BRANCH}" - -# Collect test results -log_info "Collecting test results from ${RESULTS_DIR}..." - -declare -A RESULTS -TOTAL_PASS=0 -TOTAL_FAIL=0 -FAILURES="" - -for config_dir in "${RESULTS_DIR}"/*; do - [ -d "${config_dir}" ] || continue - config_name=$(basename "${config_dir}") - - # Check validation logs for pass/fail - s1_status="unknown" - s2_status="unknown" - - # Scenario 1: profiler-only - s1_log="${config_dir}/profiler-only-${config_name}.log" - if [ -f "${s1_log}" ]; then - if grep -q "SUCCESS:" "${s1_log}" 2>/dev/null; then - s1_status="pass" - elif grep -q "VALIDATION_FAILED" "${s1_log}" 2>/dev/null; then - s1_status="fail" - fi - fi - - # Scenario 2: tracer+profiler - s2_log="${config_dir}/tracer-profiler-${config_name}.log" - if [ -f "${s2_log}" ]; then - if grep -q "SUCCESS:" "${s2_log}" 2>/dev/null; then - s2_status="pass" - elif grep -q "VALIDATION_FAILED" "${s2_log}" 2>/dev/null; then - s2_status="fail" - fi - fi - - # Determine overall status for this config - if [ "${s1_status}" = "pass" ] && [ "${s2_status}" = "pass" ]; then - RESULTS["${config_name}"]="pass" - TOTAL_PASS=$((TOTAL_PASS + 1)) - elif [ "${s1_status}" = "fail" ] || [ "${s2_status}" = "fail" ]; then - RESULTS["${config_name}"]="fail" - TOTAL_FAIL=$((TOTAL_FAIL + 1)) - # Collect failure details - FAILURES="${FAILURES}\n
${config_name}\n\n" - if [ "${s1_status}" = "fail" ] && [ -f "${s1_log}" ]; then - FAILURES="${FAILURES}**Profiler-only:**\n\`\`\`\n$(tail -20 "${s1_log}")\n\`\`\`\n" - fi - if [ "${s2_status}" = "fail" ] && [ -f "${s2_log}" ]; then - FAILURES="${FAILURES}**Tracer+profiler:**\n\`\`\`\n$(tail -20 "${s2_log}")\n\`\`\`\n" - fi - FAILURES="${FAILURES}
\n" - else - RESULTS["${config_name}"]="unknown" - fi -done - -TOTAL=$((TOTAL_PASS + TOTAL_FAIL)) - -# Determine overall status -if [ "${TOTAL_FAIL}" -gt 0 ]; then - OVERALL_STATUS="failure" - STATUS_EMOJI=":x:" - STATUS_TEXT="FAILED" -elif [ "${TOTAL_PASS}" -gt 0 ]; then - OVERALL_STATUS="success" - STATUS_EMOJI=":white_check_mark:" - STATUS_TEXT="PASSED" -else - OVERALL_STATUS="neutral" - STATUS_EMOJI=":grey_question:" - STATUS_TEXT="NO RESULTS" -fi - -log_info "Results: ${TOTAL_PASS} passed, ${TOTAL_FAIL} failed out of ${TOTAL} configurations" - -# Build the comment body -DDPROF_SHA="${DDPROF_COMMIT_SHA:-$(cat ddprof-commit-sha.txt 2>/dev/null || echo unknown)}" - -if [ "${OVERALL_STATUS}" = "success" ]; then - # All tests passed - keep it short - COMMENT_BODY=":white_check_mark: **All ${TOTAL} integration tests passed** - -:bar_chart: [Dashboard](${DASHBOARD_URL}) · :construction_worker: [Pipeline](${CI_PIPELINE_URL:-}) · :package: \`${DDPROF_SHA:0:8}\`" -else - # Some failures or unknowns - show full matrix - COMMENT_BODY="${STATUS_EMOJI} **${TOTAL_PASS}** passed, **${TOTAL_FAIL}** failed out of **${TOTAL}** configurations - -### Test Matrix - -| Platform | JDK 8 | JDK 11 | JDK 17 | JDK 21 | JDK 25 | -|----------|-------|--------|--------|--------|--------|" - - # Build matrix rows - for platform in "glibc-x64-hotspot" "glibc-x64-openj9" "glibc-arm64-hotspot" "glibc-arm64-openj9" \ - "musl-x64-hotspot" "musl-x64-openj9" "musl-arm64-hotspot" "musl-arm64-openj9"; do - row="| ${platform} |" - for jdk in 8 11 17 21 25; do - config="${platform}-jdk${jdk}" - status="${RESULTS[${config}]:-unknown}" - case "${status}" in - pass) row="${row} :white_check_mark: |" ;; - fail) row="${row} :x: |" ;; - *) row="${row} :grey_question: |" ;; - esac - done - COMMENT_BODY="${COMMENT_BODY} -${row}" - done - - # Add failure details if any - if [ -n "${FAILURES}" ]; then - COMMENT_BODY="${COMMENT_BODY} - -### Failure Details -$(echo -e "${FAILURES}")" - fi - - # Add links - COMMENT_BODY="${COMMENT_BODY} - -### Links -- :bar_chart: [Full Dashboard](${DASHBOARD_URL}) -- :construction_worker: [Pipeline](${CI_PIPELINE_URL:-}) -- :package: Commit: \`${DDPROF_SHA}\`" -fi - -# Post comment using pr-commenter -if [ "${PR_COMMENTER_AVAILABLE}" = "true" ]; then - log_info "Posting comment via pr-commenter..." - - if echo "${COMMENT_BODY}" | pr-commenter \ - --for-repo="${REPO}" \ - --for-pr="${DDPROF_COMMIT_BRANCH}" \ - --header="Integration Tests" \ - --on-duplicate=replace; then - log_info "Successfully posted comment" - else - log_error "Failed to post comment via pr-commenter" - log_info "Comment that would be posted:" - echo "${COMMENT_BODY}" - exit 1 - fi -else - log_info "Comment that would be posted to PR:" - echo "" - echo "${COMMENT_BODY}" - echo "" -fi - -# Exit with failure if tests failed (makes pipeline fail) -if [ "${OVERALL_STATUS}" = "failure" ]; then - log_error "Integration tests failed - marking pipeline as failed" - exit 1 -fi - -exit 0 diff --git a/.gitlab/dd-trace-integration/publish-gh-pages.sh b/.gitlab/dd-trace-integration/publish-gh-pages.sh deleted file mode 100755 index 78c86d04f..000000000 --- a/.gitlab/dd-trace-integration/publish-gh-pages.sh +++ /dev/null @@ -1,254 +0,0 @@ -#!/bin/bash - -# publish-gh-pages.sh - Publish integration test reports to GitHub Pages -# -# Usage: publish-gh-pages.sh [results-dir] -# -# Generates reports for all test configurations and publishes to gh-pages branch. -# Reports are available at: https://datadog.github.io/async-profiler-build/ -# -# In CI: Uses Octo-STS for secure, short-lived GitHub tokens (no secrets needed) -# Locally: Use 'devflow gitlab auth' for Octo-STS, or set GITHUB_TOKEN env var - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="${SCRIPT_DIR}/../.." -RESULTS_BASE="${1:-${PROJECT_ROOT}/integration-test-results}" -MAX_HISTORY=10 - -# GitHub repo for Pages -GITHUB_REPO="DataDog/java-profiler" -PAGES_URL="https://datadog.github.io/java-profiler" - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } -log_error() { echo -e "${RED}[ERROR]${NC} $*"; } - -# Obtain GitHub token -obtain_github_token() { - # Try dd-octo-sts CLI (works in CI with DDOCTOSTS_ID_TOKEN) - if command -v dd-octo-sts >/dev/null 2>&1 && [ -n "${DDOCTOSTS_ID_TOKEN:-}" ]; then - log_info "Obtaining GitHub token via dd-octo-sts CLI..." - # Policy name matches the .sts.yaml filename (without extension) - - # Run dd-octo-sts and capture only stdout (don't capture stderr to avoid error messages in token) - local TOKEN_OUTPUT - local TOKEN_EXIT_CODE - TOKEN_OUTPUT=$(dd-octo-sts token --scope DataDog/java-profiler --policy async-profiler-build.ci 2>/tmp/dd-octo-sts-error.log) - TOKEN_EXIT_CODE=$? - - if [ $TOKEN_EXIT_CODE -eq 0 ] && [ -n "${TOKEN_OUTPUT}" ]; then - # Validate token format (GitHub tokens start with ghs_, ghp_, or look like JWT) - if [[ "${TOKEN_OUTPUT}" =~ ^(ghs_|ghp_|v1\.|[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.) ]]; then - GITHUB_TOKEN="${TOKEN_OUTPUT}" - log_info "GitHub token obtained via dd-octo-sts (expires in 1 hour)" - return 0 - else - log_warn "dd-octo-sts returned invalid token format (first 50 chars): ${TOKEN_OUTPUT:0:50}" - fi - else - log_warn "dd-octo-sts token exchange failed (exit code: ${TOKEN_EXIT_CODE})" - if [ -s /tmp/dd-octo-sts-error.log ]; then - log_warn "dd-octo-sts error output:" - cat /tmp/dd-octo-sts-error.log | head -10 >&2 - fi - fi - fi - - # Fall back to GITHUB_TOKEN environment variable - if [ -n "${GITHUB_TOKEN:-}" ]; then - log_info "Using GITHUB_TOKEN from environment" - return 0 - fi - - return 1 -} - -if ! obtain_github_token; then - log_error "Failed to obtain GitHub token" - log_error "Options:" - log_error " 1. Run in GitLab CI with dd-octo-sts-ci-base image and DDOCTOSTS_ID_TOKEN" - log_error " 2. Set GITHUB_TOKEN env var (PAT with 'repo' scope)" - exit 1 -fi - -# Create temporary directory for gh-pages content -WORK_DIR=$(mktemp -d) -trap "rm -rf ${WORK_DIR}" EXIT - -log_info "Preparing gh-pages content in: ${WORK_DIR}" - -# Clone gh-pages branch (or create if doesn't exist) -log_info "Cloning gh-pages branch..." -cd "${WORK_DIR}" - -if git clone --depth 1 --branch gh-pages "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" pages 2>/dev/null; then - cd pages - log_info "Cloned existing gh-pages branch" -else - log_info "Creating new gh-pages branch..." - mkdir pages && cd pages - git init - git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" - git checkout -b gh-pages -fi - -# Create/update Jekyll config with baseurl for subpath deployment -cat > "_config.yml" </dev/null | wc -l)" -else - log_warn "Results directory does not exist: ${RESULTS_BASE}" -fi - -# Generate run JSON for this pipeline -RUN_JSON_FILE=$(mktemp) -trap "rm -rf ${WORK_DIR} ${RUN_JSON_FILE}" EXIT - -log_info "Running generate-run-json.sh..." -if "${SCRIPT_DIR}/generate-run-json.sh" "${RESULTS_BASE}" > "${RUN_JSON_FILE}"; then - log_info "Generated run JSON successfully" - log_info "Run JSON size: $(wc -c < "${RUN_JSON_FILE}") bytes" - log_info "Run JSON preview:" - head -20 "${RUN_JSON_FILE}" || true - - # Update history (prepend new run, keep last MAX_HISTORY) - log_info "Running update-history.sh..." - if "${SCRIPT_DIR}/../common/update-history.sh" integration "${RUN_JSON_FILE}" "."; then - log_info "Updated integration history" - else - log_warn "Failed to update history, error output above" - fi -else - log_warn "Failed to generate run JSON, error output:" - log_warn "Showing last 100 lines of output:" - tail -100 "${RUN_JSON_FILE}" || true - - log_warn "" - log_warn "Checking first config directory for debugging:" - first_config=$(find "${RESULTS_BASE}" -maxdepth 1 -type d ! -path "${RESULTS_BASE}" | head -1) - if [ -n "$first_config" ]; then - log_warn "Contents of $(basename "$first_config"):" - ls -la "$first_config" || true - log_warn "" - log_warn "Sample log file content (if exists):" - find "$first_config" -name "*.log" -type f | head -1 | xargs head -20 2>/dev/null || log_warn "No log files found" - fi -fi - -# Generate dashboard and index pages -log_info "Generating dashboard..." -if "${SCRIPT_DIR}/../common/generate-dashboard.sh" "." 2>&1; then - log_info "Generated dashboard index.md" - if [ -f "index.md" ]; then - log_info "Dashboard size: $(wc -c < "index.md") bytes" - fi -else - log_warn "Failed to generate dashboard, error output above" -fi - -log_info "Generating integration index..." -if "${SCRIPT_DIR}/../common/generate-index.sh" integration "." 2>&1; then - log_info "Generated integration/index.md" - if [ -f "integration/index.md" ]; then - log_info "Integration index size: $(wc -c < "integration/index.md") bytes" - fi -else - log_warn "Failed to generate integration index, error output above" -fi - -# ============================================ -# DETAILED REPORTS (per-config reports for current run) -# ============================================ -log_info "Generating detailed reports..." - -TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC") - -# Find all test result directories -RESULT_DIRS=() -while IFS= read -r -d '' dir; do - if [[ "${dir}" == *"/history"* ]]; then - continue - fi - if ls "${dir}"/*.log &>/dev/null 2>&1; then - RESULT_DIRS+=("${dir}") - fi -done < <(find "${RESULTS_BASE}" -mindepth 1 -maxdepth 1 -type d -print0 2>/dev/null | sort -z) - -if [ ${#RESULT_DIRS[@]} -eq 0 ]; then - log_warn "No test results found in ${RESULTS_BASE}" -fi - -log_info "Found ${#RESULT_DIRS[@]} test configuration(s)" - -# Create reports directory for detailed per-config reports -mkdir -p reports - -# Process each configuration -for dir in "${RESULT_DIRS[@]}"; do - config_name=$(basename "${dir}") - log_info "Processing: ${config_name}" - - # Generate detailed report - report_file="reports/${config_name}.md" - if "${SCRIPT_DIR}/generate-report.sh" "${dir}" "${WORK_DIR}/pages/${report_file}" 2>/dev/null; then - # Add front matter for Jekyll - tmp_file=$(mktemp) - cat > "${tmp_file}" <> "${tmp_file}" - mv "${tmp_file}" "${WORK_DIR}/pages/${report_file}" - else - log_warn "Failed to generate report for ${config_name}" - fi -done - -# Commit and push -log_info "Committing changes..." -git add -A -if git diff --staged --quiet; then - log_info "No changes to commit" -else - git config user.email "ci@datadoghq.com" - git config user.name "CI Bot" - git commit -m "Update integration test reports - ${TIMESTAMP}" - - log_info "Pushing to gh-pages..." - git push origin gh-pages --force - - log_info "✅ Reports published successfully!" - log_info "View at: ${PAGES_URL}" -fi - -echo "" -echo "PAGES_URL=${PAGES_URL}" diff --git a/.gitlab/dd-trace-integration/run-integration-test.sh b/.gitlab/dd-trace-integration/run-integration-test.sh deleted file mode 100755 index 555fd0dd6..000000000 --- a/.gitlab/dd-trace-integration/run-integration-test.sh +++ /dev/null @@ -1,628 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# run-integration-test.sh - Simplified integration tests for profiler -# -# This script runs self-contained integration tests without building dd-trace-java. -# It tests the patched dd-java-agent with both profiler-only and tracer+profiler scenarios. -# -# Expected environment variables: -# - LIBC_VARIANT: glibc or musl -# - ARCH: x64 or arm64 -# - JVM_TYPE: hotspot or openj9 -# - JAVA_VERSION: 8, 11, 17, 21, 25 -# -# Optional: -# - CI_PROJECT_DIR: Project root (auto-detected if not set) -# - JAVA_HOME: Java installation (must be set) -# - TEST_DURATION: Test duration in seconds (default: 30) - -# ======================================== -# Configuration -# ======================================== - -# Use CI_PROJECT_DIR if available, otherwise calculate from script location -if [ -n "${CI_PROJECT_DIR:-}" ]; then - PROJECT_ROOT="${CI_PROJECT_DIR}" -else - HERE=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &> /dev/null && pwd) - PROJECT_ROOT="${HERE}/../.." -fi - -# Test configuration -TEST_DURATION="${TEST_DURATION:-60}" -TEST_THREADS=4 -CPU_ITERATIONS=10000 -ALLOC_RATE=1000 - -# Colors for output -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -function log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -function log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -function log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} - -# ======================================== -# Validate Environment -# ======================================== -echo "=== Integration Test Configuration ===" -echo "LIBC_VARIANT=${LIBC_VARIANT:-}" -echo "ARCH=${ARCH:-}" -echo "JVM_TYPE=${JVM_TYPE:-}" -echo "JAVA_VERSION=${JAVA_VERSION:-}" -echo "JAVA_HOME=${JAVA_HOME:-}" -echo "TEST_DURATION=${TEST_DURATION}" -echo "" - -# Validate required variables -if [ -z "${LIBC_VARIANT:-}" ] || [ -z "${ARCH:-}" ] || [ -z "${JVM_TYPE:-}" ] || [ -z "${JAVA_VERSION:-}" ]; then - log_error "Missing required environment variables" - log_error "Required: LIBC_VARIANT, ARCH, JVM_TYPE, JAVA_VERSION" - exit 1 -fi - -# Validate JAVA_HOME -if [ -z "${JAVA_HOME:-}" ]; then - log_error "JAVA_HOME not set" - exit 1 -fi - -if [ ! -x "${JAVA_HOME}/bin/java" ]; then - log_error "Java not found at: ${JAVA_HOME}/bin/java" - exit 1 -fi - -if [ ! -x "${JAVA_HOME}/bin/javac" ]; then - log_error "javac not found at: ${JAVA_HOME}/bin/javac" - exit 1 -fi - -# ======================================== -# System Diagnostics Functions -# ======================================== - -collect_system_metrics() { - local phase="$1" # start|mid|end - local output="${RESULTS_DIR}/diagnostics/system-metrics-${phase}.json" - - mkdir -p "${RESULTS_DIR}/diagnostics" - - # Collect metrics - local cpu_count=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "1") - local cpu_quota=$(cat /sys/fs/cgroup/cpu/cpu.cfs_quota_us 2>/dev/null || echo "-1") - local cpu_period=$(cat /sys/fs/cgroup/cpu/cpu.cfs_period_us 2>/dev/null || echo "-1") - local load_avg=$(uptime | awk -F'load average:' '{print $2}' | xargs) - local container=$(test -f /.dockerenv && echo "true" || echo "false") - - # Parse throttling stats - local throttle_stats=$(cat /sys/fs/cgroup/cpu/cpu.stat 2>/dev/null || echo "") - local nr_periods=$(echo "$throttle_stats" | grep nr_periods | awk '{print $2}') - local nr_throttled=$(echo "$throttle_stats" | grep nr_throttled | awk '{print $2}') - local throttled_time=$(echo "$throttle_stats" | grep throttled_time | awk '{print $2}') - - # Calculate throttle percentage - local throttle_pct=0 - if [ -n "$nr_periods" ] && [ "$nr_periods" -gt 0 ] && [ "$cpu_period" -gt 0 ]; then - throttle_pct=$(awk "BEGIN {printf \"%.2f\", ($throttled_time / ($nr_periods * $cpu_period)) * 100}") - fi - - # Write JSON - cat > "$output" </dev/null || date +%Y-%m-%dT%H:%M:%S%z)", - "phase": "$phase", - "cpu_count": $cpu_count, - "cpu_quota": $cpu_quota, - "cpu_period": $cpu_period, - "load_average": "$load_avg", - "container": $container, - "throttling": { - "nr_periods": ${nr_periods:-0}, - "nr_throttled": ${nr_throttled:-0}, - "throttled_time_ns": ${throttled_time:-0}, - "percentage": $throttle_pct - } -} -EOF - - log_info "System metrics collected ($phase): CPU=$cpu_count, Throttled=${throttle_pct}%" -} - -start_cpu_monitor() { - # Background process sampling CPU count every 5s - local monitor_file="${RESULTS_DIR}/diagnostics/cpu-timeline.log" - mkdir -p "${RESULTS_DIR}/diagnostics" - - ( - while true; do - local cpu_now=$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo "1") - echo "$(date +%s) $cpu_now" >> "$monitor_file" - sleep 5 - done - ) & - - echo $! > "${RESULTS_DIR}/diagnostics/monitor.pid" - log_info "CPU monitor started (PID: $!)" -} - -stop_cpu_monitor() { - local pid_file="${RESULTS_DIR}/diagnostics/monitor.pid" - if [ -f "$pid_file" ]; then - local monitor_pid=$(cat "$pid_file") - kill "$monitor_pid" 2>/dev/null || true - rm "$pid_file" - log_info "CPU monitor stopped" - fi -} - -generate_diagnostics_summary() { - local output="${RESULTS_DIR}/diagnostics/summary.txt" - - # Analyze CPU timeline for changes - local cpu_changes=$(awk '{print $2}' "${RESULTS_DIR}/diagnostics/cpu-timeline.log" | sort -u | wc -l) - local cpu_min=$(awk '{print $2}' "${RESULTS_DIR}/diagnostics/cpu-timeline.log" | sort -n | head -1) - local cpu_max=$(awk '{print $2}' "${RESULTS_DIR}/diagnostics/cpu-timeline.log" | sort -n | tail -1) - - # Extract throttling from end metrics - local throttle_pct=$(grep '"percentage"' "${RESULTS_DIR}/diagnostics/system-metrics-end.json" | awk -F': ' '{print $2}' | tr -d ',') - - cat > "$output" <&1 | head -3 - -# ======================================== -# Install Prerequisites -# ======================================== -echo "" -log_info "Installing prerequisites (jbang for JFR validation)..." - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -if [ -f "${SCRIPT_DIR}/install-prerequisites.sh" ]; then - source "${SCRIPT_DIR}/install-prerequisites.sh" -else - log_warn "install-prerequisites.sh not found, skipping" -fi - -# Ensure jbang is in PATH -export PATH="$HOME/.jbang/bin:$PATH" - -# Verify jbang is available -if ! command -v jbang &> /dev/null; then - log_error "jbang not found after installation" - log_error "JFR validation will not work" - exit 1 -fi - -log_info "jbang version: $(jbang version 2>&1 | head -1)" - -# ======================================== -# Artifact Collection on Exit -# ======================================== -function collect_artifacts() { - log_info "Collecting artifacts..." - - # Copy JFR recordings from dump directories - find /tmp -type d -name 'jfr-*' -exec sh -c 'cp "$0"/*.jfr "${1}/" 2>/dev/null || true' {} "${RESULTS_DIR}" \; 2>/dev/null || true - - # Copy agent logs - find /tmp -maxdepth 1 -name '*-agent.log' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - - # Copy HotSpot JVM crash dumps - find /tmp -maxdepth 1 -name 'hs_err*' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - find . -maxdepth 1 -name 'hs_err*' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - - # Copy OpenJ9 crash dumps - find /tmp -maxdepth 1 -name 'javacore*.txt' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - find /tmp -maxdepth 1 -name 'Snap*.trc' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - find /tmp -maxdepth 1 -name 'jitdump*' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - find /tmp -maxdepth 1 -name 'core.*' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - find . -maxdepth 1 -name 'javacore*.txt' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - find . -maxdepth 1 -name 'Snap*.trc' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - - # Copy validation reports - find /tmp/jfr-validation -name '*.log' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - - # Copy any test output logs - find /tmp -maxdepth 1 -name 'test-*.log' -exec cp {} "${RESULTS_DIR}/" \; 2>/dev/null || true - - log_info "Artifacts collected to: ${RESULTS_DIR}" -} - -trap collect_artifacts EXIT - -# ======================================== -# Calculate Threshold Multiplier -# ======================================== -# Base multiplier is 1.0, adjusted for platform and JVM type - -THRESHOLD_MULTIPLIER=1.0 - -# Platform adjustments -case "${ARCH}-${LIBC_VARIANT}" in - x64-glibc) - PLATFORM_MULT=1.0 - ;; - x64-musl) - PLATFORM_MULT=1.0 - ;; - arm64-glibc) - PLATFORM_MULT=0.8 - ;; - arm64-musl) - PLATFORM_MULT=0.8 - ;; - *) - log_warn "Unknown platform: ${ARCH}-${LIBC_VARIANT}, using default multiplier" - PLATFORM_MULT=1.0 - ;; -esac - -# JVM type adjustments -case "${JVM_TYPE}" in - hotspot) - JVM_MULT=1.0 - ;; - openj9) - JVM_MULT=0.5 # OpenJ9 typically produces fewer samples - ;; - *) - log_warn "Unknown JVM type: ${JVM_TYPE}, using default multiplier" - JVM_MULT=1.0 - ;; -esac - -# Extra adjustment for musl libc (Docker-on-runner with significant overhead) -LIBC_MULT=1.0 -if [ "${LIBC_VARIANT}" = "musl" ]; then - LIBC_MULT=0.25 -fi - -# Calculate final multiplier -THRESHOLD_MULTIPLIER=$(awk "BEGIN {print ${PLATFORM_MULT} * ${JVM_MULT} * ${LIBC_MULT}}") - -log_info "Threshold multiplier: ${THRESHOLD_MULTIPLIER} (platform=${PLATFORM_MULT}, jvm=${JVM_MULT}, libc=${LIBC_MULT})" - -# ======================================== -# Find Patched Agent -# ======================================== -log_info "Locating patched dd-java-agent..." - -PATCHED_AGENT="${PROJECT_ROOT}/dd-java-agent-patched.jar" - -if [ ! -f "${PATCHED_AGENT}" ]; then - log_error "Patched agent not found: ${PATCHED_AGENT}" - log_error "Expected artifact from prepare-patched-agent job" - exit 1 -fi - -log_info "Patched agent: ${PATCHED_AGENT}" -log_info "Agent size: $(du -h "${PATCHED_AGENT}" | cut -f1)" - -# Verify ddprof is actually in the patched agent -log_info "Verifying ddprof presence in patched agent..." -NATIVE_LIBS=$(unzip -l "${PATCHED_AGENT}" | grep -c "shared/META-INF/native-libs/.*\.so$" || echo "0") -CLASSDATA=$(unzip -l "${PATCHED_AGENT}" | grep -c "shared/.*\.classdata$" || echo "0") -log_info " - Native libraries: ${NATIVE_LIBS}" -log_info " - Classdata files: ${CLASSDATA}" - -if [ "${NATIVE_LIBS}" -eq 0 ]; then - log_error "No ddprof native libraries found in patched agent!" - log_error "Patching may have failed" - exit 1 -fi - -# ======================================== -# Compile Test Application -# ======================================== -log_info "Compiling test application..." - -TEST_APP_SRC="${PROJECT_ROOT}/.gitlab/test-apps/ProfilerTestApp.java" -TEST_APP_DIR="/tmp/test-app-$$" - -if [ ! -f "${TEST_APP_SRC}" ]; then - log_error "Test app not found: ${TEST_APP_SRC}" - exit 1 -fi - -mkdir -p "${TEST_APP_DIR}" -cp "${TEST_APP_SRC}" "${TEST_APP_DIR}/" - -cd "${TEST_APP_DIR}" - -if ! "${JAVA_HOME}/bin/javac" ProfilerTestApp.java 2>&1; then - log_error "Test app compilation failed" - exit 1 -fi - -log_info "✓ Test application compiled successfully" - -# ======================================== -# Run Scenario 1: Profiler-Only -# ======================================== -echo "" -echo "========================================" -echo " Scenario 1: Profiler-Only" -echo "========================================" -echo "" - -SCENARIO1_JFR_DIR="/tmp/jfr-profiler-only-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}" -SCENARIO1_LOG="/tmp/profiler-only-agent.log" - -mkdir -p "${SCENARIO1_JFR_DIR}" - -# Collect system metrics at test start -collect_system_metrics "start" -start_cpu_monitor - -log_info "Running profiler-only test (${TEST_DURATION}s)..." -log_info "JFR dump directory: ${SCENARIO1_JFR_DIR}" -log_info "Log output: ${SCENARIO1_LOG}" - -# Log Java version details for debugging allocation profiling -log_info "Java version details:" -"${JAVA_HOME}/bin/java" -XshowSettings:properties -version 2>&1 | grep -E "java\.(version|vendor|runtime|vm)" || true -echo "" - -# Run test with profiler-only configuration -"${JAVA_HOME}/bin/java" \ - -javaagent:"${PATCHED_AGENT}" \ - -Ddd.profiling.enabled=true \ - -Ddd.profiling.ddprof.enabled=true \ - -Ddd.trace.enabled=false \ - -Ddd.profiling.start-delay=0 \ - -Ddd.profiling.start-force-first=true \ - -Ddd.profiling.upload.period=10 \ - -Ddd.profiling.debug.dump_path="${SCENARIO1_JFR_DIR}" \ - -Ddd.service.name=profiler-integration-test \ - -Ddd.trace.startup.logs=true \ - -Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug \ - -cp . ProfilerTestApp \ - --duration "${TEST_DURATION}" \ - --threads "${TEST_THREADS}" \ - --cpu-iterations "${CPU_ITERATIONS}" \ - --alloc-rate "${ALLOC_RATE}" \ - > "${SCENARIO1_LOG}" 2>&1 - -# Show agent startup logs -log_info "Agent startup log (first 100 lines):" -head -100 "${SCENARIO1_LOG}" - -log_info "✓ Profiler-only test completed" - -# Collect mid-test system metrics -collect_system_metrics "mid" - -# Find the JFR recording (profiler writes continuously) -log_info "Locating JFR recording in ${SCENARIO1_JFR_DIR}..." - -SCENARIO1_JFR=$(find "${SCENARIO1_JFR_DIR}" -name "*.jfr" -print -quit) - -if [ -z "${SCENARIO1_JFR}" ] || [ ! -f "${SCENARIO1_JFR}" ]; then - log_error "JFR recording not found in: ${SCENARIO1_JFR_DIR}" - ls -la "${SCENARIO1_JFR_DIR}" || true - exit 1 -fi - -log_info "Found JFR recording: ${SCENARIO1_JFR}" - -JFR_SIZE=$(stat -f%z "${SCENARIO1_JFR}" 2>/dev/null || stat -c%s "${SCENARIO1_JFR}" 2>/dev/null) -log_info "JFR recording size: ${JFR_SIZE} bytes" - -if [ "${JFR_SIZE}" -lt 1024 ]; then - log_error "JFR recording is too small (< 1KB)" - exit 1 -fi - -# Run JFR validation with conformance checking -VALIDATION_LOG="/tmp/jfr-validation/profiler-only-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}.log" -mkdir -p /tmp/jfr-validation - -log_info "Running conformance-based JFR validation..." - -# Use conformance wrapper with explicit configuration flags -# Since we're using the patched agent with ddprof, ddprof is enabled by default -set +e -"${PROJECT_ROOT}/test-validation/validate-jfr-conformance.sh" \ - "${SCENARIO1_JFR}" \ - --ddprof-enabled=true \ - --tracer-enabled=false \ - --test-duration="${TEST_DURATION}" \ - --arch="${ARCH}" \ - --libc="${LIBC_VARIANT}" \ - --jvm-type="${JVM_TYPE}" \ - --output="${VALIDATION_LOG}" - -VALIDATION_EXIT=$? -set -e - -# Show validation output and check exit code -if [ ${VALIDATION_EXIT} -ne 0 ]; then - echo "" - log_error "==========================================" - log_error " PROFILER-ONLY VALIDATION FAILED" - log_error " Exit code: ${VALIDATION_EXIT}" - log_error "==========================================" - log_error "" - - if [ -f "${VALIDATION_LOG}" ]; then - log_error "Validation output:" - cat "${VALIDATION_LOG}" - else - log_error "Validation log file not found: ${VALIDATION_LOG}" - fi - - exit 1 -fi - -log_info "✓ Profiler-only validation PASSED" - -# Copy validation log to results directory for artifact collection -CONFIG_NAME="${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}" -if [ -f "${VALIDATION_LOG}" ]; then - cp "${VALIDATION_LOG}" "${RESULTS_DIR}/profiler-only-${CONFIG_NAME}.log" - log_info "Copied validation log to ${RESULTS_DIR}/profiler-only-${CONFIG_NAME}.log" -fi - -# ======================================== -# Run Scenario 2: Tracer+Profiler -# ======================================== -echo "" -echo "========================================" -echo " Scenario 2: Tracer+Profiler" -echo "========================================" -echo "" - -SCENARIO2_JFR_DIR="/tmp/jfr-tracer-profiler-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}" -SCENARIO2_LOG="/tmp/tracer-profiler-agent.log" - -mkdir -p "${SCENARIO2_JFR_DIR}" - -log_info "Running tracer+profiler test (${TEST_DURATION}s)..." -log_info "JFR dump directory: ${SCENARIO2_JFR_DIR}" -log_info "Log output: ${SCENARIO2_LOG}" - -# Run test with tracer+profiler configuration -"${JAVA_HOME}/bin/java" \ - -javaagent:"${PATCHED_AGENT}" \ - -Ddd.profiling.enabled=true \ - -Ddd.profiling.ddprof.enabled=true \ - -Ddd.trace.enabled=true \ - -Ddd.profiling.start-delay=0 \ - -Ddd.profiling.start-force-first=true \ - -Ddd.profiling.upload.period=10 \ - -Ddd.profiling.debug.dump_path="${SCENARIO2_JFR_DIR}" \ - -Ddd.service.name=profiler-integration-test \ - -Ddd.trace.startup.logs=true \ - -Ddatadog.slf4j.simpleLogger.defaultLogLevel=debug \ - -cp . ProfilerTestApp \ - --duration "${TEST_DURATION}" \ - --threads "${TEST_THREADS}" \ - --cpu-iterations "${CPU_ITERATIONS}" \ - --alloc-rate "${ALLOC_RATE}" \ - > "${SCENARIO2_LOG}" 2>&1 - -log_info "✓ Tracer+profiler test completed" - -# Find the JFR recording (profiler writes continuously) -log_info "Locating JFR recording in ${SCENARIO2_JFR_DIR}..." - -SCENARIO2_JFR=$(find "${SCENARIO2_JFR_DIR}" -name "*.jfr" -print -quit) - -if [ -z "${SCENARIO2_JFR}" ] || [ ! -f "${SCENARIO2_JFR}" ]; then - log_error "JFR recording not found in: ${SCENARIO2_JFR_DIR}" - ls -la "${SCENARIO2_JFR_DIR}" || true - exit 1 -fi - -log_info "Found JFR recording: ${SCENARIO2_JFR}" - -JFR_SIZE=$(stat -f%z "${SCENARIO2_JFR}" 2>/dev/null || stat -c%s "${SCENARIO2_JFR}" 2>/dev/null) -log_info "JFR recording size: ${JFR_SIZE} bytes" - -if [ "${JFR_SIZE}" -lt 1024 ]; then - log_error "JFR recording is too small (< 1KB)" - exit 1 -fi - -# Run JFR validation with conformance checking -VALIDATION_LOG="/tmp/jfr-validation/tracer-profiler-${LIBC_VARIANT}-${ARCH}-${JVM_TYPE}-jdk${JAVA_VERSION}.log" -mkdir -p /tmp/jfr-validation - -log_info "Running conformance-based JFR validation..." - -# Use conformance wrapper with tracer enabled -set +e -"${PROJECT_ROOT}/test-validation/validate-jfr-conformance.sh" \ - "${SCENARIO2_JFR}" \ - --ddprof-enabled=true \ - --tracer-enabled=true \ - --test-duration="${TEST_DURATION}" \ - --arch="${ARCH}" \ - --libc="${LIBC_VARIANT}" \ - --jvm-type="${JVM_TYPE}" \ - --output="${VALIDATION_LOG}" - -VALIDATION_EXIT=$? -set -e - -# Show validation output and check exit code -if [ ${VALIDATION_EXIT} -ne 0 ]; then - echo "" - log_error "==========================================" - log_error " TRACER+PROFILER VALIDATION FAILED" - log_error " Exit code: ${VALIDATION_EXIT}" - log_error "==========================================" - log_error "" - - if [ -f "${VALIDATION_LOG}" ]; then - log_error "Validation output:" - cat "${VALIDATION_LOG}" - else - log_error "Validation log file not found: ${VALIDATION_LOG}" - fi - - exit 1 -fi - -log_info "✓ Tracer+profiler validation PASSED" - -# Copy validation log to results directory for artifact collection -if [ -f "${VALIDATION_LOG}" ]; then - cp "${VALIDATION_LOG}" "${RESULTS_DIR}/tracer-profiler-${CONFIG_NAME}.log" - log_info "Copied validation log to ${RESULTS_DIR}/tracer-profiler-${CONFIG_NAME}.log" -fi - -# Collect final system metrics and generate summary -collect_system_metrics "end" -stop_cpu_monitor -generate_diagnostics_summary - -# ======================================== -# Cleanup -# ======================================== -cd "${PROJECT_ROOT}" -rm -rf "${TEST_APP_DIR}" - -# ======================================== -# Summary -# ======================================== -echo "" -echo "========================================" -echo " Integration Tests Summary" -echo "========================================" -echo "" -log_info "Platform: ${LIBC_VARIANT}-${ARCH}" -log_info "JVM: ${JVM_TYPE} JDK${JAVA_VERSION}" -log_info "✓ Scenario 1: Profiler-Only - PASSED" -log_info "✓ Scenario 2: Tracer+Profiler - PASSED" -log_info "" -log_info "All integration tests PASSED" -log_info "Results saved to: ${RESULTS_DIR}" -echo "" diff --git a/.gitlab/dd-trace-integration/verify-patch-compatibility.sh b/.gitlab/dd-trace-integration/verify-patch-compatibility.sh deleted file mode 100755 index cbae6b7d4..000000000 --- a/.gitlab/dd-trace-integration/verify-patch-compatibility.sh +++ /dev/null @@ -1,397 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Verify compatibility between ddprof classes and patched dd-java-agent - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -# Input JARs -DDPROF_JAR="${DDPROF_JAR:-${PROJECT_ROOT}/ddprof.jar}" -DD_AGENT_JAR="${DD_AGENT_JAR:-${PROJECT_ROOT}/dd-java-agent-original.jar}" -PATCHED_JAR="${PATCHED_JAR:-${PROJECT_ROOT}/dd-java-agent-patched.jar}" - -# Working directory -WORK_DIR="${WORK_DIR:-/tmp/jar-verify-$$}" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -function log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -function log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -function log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} - -function log_debug() { - if [ "${DEBUG:-false}" = "true" ]; then - echo -e "${BLUE}[DEBUG]${NC} $*" - fi -} - -function cleanup() { - if [ -d "${WORK_DIR}" ]; then - log_debug "Cleaning up work directory: ${WORK_DIR}" - rm -rf "${WORK_DIR}" - fi -} - -trap cleanup EXIT - -function usage() { - cat << EOF -Usage: $0 [OPTIONS] - -Verify compatibility between ddprof and patched dd-java-agent. - -This script checks for breaking changes in public API by comparing -class signatures using javap. - -OPTIONS: - --ddprof-jar Path to ddprof.jar (default: ${DDPROF_JAR}) - --dd-agent-jar Path to original dd-java-agent.jar (default: ${DD_AGENT_JAR}) - --patched-jar Path to patched dd-java-agent.jar (default: ${PATCHED_JAR}) - --work-dir Working directory (default: /tmp/jar-verify-\$\$) - --debug Enable debug output - --help Show this help message - -ENVIRONMENT VARIABLES: - DDPROF_JAR Path to ddprof.jar - DD_AGENT_JAR Path to original dd-java-agent.jar - PATCHED_JAR Path to patched dd-java-agent.jar - WORK_DIR Working directory - DEBUG Enable debug output (true/false) - -EXIT CODES: - 0 - Compatible, safe to proceed - 1 - Incompatible, breaking changes detected or validation error - -EXAMPLES: - # Verify with default paths - $0 - - # Verify with custom paths - $0 --ddprof-jar /path/to/ddprof.jar --patched-jar /path/to/patched.jar - - # Enable debug output - DEBUG=true $0 -EOF -} - -# Parse command line arguments -while [ $# -gt 0 ]; do - case "$1" in - --ddprof-jar) - DDPROF_JAR="$2" - shift 2 - ;; - --dd-agent-jar) - DD_AGENT_JAR="$2" - shift 2 - ;; - --patched-jar) - PATCHED_JAR="$2" - shift 2 - ;; - --work-dir) - WORK_DIR="$2" - shift 2 - ;; - --debug) - DEBUG=true - shift - ;; - --help) - usage - exit 0 - ;; - *) - log_error "Unknown option: $1" - usage - exit 1 - ;; - esac -done - -# Validate input JARs exist -if [ ! -f "${DDPROF_JAR}" ]; then - log_error "ddprof JAR not found: ${DDPROF_JAR}" - exit 1 -fi - -if [ ! -f "${DD_AGENT_JAR}" ]; then - log_error "dd-java-agent JAR not found: ${DD_AGENT_JAR}" - exit 1 -fi - -if [ ! -f "${PATCHED_JAR}" ]; then - log_error "patched JAR not found: ${PATCHED_JAR}" - exit 1 -fi - -# Check for javap -if ! command -v javap &> /dev/null; then - log_error "javap not found in PATH" - log_error "javap is part of the JDK. Please ensure JAVA_HOME is set and JDK bin is in PATH" - exit 1 -fi - -log_info "Starting compatibility verification" -log_info " ddprof: ${DDPROF_JAR}" -log_info " dd-agent-orig: ${DD_AGENT_JAR}" -log_info " patched: ${PATCHED_JAR}" - -# Create working directories -mkdir -p "${WORK_DIR}/original" "${WORK_DIR}/ddprof" - -# Extract JARs once -log_info "Extracting dd-java-agent-original..." -if ! unzip -q "${DD_AGENT_JAR}" -d "${WORK_DIR}/original/"; then - log_error "Failed to extract dd-java-agent" - exit 1 -fi - -log_info "Extracting ddprof..." -if ! unzip -q "${DDPROF_JAR}" -d "${WORK_DIR}/ddprof/"; then - log_error "Failed to extract ddprof" - exit 1 -fi - -# Create symlinks for .classdata files so javap can read them -# NOTE: These symlinks are temporary and only for validation - they exist -# in WORK_DIR and are cleaned up by trap. They do NOT affect the patched JAR. -# Only symlink profiler classes (com/datadoghq/profiler) since those are the only ones ddprof provides -log_info "Creating .class symlinks for profiler .classdata files..." - -cd "${WORK_DIR}/original" -if [ -d "shared/com/datadoghq/profiler" ]; then - CLASSDATA_COUNT=$(find shared/com/datadoghq/profiler -name "*.classdata" -type f | wc -l | tr -d ' ') - log_debug "Found ${CLASSDATA_COUNT} profiler .classdata files" - - find shared/com/datadoghq/profiler -name "*.classdata" -type f | while IFS= read -r classdata; do - # URL-decode: %24 -> $ - classfile="${classdata%.classdata}.class" - classfile="${classfile//%24/$}" - - # Create directory if needed (for inner classes) - mkdir -p "$(dirname "${classfile}")" - - log_debug " Symlinking: ${classfile} -> $(basename "${classdata}")" - ln -sf "$(basename "${classdata}")" "${classfile}" - done -else - log_warn "No shared/com/datadoghq/profiler directory found in original" -fi -cd - > /dev/null - -log_info "✓ Created symlinks for javap compatibility" - -# Find all classes in ddprof (excluding META-INF) -log_info "Finding classes to verify..." -CLASSES=$(cd "${WORK_DIR}/ddprof" && find . -name "*.class" -type f | grep -v "^\\./META-INF/" | sed 's|^\./||' || true) -CLASS_COUNT=$(echo "${CLASSES}" | grep -c . || echo 0) - -if [ "${CLASS_COUNT}" -eq 0 ]; then - log_warn "No classes found in ddprof JAR (this is unusual but not necessarily an error)" - log_info "Verification passed (no classes to verify)" - exit 0 -fi - -log_info "Found ${CLASS_COUNT} classes to verify" - -# Extract and compare signatures for each class -CHECKED_COUNT=0 -SKIPPED_COUNT=0 -BREAKING_CHANGES=0 - -function extract_signature() { - local classfile="$1" - local output="$2" - - log_debug " extract_signature: classfile=${classfile}" - - if [ ! -f "${classfile}" ] && [ ! -L "${classfile}" ]; then - log_debug " extract_signature: File does not exist" - return 1 - fi - - # Run javap with -protected flag to get clean method/field signatures - # Skip first line (Compiled from) and class declaration line - if javap -protected "${classfile}" 2>/dev/null | \ - tail -n +2 | \ - grep -E "^\s+(public|protected)" | \ - sed 's/^[[:space:]]*//' | \ - sort > "${output}"; then - log_debug " extract_signature: SUCCESS - extracted signatures" - return 0 - else - log_debug " extract_signature: FAILED - no public/protected members found" - return 1 - fi -} - -function normalize_signature() { - # Normalize method signatures for comparison - # Remove whitespace variations, package names from return types, etc. - sed 's/\s\+/ /g' | \ - sed 's/^ //g' | \ - sed 's/ $//g' | \ - sort -u -} - -log_info "Comparing public/protected API signatures..." - -while IFS= read -r classfile; do - if [ -z "${classfile}" ]; then - continue - fi - - # Convert path to fully qualified class name - classname=$(echo "${classfile}" | sed 's#/#.#g' | sed 's#\.class$##') - - log_debug "Checking class: ${classname}" - - # Check if class exists in original (as .class symlink we created) - ORIGINAL_FILE="${WORK_DIR}/original/shared/${classfile}" - - if [ ! -f "${ORIGINAL_FILE}" ] && [ ! -L "${ORIGINAL_FILE}" ]; then - log_debug "Class ${classname} is new in ddprof (not in original dd-agent)" - SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) - continue - fi - - log_debug " Found in original: shared/${classfile}" - - # Extract signatures from both versions - ORIGINAL_SIG="${WORK_DIR}/original-${classname}.txt" - DDPROF_SIG="${WORK_DIR}/ddprof-${classname}.txt" - - DDPROF_FILE="${WORK_DIR}/ddprof/${classfile}" - - if ! extract_signature "${ORIGINAL_FILE}" "${ORIGINAL_SIG}"; then - log_debug "Could not extract ${classname} from dd-agent (might be non-public or package-private)" - SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) - continue - fi - - if ! extract_signature "${DDPROF_FILE}" "${DDPROF_SIG}"; then - log_warn "⚠ Could not extract ${classname} from ddprof" - SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) - continue - fi - - # Normalize both signatures for comparison - cat "${ORIGINAL_SIG}" | normalize_signature > "${ORIGINAL_SIG}.norm" - cat "${DDPROF_SIG}" | normalize_signature > "${DDPROF_SIG}.norm" - - # Check if all public/protected methods from original are present in ddprof - MISSING_METHODS="${WORK_DIR}/missing-${classname}.txt" - comm -23 "${ORIGINAL_SIG}.norm" "${DDPROF_SIG}.norm" > "${MISSING_METHODS}" - - if [ -s "${MISSING_METHODS}" ]; then - log_warn "⚠ API changes detected in ${classname}:" - log_warn " Methods in dd-java-agent-original but missing in ddprof:" - while IFS= read -r method; do - log_warn " ${method}" - done < "${MISSING_METHODS}" - - # Also check for methods in ddprof that are not in original (additions) - ADDED_METHODS="${WORK_DIR}/added-${classname}.txt" - comm -13 "${ORIGINAL_SIG}.norm" "${DDPROF_SIG}.norm" > "${ADDED_METHODS}" - if [ -s "${ADDED_METHODS}" ]; then - log_info " Methods added in ddprof (not in original):" - while IFS= read -r method; do - log_info " ${method}" - done < "${ADDED_METHODS}" - fi - - BREAKING_CHANGES=$((BREAKING_CHANGES + 1)) - else - log_debug "✓ ${classname} is API-compatible" - fi - - CHECKED_COUNT=$((CHECKED_COUNT + 1)) - -done <<< "${CLASSES}" - -log_info "API compatibility check complete:" -log_info " - ${CHECKED_COUNT} classes verified for API compatibility" -log_info " - ${SKIPPED_COUNT} classes skipped (new or non-public)" -log_info " - ${BREAKING_CHANGES} classes with breaking changes" - -# Verify native libraries are present -log_info "Verifying native libraries..." -NATIVE_IN_DDPROF=$(unzip -l "${DDPROF_JAR}" | grep -c "META-INF/native-libs/.*\.so$" || true) -if [ -z "${NATIVE_IN_DDPROF}" ] || ! [[ "${NATIVE_IN_DDPROF}" =~ ^[0-9]+$ ]]; then - NATIVE_IN_DDPROF=0 -fi - -NATIVE_IN_PATCHED=$(unzip -l "${PATCHED_JAR}" | grep -c "shared/META-INF/native-libs/.*\.so$" || true) -if [ -z "${NATIVE_IN_PATCHED}" ] || ! [[ "${NATIVE_IN_PATCHED}" =~ ^[0-9]+$ ]]; then - NATIVE_IN_PATCHED=0 -fi - -log_debug "Native libraries in ddprof: ${NATIVE_IN_DDPROF}" -log_debug "Native libraries in patched: ${NATIVE_IN_PATCHED}" - -# Note: ddprof.jar may only contain 1 lib (e.g., from debug build on one arch) -# The patched JAR gets all platform libs from libs/$TARGET/ directories, not from ddprof.jar -if [ "${NATIVE_IN_PATCHED}" -eq 0 ]; then - log_warn "No native libraries found in patched JAR" - log_warn "This may be expected if building without native libraries" -else - log_info "✓ Found ${NATIVE_IN_PATCHED} native libraries in patched JAR" -fi - -# Verify all expected platforms are present -log_info "Verifying platform coverage..." -EXPECTED_PLATFORMS=("linux-x64" "linux-x64-musl" "linux-arm64" "linux-arm64-musl") -MISSING_PLATFORMS=() - -for platform in "${EXPECTED_PLATFORMS[@]}"; do - if unzip -l "${PATCHED_JAR}" "shared/META-INF/native-libs/${platform}/libjavaProfiler.so" > /dev/null 2>&1; then - log_debug "✓ Found platform: ${platform}" - else - log_debug "✗ Missing platform: ${platform}" - MISSING_PLATFORMS+=("${platform}") - fi -done - -if [ ${#MISSING_PLATFORMS[@]} -gt 0 ]; then - log_warn "Some platforms are missing: ${MISSING_PLATFORMS[*]}" - log_warn "This may be expected if not all platforms were built" -else - log_info "✓ All expected platforms present" -fi - -# Final verdict -log_info "" -log_info "============================================" -log_info "VERIFICATION COMPLETED" -log_info "============================================" -log_info "" -log_info "Summary:" -log_info " - ${CHECKED_COUNT} classes verified for API compatibility" -log_info " - ${SKIPPED_COUNT} classes skipped (new or non-public)" -log_info " - ${BREAKING_CHANGES} classes with API changes (warnings)" -log_info " - ${NATIVE_IN_PATCHED} native libraries verified" -log_info " - ${#MISSING_PLATFORMS[@]} platforms missing (may be expected)" - -if [ "${BREAKING_CHANGES}" -gt 0 ]; then - log_warn "" - log_warn "API changes detected but not blocking (see warnings above)" - log_warn "Review warnings for methods removed from dd-java-agent" -fi - -exit 0 diff --git a/.gitlab/fuzzing/.gitlab-ci.yml b/.gitlab/fuzzing/.gitlab-ci.yml deleted file mode 100644 index 8c03caac1..000000000 --- a/.gitlab/fuzzing/.gitlab-ci.yml +++ /dev/null @@ -1,31 +0,0 @@ -variables: - FUZZ_IMAGE: registry.ddbuild.io/java-profiler-fuzz - FUZZYDOG_VERSION: "0.28.0" - -fuzz_infra: - needs: [] - extends: .retry-config - image: registry.ddbuild.io/images/docker:27.3.1 - tags: ["arch:amd64"] - stage: fuzz - timeout: 30m - allow_failure: true - id_tokens: - DDSIGN_ID_TOKEN: - aud: image-integrity - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && $RUN_FUZZ == "true"' - - when: manual - before_script: - - apt-get update -qq && apt-get install -y -qq curl unzip jq - - >- - curl -fsSL "https://binaries.ddbuild.io/fuzzing/fuzzydog/${FUZZYDOG_VERSION}/fuzzydog-tar.tar.gz" - | tar -xz -C /usr/local/bin fuzzydog-linux-amd64 && - mv /usr/local/bin/fuzzydog-linux-amd64 /usr/local/bin/fuzzydog - - >- - curl -fsSL "https://releases.hashicorp.com/vault/1.21.1/vault_1.21.1_linux_amd64.zip" - -o /tmp/vault.zip && - unzip -o /tmp/vault.zip -d /usr/local/bin vault && - rm /tmp/vault.zip - script: - - .gitlab/scripts/fuzz_infra.sh diff --git a/.gitlab/ghtools/Dockerfile b/.gitlab/ghtools/Dockerfile deleted file mode 100644 index 6170903fc..000000000 --- a/.gitlab/ghtools/Dockerfile +++ /dev/null @@ -1,5 +0,0 @@ -FROM --platform=linux/amd64 registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest -ARG CI_JOB_TOKEN - -COPY ./setup.sh /tmp/setup.sh -RUN /tmp/setup.sh $CI_JOB_TOKEN \ No newline at end of file diff --git a/.gitlab/ghtools/setup.sh b/.gitlab/ghtools/setup.sh deleted file mode 100755 index 86b41a28d..000000000 --- a/.gitlab/ghtools/setup.sh +++ /dev/null @@ -1,27 +0,0 @@ -#! /bin/bash - -set -eou pipefail - -CI_JOB_TOKEN=$1 - -if [ -z "$CI_JOB_TOKEN" ]; then - echo "Skip installation of Github tools." - exit 0 -fi - -#apt update && apt install -y hwinfo procps git curl software-properties-common build-essential libnss3-dev zlib1g-dev libgdbm-dev libncurses5-dev libssl-dev libffi-dev libreadline-dev libsqlite3-dev libbz2-dev openjdk-11-jdk -#git clone -q --depth 1 https://github.com/pyenv/pyenv.git --branch "v2.0.4" --single-branch /pyenv -#export PYENV_ROOT="/pyenv" -#export PATH="/pyenv/shims:/pyenv/bin:$PATH" -#eval "$(pyenv init -)" -#pyenv install 3.9.6 && pyenv global 3.9.6 -# -#pip3 install awscli virtualenv setuptools - -git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.ddbuild.io/DataDog/benchmarking-platform-tools.git benchmarking-tools - -cd benchmarking-tools/github-tools -./install.sh - -echo "GITHUB_TOOLS_HOME=$(pwd)" >> ~/.bashrc -echo 'PATH=${GITHUB_TOOLS_HOME}:$PATH' >> ~/.bashrc \ No newline at end of file diff --git a/.gitlab/jdk-integration/.gitlab-ci.yml b/.gitlab/jdk-integration/.gitlab-ci.yml deleted file mode 100644 index 4fe94b90f..000000000 --- a/.gitlab/jdk-integration/.gitlab-ci.yml +++ /dev/null @@ -1,53 +0,0 @@ -stages: [trigger, notify] - -benchmark-with-jdk: - stage: trigger - timeout: 6h - variables: - DOWNSTREAM: "${DOWNSTREAM}" - BENCHMARK_JDK: "${BENCHMARK_JDK}" - ITERATIONS: "${BENCHMARK_ITERATIONS:-1}" - MODES: "${BENCHMARK_MODES:-cpu,wall,alloc,memleak}" - KUBERNETES_SERVICE_ACCOUNT_OVERWRITE: java-profiler - rules: - - if: '$BENCHMARK_JDK == null || $DOWNSTREAM == null' - when: never - - when: always - tags: [ "arch:amd64" ] - image: $BENCHMARK_IMAGE_AMD64 - -test-with-jdk: - stage: trigger - variables: - JDK_VERSION: "${JDK_VERSION}" - HASH: "${HASH}" - DEBUG_LEVEL: "${DEBUG_LEVEL}" - rules: - - if: '$JDK_VERSION == null || $DEBUG_LEVEL == null || $HASH == null || $DOWNSTREAM == null' - when: never - - when: always - image: "registry.ddbuild.io/ci/openjdk-build:${JDK_VERSION}-${DEBUG_LEVEL}-${HASH}" - tags: [ "arch:amd64" ] - script: - - apt update && DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends curl git moreutils awscli amazon-ecr-credential-helper gnupg2 build-essential g++ zip unzip cmake - - curl -s "https://get.sdkman.io" | bash - - source "$HOME/.sdkman/bin/sdkman-init.sh" - - sdk install java 11.0.18-tem - - export TEST_JAVA_HOME=/usr/lib/jvm - - export JAVA_HOME=/root/.sdkman/candidates/java/11.0.18-tem - - export JDK_TOOL_OPTIONS="-XX:ErrorFile=/tmp/hs_err_pid_%p.log" - - ./gradlew :ddprof-test:testDebug --max-workers=1 --no-build-cache --stacktrace --info --no-watch-fs --no-daemon - artifacts: - when: always - name: "test-with-jdk-${JDK_VERSION}-${DEBUG_LEVEL}-${HASH}.zip" - paths: - - ddprof-test/build/reports - - /tmp/hs_err*.log - -notify-slack: - stage: notify - when: on_failure - image: registry.ddbuild.io/slack-notifier:latest - tags: ["arch:amd64"] - script: - - .gitlab/jdk-integration/notify_channel.sh "${JDK_VERSION}" "${DEBUG_LEVEL}" "${HASH}" diff --git a/.gitlab/jdk-integration/notify_channel.sh b/.gitlab/jdk-integration/notify_channel.sh deleted file mode 100755 index 77dedb6a4..000000000 --- a/.gitlab/jdk-integration/notify_channel.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -if [ ! -z "${CANCELLED:-}" ]; then - exit 0 -fi - -# Source centralized configuration -HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source "${HERE}/../../.gitlab/config.env" - -PIPELINE_URL="$CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID" -PIPELINE_LINK="<$PIPELINE_URL|pipeline #$CI_PIPELINE_ID>" - -# get the JDK build args -JDK_VERSION=$1 -DEBUG_LEVEL=$2 -HASH=$3 - -JDK_IMAGE="registry.ddbuild.io/ci/openjdk-build:${JDK_VERSION}-${DEBUG_LEVEL}-${HASH}" - -MESSAGE_TEXT=":nuke: JDK Integration tests failed (image=\"${JDK_IMAGE}\", pipeline=$PIPELINE_LINK)" - -postmessage "$SLACK_CHANNEL" "$MESSAGE_TEXT" "alert" diff --git a/.gitlab/reliability/.gitlab-ci.yml b/.gitlab/reliability/.gitlab-ci.yml deleted file mode 100644 index e8740b752..000000000 --- a/.gitlab/reliability/.gitlab-ci.yml +++ /dev/null @@ -1,161 +0,0 @@ -variables: - PREPARE_IMAGE: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:latest - DD_OCTO_STS_IMAGE: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - -# Shared template — concrete jobs supply image, tags, and ARCH variable -.reliability_job: - stage: reliability - timeout: 6h - variables: - RUNTIME: "${RUNTIME}" - needs: - - get-versions - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: always - - when: never - parallel: - matrix: - - CONFIG: ["profiler", "profiler+tracer"] - VARIANT: ["jit", "memory"] - ALLOCATOR: ["gmalloc", "jemalloc", "tcmalloc"] - script: - - set +e - - echo "runtime=${RUNTIME}, config=${CONFIG}, variant=${VARIANT}, allocator=${ALLOCATOR}, arch=${ARCH}" - - .gitlab/reliability/run.sh "$RUNTIME" "$CONFIG" "$VARIANT" "$ALLOCATOR" "$ARCH" 2>err.log 1>out.log - - REASON=$(cat err.log | grep "FAIL:" | cut -f2 -d':') || true - - if [ -n "${REASON}" ]; then echo "REASON_${CONFIG}_${ALLOCATOR}_${ARCH}X${VARIANT}=${REASON}" | tr '+' '_' >> build.env; exit 1; fi - after_script: - - | - if [[ "$CI_JOB_STATUS" == "failed" ]]; then - grep -q "$(printf 'REASON_%s_%s_%sX%s=' "${CONFIG}" "${ALLOCATOR}" "${ARCH}" "${VARIANT}" | tr '+' '_')" build.env 2>/dev/null || echo "REASON_${CONFIG}_${ALLOCATOR}_${ARCH}X${VARIANT}=Unknown failure, perhaps timeout" | tr '+' '_' >> build.env - fi - artifacts: - name: "results-${ARCH}" - when: always - paths: - - memwatch.log - - memwatch-trend.png - - hs_err.log - - err.log - - out.log - reports: - dotenv: build.env - expire_in: 1 day - -reliability-amd64: - extends: .reliability_job - tags: [ "arch:amd64" ] - image: $BENCHMARK_IMAGE_AMD64 - variables: - ARCH: amd64 - -reliability-aarch64: - extends: .reliability_job - tags: [ "arch:arm64" ] - image: $BENCHMARK_IMAGE_ARM64 - variables: - ARCH: aarch64 - -# Chaos variant runs antagonists under a patched dd-java-agent. Pulls the -# precomputed chaos.jar artifact from the chaos:build job (stresstest stage). -.reliability_chaos_job: - stage: reliability - timeout: 6h - variables: - RUNTIME: "${RUNTIME}" - needs: - - get-versions - - job: chaos:build - artifacts: true - - job: build-artifact - artifacts: true - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: always - - when: never - parallel: - matrix: - - CONFIG: ["profiler", "profiler+tracer"] - ALLOCATOR: ["gmalloc", "jemalloc", "tcmalloc"] - CHAOS_JDK: ["21.0.3-tem", "25.0.3-tem"] - script: - - set +e - - echo "runtime=${RUNTIME}, config=${CONFIG}, allocator=${ALLOCATOR}, arch=${ARCH}, jdk=${CHAOS_JDK}" - - CHAOS_JDK="${CHAOS_JDK}" .gitlab/reliability/chaos_check.sh "$RUNTIME" "$CONFIG" "$ALLOCATOR" 2>err.log 1>out.log - - REASON=$(cat err.log | grep "FAIL:" | cut -f2 -d':') || true - - if [ -n "${REASON}" ]; then echo "REASON_${CONFIG}_${ALLOCATOR}_${ARCH}_${CHAOS_JDK//[.-]/_}Xchaos=${REASON}" | tr '+' '_' >> build.env; exit 1; fi - after_script: - - | - if [[ "$CI_JOB_STATUS" == "failed" ]]; then - grep -q "$(printf 'REASON_%s_%s_%s_%sXchaos=' "${CONFIG}" "${ALLOCATOR}" "${ARCH}" "${CHAOS_JDK//[.-]/_}" | tr '+' '_')" build.env 2>/dev/null || echo "REASON_${CONFIG}_${ALLOCATOR}_${ARCH}_${CHAOS_JDK//[.-]/_}Xchaos=Unknown failure, perhaps timeout" | tr '+' '_' >> build.env - fi - artifacts: - name: "chaos-results-${ARCH}" - when: always - paths: - - hs_err.log - - err.log - - out.log - reports: - dotenv: build.env - expire_in: 1 day - -reliability-chaos-amd64: - extends: .reliability_chaos_job - tags: [ "arch:amd64" ] - image: $BENCHMARK_IMAGE_AMD64 - variables: - ARCH: amd64 - -reliability-chaos-aarch64: - extends: .reliability_chaos_job - tags: [ "arch:arm64" ] - image: $BENCHMARK_IMAGE_ARM64 - variables: - ARCH: aarch64 - -notify-slack: - stage: notify - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule"' - when: on_failure - - when: never - image: registry.ddbuild.io/slack-notifier:latest - tags: [ "arch:amd64" ] - needs: - - get-versions - - reliability-amd64 - - reliability-aarch64 - - reliability-chaos-amd64 - - reliability-chaos-aarch64 - script: - - .gitlab/reliability/notify_channel.sh "${CURRENT_VERSION}" - -publish-reliability-gh-pages: - stage: notify - rules: - - if: '$CI_PIPELINE_SOURCE == "schedule" && ($CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH || $CI_COMMIT_BRANCH == "main")' - when: always - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - tags: [ "arch:arm64" ] - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - needs: - - job: reliability-amd64 - artifacts: true - - job: reliability-aarch64 - artifacts: true - - job: reliability-chaos-amd64 - artifacts: true - - job: reliability-chaos-aarch64 - artifacts: true - timeout: 10m - script: - - ./.gitlab/reliability/publish-gh-pages.sh - allow_failure: true - -include: - - local: .gitlab/common.yml - - local: .gitlab/benchmarks/images.yml diff --git a/.gitlab/reliability/chaos_check.sh b/.gitlab/reliability/chaos_check.sh deleted file mode 100755 index 844a8522b..000000000 --- a/.gitlab/reliability/chaos_check.sh +++ /dev/null @@ -1,173 +0,0 @@ -#!/usr/bin/env bash - -set +e # Disable exit on error - -HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -ROOT="$( cd "${HERE}/../.." >/dev/null 2>&1 && pwd )" - -RUNTIME=${1} -CONFIG=${2:-profiler+tracer} -ALLOCATOR=${3:-gmalloc} - -echo "Chaos run: runtime=${RUNTIME}s config=${CONFIG} allocator=${ALLOCATOR}" - -CHAOS_JDK="${CHAOS_JDK:-21.0.3-tem}" -# CHAOS_JDK uses sdkman notation (-); extract major for Adoptium API. -JDK_MAJOR="${CHAOS_JDK%%.*}" -JDK_ARCH=$(uname -m | sed 's/x86_64/x64/') -JDK_INSTALL_DIR="/opt/jdk-${CHAOS_JDK}" - -if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then - TMP=$(mktemp -d) - DL_URL="https://api.adoptium.net/v3/binary/latest/${JDK_MAJOR}/ga/linux/${JDK_ARCH}/jdk/hotspot/normal/eclipse" - echo "Downloading JDK ${CHAOS_JDK} (major ${JDK_MAJOR}) from Adoptium..." - if ! curl -fsSL --max-time 300 "${DL_URL}" -o "${TMP}/jdk.tar.gz"; then - echo "FAIL:JDK ${CHAOS_JDK} download failed" >&2 - rm -rf "${TMP}" - exit 1 - fi - mkdir -p "${JDK_INSTALL_DIR}" - tar -xzf "${TMP}/jdk.tar.gz" -C "${JDK_INSTALL_DIR}" --strip-components=1 - rm -rf "${TMP}" -fi - -if [ ! -x "${JDK_INSTALL_DIR}/bin/java" ]; then - echo "FAIL:JDK ${CHAOS_JDK} not available after install" >&2 - exit 1 -fi -export JAVA_HOME="${JDK_INSTALL_DIR}" -export PATH="${JAVA_HOME}/bin:${PATH}" -ACTIVE_JDK=$(java -version 2>&1 | head -1) -# Check major version only — patch may differ from the Adoptium latest GA. -if ! echo "${ACTIVE_JDK}" | grep -qE "\"${JDK_MAJOR}\."; then - echo "FAIL:wrong JDK active (expected major ${JDK_MAJOR}, got: ${ACTIVE_JDK})" >&2 - exit 1 -fi - -# Resolve ddprof.jar: prefer local build artifact, fall back to Maven snapshot. -# Running mvn from /tmp avoids the empty pom.xml at the repo root. -DDPROF_JAR_LOCAL=$(ls "${ROOT}/ddprof-lib/build/libs/ddprof-"*.jar 2>/dev/null | head -1) -if [ -n "${DDPROF_JAR_LOCAL}" ] && [ -f "${DDPROF_JAR_LOCAL}" ]; then - DDPROF_JAR="${DDPROF_JAR_LOCAL}" - echo "Using local ddprof jar: ${DDPROF_JAR}" -else - if [ -z "${CURRENT_VERSION:-}" ]; then - echo "FAIL:CURRENT_VERSION is empty and no local jar found (get-versions dotenv missing)" >&2 - exit 1 - fi - echo "Local ddprof jar not found — downloading ${CURRENT_VERSION} from Maven snapshots" - (cd /tmp && mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ - -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ - -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION}) - DDPROF_JAR="/root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar" -fi - -if [ ! -f "${DDPROF_JAR}" ]; then - echo "FAIL:ddprof jar unavailable" >&2 - exit 1 -fi - -mkdir -p /var/lib/datadog -wget -q --timeout=120 --tries=3 -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' - -# chaos.jar is produced once per pipeline by the chaos:build job (stresstest -# stage) and pulled here as an artifact. Fall back to an inline build if the -# artifact is absent (e.g. local repro outside CI). -CHAOS_JAR="${ROOT}/ddprof-stresstest/build/libs/chaos.jar" -if [ ! -f "${CHAOS_JAR}" ]; then - echo "chaos.jar artifact not present — building inline" - ( cd "${ROOT}" && ./gradlew :ddprof-stresstest:chaosJar -q ) -fi - -if [ ! -f "${CHAOS_JAR}" ]; then - echo "FAIL:chaos.jar unavailable" >&2 - exit 1 -fi - -# Patch dd-java-agent.jar with the locally built ddprof contents so the agent's -# (relocated) profiler classes match the version under test. -DD_AGENT_JAR=/var/lib/datadog/dd-java-agent.jar \ -DDPROF_JAR=${DDPROF_JAR} \ -OUTPUT_JAR=/var/lib/datadog/dd-java-agent-patched.jar \ -"${ROOT}/utils/patch-dd-java-agent.sh" - -if [ ! -f /var/lib/datadog/dd-java-agent-patched.jar ]; then - echo "FAIL:dd-java-agent patching failed" >&2 - exit 1 -fi - -PATCHED_AGENT=/var/lib/datadog/dd-java-agent-patched.jar - -case $CONFIG in - profiler) - echo "Running with profiler only" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" - # @Trace is a no-op without the tracer, so trace-context is excluded here. - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" - ;; - profiler+tracer) - echo "Running with profiler and tracer" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - ANTAGONISTS="thread-churn,alloc-storm,vthread-churn,classloader-churn,trace-context,bounded-pool,context-hop,consumer-group,hidden-class-churn,direct-memory,weakref-wave,dump-storm" - ;; - *) - echo "Unknown configuration: $CONFIG" - exit 1 - ;; -esac - -case $ALLOCATOR in - gmalloc) - echo "Running with gmalloc" - ;; - tcmalloc) - echo "Running with tcmalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4') - ;; - jemalloc) - echo "Running with jemalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so') - ;; - *) - echo "Unknown allocator: $ALLOCATOR" - echo "Valid values are: gmalloc, tcmalloc, jemalloc" - exit 1 - ;; -esac - -echo "LD_PRELOAD=$LD_PRELOAD" - -timeout "$((RUNTIME + 300))" \ -java -javaagent:${PATCHED_AGENT} \ - ${ENABLEMENT} \ - -Ddd.profiling.upload.period=10 \ - -Ddd.profiling.start-force-first=true \ - -Ddd.profiling.ddprof.liveheap.enabled=true \ - -Ddd.profiling.ddprof.alloc.enabled=true \ - -Ddd.profiling.ddprof.wall.enabled=true \ - -Ddd.profiling.ddprof.nativemem.enabled=true \ - -Ddd.env=java-profiler-stability \ - -Ddd.service=java-profiler-chaos \ - -Xmx2g -Xms2g \ - -XX:MaxMetaspaceSize=384m \ - -XX:ErrorFile=${HERE}/../../hs_err.log \ - -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \ - -jar ${CHAOS_JAR} \ - --duration ${RUNTIME}s \ - --antagonists ${ANTAGONISTS} - -RC=$? -echo "RC=$RC" - -if [ $RC -ne 0 ]; then - CRASH_MSG="Chaos harness crashed (RC=${RC})" - HS_ERR="${HERE}/../../hs_err.log" - if [ -f "${HS_ERR}" ]; then - SIG=$(grep -m1 '^siginfo:' "${HS_ERR}" 2>/dev/null | tr -d '\n' | cut -c1-120) - FRAME=$(grep -m1 'libjavaProfiler\|AsyncProfiler' "${HS_ERR}" 2>/dev/null | sed 's/^[[:space:]]*//' | tr -d '\n' | cut -c1-120) - [ -n "${SIG}" ] && CRASH_MSG="${CRASH_MSG};${SIG}" - [ -n "${FRAME}" ] && CRASH_MSG="${CRASH_MSG};${FRAME}" - fi - echo "FAIL:${CRASH_MSG}" >&2 - exit 1 -fi diff --git a/.gitlab/reliability/generate-run-json.sh b/.gitlab/reliability/generate-run-json.sh deleted file mode 100755 index a460cb0cd..000000000 --- a/.gitlab/reliability/generate-run-json.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash - -# generate-run-json.sh - Generate run JSON for reliability tests -# -# Usage: generate-run-json.sh [artifacts-dir] -# -# Parses build.env files for failure reasons and outputs a JSON object -# suitable for update-history.sh. Reads CI environment variables for metadata. - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="${SCRIPT_DIR}/../.." -ARTIFACTS_DIR="${1:-${PROJECT_ROOT}}" - -# Read metadata from environment or defaults -TIMESTAMP=$(date -u +"%Y-%m-%dT%H:%M:%SZ") -PIPELINE_ID="${CI_PIPELINE_ID:-0}" -PIPELINE_URL="${CI_PIPELINE_URL:-#}" -DDPROF_BRANCH="${DDPROF_COMMIT_BRANCH:-main}" -DDPROF_SHA="${DDPROF_COMMIT_SHA:-unknown}" - -# Read version from environment or version.txt -LIB_VERSION="${CURRENT_VERSION:-unknown}" -if [ "${LIB_VERSION}" = "unknown" ] && [ -f "${PROJECT_ROOT}/version.txt" ]; then - LIB_VERSION=$(awk -F: '{print $NF}' "${PROJECT_ROOT}/version.txt" | tr -d ' ') -fi - -# Lookup PR for branch -PR_JSON="{}" -if [ -x "${SCRIPT_DIR}/../common/lookup-pr.sh" ]; then - PR_JSON=$("${SCRIPT_DIR}/../common/lookup-pr.sh" "${DDPROF_BRANCH}" 2>/dev/null) || PR_JSON="{}" -fi - -# Non-chaos: 2 configs x 2 variants x 3 allocators x 2 architectures = 24 -# Chaos: 2 configs x 3 allocators x 2 architectures = 12 -# Total = 36 -EXPECTED_CONFIGS=36 - -# Parse reliability results -python3 </dev/null 2>&1 && pwd )" - -RUNTIME=${1} -CONFIG=${2:-profiler} -ALLOCATOR=${3:-gmalloc} - -echo "JIT Stability Check with runtime: ${RUNTIME} seconds, config=${CONFIG}" - -curl -s "https://get.sdkman.io" | bash -source "/root/.sdkman/bin/sdkman-init.sh" 1>/dev/null 2>/dev/null - -sdk install java 21.0.3-tem 1>/dev/null 2>/dev/null - -mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ - -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ - -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION} 1>/dev/null 2>/dev/null - -mkdir -p /var/lib/datadog/${CURRENT_VERSION} -rm -rf /var/lib/datadog/${CURRENT_VERSION}/* - -unzip -q -d /var/lib/datadog/${CURRENT_VERSION} /root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar -AGENT_LIB=$(find /var/lib/datadog/${CURRENT_VERSION} -name 'libjavaProfiler.so' | fgrep '/linux-x64/') - -echo "Agent lib: ${AGENT_LIB}" -uname -a -echo "Run duration: ${RUNTIME} seconds" - -wget -q -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' - -case $CONFIG in - profiler) - echo "Running with profiler" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" - ;; - profiler+tracer) - echo "Running with profiler and tracer" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - ;; - *) - echo "Unknown configuration: $CONFIG" - exit 1 - ;; -esac - -case $ALLOCATOR in - gmalloc) - echo "Running with gmalloc" - ;; - tcmalloc) - echo "Running with tcmalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4') - ;; - jemalloc) - echo "Running with jemalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so') - ;; - *) - echo "Unknown allocator: $ALLOCATOR" - echo "Valid values are: gmalloc, tcmalloc, jemalloc" - exit 1 - ;; -esac - -echo "LD_PRELOAD=$LD_PRELOAD" - -start_time=$(date +%s) - -while true; do - # Get the current time - current_time=$(date +%s) - - # Calculate the elapsed time - elapsed_time=$((current_time - start_time)) - - # Break the loop if the elapsed time is greater than or equal to the duration - if ((elapsed_time >= ${RUNTIME})); then - break - fi - - java -javaagent:/var/lib/datadog/dd-java-agent.jar \ - $ENABLEMENT \ - -Ddd.profiling.upload.period=1 \ - -Ddd.profiling.start-force-first=true \ - -Ddd.profiling.ddprof.liveheap.enabled=true \ - -Ddd.profiling.ddprof.alloc.enabled=true \ - -Ddd.profiling.dprof.wall.enabled=true \ - -Ddd.integration.renaissance.enabled=true \ - -Ddd.env=java-profiler-stability \ - -Ddd.service=java-profiler-jit-stability \ - -Ddd.profiling.ddprof.debug.lib="${AGENT_LIB}" \ - -Xmx800m -Xms800m \ - -Dctl=$CONTROL_FILE \ - -XX:ErrorFile=${HERE}/../../hs_err.log \ - -XX:OnError="${HERE}/../../dd_crash_uploader.sh %p" \ - -jar /var/lib/benchmarks/renaissance.jar akka-uct -t 30 - - RC=$? - echo "RC=$RC" - - if [ $RC -ne 0 ]; then - echo "FAIL:Benchmark crashed" >&2 - exit 1 - fi -done diff --git a/.gitlab/reliability/memory_trend_check.py b/.gitlab/reliability/memory_trend_check.py deleted file mode 100644 index d56281f75..000000000 --- a/.gitlab/reliability/memory_trend_check.py +++ /dev/null @@ -1,70 +0,0 @@ -import argparse -import numpy as np -import matplotlib.pyplot as plt - -def detect_memory_growth(memory_log, window_size=5, threshold=0.01): - growth_trend = [] - for i in range(len(memory_log) - window_size + 1): - window = memory_log[i:i + window_size] - growth = np.diff(window) - avg_growth = np.mean(growth) - growth_trend.append(avg_growth > threshold * window[0]) - - persistent_growth = sum(growth_trend) / len(growth_trend) > 0.5 # More than 50% of the windows show growth - return persistent_growth, growth_trend - -def read_memory_log(file_path): - with open(file_path, 'r') as file: - lines = file.readlines() - memory_log = [int(line.split(":")[1].strip()) for line in lines if line.startswith("mem:")] - return memory_log - -def plot_memory_log(memory_log, growth_trend, window_size, output=None, show=False): - print("Plotting memory log to " + output) - - plt.figure(figsize=(10, 5)) - plt.plot(range(0, len(memory_log)), memory_log, label='Memory Usage') - plt.title('Memory Usage Over Time') - plt.xlabel('Sample') - plt.ylabel('Memory (units)') - - # Highlight the windows where growth is detected - for i in range(len(growth_trend)): - if growth_trend[i]: - plt.axvspan(i, i + window_size, color='red', alpha=0.3) - - plt.legend() - if (output): - plt.savefig(output) - plt.close() - if (show): - plt.show() - -def main(): - # Set up the argument parser - parser = argparse.ArgumentParser(description="Calculate moving average and check if the trend is increasing.") - parser.add_argument("filename", type=str, help="The path to the data file.") - parser.add_argument("--plot", action="store_true", help="Plot the memory usage over time.") - parser.add_argument("--output", type=str, help="Output file for the plot.") - - print("Arguments: ", parser.parse_args()) - - # Parse the command line arguments - args = parser.parse_args() - - memory_log = read_memory_log(args.filename ) - - window_size = 5 - threshold = 0.01 - persistent_growth, growth_trend = detect_memory_growth(memory_log, window_size, threshold) - - if persistent_growth: - print("Persistent and significant memory growth detected.") - print("TREND:up") - else: - print("No persistent and significant memory growth detected.") - - plot_memory_log(memory_log, growth_trend, window_size, args.output, args.plot) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/.gitlab/reliability/memory_trend_check.sh b/.gitlab/reliability/memory_trend_check.sh deleted file mode 100755 index e08c35c83..000000000 --- a/.gitlab/reliability/memory_trend_check.sh +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/bin/env bash - -set +e # Disable exit on error - -HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -RUNTIME=${1} -CONFIG=${2:-profiler} -ALLOCATOR=${3:-gmalloc} - -# Function to count CPUs from ranges and individual numbers -count_cpus() { - local cpus=$(cat /sys/fs/cgroup/cpuset/cpuset.cpus) - local cpu_count=0 - local IFS=',' # Use comma as delimiter to split ranges and numbers - - # Iterate over ranges and numbers - for range in $cpus; do - if [[ "$range" =~ - ]]; then - # It's a range, calculate the number of CPUs in the range - local start_cpu=$(echo $range | cut -d '-' -f 1) - local end_cpu=$(echo $range | cut -d '-' -f 2) - local num_cpus=$((end_cpu - start_cpu + 1)) - cpu_count=$((cpu_count + num_cpus)) - else - # It's a single CPU number - cpu_count=$((cpu_count + 1)) - fi - done - - echo $cpu_count -} - -curl -s "https://get.sdkman.io" | bash -source "/root/.sdkman/bin/sdkman-init.sh" 1>/dev/null 2>/dev/null - -sdk install java 21.0.3-tem 1>/dev/null 2>/dev/null - -mvn org.apache.maven.plugins:maven-dependency-plugin:2.1:get \ - -DrepoUrl=https://central.sonatype.com/repository/maven-snapshots/ \ - -Dartifact=com.datadoghq:ddprof:${CURRENT_VERSION} 1>/dev/null 2>/dev/null - -mkdir -p /var/lib/datadog/${CURRENT_VERSION} -rm -rf /var/lib/datadog/${CURRENT_VERSION}/* - -unzip -q -d /var/lib/datadog/${CURRENT_VERSION} /root/.m2/repository/com/datadoghq/ddprof/${CURRENT_VERSION}/ddprof-${CURRENT_VERSION}.jar -AGENT_LIB=$(find /var/lib/datadog/${CURRENT_VERSION} -name 'libjavaProfiler.so' | fgrep '/linux-x64/') - -echo "Agent lib: ${AGENT_LIB}" -uname -a -echo "CPU Cores: $(count_cpus)" -echo "Run duration: ${RUNTIME} seconds" - -wget -q -O /var/lib/datadog/dd-java-agent.jar 'https://dtdg.co/latest-java-tracer' - -CONTROL_FILE=".running" -touch $CONTROL_FILE -sh ./benchmarks/steps/mem_watch.sh $CONTROL_FILE ${HERE}/../../memwatch.log & - -case $CONFIG in - profiler) - echo "Running with profiler" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=false" - ;; - profiler+tracer) - echo "Running with profiler and tracer" - ENABLEMENT="-Ddd.profiling.enabled=true -Ddd.trace.enabled=true" - ;; - *) - echo "Unknown configuration: $CONFIG" - exit 1 - ;; -esac - -case $ALLOCATOR in - gmalloc) - echo "Running with gmalloc" - ;; - tcmalloc) - echo "Running with tcmalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libtcmalloc_minimal.so.4') - ;; - jemalloc) - echo "Running with jemalloc" - export LD_PRELOAD=$(find /usr/lib/ -name 'libjemalloc.so') - ;; - *) - echo "Unknown allocator: $ALLOCATOR" - echo "Valid values are: gmalloc, tcmalloc, jemalloc" - exit 1 - ;; -esac - -echo "LD_PRELOAD=$LD_PRELOAD" - -java -javaagent:/var/lib/datadog/dd-java-agent.jar \ - $ENABLEMENT \ - -Ddd.profiling.upload.period=5 \ - -Ddd.profiling.start-force-first=true \ - -Ddd.profiling.ddprof.liveheap.enabled=true \ - -Ddd.profiling.ddprof.alloc.enabled=true \ - -Ddd.profiling.dprof.wall.enabled=true \ - -Ddd.integration.renaissance.enabled=true \ - -Ddd.env=java-profiler-stability \ - -Ddd.service=java-profiler-memory-trend \ - -Ddd.profiling.ddprof.debug.lib="${AGENT_LIB}" \ - -Xmx800m -Xms800m \ - -XX:MaxMetaspaceSize=384m \ - -Dctl=$CONTROL_FILE \ - -XX:ErrorFile=${HERE}/../../hs_err.log \ - -jar /var/lib/benchmarks/renaissance.jar akka-uct -t ${RUNTIME} - -RC=$? -echo "RC=$RC" -rm -f $CONTROL_FILE > /dev/null - -if [ $RC -ne 0 ]; then - echo "FAIL:Benchmark crashed" >&2 -fi - -LOG=$(python3 ${HERE}/memory_trend_check.py --output ${HERE}/../../memwatch-trend.png ${HERE}/../../memwatch.log) -echo "$LOG" - -TREND=$(echo "$LOG" | grep "TREND:" | cut -f2 -d':') - -if [ "$TREND" == "up" ]; then - echo "FAIL:Memory usage is trending up" >&2 -fi \ No newline at end of file diff --git a/.gitlab/reliability/notify_channel.sh b/.gitlab/reliability/notify_channel.sh deleted file mode 100755 index 80675d5d9..000000000 --- a/.gitlab/reliability/notify_channel.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/bash -set -euxo pipefail - -if [ ! -z "${CANCELLED:-}" ]; then - exit 0 -fi - -# Source centralized configuration -HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -source "${HERE}/../../.gitlab/config.env" - -PIPELINE_URL="$CI_PROJECT_URL/pipelines/$CI_PIPELINE_ID" -PIPELINE_LINK="<$PIPELINE_URL|pipeline #$CI_PIPELINE_ID>" - -VERSION=$1 - -FOUND= -env | while IFS='=' read -r name value; do - # Check if the variable name starts with 'REASON_' - if [[ $name == REASON_* ]]; then - STRIPPED="${name#REASON_}" - CONFIG="${STRIPPED%%X*}" - VARIANT="${STRIPPED#*X}" - REASON=$value - MESSAGE_TEXT=":bomb: Reliability test failed for ${VERSION} (pipeline=$PIPELINE_LINK, reason=\"$REASON\", config=\"$CONFIG\", variant=\"$VARIANT\")" - postmessage "$SLACK_CHANNEL" "$MESSAGE_TEXT" "alert" - found="true" - fi -done diff --git a/.gitlab/reliability/post-pr-comment.sh b/.gitlab/reliability/post-pr-comment.sh deleted file mode 100755 index 8be574a9c..000000000 --- a/.gitlab/reliability/post-pr-comment.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -# Post aggregated reliability + chaos test results as a single PR comment. -# -# Reads REASON_* variables written to build.env by the reliability/chaos jobs -# and emits a ✅/❌ matrix with failure
blocks. -# -# Required env: -# DDPROF_COMMIT_BRANCH – branch name used to locate the open PR -# Optional env: -# CI_PIPELINE_URL - -set -euo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# ── Collect failures from REASON_* env vars ──────────────────────────────────── -rel_fail=0; rel_failures="" -chaos_fail=0; chaos_failures="" - -for key in $(compgen -v | grep -E '^REASON_.*X(jit|memory)$' | sort); do - reason="${!key}" - label="${key#REASON_}" - rel_fail=$((rel_fail + 1)) - detail=$(printf '%s' "${reason//\`/}" | tr ';' '\n') - rel_failures="${rel_failures} -
❌ ${label//_/ } - -\`\`\` -${detail} -\`\`\` - -
" -done - -for key in $(compgen -v | grep -E '^REASON_.*Xchaos$' | sort); do - reason="${!key}" - label="${key#REASON_}" - chaos_fail=$((chaos_fail + 1)) - detail=$(printf '%s' "${reason//\`/}" | tr ';' '\n') - chaos_failures="${chaos_failures} -
❌ chaos: ${label//_/ } - -\`\`\` -${detail} -\`\`\` - -
" -done - -# ── Assemble comment ──────────────────────────────────────────────────────────── -total_fail=$((rel_fail + chaos_fail)) -if [ "${total_fail}" -gt 0 ]; then - overall="❌ **${total_fail} failure(s) detected**" -else - overall="✅ **All reliability & chaos checks passed**" -fi - -BODY_FILE=$(mktemp) -trap 'rm -f "${BODY_FILE}"' EXIT -cat > "${BODY_FILE}" <err.log 1>out.log - - REASON=$(grep -m1 'FAIL:' err.log | cut -f2- -d':' | tr -d '\n') || true - - if [ -n "${REASON}" ]; then _key=$(printf 'REASON_%s_%s_%sX%s' "${CONFIG}" "${ALLOCATOR}" "${ARCH}" "${VARIANT}" | tr '+' '_'); echo "${_key}=${REASON}" >> build.env; exit 1; fi - after_script: - - | - if [[ "$CI_JOB_STATUS" == "failed" ]]; then - _key=$(printf 'REASON_%s_%s_%sX%s' "${CONFIG}" "${ALLOCATOR}" "${ARCH}" "${VARIANT}" | tr '+' '_') - grep -q "${_key}=" build.env 2>/dev/null || echo "${_key}=Unknown failure, perhaps timeout" >> build.env - fi - artifacts: - name: "results-${ARCH}" - when: always - paths: - - memwatch.log - - memwatch-trend.png - - hs_err.log - - err.log - - out.log - reports: - dotenv: build.env - expire_in: 1 day - -reliability-amd64: - extends: .reliability_pr_job - tags: ["arch:amd64"] - image: $BENCHMARK_IMAGE_AMD64 - variables: - ARCH: amd64 - -reliability-aarch64: - extends: .reliability_pr_job - tags: ["arch:arm64"] - image: $BENCHMARK_IMAGE_ARM64 - variables: - ARCH: aarch64 - -# ── Chaos ──────────────────────────────────────────────────────────────────── -# chaos_check.sh builds chaos.jar inline (via Gradle) when the artifact is -# absent, and downloads ddprof from Maven snapshots when no local jar exists. - -.reliability_chaos_pr_job: - stage: test - timeout: 6h - variables: - RUNTIME: "120" - needs: - - job: get-versions - artifacts: true - rules: - - when: on_success - parallel: - matrix: - - CONFIG: ["profiler", "profiler+tracer"] - ALLOCATOR: ["gmalloc", "jemalloc", "tcmalloc"] - CHAOS_JDK: ["21.0.3-tem", "25.0.3-tem"] - script: - - set +e - - echo "runtime=${RUNTIME}, config=${CONFIG}, allocator=${ALLOCATOR}, arch=${ARCH}, jdk=${CHAOS_JDK}" - - CHAOS_JDK="${CHAOS_JDK}" .gitlab/reliability/chaos_check.sh "$RUNTIME" "$CONFIG" "$ALLOCATOR" 2>err.log 1>out.log - - REASON=$(grep -m1 'FAIL:' err.log | cut -f2- -d':' | tr -d '\n') || true - - if [ -n "${REASON}" ]; then _key=$(printf 'REASON_%s_%s_%s_%sXchaos' "${CONFIG}" "${ALLOCATOR}" "${ARCH}" "${CHAOS_JDK//[.-]/_}" | tr '+' '_'); echo "${_key}=${REASON}" >> build.env; exit 1; fi - after_script: - - | - if [[ "$CI_JOB_STATUS" == "failed" ]]; then - _key=$(printf 'REASON_%s_%s_%s_%sXchaos' "${CONFIG}" "${ALLOCATOR}" "${ARCH}" "${CHAOS_JDK//[.-]/_}" | tr '+' '_') - grep -q "${_key}=" build.env 2>/dev/null || echo "${_key}=Unknown failure, perhaps timeout" >> build.env - fi - artifacts: - name: "chaos-results-${ARCH}" - when: always - paths: - - hs_err.log - - err.log - - out.log - reports: - dotenv: build.env - expire_in: 1 day - -reliability-chaos-amd64: - extends: .reliability_chaos_pr_job - tags: ["arch:amd64"] - image: $BENCHMARK_IMAGE_AMD64 - variables: - ARCH: amd64 - -reliability-chaos-aarch64: - extends: .reliability_chaos_pr_job - tags: ["arch:arm64"] - image: $BENCHMARK_IMAGE_ARM64 - variables: - ARCH: aarch64 - -# ── PR comment ─────────────────────────────────────────────────────────────── - -post-reliability-pr-comment: - extends: .retry-config - stage: notify - tags: ["arch:arm64"] - image: registry.ddbuild.io/images/dd-octo-sts-ci-base:2025.06-1 - id_tokens: - DDOCTOSTS_ID_TOKEN: - aud: dd-octo-sts - needs: - - job: reliability-amd64 - artifacts: true - - job: reliability-aarch64 - artifacts: true - - job: reliability-chaos-amd64 - artifacts: true - - job: reliability-chaos-aarch64 - artifacts: true - rules: - - when: always - timeout: 5m - script: - - .gitlab/reliability/post-pr-comment.sh - allow_failure: true diff --git a/.gitlab/reliability/publish-gh-pages.sh b/.gitlab/reliability/publish-gh-pages.sh deleted file mode 100755 index 9f79e5d22..000000000 --- a/.gitlab/reliability/publish-gh-pages.sh +++ /dev/null @@ -1,170 +0,0 @@ -#!/bin/bash - -# publish-gh-pages.sh - Publish reliability test reports to GitHub Pages -# -# Usage: publish-gh-pages.sh [artifacts-dir] -# -# Updates reliability history and regenerates GitHub Pages site. -# Reports are available at: https://datadog.github.io/async-profiler-build/reliability/ -# -# In CI: Uses Octo-STS for secure, short-lived GitHub tokens (no secrets needed) -# Locally: Use 'devflow gitlab auth' for Octo-STS, or set GITHUB_TOKEN env var - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="${SCRIPT_DIR}/../.." -ARTIFACTS_DIR="${1:-${PROJECT_ROOT}}" -export MAX_HISTORY=10 - -# GitHub repo for Pages -GITHUB_REPO="DataDog/java-profiler" -PAGES_URL="https://datadog.github.io/java-profiler" - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $*"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $*"; } -log_error() { echo -e "${RED}[ERROR]${NC} $*"; } - -# Obtain GitHub token -obtain_github_token() { - # Try dd-octo-sts CLI (works in CI with DDOCTOSTS_ID_TOKEN) - if command -v dd-octo-sts >/dev/null 2>&1 && [ -n "${DDOCTOSTS_ID_TOKEN:-}" ]; then - log_info "Obtaining GitHub token via dd-octo-sts CLI..." - # Policy name matches the .sts.yaml filename (without extension) - - # Run dd-octo-sts and capture only stdout (don't capture stderr to avoid error messages in token) - local TOKEN_OUTPUT - local TOKEN_EXIT_CODE - TOKEN_OUTPUT=$(dd-octo-sts token --scope DataDog/java-profiler --policy async-profiler-build.ci 2>/tmp/dd-octo-sts-error.log) - TOKEN_EXIT_CODE=$? - - if [ $TOKEN_EXIT_CODE -eq 0 ] && [ -n "${TOKEN_OUTPUT}" ]; then - # Validate token format (GitHub tokens start with ghs_, ghp_, or look like JWT) - if [[ "${TOKEN_OUTPUT}" =~ ^(ghs_|ghp_|v1\.|[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.) ]]; then - GITHUB_TOKEN="${TOKEN_OUTPUT}" - log_info "GitHub token obtained via dd-octo-sts (expires in 1 hour)" - return 0 - else - log_warn "dd-octo-sts returned invalid token format (first 50 chars): ${TOKEN_OUTPUT:0:50}" - fi - else - log_warn "dd-octo-sts token exchange failed (exit code: ${TOKEN_EXIT_CODE})" - if [ -s /tmp/dd-octo-sts-error.log ]; then - log_warn "dd-octo-sts error output:" - cat /tmp/dd-octo-sts-error.log | head -10 >&2 - fi - fi - fi - - # Fall back to GITHUB_TOKEN environment variable - if [ -n "${GITHUB_TOKEN:-}" ]; then - log_info "Using GITHUB_TOKEN from environment" - return 0 - fi - - return 1 -} - -if ! obtain_github_token; then - log_error "Failed to obtain GitHub token" - log_error "Options:" - log_error " 1. Run in GitLab CI with dd-octo-sts-ci-base image and DDOCTOSTS_ID_TOKEN" - log_error " 2. Set GITHUB_TOKEN env var (PAT with 'repo' scope)" - exit 1 -fi - -# Create temporary directory for gh-pages content -WORK_DIR=$(mktemp -d) -RUN_JSON_FILE=$(mktemp) -# shellcheck disable=SC2064 # Intentional: capture values at setup time -trap "rm -rf ${WORK_DIR} ${RUN_JSON_FILE}" EXIT - -log_info "Preparing gh-pages content in: ${WORK_DIR}" - -# Clone gh-pages branch (or create if doesn't exist) -log_info "Cloning gh-pages branch..." -cd "${WORK_DIR}" - -if git clone --depth 1 --branch gh-pages "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" pages 2>/dev/null; then - cd pages - log_info "Cloned existing gh-pages branch" -else - log_info "Creating new gh-pages branch..." - mkdir pages && cd pages - git init - git remote add origin "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPO}.git" - git checkout -b gh-pages -fi - -# Create Jekyll config if not exists -if [ ! -f "_config.yml" ]; then - cat > "_config.yml" < "${RUN_JSON_FILE}" 2>/dev/null; then - log_info "Generated run JSON" - - # Update history (prepend new run, keep last MAX_HISTORY) - if "${SCRIPT_DIR}/../common/update-history.sh" reliability "${RUN_JSON_FILE}" "." 2>/dev/null; then - log_info "Updated reliability history" - else - log_warn "Failed to update history" - fi -else - log_warn "Failed to generate run JSON" -fi - -# Generate dashboard and index pages -log_info "Generating dashboard..." -if "${SCRIPT_DIR}/../common/generate-dashboard.sh" "." 2>&1; then - log_info "Generated dashboard index.md" -else - log_warn "Failed to generate dashboard" -fi - -log_info "Generating reliability index..." -if "${SCRIPT_DIR}/../common/generate-index.sh" reliability "." 2>/dev/null; then - log_info "Generated reliability/index.md" -else - log_warn "Failed to generate reliability index" -fi - -# Commit and push -TIMESTAMP=$(date -u +"%Y-%m-%d %H:%M UTC") -log_info "Committing changes..." -git add -A -if git diff --staged --quiet; then - log_info "No changes to commit" -else - git config user.email "ci@datadoghq.com" - git config user.name "CI Bot" - git commit -m "Update reliability reports - ${TIMESTAMP}" - - log_info "Pushing to gh-pages..." - git push origin gh-pages --force - - log_info "Reports published successfully!" - log_info "View at: ${PAGES_URL}/reliability/" -fi - -echo "" -echo "PAGES_URL=${PAGES_URL}/reliability/" diff --git a/.gitlab/reliability/run.sh b/.gitlab/reliability/run.sh deleted file mode 100755 index ae3174b7c..000000000 --- a/.gitlab/reliability/run.sh +++ /dev/null @@ -1,32 +0,0 @@ -#!/usr/bin/env bash - -set -e - -HERE="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" - -RUNTIME=${1} -CONFIG=${2:-profiler} -VARIANT=${3:-jit} -ALLOCATOR=${4:-gmalloc} -ARCH=${5:-AMD64} - -echo "Running with runtime: ${RUNTIME} seconds, config=${CONFIG}, variant=${VARIANT}, allocator=${ALLOCATOR}, arch=${ARCH}" - -case $VARIANT in - jit) - # Short variant - ${HERE}/jit_stability_check.sh $RUNTIME $CONFIG $ALLOCATOR - ;; - memory) - # Long variant - ${HERE}/memory_trend_check.sh $RUNTIME $CONFIG $ALLOCATOR - ;; - chaos) - # Reliability chaos harness — runs antagonists under a patched dd-java-agent - ${HERE}/chaos_check.sh $RUNTIME $CONFIG $ALLOCATOR - ;; - *) - echo "Unknown variant: $VARIANT" - exit 1 - ;; -esac \ No newline at end of file diff --git a/.gitlab/sanitizer-tests/.gitlab-ci.yml b/.gitlab/sanitizer-tests/.gitlab-ci.yml deleted file mode 100644 index 1bf7ca232..000000000 --- a/.gitlab/sanitizer-tests/.gitlab-ci.yml +++ /dev/null @@ -1,116 +0,0 @@ -# C++ unit tests under ASan and TSan. -# -# These run on every branch push (not MR pipelines — GitHub Actions handles those). -# -# Strategy: use Gradle only for compile+link (buildGtest{Config}), then run -# each binary directly from the shell. This bypasses Gradle's daemon I/O -# which swallows child process output when fd 1/2 are not the terminal. - -.sanitizer_job: - stage: sanitizer - extends: .cache-config - needs: [] - timeout: 30m - variables: - GRADLE_USER_HOME: .gradle - rules: - - if: '$JDK_VERSION != null || $DEBUG_LEVEL != null || $HASH != null || $DOWNSTREAM != null' - when: never - - if: '$CI_PIPELINE_SOURCE == "merge_request_event"' - when: never - - when: on_success - interruptible: true - before_script: - - apt-get update -qq - - apt-get install -y -qq cmake libgtest-dev libgmock-dev binutils libc6-dbg llvm - script: - - ./gradlew :ddprof-lib:buildGtest${SANITIZER_CONFIG} --no-daemon --parallel --build-cache - - | - find ddprof-lib/build/bin/gtest -mindepth 2 -maxdepth 2 -type f -executable \ - | grep "/${SANITIZER_LC}_" \ - | sort \ - | while read binary; do - echo "" - echo "=== $(basename $binary) ===" - "$binary" - rc=$? - if [ $rc -ne 0 ]; then - echo "FAILED: $(basename $binary) exited $rc" - exit $rc - fi - done - artifacts: - when: always - paths: - - ddprof-lib/build/bin/gtest/${SANITIZER_LC}*/ - expire_in: 1 day - -gtest-asan-amd64: - extends: .sanitizer_job - tags: [ "arch:amd64" ] - image: $BUILD_IMAGE_X64 - variables: - SANITIZER_CONFIG: Asan - SANITIZER_LC: asan - -gtest-tsan-amd64: - extends: .sanitizer_job - # docker-in-docker:amd64 = Kata Containers (kata-qemu micro VMs). - # Kata maps host-guest communication structures at fixed high addresses - # that land in TSan's shadow region regardless of LLVM version or sysctl. - # TSan on amd64 requires a non-Kata runner (EC2 or bare metal). - tags: [ "docker-in-docker:amd64" ] - image: $BUILD_IMAGE_X64 - variables: - SANITIZER_CONFIG: Tsan - SANITIZER_LC: tsan - script: - - ./gradlew :ddprof-lib:buildGtest${SANITIZER_CONFIG} --no-daemon --parallel --build-cache - - | - find ddprof-lib/build/bin/gtest -mindepth 2 -maxdepth 2 -type f -executable \ - | grep "/${SANITIZER_LC}_" | sort | while read binary; do - echo "=== $(basename $binary) ===" - GTEST_DEATH_TEST_STYLE=threadsafe "$binary" - rc=$? - if [ $rc -ne 0 ]; then - echo "FAILED: $(basename $binary) exited $rc" - exit $rc - fi - done - -gtest-asan-arm64: - extends: .sanitizer_job - tags: [ "arch:arm64" ] - image: $BUILD_IMAGE_ARM64 - variables: - SANITIZER_CONFIG: Asan - SANITIZER_LC: asan - -gtest-tsan-arm64: - extends: .sanitizer_job - # docker-in-docker:arm64 = EC2 VM. sysctl works directly. - # vm.mmap_rnd_bits=28 is sufficient — TSan's LLVM re-exec handles the rare - # case where a library lands in the shadow region by re-running the process - # via personality(ADDR_NO_RANDOMIZE). - # Do NOT set kernel.randomize_va_space=0: with ASLR fully off, ld-linux-aarch64.so - # loads at its fixed default address (0x002000000000) which is exactly TSan's - # 39-bit shadow start — guaranteed conflict every time. - tags: [ "docker-in-docker:arm64" ] - image: $BUILD_IMAGE_ARM64 - variables: - SANITIZER_CONFIG: Tsan - SANITIZER_LC: tsan - script: - - ./gradlew :ddprof-lib:buildGtest${SANITIZER_CONFIG} --no-daemon --parallel --build-cache - - | - sysctl -w vm.mmap_rnd_bits=28 2>/dev/null || true - find ddprof-lib/build/bin/gtest -mindepth 2 -maxdepth 2 -type f -executable \ - | grep "/${SANITIZER_LC}_" | sort | while read binary; do - echo "=== $(basename $binary) ===" - GTEST_DEATH_TEST_STYLE=threadsafe "$binary" - rc=$? - if [ $rc -ne 0 ]; then - echo "FAILED: $(basename $binary) exited $rc" - exit $rc - fi - done diff --git a/.gitlab/scripts/build.sh b/.gitlab/scripts/build.sh deleted file mode 100755 index f295e7f4f..000000000 --- a/.gitlab/scripts/build.sh +++ /dev/null @@ -1,62 +0,0 @@ -#! /bin/bash - -set -eo pipefail # exit on any failure, including mid-pipeline -set -x - -if [ ! -z "${CANCELLED:-}" ]; then - exit 0 -fi - -if [ -z "$TARGET" ]; then - echo "Expecting the TARGET variable to be set" - exit 1 -fi - -HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -REPO_ROOT=$( cd "${HERE}/../.." && pwd ) - -if [ -z "${JAVA_HOME}" ] || [ ! -x "${JAVA_HOME}/bin/java" ]; then - # JAVA_HOME is unset or points to a non-existent binary; try the SDKMAN default. - if [ -x ~/.sdkman/candidates/java/current/bin/java ]; then - export JAVA_HOME=~/.sdkman/candidates/java/current - else - echo "ERROR: JAVA_HOME=${JAVA_HOME:-} does not point to a valid Java installation." - exit 1 - fi -fi - -echo "Using Java @ ${JAVA_HOME}" - -source .gitlab/scripts/includes.sh - -# Remove corrupt JARs from Gradle caches (non-ZIP content from failed/proxied downloads) -# JARs not starting with PK magic bytes are invalid and must be deleted before Gradle reads them -for cache_dir in .gradle ~/.gradle; do - if [ -d "$cache_dir" ]; then - find "$cache_dir" -name "*.jar" | while IFS= read -r jar; do - magic=$(head -c 2 "$jar" 2>/dev/null | od -A n -t x1 | tr -d ' \n') - if [ "$magic" != "504b" ]; then - echo "WARN: removing corrupt JAR (magic=${magic:-empty}): $jar" - rm -f "$jar" - fi - done - fi -done - -function onexit { - mkdir -p "${REPO_ROOT}/test/${TARGET}/reports" - mkdir -p "${REPO_ROOT}/test/${TARGET}/logs" - mv "${REPO_ROOT}/ddprof-test/build/reports" "${REPO_ROOT}/test/${TARGET}/" 2>/dev/null || true - mv /tmp/*.jfr "${REPO_ROOT}/test/${TARGET}/logs" 2>/dev/null || true - mv /tmp/*.json "${REPO_ROOT}/test/${TARGET}/logs" 2>/dev/null || true - mv /tmp/*.txt "${REPO_ROOT}/test/${TARGET}/logs" 2>/dev/null || true - find . -name 'hs_err*' | xargs -I {} cp {} "${REPO_ROOT}/test/${TARGET}/logs" 2>/dev/null || true -} - -trap onexit EXIT - -set -x -./gradlew -Pddprof_version="$(get_version)" -Pskip-cpp-tests :ddprof-lib:assembleReleaseJar --no-build-cache --stacktrace --info --no-watch-fs --no-daemon - -mkdir -p "${REPO_ROOT}/libs" -cp -r "${REPO_ROOT}/ddprof-lib/build/native/release/META-INF/native-libs/"* "${REPO_ROOT}/libs/" diff --git a/.gitlab/scripts/check-image-updates.sh b/.gitlab/scripts/check-image-updates.sh deleted file mode 100755 index 1255d3adc..000000000 --- a/.gitlab/scripts/check-image-updates.sh +++ /dev/null @@ -1,282 +0,0 @@ -#!/bin/bash -# check-image-updates.sh - Detect available image updates in registry -# -# Compares current image references in YAML files against the registry -# and outputs a JSON array of needed updates. -# -# Usage: ./scripts/check-image-updates.sh -# Output: JSON array to stdout -# -# Required environment: -# CI_JOB_TOKEN - GitLab CI job token (for API access) -# CI_PROJECT_ID - GitLab project ID (auto-set in CI) -# -# Optional environment: -# GITLAB_URL - GitLab instance URL (default: https://gitlab.ddbuild.io) - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" - -# Configuration -REGISTRY="registry.ddbuild.io" -GITLAB_URL="${GITLAB_URL:-https://gitlab.ddbuild.io}" -GITLAB_PROJECT_PATH="DataDog/java-profiler" - -# Colors for stderr output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } -log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } - -# Image definitions: var_name|yaml_file|tag_suffix|job_name|registry_path -# registry_path is relative to REGISTRY/ci/ -IMAGE_DEFS=( - "BUILD_IMAGE_X64|.gitlab/build-deploy/.gitlab-ci.yml|x64-base|image-base-build-x64|async-profiler-build" - "BUILD_IMAGE_X64_2_17|.gitlab/build-deploy/.gitlab-ci.yml|x64-2.17-base|image-base-build-x64-2.17|async-profiler-build" - "BUILD_IMAGE_X64_MUSL|.gitlab/build-deploy/.gitlab-ci.yml|x64-musl-base|image-base-build-x64-musl|async-profiler-build" - "BUILD_IMAGE_ARM64|.gitlab/build-deploy/.gitlab-ci.yml|arm64-base|image-base-build-arm64|async-profiler-build" - "BUILD_IMAGE_ARM64_MUSL|.gitlab/build-deploy/.gitlab-ci.yml|arm64-musl-base|image-base-build-arm64-musl|async-profiler-build" - "DATADOG_CI_IMAGE|.gitlab/build-deploy/.gitlab-ci.yml|datadog-ci|image-datadog-ci|async-profiler-build" - "BENCHMARK_IMAGE_AMD64|.gitlab/benchmarks/images.yml|amd64-benchmarks|image-benchmarks-amd64|async-profiler-build-amd64" - "BENCHMARK_IMAGE_ARM64|.gitlab/benchmarks/images.yml|arm64-benchmarks|image-benchmarks-arm64|async-profiler-build-arm64" -) - -# Extract current image reference from YAML file -# Returns: full image reference (e.g., registry.../v12345-x64-base@sha256:abc...) -get_current_ref() { - local var_name="$1" - local yaml_file="$2" - local full_path="${PROJECT_ROOT}/${yaml_file}" - - if [[ ! -f "$full_path" ]]; then - log_error "YAML file not found: $full_path" - return 1 - fi - - # Match lines like: BUILD_IMAGE_X64: registry.../... - # Handle both with and without quotes - # Use awk to split on first colon after variable name - grep -E "^\s*${var_name}:" "$full_path" | \ - sed "s/^[[:space:]]*${var_name}:[[:space:]]*//" | \ - tr -d ' "'"'" | \ - head -1 -} - -# Extract pipeline ID from image tag -# Input: v95799584-x64-base or registry.../v95799584-x64-base@sha256:... -# Output: 95799584 -extract_pipeline_id() { - local ref="$1" - echo "$ref" | grep -oE 'v[0-9]+' | head -1 | sed 's/^v//' -} - -# Extract tag from full image reference -# Input: registry.../v95799584-x64-base@sha256:abc... -# Output: v95799584-x64-base -extract_tag() { - local ref="$1" - echo "$ref" | sed 's/@sha256:.*//' | rev | cut -d':' -f1 | rev -} - -# Query registry for latest tag matching pattern -# Returns the tag with highest pipeline ID -get_latest_tag() { - local registry_path="$1" - local tag_suffix="$2" - local full_registry="${REGISTRY}/ci/${registry_path}" - - log_info "Querying registry: ${full_registry} for *-${tag_suffix}" - - # List all tags, filter by suffix, sort by pipeline ID (numeric), get latest - local tags - tags=$(crane ls "${full_registry}" 2>/dev/null || echo "") - - if [[ -z "$tags" ]]; then - log_warn "No tags found in ${full_registry}" - return 1 - fi - - # Filter tags matching pattern v- - local matching_tags - matching_tags=$(echo "$tags" | grep -E "^v[0-9]+-${tag_suffix}$" || true) - - if [[ -z "$matching_tags" ]]; then - log_warn "No tags matching pattern *-${tag_suffix}" - return 1 - fi - - # Sort by pipeline ID (extract number after 'v', sort numerically) - echo "$matching_tags" | \ - awk -F'-' '{print $1}' | \ - sed 's/^v//' | \ - sort -n | \ - tail -1 | \ - xargs -I{} echo "v{}-${tag_suffix}" -} - -# Get SHA256 digest for an image tag -get_digest() { - local registry_path="$1" - local tag="$2" - local full_image="${REGISTRY}/ci/${registry_path}:${tag}" - - crane digest "${full_image}" 2>/dev/null -} - -# Query GitLab API to find job URL within a pipeline -get_job_url() { - local pipeline_id="$1" - local job_name="$2" - - if [[ -z "${CI_JOB_TOKEN:-}" ]]; then - log_warn "CI_JOB_TOKEN not set, cannot query GitLab API for job URL" - echo "" - return 0 - fi - - # URL-encode the project path - local encoded_project - encoded_project=$(echo -n "${GITLAB_PROJECT_PATH}" | jq -sRr @uri) - - local api_url="${GITLAB_URL}/api/v4/projects/${encoded_project}/pipelines/${pipeline_id}/jobs" - - log_info "Querying GitLab API for job '${job_name}' in pipeline ${pipeline_id}" - - local response - response=$(curl -s --fail \ - -H "JOB-TOKEN: ${CI_JOB_TOKEN}" \ - "${api_url}" 2>/dev/null || echo "[]") - - # Find the job with matching name and extract web_url - local job_url - job_url=$(echo "$response" | jq -r ".[] | select(.name == \"${job_name}\") | .web_url" 2>/dev/null | head -1) - - if [[ -n "$job_url" && "$job_url" != "null" ]]; then - echo "$job_url" - else - log_warn "Job '${job_name}' not found in pipeline ${pipeline_id}" - # Fallback to pipeline URL - echo "${GITLAB_URL}/${GITLAB_PROJECT_PATH}/-/pipelines/${pipeline_id}" - fi -} - -# Main detection logic -main() { - log_info "Starting image update detection..." - - # Verify crane is available - if ! command -v crane &>/dev/null; then - log_error "crane command not found. Please install google/go-containerregistry" - exit 1 - fi - - # Verify jq is available - if ! command -v jq &>/dev/null; then - log_error "jq command not found. Please install jq" - exit 1 - fi - - cd "$PROJECT_ROOT" - - local updates="[]" - local checked=0 - local found=0 - - for def in "${IMAGE_DEFS[@]}"; do - IFS='|' read -r var_name yaml_file tag_suffix job_name registry_path <<< "$def" - ((checked++)) - - log_info "Checking ${var_name}..." - - # Get current reference from YAML - local current_ref - current_ref=$(get_current_ref "$var_name" "$yaml_file" || echo "") - if [[ -z "$current_ref" ]]; then - log_warn "Could not find ${var_name} in ${yaml_file}, skipping" - continue - fi - - local current_tag - current_tag=$(extract_tag "$current_ref") - local current_pipeline_id - current_pipeline_id=$(extract_pipeline_id "$current_ref") - - log_info " Current: ${current_tag} (pipeline ${current_pipeline_id})" - - # Get latest tag from registry - local latest_tag - latest_tag=$(get_latest_tag "$registry_path" "$tag_suffix" || echo "") - if [[ -z "$latest_tag" ]]; then - log_warn " Could not determine latest tag, skipping" - continue - fi - - local latest_pipeline_id - latest_pipeline_id=$(extract_pipeline_id "$latest_tag") - - log_info " Latest: ${latest_tag} (pipeline ${latest_pipeline_id})" - - # Compare pipeline IDs - if [[ "$latest_pipeline_id" -gt "$current_pipeline_id" ]]; then - log_info " UPDATE AVAILABLE: ${current_tag} -> ${latest_tag}" - ((found++)) - - # Get digest for new image - local new_digest - new_digest=$(get_digest "$registry_path" "$latest_tag" || echo "") - if [[ -z "$new_digest" ]]; then - log_warn " Could not get digest for ${latest_tag}, skipping" - continue - fi - - # Get job URL from GitLab API - local job_url - job_url=$(get_job_url "$latest_pipeline_id" "$job_name") - - # Build full image reference - local new_full_ref="${REGISTRY}/ci/${registry_path}:${latest_tag}@${new_digest}" - - # Extract current digest for comparison - local current_digest - current_digest=$(echo "$current_ref" | grep -oE 'sha256:[a-f0-9]+' || echo "") - - # Add to updates array - updates=$(echo "$updates" | jq \ - --arg var_name "$var_name" \ - --arg yaml_file "$yaml_file" \ - --arg current_tag "$current_tag" \ - --arg current_digest "$current_digest" \ - --arg new_tag "$latest_tag" \ - --arg new_digest "$new_digest" \ - --arg new_full_ref "$new_full_ref" \ - --arg job_url "$job_url" \ - --arg job_name "$job_name" \ - '. + [{ - var_name: $var_name, - yaml_file: $yaml_file, - current_tag: $current_tag, - current_digest: $current_digest, - new_tag: $new_tag, - new_digest: $new_digest, - new_full_ref: $new_full_ref, - job_url: $job_url, - job_name: $job_name - }]') - else - log_info " Up to date" - fi - done - - log_info "Detection complete: checked ${checked} images, found ${found} updates" - - # Output JSON to stdout - echo "$updates" | jq . -} - -main "$@" diff --git a/.gitlab/scripts/create-image-update-pr.sh b/.gitlab/scripts/create-image-update-pr.sh deleted file mode 100755 index 032644658..000000000 --- a/.gitlab/scripts/create-image-update-pr.sh +++ /dev/null @@ -1,394 +0,0 @@ -#!/bin/bash -# create-image-update-pr.sh - Create GitHub PR for image updates -# -# Takes JSON output from check-image-updates.sh and creates a PR on GitHub -# with the updated image references. -# -# Usage: ./scripts/create-image-update-pr.sh [updates.json] -# cat updates.json | ./scripts/create-image-update-pr.sh -# -# Required environment: -# DDOCTOSTS_ID_TOKEN - GitLab OIDC token for Octo-STS (CI provides this) -# OR -# GITHUB_TOKEN - GitHub token with repo write access (for bootstrap) -# -# Optional environment: -# DRY_RUN - Set to 'true' to skip PR creation - -set -euo pipefail - -# Configuration -GITHUB_REPO="DataDog/java-profiler" -OCTO_STS_SCOPE="DataDog/java-profiler" -OCTO_STS_POLICY="update-images" - -# Colors for stderr output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } -log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } - -# Obtain GitHub token -obtain_github_token() { - # Try Octo-STS first (preferred for CI) - if command -v dd-octo-sts &>/dev/null && [[ -n "${DDOCTOSTS_ID_TOKEN:-}" ]]; then - log_info "Obtaining GitHub token via Octo-STS..." - local token - token=$(dd-octo-sts token --scope "${OCTO_STS_SCOPE}" --policy "${OCTO_STS_POLICY}" 2>/dev/null || echo "") - if [[ -n "$token" ]]; then - log_info "Successfully obtained GitHub token via Octo-STS" - echo "$token" - return 0 - fi - log_warn "Octo-STS token exchange failed (policy may not exist yet)" - fi - - # Fall back to GITHUB_TOKEN environment variable - if [[ -n "${GITHUB_TOKEN:-}" ]]; then - log_info "Using GITHUB_TOKEN from environment" - echo "${GITHUB_TOKEN}" - return 0 - fi - - log_error "No GitHub authentication available" - log_error "Either:" - log_error " 1. Run in GitLab CI with DDOCTOSTS_ID_TOKEN configured, OR" - log_error " 2. Set GITHUB_TOKEN environment variable (for bootstrap)" - return 1 -} - -# Check if STS policy exists in the cloned repo -check_sts_policy_exists() { - local repo_dir="$1" - [[ -f "${repo_dir}/.github/chainguard/update-images.sts.yaml" ]] -} - -# Create STS policy file content -create_sts_policy_content() { - cat <<'EOF' -# Octo-STS Trust Policy for Image Update PRs -# -# This policy allows the GitLab CI check-image-updates job to: -# - Push branches to the repository -# - Create pull requests for image updates -# -# Trust Policy Location: .github/chainguard/update-images.sts.yaml -# Documentation: https://edu.chainguard.dev/open-source/octo-sts/ - -# GitLab OIDC issuer -issuer: https://gitlab.ddbuild.io - -# Match GitLab CI jobs from the async-profiler-build project -# The scheduled job runs on main, but we allow any branch for manual triggers -subject_pattern: project_path:DataDog/java-profiler:ref_type:branch:ref:.* - -# GitHub API permissions -permissions: - # Required to push branches - contents: write - # Required to create PRs - pull_requests: write -EOF -} - -# Update a YAML file with new image reference -update_yaml_file() { - local yaml_file="$1" - local var_name="$2" - local new_full_ref="$3" - local job_url="$4" - - log_info "Updating ${var_name} in ${yaml_file}" - - # Comment format: # Generated by - # For .gitlab-ci.yml (GHTOOLS_IMAGE), format is just: # - local comment_prefix="# Generated by" - if [[ "$yaml_file" == ".gitlab-ci.yml" ]]; then - comment_prefix="#" - fi - - # First, update or add the comment line before the variable - # Look for existing comment and variable pattern - if grep -q "^\s*${comment_prefix}.*${var_name}" "$yaml_file" 2>/dev/null || \ - grep -B1 "^\s*${var_name}:" "$yaml_file" | grep -q "^#"; then - # Update existing comment (line before the variable) - # Use sed with pattern matching: find the line before VAR_NAME and update it - sed -i.bak -E "/^\s*${var_name}:/{ - x - s|^.*$| ${comment_prefix} ${job_url}| - x - }" "$yaml_file" - rm -f "${yaml_file}.bak" - fi - - # Update the variable value - # Match: " VAR_NAME: old_value" and replace with " VAR_NAME: new_value" - sed -i.bak -E "s|^(\s*${var_name}:).*$|\1 ${new_full_ref}|" "$yaml_file" - rm -f "${yaml_file}.bak" -} - -# More robust YAML update using line-by-line processing -update_yaml_robust() { - local yaml_file="$1" - local var_name="$2" - local new_full_ref="$3" - local job_url="$4" - - log_info "Updating ${var_name} in ${yaml_file}" - - local temp_file - temp_file=$(mktemp) - - # Determine comment format - local comment_line - if [[ "$yaml_file" == ".gitlab-ci.yml" ]]; then - comment_line=" # ${job_url}" - else - comment_line=" # Generated by ${job_url}" - fi - - local prev_was_comment=false - - while IFS= read -r line || [[ -n "$line" ]]; do - # Check if this line is the target variable - if [[ "$line" =~ ^[[:space:]]*${var_name}: ]]; then - # Check if previous line was a comment we should replace - if $prev_was_comment; then - # Remove the last line from temp_file (the old comment) - head -n -1 "$temp_file" > "${temp_file}.tmp" - mv "${temp_file}.tmp" "$temp_file" - fi - # Write new comment and updated variable - echo "$comment_line" >> "$temp_file" - # Extract indentation from original line - local indent - indent=$(echo "$line" | sed 's/^\([[:space:]]*\).*/\1/') - echo "${indent}${var_name}: ${new_full_ref}" >> "$temp_file" - prev_was_comment=false - continue - fi - - # Track if this line is a comment (for next iteration) - if [[ "$line" =~ ^[[:space:]]*# ]]; then - prev_was_comment=true - else - prev_was_comment=false - fi - - echo "$line" >> "$temp_file" - done < "$yaml_file" - - mv "$temp_file" "$yaml_file" -} - -# Build PR description -build_pr_description() { - local updates_json="$1" - local include_sts_policy="$2" - - cat </dev/null || echo "") - - if [[ -n "$existing_pr" ]]; then - echo "$existing_pr" - fi -} - -main() { - local updates_file="${1:-}" - - # Read updates JSON - local updates_json - if [[ -n "$updates_file" ]]; then - if [[ ! -f "$updates_file" ]]; then - log_error "Updates file not found: $updates_file" - exit 1 - fi - updates_json=$(cat "$updates_file") - else - log_info "Reading updates from stdin..." - updates_json=$(cat) - fi - - # Validate JSON - if ! echo "$updates_json" | jq empty 2>/dev/null; then - log_error "Invalid JSON input" - exit 1 - fi - - # Reject any entry missing a valid sha256 digest in new_full_ref - local bad_refs - bad_refs=$(echo "$updates_json" | jq -r '.[] | select(.new_full_ref | test("@sha256:[a-f0-9]{64}$") | not) | .var_name') - if [[ -n "$bad_refs" ]]; then - log_error "Refusing to create PR: the following entries have missing or invalid digest in new_full_ref:" - echo "$bad_refs" | while read -r v; do log_error " $v"; done - exit 1 - fi - - # Check if there are any updates - local update_count - update_count=$(echo "$updates_json" | jq 'length') - if [[ "$update_count" -eq 0 ]]; then - log_info "No updates to apply" - exit 0 - fi - - log_info "Processing ${update_count} image update(s)..." - - # Build PR title and description early (for dry run) - local pr_title - pr_title="ci: Update CI images ($(date +%Y-%m-%d))" - local branch_name - branch_name="ci/update-images-$(date +%Y%m%d-%H%M%S)" - - # For dry run, show what would happen without cloning - if [[ "${DRY_RUN:-}" == "true" ]]; then - # Assume STS policy doesn't exist for dry run display - local include_sts_policy="true" - local pr_body - pr_body=$(build_pr_description "$updates_json" "$include_sts_policy") - - log_info "DRY RUN: Would create PR with:" - log_info " Title: ${pr_title}" - log_info " Branch: ${branch_name}" - log_info " Files that would be changed:" - echo "$updates_json" | jq -r '.[] | " - \(.yaml_file): \(.var_name)"' - log_info " - .github/chainguard/update-images.sts.yaml (bootstrap, if not exists)" - log_info "" - log_info " PR Description:" - echo "$pr_body" - exit 0 - fi - - # Obtain GitHub token - local gh_token - gh_token=$(obtain_github_token) || exit 1 - - # Check for existing PR - local existing_pr - existing_pr=$(check_existing_pr "$gh_token") - if [[ -n "$existing_pr" ]]; then - log_warn "An open PR already exists: ${existing_pr}" - log_warn "Please close or merge it before creating a new one" - exit 0 - fi - - # Create temp directory for clone - local work_dir - work_dir=$(mktemp -d) - # shellcheck disable=SC2064 - trap "rm -rf '$work_dir'" EXIT - - log_info "Cloning ${GITHUB_REPO}..." - cd "$work_dir" - git clone --depth 1 "https://x-access-token:${gh_token}@github.com/${GITHUB_REPO}.git" repo - cd repo - - # Check if STS policy exists - local include_sts_policy="false" - if ! check_sts_policy_exists "."; then - log_info "STS policy not found, will include in PR (bootstrap)" - include_sts_policy="true" - mkdir -p .github/chainguard - create_sts_policy_content > .github/chainguard/update-images.sts.yaml - fi - - # Create branch - log_info "Creating branch: ${branch_name}" - git checkout -b "$branch_name" - - # Apply updates to YAML files - echo "$updates_json" | jq -c '.[]' | while read -r update; do - local var_name yaml_file new_full_ref job_url - var_name=$(echo "$update" | jq -r '.var_name') - yaml_file=$(echo "$update" | jq -r '.yaml_file') - new_full_ref=$(echo "$update" | jq -r '.new_full_ref') - job_url=$(echo "$update" | jq -r '.job_url // ""') - - update_yaml_robust "$yaml_file" "$var_name" "$new_full_ref" "$job_url" - done - - # Build final PR description - local pr_body - pr_body=$(build_pr_description "$updates_json" "$include_sts_policy") - - # Commit changes - git config user.email "ci@datadoghq.com" - git config user.name "CI Bot" - git add -A - - # Check if there are changes to commit - if git diff --cached --quiet; then - log_warn "No changes to commit" - exit 0 - fi - - git commit -m "$pr_title" - - # Push branch - log_info "Pushing branch to origin..." - git push -u origin "$branch_name" - - # Create PR - log_info "Creating pull request..." - local pr_url - pr_url=$(GH_TOKEN="$gh_token" gh pr create \ - --repo "${GITHUB_REPO}" \ - --title "$pr_title" \ - --body "$pr_body" \ - --base main \ - --head "$branch_name" \ - --draft) - - log_info "PR created: ${pr_url}" - - # Try to add label (may fail if label doesn't exist) - GH_TOKEN="$gh_token" gh pr edit "$pr_url" --add-label "CI" 2>/dev/null || true - GH_TOKEN="$gh_token" gh pr edit "$pr_url" --add-label "AI" 2>/dev/null || true - - echo "$pr_url" -} - -main "$@" diff --git a/.gitlab/scripts/deploy.sh b/.gitlab/scripts/deploy.sh deleted file mode 100755 index c0e74f676..000000000 --- a/.gitlab/scripts/deploy.sh +++ /dev/null @@ -1,51 +0,0 @@ -#! /bin/bash - -set -eo pipefail # exit on any failure, including mid-pipeline -set +x - -if [ ! -z "${CANCELLED:-}" ]; then - exit 0 -fi - -# NEW: Mode parameter -MODE="${1:-all}" # Options: all, assemble, publish - -HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -# Load centralized configuration -source "${HERE}/../../.gitlab/config.env" - -# debug the CI env -echo "CI_COMMIT_TAG=${CI_COMMIT_TAG}" -echo "CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH}" -echo "CI_DEFAULT_BRANCH=${CI_DEFAULT_BRANCH}" -echo "MODE=${MODE}" - -# Only fetch AWS SSM secrets when publishing -if [ "$MODE" = "publish" ] || [ "$MODE" = "all" ]; then - export SONATYPE_USERNAME=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.sonatype_token_user --with-decryption --query "Parameter.Value" --out text) - export SONATYPE_PASSWORD=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.sonatype_token --with-decryption --query "Parameter.Value" --out text) - export GPG_PRIVATE_KEY=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.signing.gpg_private_key --with-decryption --query "Parameter.Value" --out text) - export GPG_PASSWORD=$(aws ssm get-parameter --region ${AWS_REGION} --name ${SSM_PREFIX}.signing.gpg_passphrase --with-decryption --query "Parameter.Value" --out text) -fi - -source .gitlab/scripts/includes.sh - -LIB_VERSION=$(get_version) -echo "com.datadoghq:ddprof:${LIB_VERSION}" > version.txt - -# Assemble task (always needed for artifact creation) -if [ "$MODE" = "assemble" ] || [ "$MODE" = "all" ]; then - echo "=== Assembling artifact ===" - ./gradlew -Pskip-native -Pskip-tests -Pddprof_version="${LIB_VERSION}" -PbuildInfo.build.number=$CI_JOB_ID -Pwith-libs="$(pwd)/libs" :ddprof-lib:jar assembleAll --exclude-task compileFuzzer --exclude-task sign --max-workers=1 --no-build-cache --stacktrace --info --no-watch-fs --no-daemon -fi - -# Publish task (only when publishing to Maven Central) -if [ "$MODE" = "publish" ] || [ "$MODE" = "all" ]; then - echo "=== Publishing to Sonatype ===" - if [ -z "${GPG_PRIVATE_KEY:-}" ]; then - echo "ERROR: GPG_PRIVATE_KEY is not set — run the create_key CI job first to provision the signing key in SSM (ci.java-profiler.signing.gpg_private_key)" - exit 1 - fi - ./gradlew -Pskip-native -Pskip-tests -Pddprof_version="${LIB_VERSION}" -PbuildInfo.build.number=$CI_JOB_ID -Pwith-libs="$(pwd)/libs" publishToSonatype closeAndReleaseSonatypeStagingRepository --exclude-task compileFuzzer --max-workers=1 --no-build-cache --stacktrace --info --no-watch-fs --no-daemon -fi diff --git a/.gitlab/scripts/fuzz_infra.sh b/.gitlab/scripts/fuzz_infra.sh deleted file mode 100755 index 659592f18..000000000 --- a/.gitlab/scripts/fuzz_infra.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -FUZZ_IMAGE="${FUZZ_IMAGE:-registry.ddbuild.io/java-profiler-fuzz}" -GIT_SHA="${CI_COMMIT_SHORT_SHA:-$(git rev-parse --short HEAD)}" - -export FUZZYDOG_AUTH_TOKEN -FUZZYDOG_AUTH_TOKEN=$(vault read -field=token identity/oidc/token/security-fuzzing-platform) - -# Build and push the compiled image (all fuzz binaries + fuzzydog) -docker buildx build \ - --target build \ - -f docker/Dockerfile.fuzz \ - --build-arg "FUZZYDOG_VERSION=${FUZZYDOG_VERSION}" \ - -t "${FUZZ_IMAGE}:${GIT_SHA}" \ - --push \ - --metadata-file compiled-metadata.json \ - . - -COMPILED_DIGEST=$(jq -r '."containerimage.digest"' compiled-metadata.json) - -# Extract binary list via the manifest target -docker buildx build \ - --target manifest \ - -f docker/Dockerfile.fuzz \ - --build-arg "FUZZYDOG_VERSION=${FUZZYDOG_VERSION}" \ - --output "type=local,dest=manifest-out" \ - . - -# For each binary: build thin per-binary image, sign, replicate, register -while IFS= read -r binary; do - [ -z "${binary}" ] && continue - # Normalize to k8s-safe label: camelCase -> lowercase-hyphenated, prefixed with repo name - normalized=$(printf '%s' "${binary}" | sed 's/[A-Z]/-&/g' | tr '[:upper:]' '[:lower:]' | sed 's/^-//') - fuzz_app="java-profiler-${normalized}" - IMAGE_REF="${FUZZ_IMAGE}:${GIT_SHA}-${normalized}" - - printf 'FROM %s@%s\nENV FUZZ_APP=%s\nENV FUZZ_BUILD_ID=%s\nRUN ln -sf /fuzzer/builds/%s /fuzzer/builds/%s\n' \ - "${FUZZ_IMAGE}" "${COMPILED_DIGEST}" "${fuzz_app}" "${GIT_SHA}" "${binary}" "${GIT_SHA}" \ - | docker buildx build - \ - -t "${IMAGE_REF}" \ - --push \ - --metadata-file "meta-${binary}.json" - - ddsign sign "${IMAGE_REF}" --docker-metadata-file "meta-${binary}.json" - ddsign replicate --to us1.ddbuild.io \ - "${FUZZ_IMAGE}@$(jq -r '."containerimage.digest"' "meta-${binary}.json")" - - fuzzydog fuzzer create "${fuzz_app}" \ - --image "${IMAGE_REF}" \ - --version "${GIT_SHA}" \ - --type libfuzzer \ - --team profiling \ - --slack-channel profiling-java \ - --repository-url https://github.com/DataDog/java-profiler -done < manifest-out/fuzz_binaries.txt diff --git a/.gitlab/scripts/get-github-token-via-octo-sts.sh b/.gitlab/scripts/get-github-token-via-octo-sts.sh deleted file mode 100755 index 0bb85011f..000000000 --- a/.gitlab/scripts/get-github-token-via-octo-sts.sh +++ /dev/null @@ -1,101 +0,0 @@ -#!/bin/bash - -# Exchange GitLab CI OIDC Token for GitHub Token via Datadog Octo-STS -# This script uses GitLab's OIDC provider to obtain a short-lived GitHub token -# without requiring any stored secrets in the GitLab project. - -set -euo pipefail - -# Configuration -OCTO_STS_DOMAIN="${OCTO_STS_DOMAIN:-octo-sts.chainguard.dev}" -OCTO_STS_AUDIENCE="${OCTO_STS_AUDIENCE:-dd-octo-sts}" -OCTO_STS_SCOPE="${OCTO_STS_SCOPE:-DataDog/java-profiler}" -OCTO_STS_POLICY="${OCTO_STS_POLICY:-gist-update}" - -# Colors for output -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -RED='\033[0;31m' -NC='\033[0m' # No Color - -function log_info() { - echo -e "${GREEN}[INFO]${NC} $*" >&2 -} - -function log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" >&2 -} - -function log_error() { - echo -e "${RED}[ERROR]${NC} $*" >&2 -} - -# Validate GitLab CI environment -if [ -z "${CI:-}" ]; then - log_error "This script must run in GitLab CI environment" - log_error "CI environment variable not set" - exit 1 -fi - -# Check for GitLab OIDC token -if [ -z "${CI_JOB_JWT_V2:-}" ]; then - log_error "GitLab OIDC token (CI_JOB_JWT_V2) not available" - log_error "Ensure the CI job has 'id_tokens:' configured" - exit 1 -fi - -log_info "Exchanging GitLab OIDC token for GitHub token via Octo-STS..." -log_info "Scope: ${OCTO_STS_SCOPE}" -log_info "Policy: ${OCTO_STS_POLICY}" - -# Build Octo-STS exchange URL -EXCHANGE_URL="https://${OCTO_STS_DOMAIN}/sts/exchange?scope=${OCTO_STS_SCOPE}&identity=${OCTO_STS_POLICY}" - -log_info "Exchange URL: ${EXCHANGE_URL}" - -# Exchange OIDC token for GitHub token -response=$(curl -s -w "\n%{http_code}" \ - -X POST \ - -H "Authorization: Bearer ${CI_JOB_JWT_V2}" \ - -H "Accept: application/json" \ - "${EXCHANGE_URL}") - -# Split response into body and status code -http_code=$(echo "${response}" | tail -n1) -body=$(echo "${response}" | sed '$d') - -log_info "HTTP status: ${http_code}" -log_info "Response body: ${body}" - -# Check HTTP status code -if [ "${http_code}" -ne 200 ]; then - log_error "Octo-STS token exchange failed (HTTP ${http_code})" - log_error "Response: ${body}" - - if [ "${http_code}" -eq 401 ]; then - log_error "Authentication failed - OIDC token was rejected" - log_error "Possible causes:" - log_error " 1. Trust policy not configured for this repository" - log_error " 2. Trust policy doesn't match GitLab CI claims" - log_error " 3. Octo-STS configuration issue" - elif [ "${http_code}" -eq 404 ]; then - log_error "Trust policy not found" - log_error "Expected location: https://github.com/${OCTO_STS_SCOPE}/.github/chainguard/${OCTO_STS_POLICY}.sts.yaml" - fi - - exit 1 -fi - -# Extract token from response (expecting JSON: {"token": "ghs_..."}) -github_token=$(echo "${body}" | grep -o '"token":"[^"]*"' | cut -d'"' -f4) - -if [ -z "${github_token}" ]; then - log_error "Failed to extract GitHub token from response" - log_error "Response: ${body}" - exit 1 -fi - -log_info "✅ Successfully obtained GitHub token (expires in 1 hour)" - -# Output token to stdout (caller can capture with TOKEN=$(./get-github-token-via-octo-sts.sh)) -echo "${github_token}" diff --git a/.gitlab/scripts/includes.sh b/.gitlab/scripts/includes.sh deleted file mode 100755 index 676b4f29e..000000000 --- a/.gitlab/scripts/includes.sh +++ /dev/null @@ -1,75 +0,0 @@ -function get_version() { - rm -f .version - - if [[ "${CI_COMMIT_TAG}" =~ ^v_[0-9.]+(-SNAPSHOT)?$ ]]; then - echo "${CI_COMMIT_TAG//v_/}" - return - fi - - local gradlecmd=$GRADLE_CMD - if [ -z "$gradlecmd" ]; then - gradlecmd="./gradlew" - fi - ${gradlecmd} printVersion --max-workers=1 --no-build-cache --stacktrace --info --no-watch-fs --no-daemon | grep 'Version:' | cut -f2 -d' ' > .version - local version=$(cat .version) - if [ -z "$version" ]; then - echo "ERROR: Failed to determine version from Gradle printVersion task" >&2 - return 1 - fi - - local branch="${CI_COMMIT_BRANCH:-${CI_COMMIT_REF_NAME:-}}" - local default_branch="${CI_DEFAULT_BRANCH:-main}" - if [ -n "${branch}" ] && [ "${branch}" != "${default_branch}" ] && [ "${branch}" != "main" ]; then - local suffix=$(echo "$branch" | tr '/' '_') - version=$(echo "$version" | sed "s#-SNAPSHOT#-${suffix}-SNAPSHOT#g") - fi - echo "${version}" -} - -function get_current_version() { - get_version -} - -function get_previous_version() { - CURRENT=$(get_current_version) - LOOKBACK=1 - if [[ ! $CURRENT =~ ^.*?-SNAPSHOT$ ]]; then - # current version is not a snapshot; need to look at the previous tag - LOOKBACK=2 - fi - git tag | grep v_ | sort -t_ -k2,2V | tail -n ${LOOKBACK} | head -n 1 | sed -e "s#v_##g" -} - -function setup_java_home() { - if [ -z "${JAVA_HOME}" ] || [ ! -x "${JAVA_HOME}/bin/java" ]; then - if [ -x ~/.sdkman/candidates/java/current/bin/java ]; then - export JAVA_HOME=~/.sdkman/candidates/java/current - else - echo "ERROR: JAVA_HOME=${JAVA_HOME:-} does not point to a valid Java installation." - exit 1 - fi - fi - - echo "Using Java @ ${JAVA_HOME}" -} - -function collect_artifacts() { - local target=$1 - local artifact_type=$2 # "test" or "stresstest" - local source_dir=$3 - local base_dir=${4:-${HERE:-$(pwd)}} - - mkdir -p "${base_dir}/${artifact_type}/${target}/reports" - mkdir -p "${base_dir}/${artifact_type}/${target}/logs" - - # Collect reports - if [ -d "${source_dir}/build/reports" ]; then - cp -r "${source_dir}/build/reports" "${base_dir}/${artifact_type}/${target}/" || echo "WARNING: No reports found" - fi - - # Collect logs from /tmp - find /tmp -maxdepth 1 \( -name "*.jfr" -o -name "*.json" -o -name "*.txt" \) -exec cp {} "${base_dir}/${artifact_type}/${target}/logs/" \; 2>/dev/null || true - - # Collect crash logs (limit search depth to avoid long searches) - find . -maxdepth 2 -name 'hs_err*' -exec cp {} "${base_dir}/${artifact_type}/${target}/logs/" \; 2>/dev/null || true -} diff --git a/.gitlab/scripts/prepare.sh b/.gitlab/scripts/prepare.sh deleted file mode 100755 index 38f841730..000000000 --- a/.gitlab/scripts/prepare.sh +++ /dev/null @@ -1,47 +0,0 @@ -#! /bin/bash - -set -eo pipefail # exit on any failure, including mid-pipeline - -# Normalize CI_COMMIT_BRANCH — it may be empty for trigger/pipeline sources; -# fall back to CI_COMMIT_REF_NAME which is always populated by GitLab. -CI_COMMIT_BRANCH="${CI_COMMIT_BRANCH:-${CI_COMMIT_REF_NAME:-}}" -export CI_COMMIT_BRANCH - -# Check if we should skip non-PR branch builds -# Allow: default branch pushes, scheduled runs, and web (manual) triggers -# Gate: push/trigger/pipeline sources on non-default branches must have an open GitHub PR -if [ "${CI_PIPELINE_SOURCE}" == "push" ] || [ "${CI_PIPELINE_SOURCE}" == "trigger" ] || [ "${CI_PIPELINE_SOURCE}" == "pipeline" ]; then - if [ -n "${CI_COMMIT_BRANCH}" ] && \ - [ "${CI_COMMIT_BRANCH}" != "${CI_DEFAULT_BRANCH:-main}" ] && \ - [[ ! ${CI_COMMIT_TAG} =~ ^v_[1-9][0-9]*\.[0-9]+\.[0-9]+(-SNAPSHOT)?$ ]] && \ - [[ ! ${CI_COMMIT_BRANCH} =~ ^release/[0-9]+\.[0-9]+\._$ ]]; then - # Check if the branch has an open PR in DataDog/java-profiler - API_RESPONSE=$(curl -sf "https://api.github.com/repos/DataDog/java-profiler/pulls?head=DataDog:${CI_COMMIT_BRANCH}&state=open&per_page=1" 2>/dev/null || echo "") - if [ -n "${API_RESPONSE}" ] && ! echo "${API_RESPONSE}" | grep -q '"number"'; then - echo "No open PR for branch ${CI_COMMIT_BRANCH}, skipping pipeline" - echo "CANCELLED=true" >> build.env - exit 0 - fi - # Detect PR labels and export flags for downstream jobs - if command -v jq >/dev/null 2>&1; then - if echo "${API_RESPONSE}" | jq -e '[.[0].labels[].name] | any(. == "test:reliability")' >/dev/null 2>&1; then - echo "RUN_RELIABILITY=true" >> build.env - fi - elif echo "${API_RESPONSE}" | grep -q '"test:reliability"'; then - echo "RUN_RELIABILITY=true" >> build.env - fi - fi -fi - -apt-get update -qq && apt-get install -y -qq openjdk-21-jdk-headless - -source .gitlab/scripts/includes.sh - -LIB_VERSION=$(get_version) -echo "com.datadoghq:ddprof:${LIB_VERSION}" > version.txt - -# Export CI_COMMIT_BRANCH so downstream jobs inherit the resolved value -echo "CI_COMMIT_BRANCH=${CI_COMMIT_BRANCH}" >> build.env -# Export DDPROF_COMMIT_BRANCH/SHA for integration test scripts (legacy variable names) -echo "DDPROF_COMMIT_BRANCH=${CI_COMMIT_BRANCH}" >> build.env -echo "DDPROF_COMMIT_SHA=${CI_COMMIT_SHA}" >> build.env diff --git a/.gitlab/scripts/rebuild-images.sh b/.gitlab/scripts/rebuild-images.sh deleted file mode 100755 index 1ed6bbc2d..000000000 --- a/.gitlab/scripts/rebuild-images.sh +++ /dev/null @@ -1,276 +0,0 @@ -#!/bin/bash -# rebuild-images.sh - Build and push CI images, then create a GitHub PR with updated references -# -# Usage: REBUILD_IMAGES="x64,arm64" ./scripts/rebuild-images.sh -# REBUILD_IMAGES="" ./scripts/rebuild-images.sh # builds all images -# -# Required environment (auto-set in GitLab CI): -# CI_PIPELINE_ID - Pipeline ID used as image tag prefix -# CI_JOB_URL - URL of the CI job (included in PR comment) -# -# Optional environment: -# REBUILD_IMAGES - Comma/space-separated short names; empty = all -# DRY_RUN - Set to 'true' to print plan and exit without building -# REGISTRY - Override registry (default: registry.ddbuild.io) -# -# Base image variables come from .gitlab/build-deploy/images.yml which is -# included in the root pipeline and available as CI variables: -# OPENJDK_BASE_IMAGE, OPENJDK_BASE_IMAGE_ARM64, OPENJDK_BASE_IMAGE_MUSL, -# OPENJDK_BASE_IMAGE_ARM64_MUSL, BASE_IMAGE_LIBC_2_17, BASE_BENCHMARK_IMAGE_NAME, -# DOCKER_IMAGE - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" - -REGISTRY="${REGISTRY:-registry.ddbuild.io}" - -# Colors for stderr -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -NC='\033[0m' - -log_info() { echo -e "${GREEN}[INFO]${NC} $*" >&2; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $*" >&2; } -log_error() { echo -e "${RED}[ERROR]${NC} $*" >&2; } - -usage() { - echo "Usage: REBUILD_IMAGES=\"\" $0" >&2 - echo "" >&2 - echo "Valid short names:" >&2 - echo " x64 glibc x86_64 build image" >&2 - echo " x64-2.17 glibc 2.17 (centos7) x86_64 build image" >&2 - echo " x64-musl musl x86_64 build image" >&2 - echo " arm64 glibc arm64 build image" >&2 - echo " arm64-musl musl arm64 build image" >&2 - echo " datadog-ci datadog-ci utility image" >&2 - echo " benchmarks-amd64 benchmark runner image for amd64" >&2 - echo " benchmarks-arm64 benchmark runner image for arm64" >&2 - echo "" >&2 - echo "Leave REBUILD_IMAGES empty to rebuild all images." >&2 -} - -# IMAGE_DEFS format (pipe-delimited): -# short_name | VAR_NAME | yaml_file | tag_suffix | dockerfile | platform | registry_path | base_image_var -# -# base_image_var is the name of the env var holding the base image (empty if none needed). -IMAGE_DEFS=( - "x64|BUILD_IMAGE_X64|.gitlab/build-deploy/.gitlab-ci.yml|x64-base|.gitlab/base/Dockerfile|linux/amd64|async-profiler-build|OPENJDK_BASE_IMAGE" - "x64-2.17|BUILD_IMAGE_X64_2_17|.gitlab/build-deploy/.gitlab-ci.yml|x64-2.17-base|.gitlab/base/centos7/Dockerfile|linux/amd64|async-profiler-build|BASE_IMAGE_LIBC_2_17" - "x64-musl|BUILD_IMAGE_X64_MUSL|.gitlab/build-deploy/.gitlab-ci.yml|x64-musl-base|.gitlab/base/Dockerfile.musl|linux/amd64|async-profiler-build|OPENJDK_BASE_IMAGE_MUSL" - "arm64|BUILD_IMAGE_ARM64|.gitlab/build-deploy/.gitlab-ci.yml|arm64-base|.gitlab/base/Dockerfile|linux/arm64|async-profiler-build|OPENJDK_BASE_IMAGE_ARM64" - "arm64-musl|BUILD_IMAGE_ARM64_MUSL|.gitlab/build-deploy/.gitlab-ci.yml|arm64-musl-base|.gitlab/base/Dockerfile.musl|linux/arm64|async-profiler-build|OPENJDK_BASE_IMAGE_ARM64_MUSL" - "datadog-ci|DATADOG_CI_IMAGE|.gitlab/build-deploy/.gitlab-ci.yml|datadog-ci|.gitlab/Dockerfile.datadog-ci|linux/amd64|async-profiler-build|" - "benchmarks-amd64|BENCHMARK_IMAGE_AMD64|.gitlab/benchmarks/images.yml|amd64-benchmarks|.gitlab/benchmarks/docker/Dockerfile|linux/amd64|async-profiler-build-amd64|BASE_BENCHMARK_IMAGE_NAME" - "benchmarks-arm64|BENCHMARK_IMAGE_ARM64|.gitlab/benchmarks/images.yml|arm64-benchmarks|.gitlab/benchmarks/docker/Dockerfile|linux/arm64|async-profiler-build-arm64|BASE_BENCHMARK_IMAGE_NAME" -) - -# Extract current image reference from YAML file (reused from check-image-updates.sh) -get_current_ref() { - local var_name="$1" - local yaml_file="$2" - local full_path="${PROJECT_ROOT}/${yaml_file}" - - if [[ ! -f "$full_path" ]]; then - log_error "YAML file not found: $full_path" - return 1 - fi - - grep -E "^\s*${var_name}:" "$full_path" | \ - sed "s/^[[:space:]]*${var_name}:[[:space:]]*//" | \ - tr -d ' "'"'" | \ - head -1 -} - -# Extract tag from full image reference (reused from check-image-updates.sh) -extract_tag() { - local ref="$1" - echo "$ref" | sed 's/@sha256:.*//' | rev | cut -d':' -f1 | rev -} - -# Build one image and return its digest. -# Arguments: short_name tag dockerfile platform registry_path base_image -# Prints the sha256 digest to stdout on success. -build_image() { - local short_name="$1" - local tag="$2" - local dockerfile="$3" - local platform="$4" - local registry_path="$5" - local base_image="$6" - - local full_tag="${REGISTRY}/ci/${registry_path}:${tag}" - local meta_file - meta_file=$(mktemp) - - local build_args=() - [[ -n "$base_image" ]] && build_args+=(--build-arg "BASE_IMAGE=${base_image}") - [[ -n "${CI_JOB_TOKEN:-}" ]] && build_args+=(--build-arg "CI_JOB_TOKEN=${CI_JOB_TOKEN}") - - # benchmarks images change into the docker sub-directory before building - local build_context="." - local dockerfile_flag="-f ${PROJECT_ROOT}/${dockerfile}" - if [[ "$short_name" == benchmarks-* ]]; then - build_context="${PROJECT_ROOT}/.gitlab/benchmarks/docker" - dockerfile_flag="-f ${PROJECT_ROOT}/${dockerfile}" - elif [[ "$dockerfile" == */ ]]; then - # dockerfile is a directory (build context) - build_context="${PROJECT_ROOT}/${dockerfile}" - dockerfile_flag="" - fi - - log_info " Running: docker buildx build --platform=${platform} --tag=${full_tag} ... --push" - - # shellcheck disable=SC2086 - docker buildx build \ - --platform "${platform}" \ - --tag "${full_tag}" \ - "${build_args[@]}" \ - --push \ - --metadata-file "${meta_file}" \ - ${dockerfile_flag} \ - "${build_context}" - - ddsign sign "${full_tag}" --docker-metadata-file "${meta_file}" >&2 - - # Get manifest digest from registry: more reliable than --metadata-file - # which ddsign corrupts for some image types. - docker buildx imagetools inspect "${full_tag}" 2>/dev/null \ - | awk '/Digest:/{print $NF; exit}' -} - -find_def() { - local target_name="$1" - for def in "${IMAGE_DEFS[@]}"; do - local short_name - short_name=$(cut -d'|' -f1 <<< "$def") - if [[ "$short_name" == "$target_name" ]]; then - echo "$def" - return 0 - fi - done - return 1 -} - -main() { - cd "$PROJECT_ROOT" - - # Build list of all valid short names - local all_names=() - for def in "${IMAGE_DEFS[@]}"; do - all_names+=("$(cut -d'|' -f1 <<< "$def")") - done - - # Parse REBUILD_IMAGES (split on comma and/or whitespace) - local selected=() - if [[ -n "${REBUILD_IMAGES:-}" ]]; then - IFS=', ' read -r -a selected <<< "${REBUILD_IMAGES}" - # Validate - for name in "${selected[@]}"; do - if ! find_def "$name" > /dev/null 2>&1; then - log_error "Unknown image name: '${name}'" - usage - exit 1 - fi - done - else - selected=("${all_names[@]}") - fi - - log_info "Images to build: ${selected[*]}" - - if [[ "${DRY_RUN:-}" == "true" ]]; then - log_info "DRY RUN: would build the following images:" - for name in "${selected[@]}"; do - local def - def=$(find_def "$name") - IFS='|' read -r s_name var_name yaml_file tag_suffix dockerfile platform registry_path base_image_var <<< "$def" - local tag="v${CI_PIPELINE_ID:-DRY_RUN}-${tag_suffix}" - log_info " ${name}: ${REGISTRY}/ci/${registry_path}:${tag} (${platform})" - done - exit 0 - fi - - local updates="[]" - local failed=0 - - for name in "${selected[@]}"; do - local def - def=$(find_def "$name") - IFS='|' read -r s_name var_name yaml_file tag_suffix dockerfile platform registry_path base_image_var <<< "$def" - - local base_image="" - if [[ -n "$base_image_var" ]]; then - base_image="${!base_image_var:-}" - if [[ -z "$base_image" ]]; then - log_warn "Base image variable ${base_image_var} is not set for ${name}, building without BASE_IMAGE arg" - fi - fi - - local new_tag="v${CI_PIPELINE_ID}-${tag_suffix}" - log_info "Building ${name} (${new_tag})..." - - local digest - if ! digest=$(build_image "$name" "$new_tag" "$dockerfile" "$platform" "$registry_path" "$base_image"); then - log_error "Build failed for: ${name}" - (( failed++ )) || true - continue - fi - - if [[ -z "$digest" || "$digest" == "null" ]]; then - log_error "Build succeeded but digest is empty for ${name} (ddsign may have corrupted metadata)" - (( failed++ )) || true - continue - fi - - local new_full_ref="${REGISTRY}/ci/${registry_path}:${new_tag}@${digest}" - local current_ref - current_ref=$(get_current_ref "$var_name" "$yaml_file" || echo "") - local current_tag="" - [[ -n "$current_ref" ]] && current_tag=$(extract_tag "$current_ref") - local current_digest - current_digest=$(echo "$current_ref" | grep -oE 'sha256:[a-f0-9]+' || echo "") - local job_url="${CI_JOB_URL:-}" - - updates=$(echo "$updates" | jq \ - --arg var_name "$var_name" \ - --arg yaml_file "$yaml_file" \ - --arg current_tag "$current_tag" \ - --arg current_digest "$current_digest" \ - --arg new_tag "$new_tag" \ - --arg new_digest "$digest" \ - --arg new_full_ref "$new_full_ref" \ - --arg job_url "$job_url" \ - --arg job_name "rebuild-images" \ - '. + [{ - var_name: $var_name, - yaml_file: $yaml_file, - current_tag: $current_tag, - current_digest: $current_digest, - new_tag: $new_tag, - new_digest: $new_digest, - new_full_ref: $new_full_ref, - job_url: $job_url, - job_name: $job_name - }]') - - done - - local update_count - update_count=$(echo "$updates" | jq 'length') - - # Always write updates.json before exiting so the PR job always has valid JSON - echo "$updates" | jq . > "${PROJECT_ROOT}/updates.json" - log_info "Wrote ${update_count} update(s) to updates.json" - - if [[ "$update_count" -eq 0 ]]; then - log_error "No successful builds; nothing to create a PR for" - exit 1 - fi - - exit $failed -} - -main "$@" diff --git a/.gitlab/scripts/stresstests.sh b/.gitlab/scripts/stresstests.sh deleted file mode 100755 index 7efb1e6db..000000000 --- a/.gitlab/scripts/stresstests.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/bash - -set -eo pipefail # exit on any failure, including mid-pipeline -set -x - -if [ ! -z "${CANCELLED:-}" ]; then - exit 0 -fi - -if [ -z "$TARGET" ]; then - echo "Expecting the TARGET variable to be set" - exit 1 -fi - -HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -REPO_ROOT=$( cd "${HERE}/../.." && pwd ) - -if [ -z "${JAVA_HOME}" ]; then - # workaround for CI when JAVA_HOME is not properly defined - export JAVA_HOME=~/.sdkman/candidates/java/current -fi - -echo "Using Java @ ${JAVA_HOME}" - -source .gitlab/scripts/includes.sh - -function onexit { - mkdir -p "${REPO_ROOT}/stresstest/${TARGET}/logs" - mkdir -p "${REPO_ROOT}/stresstest/${TARGET}/results" - mv "${REPO_ROOT}/ddprof-stresstest/jmh-result.html" "${REPO_ROOT}/stresstest/${TARGET}/results" 2>/dev/null || true - find . -name 'hs_err*' | xargs -I {} cp {} "${REPO_ROOT}/stresstest/${TARGET}/logs" 2>/dev/null || true -} - -trap onexit EXIT - -./gradlew -Pddprof_version="$(get_version)" -Pskip-native -Pskip-tests -Pwith-libs="$(pwd)/libs" assembleDebugJar --max-workers=1 --build-cache --stacktrace --info --no-watch-fs --no-daemon -./gradlew -Pddprof_version="$(get_version)" -Pskip-native -Pwith-libs="$(pwd)/libs" -x gtestDebug runStressTests --max-workers=1 --build-cache --stacktrace --info --no-watch-fs --no-daemon diff --git a/.gitlab/scripts/upload.sh b/.gitlab/scripts/upload.sh deleted file mode 100755 index 267abe891..000000000 --- a/.gitlab/scripts/upload.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -# Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. -# This product includes software developed at Datadog (https://www.datadoghq.com/). Copyright 2021-Present Datadog, Inc. - -# http://redsymbol.net/articles/unofficial-bash-strict-mode/ -set -euo pipefail -IFS=$'\n\t' - -# Load centralized configuration -SCRIPT_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" -if [ -f "${SCRIPT_DIR}/../../.gitlab/config.env" ]; then - source "${SCRIPT_DIR}/../../.gitlab/config.env" -fi - -# Helper functions -print_help() { - echo "binary artifact upload tool - - -f - path to local file - -n - what to name the object in S3 - -p - prefix for key (e.g., jplib/) - -b - S3 bucket (default: ${S3_BUCKET:-binaries.ddbuild.io}) - -h - print this help and exit -" -} - -# Parameters -BUCKET="${S3_BUCKET:-binaries.ddbuild.io}" -PRE="" -NAME="" -FILE="" -DRY="no" -CMD="aws s3 cp --region ${AWS_REGION:-us-east-1} --sse AES256 --acl bucket-owner-full-control" - -if [ $# -eq 0 ]; then print_help && exit 0; fi -while getopts ":f:n:b:p:dh" arg; do - case $arg in - f) - FILE=${OPTARG} - ;; - n) - NAME=${OPTARG} - ;; - b) - BUCKET=${OPTARG} - ;; - p) - PRE=${OPTARG} - ;; - d) - DRY="yes" - ;; - h) - print_help - exit 0 - ;; - esac -done - -if [ -z "$NAME" ]; then - echo "No name (-n) given, error" - exit -1 -fi - -if [ -z "$FILE" ]; then - echo "No file (-f) given, error" - exit -1 -fi - -if [ ! -f "$FILE" ]; then - echo "File ($FILE) does not exist, error" - exit -1 -fi - -SHA_FILE="/tmp/$(basename ${FILE}).sha" -sha256sum ${FILE} > ${SHA_FILE} - -if [ "yes" == ${DRY} ]; then - echo ${CMD} ${FILE} s3://${BUCKET}/${PRE}/${NAME} - echo ${CMD} ${SHA_FILE} s3://${BUCKET}/${PRE}/${NAME}.sha -else - eval ${CMD} ${FILE} s3://${BUCKET}/${PRE}/${NAME} - eval ${CMD} ${SHA_FILE} s3://${BUCKET}/${PRE}/${NAME}.sha -fi \ No newline at end of file diff --git a/.gitlab/scripts/upsert-github-pr-comment.sh b/.gitlab/scripts/upsert-github-pr-comment.sh deleted file mode 100755 index c7a60a738..000000000 --- a/.gitlab/scripts/upsert-github-pr-comment.sh +++ /dev/null @@ -1,99 +0,0 @@ -#!/usr/bin/env bash -# Upsert a comment on the java-profiler GitHub PR for the current branch. -# -# Posts (or replaces) a single marker-tagged comment using a short-lived GitHub -# token obtained via dd-octo-sts. No pr-commenter / benchmarking-platform clone -# is required — only dd-octo-sts (present in dd-octo-sts-ci-base) plus curl/jq. -# -# Usage: -# upsert-github-pr-comment.sh -# -# comment-id : unique slug used as an HTML marker to find/replace the comment -# branch : head branch name used to locate the open PR -# body-file : path to a file holding the markdown comment body -# -# Requires in CI: dd-octo-sts CLI + DDOCTOSTS_ID_TOKEN id_token, curl, jq. -# Token policy async-profiler-build.ci grants issues:write + pull_requests:read. - -set -euo pipefail - -COMMENT_ID="${1:?comment-id required}" -BRANCH="${2:?branch required}" -BODY_FILE="${3:?body-file required}" -REPO="DataDog/java-profiler" -API="https://api.github.com/repos/${REPO}" - -log() { echo "[upsert-pr-comment] $*" >&2; } - -# gh_api [data] — performs a GitHub API call, capturing both the -# response body and HTTP status. On HTTP >= 400 it logs the status and body -# (turning opaque "curl 403" failures into actionable diagnostics) and returns 1. -# On success the response body is written to stdout. -gh_api() { - local method="$1" url="$2" data="${3:-}" - local args=(-sS -X "${method}" - -H "Authorization: Bearer ${TOKEN}" - -H "Accept: application/vnd.github+json" - -H "X-GitHub-Api-Version: 2022-11-28" - -H "User-Agent: java-profiler-ci" - -w $'\n%{http_code}') - [ -n "${data}" ] && args+=(-d "${data}") - local resp status body - resp=$(curl "${args[@]}" "${url}") || { log "curl failed for ${method} ${url}"; return 1; } - status="${resp##*$'\n'}" - body="${resp%$'\n'*}" - if [ "${status}" -ge 400 ]; then - log "GitHub API ${method} ${url} -> HTTP ${status}" - log "Response: ${body}" - return 1 - fi - printf '%s' "${body}" -} - -if [ -z "${BRANCH}" ] || [ "${BRANCH}" = "main" ] || [ "${BRANCH}" = "master" ]; then - log "Skipping PR comment for branch: ${BRANCH:-}" - exit 0 -fi -if [ ! -s "${BODY_FILE}" ]; then - log "Empty body file (${BODY_FILE}) — nothing to post" - exit 0 -fi - -# 1. Obtain a GitHub token via dd-octo-sts (no stored secrets). Trim whitespace -# and validate the format, mirroring publish-gh-pages.sh — a token polluted -# with log noise/newlines produces a malformed header and a GitHub 403. -TOKEN=$(dd-octo-sts token --scope "${REPO}" --policy async-profiler-build.ci 2>/tmp/octo-sts.err || true) -TOKEN="${TOKEN//[$'\t\r\n ']/}" -if [ -z "${TOKEN}" ]; then - log "Failed to obtain GitHub token via dd-octo-sts — skipping comment" - [ -s /tmp/octo-sts.err ] && log "dd-octo-sts: $(head -3 /tmp/octo-sts.err)" - exit 0 -fi -if [[ ! "${TOKEN}" =~ ^(ghs_|ghp_|github_pat_|v1\.|[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.) ]]; then - log "dd-octo-sts returned an unexpected token format (first 8 chars: ${TOKEN:0:8}) — skipping" - exit 0 -fi - -# 2. Resolve the open PR for this branch. -PR=$(gh_api GET "${API}/pulls?head=DataDog:${BRANCH}&state=open&per_page=1" | jq -r '.[0].number // empty') -if [ -z "${PR}" ]; then - log "No open PR found for branch ${BRANCH} — skipping comment" - exit 0 -fi - -# 3. Prepend a stable marker and build the JSON payload safely. -MARKER="" -BODY="${MARKER}"$'\n'"$(cat "${BODY_FILE}")" -PAYLOAD=$(jq -n --arg body "${BODY}" '{body: $body}') - -# 4. Find an existing marker comment and PATCH it, otherwise POST a new one. -CID=$(gh_api GET "${API}/issues/${PR}/comments?per_page=100" \ - | jq -r --arg m "${MARKER}" '.[] | select(.body | contains($m)) | .id' | head -n1) - -if [ -n "${CID}" ]; then - gh_api PATCH "${API}/issues/comments/${CID}" "${PAYLOAD}" >/dev/null - log "Updated comment ${CID} on PR #${PR}" -else - gh_api POST "${API}/issues/${PR}/comments" "${PAYLOAD}" >/dev/null - log "Created comment on PR #${PR}" -fi diff --git a/.gitlab/test-apps/ProfilerTestApp.java b/.gitlab/test-apps/ProfilerTestApp.java deleted file mode 100644 index 05925f1bf..000000000 --- a/.gitlab/test-apps/ProfilerTestApp.java +++ /dev/null @@ -1,353 +0,0 @@ -import java.util.ArrayList; -import java.util.List; -import java.util.Random; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Profiler Test Application - * - * Generates workloads to validate profiler functionality: - * - CPU activity (ExecutionSample events) - * - Memory allocations (ObjectAllocationSample events) - * - Thread activity (multiple threads for sampling) - * - * Compatible with JDK 8-25. Compiles with plain javac. - * - * Expected JFR Events: - * - jdk.ExecutionSample: CPU profiling samples - * - jdk.ObjectAllocationSample: Allocation events - * - jdk.ThreadAllocationStatistics: Per-thread allocation stats - * - * Usage: - * javac ProfilerTestApp.java - * java ProfilerTestApp [options] - * - * Options: - * --duration Duration to run (default: 30) - * --threads Number of worker threads (default: 4) - * --cpu-iterations CPU work iterations (default: 10000) - * --alloc-rate Allocations per second (default: 1000) - */ -public class ProfilerTestApp { - - // Configuration - private int durationSeconds = 30; - private int threadCount = 4; - private int cpuIterations = 10000; - private int allocationsPerSecond = 1000; - - // Runtime state - private final AtomicBoolean running = new AtomicBoolean(true); - private final AtomicLong totalIterations = new AtomicLong(0); - private final AtomicLong totalAllocations = new AtomicLong(0); - private final List threads = new ArrayList(); - - /** - * Metrics task - monitors system resources and detects CPU changes - */ - private class MetricsTask implements Runnable { - private int lastCpuCount; - - public MetricsTask() { - this.lastCpuCount = Runtime.getRuntime().availableProcessors(); - } - - public void run() { - while (running.get()) { - try { - Thread.sleep(5000); - - int cpus = Runtime.getRuntime().availableProcessors(); - long freeMemory = Runtime.getRuntime().freeMemory(); - long totalMemory = Runtime.getRuntime().totalMemory(); - - // Structured logging for parsing - System.out.printf("[METRICS] timestamp=%d cpus=%d free_mb=%d total_mb=%d%n", - System.currentTimeMillis() / 1000, - cpus, - freeMemory / 1024 / 1024, - totalMemory / 1024 / 1024); - - // Detect CPU changes - if (cpus != lastCpuCount) { - System.err.printf("[WARN] CPU count changed: %d -> %d%n", lastCpuCount, cpus); - lastCpuCount = cpus; - } - } catch (InterruptedException e) { - break; - } - } - } - } - - public static void main(String[] args) throws Exception { - ProfilerTestApp app = new ProfilerTestApp(); - app.parseArgs(args); - app.run(); - } - - private void parseArgs(String[] args) { - for (int i = 0; i < args.length; i++) { - String arg = args[i]; - - if (arg.equals("--duration") && i + 1 < args.length) { - durationSeconds = Integer.parseInt(args[++i]); - } else if (arg.equals("--threads") && i + 1 < args.length) { - threadCount = Integer.parseInt(args[++i]); - } else if (arg.equals("--cpu-iterations") && i + 1 < args.length) { - cpuIterations = Integer.parseInt(args[++i]); - } else if (arg.equals("--alloc-rate") && i + 1 < args.length) { - allocationsPerSecond = Integer.parseInt(args[++i]); - } else if (arg.equals("--help") || arg.equals("-h")) { - printUsage(); - System.exit(0); - } else { - System.err.println("Unknown argument: " + arg); - printUsage(); - System.exit(1); - } - } - } - - private void printUsage() { - System.out.println("Usage: java ProfilerTestApp [options]"); - System.out.println(); - System.out.println("Options:"); - System.out.println(" --duration Duration to run (default: 30)"); - System.out.println(" --threads Number of worker threads (default: 4)"); - System.out.println(" --cpu-iterations CPU work iterations (default: 10000)"); - System.out.println(" --alloc-rate Allocations per second (default: 1000)"); - System.out.println(" --help, -h Show this help message"); - } - - private void run() throws Exception { - System.out.println("=== Profiler Test Application ==="); - System.out.println("Configuration:"); - System.out.println(" Duration: " + durationSeconds + " seconds"); - System.out.println(" Threads: " + threadCount); - System.out.println(" CPU iterations: " + cpuIterations); - System.out.println(" Allocation rate: " + allocationsPerSecond + " per second"); - System.out.println(); - - // Set up shutdown handler - Runtime.getRuntime().addShutdownHook(new Thread() { - public void run() { - shutdown(); - } - }); - - // Start timer thread - Thread timerThread = new Thread(new TimerTask(), "timer-thread"); - timerThread.setDaemon(false); - timerThread.start(); - - // Start metrics thread - Thread metricsThread = new Thread(new MetricsTask(), "metrics-thread"); - metricsThread.setDaemon(true); - metricsThread.start(); - - // Create barrier for synchronized start - final CyclicBarrier startBarrier = new CyclicBarrier(threadCount); - - // Start worker threads - for (int i = 0; i < threadCount; i++) { - Thread workerThread = new Thread(new WorkerTask(i, startBarrier), "worker-" + i); - workerThread.setDaemon(false); - threads.add(workerThread); - workerThread.start(); - } - - System.out.println("Started " + threadCount + " worker threads"); - System.out.println("Test running..."); - System.out.println(); - - // Wait for timer to expire - timerThread.join(); - - // Stop workers - running.set(false); - - // Wait for all workers to finish - for (Thread thread : threads) { - thread.join(5000); // 5 second timeout per thread - } - - // Print summary - System.out.println(); - System.out.println("=== Test Complete ==="); - System.out.println("Total iterations: " + totalIterations.get()); - System.out.println("Total allocations: " + totalAllocations.get()); - System.out.println(); - } - - private void shutdown() { - running.set(false); - } - - /** - * Timer task - runs for specified duration - */ - private class TimerTask implements Runnable { - public void run() { - try { - Thread.sleep(durationSeconds * 1000L); - } catch (InterruptedException e) { - // Expected - } - } - } - - /** - * Worker task - generates CPU and allocation workload - */ - private class WorkerTask implements Runnable { - private final int workerId; - private final CyclicBarrier startBarrier; - private final Random random = new Random(); - - WorkerTask(int workerId, CyclicBarrier startBarrier) { - this.workerId = workerId; - this.startBarrier = startBarrier; - } - - public void run() { - try { - // Wait for all threads to be ready - startBarrier.await(); - - long iterations = 0; - long allocations = 0; - long lastReportTime = System.currentTimeMillis(); - - while (running.get()) { - // CPU-intensive work - performCPUWork(); - iterations++; - - // Memory allocations - performAllocations(); - // Note: Division by 100 assumes ~100 iterations/second (10ms sleep + work time) - // This is an approximation based on typical execution timing - allocations += allocationsPerSecond / 100; // Per iteration - - // Small sleep to avoid spinning - Thread.sleep(10); - - // Periodic reporting - long now = System.currentTimeMillis(); - if (now - lastReportTime >= 5000) { - System.out.println("Worker " + workerId + ": " + iterations + " iterations, " + allocations + " allocations"); - lastReportTime = now; - } - } - - totalIterations.addAndGet(iterations); - totalAllocations.addAndGet(allocations); - - } catch (Exception e) { - System.err.println("Worker " + workerId + " failed: " + e.getMessage()); - e.printStackTrace(); - } - } - - /** - * Perform CPU-intensive work to generate ExecutionSample events - */ - private void performCPUWork() { - // Mix of different CPU operations - - // 1. Math operations - double result = 0; - for (int i = 0; i < cpuIterations; i++) { - result += Math.sqrt(i); - result *= Math.sin(i * 0.001); - } - - // 2. Prime number calculation - int primeCount = 0; - for (int num = 2; num < cpuIterations && num < 1000; num++) { - if (isPrime(num)) { - primeCount++; - } - } - - // 3. String operations - String text = "profiler-test-" + result + "-" + primeCount; - int hash = text.hashCode(); - - // Prevent optimization - if (hash == Integer.MAX_VALUE) { - System.out.println("Unlikely: " + hash); - } - } - - /** - * Check if number is prime (CPU-intensive) - */ - private boolean isPrime(int n) { - if (n <= 1) return false; - if (n <= 3) return true; - if (n % 2 == 0 || n % 3 == 0) return false; - - for (int i = 5; i * i <= n; i += 6) { - if (n % i == 0 || n % (i + 2) == 0) { - return false; - } - } - return true; - } - - /** - * Perform memory allocations to generate ObjectAllocationSample events - */ - private void performAllocations() { - int allocsPerCall = allocationsPerSecond / 100; // Called ~100 times per second - - // 1. String allocations - List strings = new ArrayList(allocsPerCall); - for (int i = 0; i < allocsPerCall / 3; i++) { - strings.add("allocation-worker-" + workerId + "-iteration-" + i + "-" + random.nextInt()); - } - - // 2. Array allocations - List arrays = new ArrayList(allocsPerCall / 3); - for (int i = 0; i < allocsPerCall / 3; i++) { - arrays.add(new byte[1024]); // 1KB allocations - } - - // 3. Object allocations - List objects = new ArrayList(allocsPerCall / 3); - for (int i = 0; i < allocsPerCall / 3; i++) { - objects.add(new WorkerData(workerId, i, System.nanoTime())); - } - - // Prevent optimization - occasionally use the data - if (random.nextInt(1000) == 0) { - System.out.println("Allocated: " + strings.size() + " strings, " + arrays.size() + " arrays, " + objects.size() + " objects"); - } - } - } - - /** - * Data class for allocation testing - */ - private static class WorkerData { - private final int workerId; - private final int iteration; - private final long timestamp; - private final String description; - - WorkerData(int workerId, int iteration, long timestamp) { - this.workerId = workerId; - this.iteration = iteration; - this.timestamp = timestamp; - this.description = "Worker " + workerId + " iteration " + iteration; - } - - public String getDescription() { - return description + " at " + timestamp; - } - } -} diff --git a/.vdiff/annotations.yaml b/.vdiff/annotations.yaml deleted file mode 100644 index acbc8beba..000000000 --- a/.vdiff/annotations.yaml +++ /dev/null @@ -1,29 +0,0 @@ -- id: 27fc7941-fac1-447c-9668-30f25bab5ad9 - file: build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt - base_ref: v_1.39.0 - head_ref: main - line_start: 255 - line_end: 258 - side: head - context: '' - text: my annotation - author: human - created: 2026-03-21T17:00:35.360904Z - resolved: false - replies: - - author: human - text: yeah - created: 2026-03-21T18:03:34.224545Z -- id: f1e5e0a2-d4f5-44d1-94cd-5e429f450ee2 - file: build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt - base_ref: v_1.39.0 - head_ref: main - line_start: 260 - line_end: 260 - side: head - context: '' - text: interesting stuff - author: human - created: 2026-03-21T17:00:51.447846Z - resolved: false - replies: [] diff --git a/AGENTS.md b/AGENTS.md deleted file mode 100644 index f3a676a72..000000000 --- a/AGENTS.md +++ /dev/null @@ -1,735 +0,0 @@ - - -# AGENTS.md - -This file provides guidance to AI coding assistants when working with code in this repository. - -## Project Overview - -This is the Datadog Java Profiler Library, a specialized profiler derived from async-profiler but tailored for Datadog's needs. It's a multi-language project combining Java, C++, and Gradle build system with native library compilation. - -**Key Technologies:** -- Java 8+ (main API and library loading) -- C++17 (native profiling engine) -- Gradle (build system with custom native compilation tasks) -- JNI (Java Native Interface for C++ integration) -- Google Test (for C++ unit tests, compiled via custom Gradle tasks) - -## Project Operating Guide (Main Session) - -You are the **Main Orchestrator** for this repository. - -### Goals -- When I ask you to build, you MUST: - 1) run the Gradle task with plain console and increased verbosity, - 2) capture stdout into `build/logs/-.log`, - 3) **delegate** parsing to the sub-agent `gradle-log-analyst`, - 4) respond in chat with only a short status and the two output file paths: - - `build/reports/claude/gradle-summary.md` - - `build/reports/claude/gradle-summary.json` - -### Rules -- **Never** paste large log chunks into the chat. -- Prefer shell over long in-chat output. If more than ~30 lines are needed, write to a file. -- If no log path is provided, use the newest `build/logs/*.log`. -- Assume macOS/Linux unless I explicitly say Windows; if Windows, use PowerShell equivalents. -- If a step fails, print the failing command and a one-line hint, then stop. - -### Implementation Hints -- For builds, always use: `--console=plain -i` (or `-d` if I ask for debug). -- Use `mkdir -p build/logs build/reports/claude` before writing. -- Timestamp format: `$(date +%Y%m%d-%H%M%S)`. -- After the build finishes, call the sub-agent like: - "Use `gradle-log-analyst` to parse LOG_PATH; write the two reports; reply with only a 3–6 line status and the two relative file paths." - -### Shortcuts I Expect -- `./gradlew ` to do everything in one step. -- If I just say "build assembleDebugJar", interpret that as the shortcut above. - -## Build Commands -Never use 'gradle' or 'gradlew' directly. Instead, use the '/build-and-summarize' command. - -### Main Build Tasks -```bash -# Build release version (primary artifact) -./gradlew buildRelease - -# Build all configurations -./gradlew assembleAll - -# Clean build -./gradlew clean -``` - -### Development Builds -```bash -# Debug build with symbols -./gradlew buildDebug - -# ASan build (if available) -./gradlew buildAsan - -# TSan build (if available) -./gradlew buildTsan -``` - -### Testing -```bash -# Run specific test configurations -./gradlew testRelease -./gradlew testDebug -./gradlew testAsan -./gradlew testTsan - -# Run C++ unit tests only (via GtestPlugin) -./gradlew :ddprof-lib:gtestDebug # All debug tests -./gradlew :ddprof-lib:gtestRelease # All release tests -./gradlew :ddprof-lib:gtest # All tests, all configs - -# Run individual C++ test -./gradlew :ddprof-lib:gtestDebug_test_callTraceStorage - -# Cross-JDK testing -JAVA_TEST_HOME=/path/to/test/jdk ./gradlew testDebug -``` - -#### Google Test Plugin - -The project uses a custom `GtestPlugin` (in `build-logic/`) for C++ unit testing with Google Test. The plugin automatically: -- Discovers `.cpp` test files in `src/test/cpp/` -- Creates compilation, linking, and execution tasks for each test -- Filters configurations by current platform/architecture -- Integrates with NativeCompileTask and NativeLinkExecutableTask - -**Key features:** -- Platform-aware: Only creates tasks for matching OS/arch -- Assertion control: Removes `-DNDEBUG` to enable assertions in tests -- Symbol preservation: Keeps debug symbols in release test builds -- Task aggregation: Per-config (`gtestDebug`) and master (`gtest`) tasks -- Shared configurations: Uses BuildConfiguration from NativeBuildPlugin - -**Configuration example (ddprof-lib/build.gradle.kts):** -```kotlin -plugins { - id("com.datadoghq.native-build") - id("com.datadoghq.gtest") -} - -gtest { - testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) - mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) - includes.from( - "src/main/cpp", - "${javaHome}/include", - "${javaHome}/include/${platformInclude}" - ) - // Optional - enableAssertions.set(true) // Remove -DNDEBUG (default: true) - keepSymbols.set(true) // Keep symbols in release (default: true) - failFast.set(false) // Stop on first failure (default: false) -} -``` - -**See:** `build-logic/README.md` for full documentation - -#### Debug Symbol Extraction - -Release builds automatically extract debug symbols via `NativeLinkTask`, reducing production binary size (~69% smaller) while maintaining separate debug files for offline debugging. - -**Key features:** -- Platform-aware: Uses `objcopy`/`strip` on Linux, `dsymutil`/`strip` on macOS -- Automatic workflow: Extract symbols → Add GNU debuglink (Linux) → Strip library → Copy artifacts -- Size optimization: Stripped ~1.2MB production library from ~6.1MB with embedded debug info -- Debug preservation: Separate `.debug` files (Linux) or `.dSYM` bundles (macOS) - -**Tool requirements:** -- Linux: `binutils` package (objcopy, strip) -- macOS: Xcode Command Line Tools (dsymutil, strip) - -**Skip extraction:** -```bash -./gradlew buildRelease -Pskip-debug-extraction=true -``` - -**See:** `build-logic/README.md` for full documentation - -### Container-based Testing (Recommended for ASan/Non-Local Environments) - -**When to use**: For ASan testing, cross-architecture testing (aarch64), different libc variants (musl), or reproducing CI environment issues. The script defaults to Podman; use `--container=docker` to use Docker. - -```bash -# ASan tests on aarch64 Linux -./utils/run-containers-tests.sh --arch=aarch64 --config=asan --libc=glibc --jdk=21 - -# Run specific test pattern -./utils/run-containers-tests.sh --arch=aarch64 --tests="*SpecificTest*" - -# Enable C++ gtests -./utils/run-containers-tests.sh --arch=aarch64 --gtest - -# Run one C++ gtest binary -./utils/run-containers-tests.sh --config=asan --gtest-task=elfparser_ut - -# Use Docker instead of the default Podman runtime -./utils/run-containers-tests.sh --container=docker --libc=glibc --jdk=21 - -# Drop to shell for debugging -./utils/run-containers-tests.sh --arch=aarch64 --shell - -# Test with musl libc -./utils/run-containers-tests.sh --libc=musl --jdk=21 - -# Test with OpenJ9 -./utils/run-containers-tests.sh --jdk=21-j9 - -# Use mounted repo (faster, but may have stale artifacts) -./utils/run-containers-tests.sh --mount - -# Rebuild container images -./utils/run-containers-tests.sh --rebuild -``` - -**Note**: The container script supports `--config=debug|release|asan|tsan`. Use this for cross-architecture testing and reproducing CI environments. For local development, use `./gradlew testAsan` directly. - -### Build Options -```bash -# Skip native compilation -./gradlew buildDebug -Pskip-native - -# Skip all tests -./gradlew buildDebug -Pskip-tests - -# Skip C++ tests -./gradlew buildDebug -Pskip-gtest - -# Keep JFR recordings after tests -./gradlew testDebug -PkeepJFRs - -# Skip debug symbol extraction -./gradlew buildRelease -Pskip-debug-extraction=true - -# Force specific compiler (auto-detects clang++ or g++ by default) -./gradlew build -Pnative.forceCompiler=clang++ -./gradlew build -Pnative.forceCompiler=g++ -./gradlew build -Pnative.forceCompiler=/usr/bin/g++-13 -``` - -### Code Quality -```bash -# Format code -./gradlew spotlessApply - -# Static analysis -./gradlew scanBuild - -# Run stress tests -./gradlew :ddprof-stresstest:runStressTests - -# Run benchmarks -./gradlew runBenchmarks -``` - -## Architecture - -### Module Structure -- **ddprof-lib**: Main profiler library (Java + C++) -- **ddprof-test**: Integration tests -- **ddprof-test-tracer**: Tracing context tests -- **ddprof-stresstest**: JMH-based performance tests -- **malloc-shim**: Memory allocation interceptor (Linux only) - -### Build Configurations -The project supports multiple build configurations per platform: -- **release**: Optimized production build with stripped symbols -- **debug**: Debug build with full symbols -- **asan**: AddressSanitizer build for memory error detection -- **tsan**: ThreadSanitizer build for thread safety validation - -### Key Source Locations -- Java API: `ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java` -- C++ engine: `ddprof-lib/src/main/cpp/` -- Native libraries: `ddprof-lib/build/lib/main/{config}/{os}/{arch}/` -- Test resources: `ddprof-test/src/test/java/` - -### Platform Support -- **Linux**: x64, arm64 (primary platforms) -- **macOS**: arm64, x64 -- **Architecture detection**: Automatic via `PlatformUtils` in build-logic -- **musl libc detection**: Automatic detection and handling - -### Debug Information Handling -Release builds automatically extract debug symbols: -- Stripped libraries (~1.2MB) for production -- Separate debug files (~6.1MB) with full symbols -- GNU debuglink sections connect stripped libraries to debug files - -## Development Workflow - -### Running Single Tests -Use the `-Ptests` property across all platforms: -```bash -./gradlew :ddprof-test:testDebug -Ptests=ClassName.methodName # Single method -./gradlew :ddprof-test:testDebug -Ptests=ClassName # Entire class -./gradlew :ddprof-test:testDebug -Ptests="*.ClassName" # Pattern matching -``` - -**Platform Implementation Details:** -- **glibc/macOS**: Test tasks use Gradle's native Test task type with JUnit Platform integration -- **musl (Alpine)**: Exec tasks with custom ProfilerTestRunner (bypasses Gradle 9 toolchain probe issues) -- **Custom Test Runner**: Uses JUnit Platform Launcher API directly (same API used by IDEs and Gradle internally) -- **Result**: Unified `-Ptests` property works identically across all platforms, no platform-specific syntax required - -**Why `-Ptests` instead of `--tests`?** -The `-Ptests` property works consistently across both Test and Exec task types, while `--tests` only works with Test tasks. This ensures a truly unified interface across all platforms. - -### Working with Native Code -Native compilation is automatic during build. C++ code changes require: -1. Full rebuild: `/build-and-summarize clean build` -2. The build system automatically handles JNI headers and platform detection - -### Debugging Native Issues -- Use `buildDebug` for debug symbols -- Use `buildAsan` for memory error detection -- Check `gradle/sanitizers/*.supp` for suppressions -- Set `sudo sysctl vm.mmap_rnd_bits=28` if ASan crashes occur - -### Cross-Platform Development -- Use `osIdentifier()` and `archIdentifier()` functions for platform detection -- Platform-specific code goes in `os_linux.cpp`, `os_macos.cpp`, etc. -- Build configurations automatically select appropriate compiler/linker flags - -## Publishing and Artifacts - -The main artifact is `ddprof-.jar` containing: -- Java classes -- Native libraries for all supported platforms -- Metadata for library loading - -Build artifacts structure: -``` -ddprof-lib/build/ -├── lib/main/{config}/{os}/{arch}/ -│ ├── libjavaProfiler.{so|dylib} # Full library -│ ├── stripped/ → production binary -│ └── debug/ → debug symbols -└── native/{config}/META-INF/native-libs/ - └── {os}-{arch}/ → final packaged libraries -``` - -## Core Architecture Components - -### Double-Buffered Call Trace Storage -The profiler uses a sophisticated double-buffered storage system for call traces: -- **Active Storage**: Currently accepting new traces from profiling events -- **Standby Storage**: Background storage for JFR serialization and trace preservation -- **Instance-based Trace IDs**: 64-bit IDs combining instance ID (upper 32 bits) and slot (lower 32 bits) -- **Liveness Checkers**: Functions that determine which traces to preserve across storage swaps -- **Atomic Swapping**: Lock-free swap operations to minimize profiling overhead - -### JFR Integration Architecture -- **FlightRecorder**: Central JFR event recording and buffer management -- **Metadata Generation**: Dynamic JFR metadata for stack traces, methods, and classes -- **Constant Pools**: Efficient deduplication of strings, methods, and stack traces -- **Buffer Management**: Thread-local recording buffers with configurable flush thresholds - -### Native Integration Patterns -- **Signal Handler Safety**: Careful memory management in signal handler contexts - -### Multi-Engine Profiling System -- **CPU Profiling**: SIGPROF-based sampling with configurable intervals -- **Wall Clock**: SIGALRM-based sampling for blocking I/O and sleep detection -- **Allocation Profiling**: TLAB-based allocation tracking and sampling -- **Live Heap**: Object liveness tracking with weak references and GC integration - -## Critical Implementation Details - -### JVM support -- **Three supported JVM implementations**: Hotspot, J9 and Zing -- **JVM implementation specific code**: Implementation-specific code is organized under hotspot, j9 and zing subdirectories respectively -- **Shared code**: Shared code is JVM implementation independent, must not refer to JVM implementation specific code directly, but through abstraction files - jvmSupport.h, jvmSupport.inline.h, jvmSupport.cpp, jvmThread.h, jvmThread.cpp, vmEntry.h, vmEntry.cpp and javaApi.cpp - -### Thread Safety and Performance -- **Lock-free Hot Paths**: Signal handlers avoid blocking operations -- **Thread-local Buffers**: Per-thread recording buffers minimize contention -- **Atomic Operations**: Instance ID management and counter updates use atomics -- **Memory Allocation**: Minimize malloc() in hot paths, use pre-allocated containers - -### Atomic Memory Ordering (Critical for arm64) -arm64 has a weakly-ordered memory model (unlike x86 TSO). Incorrect ordering causes real lockups on arm64 that never reproduce on x86. -- **Cross-thread reads**: Always use `__ATOMIC_ACQUIRE` for loads that must see stores from another thread. Never use `__ATOMIC_RELAXED` for cross-thread visibility unless you can prove no ordering dependency exists. -- **Cross-thread writes**: Use `__ATOMIC_RELEASE` for stores that must be visible to other threads. Pair with `__ATOMIC_ACQUIRE` loads. -- **Count + pointer patterns**: When a data structure publishes a count and a separate pointer (two-phase add), both the count load and pointer load need acquire semantics so the reader sees the pointer store that preceded the count increment. -- **Default stance**: When in doubt, use acquire/release. The performance cost is negligible; the correctness cost of relaxed ordering bugs is enormous (silent arm64-only lockups). - -### Concurrent Data Structure Iteration -- **NULL gaps**: When iterating a concurrent array (e.g., `CodeCacheArray`), always NULL-check each slot — a slot may be count-allocated but pointer-not-yet-stored. -- **Cursor advancement**: Never permanently advance an iteration cursor past NULL gaps. Stop at the first NULL or track the last contiguous non-NULL entry, so missing entries are retried on the next pass. - -### 64-bit Trace ID System -- **Collision Avoidance**: Instance-based IDs prevent collisions across storage swaps -- **JFR Compatibility**: 64-bit IDs work with JFR constant pool indices -- **Stability**: Trace IDs remain stable during liveness preservation -- **Performance**: Bit-packing approach avoids atomic operations in hot paths - -### Platform-Specific Handling -- **musl libc Detection**: Automatic detection and symbol resolution adjustments -- **Architecture Support**: x64, arm64 with architecture-specific stack walking -- **Debug Symbol Handling**: Split debug information for production deployments - -## Development Guidelines - -### Code Organization Principles -- **Code Integration**: Datadog-specific extensions are integrated directly into base files (e.g., `stackWalker.h`) -- **External Dependencies**: Local code in `cpp/` - -### Performance Constraints -- **Algorithmic Complexity**: Use O(N) or better, max 256 elements for linear scans -- **Memory Fragmentation**: Minimize allocations to avoid malloc arena issues -- **Signal Handler Safety**: No blocking operations, mutex locks, or malloc() in handlers - -### Testing Strategy -- **Multi-configuration Testing**: Test across debug, release, ASan, and TSan builds -- **Cross-JDK Compatibility**: Test with Oracle JDK, OpenJDK, and OpenJ9 -- **Native-Java Integration**: Both C++ unit tests (gtest) and Java integration tests -- **Stress Testing**: JMH-based performance and stability testing - -### Debugging and Analysis -- **Debug Builds**: Use `buildDebug` for full symbols and debugging information -- **Sanitizer Builds**: ASan for memory errors, TSan for threading issues -- **Static Analysis**: `scanBuild` for additional code quality checks -- **Test Logging**: Use `TEST_LOG` macro for debug output in tests - -## Build System Architecture - -### Gradle Multi-project Structure -- **ddprof-lib**: Core profiler with native compilation -- **ddprof-test**: Integration and Java unit tests -- **ddprof-test-tracer**: Tracing context integration tests -- **ddprof-stresstest**: JMH performance benchmarks -- **malloc-shim**: Linux memory allocation interceptor - -### Native Compilation Pipeline -- **Platform Detection**: Automatic OS and architecture detection via `PlatformUtils` in build-logic -- **Configuration Matrix**: Multiple build configs (release/debug/asan/tsan) per platform -- **Symbol Processing**: Automatic debug symbol extraction for release builds -- **Library Packaging**: Final JAR contains all platform-specific native libraries -- **Compiler Detection**: Auto-detects clang++ (preferred) or g++ (fallback); override with `-Pnative.forceCompiler` - -### Native Build Plugin (build-logic) -The project includes a Kotlin-based native build plugin (`build-logic/`) for type-safe C++ compilation: -- **Composite Build**: Independent Gradle project for build logic versioning -- **Type-Safe DSL**: Kotlin-based configuration with compile-time checking -- **Auto Task Generation**: Creates compile, link, and assemble tasks per configuration -- **Debug Symbol Extraction**: Automatic split debug info for release builds (69% size reduction) -- **Source Sets**: Per-directory compiler flags for legacy/third-party code -- **Symbol Visibility**: Linux version scripts and macOS exported symbols lists - -**See:** `build-logic/README.md` for full documentation - -### Custom Native Build Plugin (build-logic) -The project uses a custom Kotlin-based native build plugin in `build-logic/` instead of Gradle's `cpp-library` and `cpp-application` plugins. This is intentional: - -**Why not cpp-library/cpp-application plugins:** -- Gradle's native plugins parse compiler version strings which breaks with newer gcc/clang versions -- JNI header detection has issues with non-standard JAVA_HOME layouts -- Plugin maintainers are unresponsive to fixes -- The plugins use undocumented internals that change between Gradle versions - -**Plugin components (`com.datadoghq.native-build`):** -- `NativeCompileTask` - Parallel C++ compilation with source sets support -- `NativeLinkTask` - Links shared libraries (.so/.dylib) with symbol visibility -- `PlatformUtils` - Platform detection and compiler location - -**Plugin components (`com.datadoghq.gtest`):** -- `NativeLinkExecutableTask` - Links executables (for gtest) -- `GtestPlugin` - Google Test integration and task generation - -**Key principle:** Direct compiler invocation without version parsing. The tasks simply find `clang++` or `g++` on PATH and invoke them with the configured flags. - -#### Configuring Build Tasks - -All build tasks support industry-standard configuration options. Configuration is done using Kotlin DSL: - -**Basic compilation:** -```kotlin -tasks.register("compileLib", NativeCompileTask::class) { - compiler.set("clang++") - compilerArgs.set(listOf("-O3", "-std=c++17", "-fPIC")) - sources.from(fileTree("src/main/cpp") { include("**/*.cpp") }) - includes.from("src/main/cpp", "${System.getenv("JAVA_HOME")}/include") - objectFileDir.set(file("build/obj")) -} -``` - -**Advanced configuration with source sets:** -```kotlin -tasks.register("compileLib", NativeCompileTask::class) { - compiler.set("clang++") - compilerArgs.set(listOf("-Wall", "-O3")) // Base flags for all files - - // Multiple source directories with per-directory flags - sourceSets { - create("main") { - sources.from(fileTree("src/main/cpp")) - compilerArgs.add("-fPIC") - } - create("legacy") { - sources.from(fileTree("src/legacy")) - compilerArgs.addAll("-Wno-deprecated", "-std=c++11") - excludes.add("**/broken/*.cpp") - } - } - - // Logging - logLevel.set(LogLevel.VERBOSE) - - objectFileDir.set(file("build/obj")) -} -``` - -**Linking shared libraries with symbol management:** -```kotlin -tasks.register("linkLib", NativeLinkTask::class) { - linker.set("clang++") - linkerArgs.set(listOf("-O3")) - objectFiles.from(fileTree("build/obj") { include("*.o") }) - outputFile.set(file("build/lib/libjavaProfiler.so")) - - // Symbol visibility control - exportSymbols.set(listOf("Java_*", "JNI_OnLoad", "JNI_OnUnload")) - hideSymbols.set(listOf("*_internal*")) - - // Libraries - lib("pthread", "dl", "m") - libPath("/usr/local/lib") - - logLevel.set(LogLevel.VERBOSE) -} -``` - -**Executable linking (for gtest):** -```kotlin -tasks.register("linkTest", NativeLinkExecutableTask::class) { - linker.set("clang++") - objectFiles.from(fileTree("build/obj/gtest") { include("*.o") }) - outputFile.set(file("build/bin/callTrace_test")) - - // Library management - lib("gtest", "gtest_main", "pthread") - libPath("/usr/local/lib") - runtimePath("/opt/lib", "/usr/local/lib") - - logLevel.set(LogLevel.VERBOSE) -} -``` - -**Task properties:** - -*NativeCompileTask:* -- `compiler`, `compilerArgs` - Compiler and flags -- `sources`, `includes` - Source files and include paths -- `sourceSets` - Per-directory compiler flag overrides -- `objectFileDir` - Output directory for object files -- `logLevel` - QUIET, NORMAL, VERBOSE, DEBUG - -*NativeLinkTask:* -- `linker`, `linkerArgs` - Linker and flags -- `objectFiles`, `outputFile` - Input objects and output library -- `exportSymbols`, `hideSymbols` - Symbol visibility control -- `lib()`, `libPath()` - Library convenience methods -- `logLevel`, `showCommandLine` - Logging options - -*NativeLinkExecutableTask:* -- `linker`, `linkerArgs` - Linker and flags -- `objectFiles`, `outputFile` - Input objects and output executable -- `lib()`, `libPath()`, `runtimePath()` - Library and rpath convenience methods -- `logLevel`, `showCommandLine` - Logging options - -### Artifact Structure -Final artifacts maintain a specific structure for deployment: -``` -META-INF/native-libs/{os}-{arch}/libjavaProfiler.{so|dylib} -``` -With separate debug symbol packages for production debugging support. - -## Build System Maintenance - -> **Detailed guide**: [doc/build/BuildSystemGuide.md](doc/build/BuildSystemGuide.md) - -### Quick Reference - -**Convention plugins** (in `build-logic/conventions/`): -- `com.datadoghq.native-build` - Multi-config C++ compilation -- `com.datadoghq.gtest` - Google Test integration -- `com.datadoghq.profiler-test` - Multi-config Java test generation -- `com.datadoghq.simple-native-lib` - Simple single-library builds - -**Key principle**: Build configurations (release/debug/asan/tsan/fuzzer) are **discovered dynamically**, not hardcoded. Add new configs in `ConfigurationPresets.kt` only. - -**Key files**: -- `ConfigurationPresets.kt` - Defines all build configurations and their flags -- `PlatformUtils.kt` - Platform detection and compiler finding -- `NativeBuildPlugin.kt` - Creates compile/link tasks per configuration - -### Common Tasks - -| Task | Description | -|------|-------------| -| Add compiler flag to all configs | Edit `commonLinuxCompilerArgs()` in `ConfigurationPresets.kt` | -| Add new build configuration | Add `register("name")` block in `ConfigurationPresets.kt` | -| Create new convention plugin | Create class, register in `build.gradle.kts`, see [guide](doc/BUILD-SYSTEM-GUIDE.md#creating-a-new-convention-plugin) | - -### Gradle Properties - -See `gradle.properties.template` for all options. Key ones: -- `skip-tests`, `skip-native`, `skip-gtest` - Skip build phases -- `native.forceCompiler` - Override compiler detection -- `ddprof_version` - Override version - -### Troubleshooting - -- **Plugin changes not taking effect**: Run `./gradlew --stop` -- **"Task not found"**: Tasks are created dynamically; check `PlatformUtils` detection -- **"Configuration not found"**: Access configurations in `afterEvaluate` - -## Legacy and Compatibility - -- Java 8 compatibility maintained throughout -- JNI interface follows async-profiler conventions -- Supports Oracle JDK, OpenJDK and OpenJ9 implementations -- Always test with /build-and-summarize testDebug -- Always consult openjdk source codes when analyzing profiler issues and looking for proposed solutions -- For OpenJ9 specific issues consul the openj9 github project -- don't use assemble task. Use assembleDebug or assembleRelease instead -- gtest tests are located in ddprof-lib/src/test/cpp -- GtestPlugin in build-logic handles gtest build setup -- Java unit tests are in ddprof-test module -- Always run /build-and-summarize spotlessApply before commiting the changes - -- When you are adding copyright - like 'Copyright 2021, 2023 Datadog, Inc' do the current year -> 'Copyright , Datadog, Inc' - When you are modifying copyright already including 'Datadog' update the 'until year' ('Copyright from year, until year') to the current year -- If modifying a file that does not contain Datadog copyright, add one -- When proposing solutions try minimizing allocations. We are fighting hard to avoid fragmentation and malloc arena issues -- Use O(N) or worse only in small amounts of elements. A rule of thumb cut-off is 256 elements. Anything larger requires either index or binary search to get better than linear performance - -- Always run /build-and-summarize spotlessApply before committing changes - -- Always create a commit message based solely on the actual changes visible in the diff - -- You can use TEST_LOG macro to log debug info which can then be used in ddprof-test tests to assert correct execution. The macro is defined in 'common.h' - -- If a file is containing copyright, make sure it is preserved. The only exception is if it mentions Datadog - then you can update the years, if necessary -- Always challange my proposals. Use deep analysis and logic to find flaws in what I am proposing - -- Exclude ddprof-lib/build/async-profiler from searches of active usage - -- Run tests with 'testDebug' gradle task - -## Build JDK Configuration - -The project uses a **two-JDK pattern**: -- **Build JDK** (`JAVA_HOME`): Used to run Gradle itself. Must be JDK 17+ for Gradle 9. -- **Test JDK** (`JAVA_TEST_HOME`): Used to run tests against different Java versions. - -**Current requirement:** JDK 21 (LTS) for building, targeting Java 8 bytecode via `--release 8`. - -### Files to Modify When Changing Build JDK Version - -When upgrading the build JDK (e.g., from JDK 21 to JDK 25), update these files: - -| File | What to Change | -|------|----------------| -| `README.md` | Update "Prerequisites" section with new JDK version | -| `.github/actions/setup_cached_java/action.yml` | Change `build_jdk=jdk21` to new version (line ~25) | -| `.github/workflows/ci.yml` | Update `java-version` in `check-formatting` job's Setup Java step | -| `utils/run-containers-tests.sh` | Update `BUILD_JDK_VERSION="21"` constant | -| `build-logic/.../JavaConventionsPlugin.kt` | Update documentation comment if minimum changes | - -### Files to Modify When Changing Target JDK Version - -When changing the target bytecode version (e.g., from Java 8 to Java 11): - -| File | What to Change | -|------|----------------| -| `build-logic/.../JavaConventionsPlugin.kt` | Change `--release 8` to new version | -| `ddprof-lib/build.gradle.kts` | Change `sourceCompatibility`/`targetCompatibility` | -| `README.md` | Update minimum Java runtime version | - -### Gradle 9 API Changes Reference - -When upgrading Gradle major versions, watch for these breaking changes: - -| Old API | New API (Gradle 9+) | Affected Files | -|---------|---------------------|----------------| -| `project.exec { }` in task actions | `ProcessBuilder` directly | `GtestPlugin.kt` | -| `String.capitalize()` | `replaceFirstChar { it.uppercaseChar() }` | Kotlin plugins | -| `createTempFile()` | `kotlin.io.path.createTempFile()` | `PlatformUtils.kt` | -| Spotless `userData()` | `editorConfigOverride()` | `SpotlessConventionPlugin.kt` | -| Spotless `indentWithSpaces()` | `leadingTabsToSpaces()` | `SpotlessConventionPlugin.kt` | - -### CI JDK Caching - -The CI caches JDKs via `.github/workflows/cache_java.yml`. When adding a new JDK version: -1. Add version URLs to `cache_java.yml` environment variables -2. Add to the `java_variant` matrix in cache jobs -3. Run the `cache_java.yml` workflow manually to populate caches - -## Agentic Work - -- Never run `./gradlew` directly. -- Always invoke the wrapper command: `./.claude/commands/build-and-summarize`. -- Pass through all arguments exactly as you would to `./gradlew`. -- Examples: - - Instead of: - ```bash - ./gradlew build - ``` - use: - ```bash - ./.claude/commands/build-and-summarize build - ``` - - Instead of: - ```bash - ./gradlew :ddprof-test:testDebug -Ptests=MuslDetectionTest - ``` - use: - ```bash - ./.claude/commands/build-and-summarize :ddprof-test:testdebug -Ptests=MuslDetectionTest - ``` - -- This ensures the full build log is captured to a file and only a summary is shown in the main session. - -## Documentation Rules -- All documentation files in `doc/` must use **PascalCase** naming (e.g., `BuildSystemGuide.md`, `CallTraceStorage.md`) -- The `doc/README.md` index file is the only lowercase exception - -## Ground rules -- Never replace the code you work on with stubs -- Never 'fix' the tests by testing constants against constants -- Never claim success until all affected tests are passing -- Always provide javadoc for public classes and methods -- Provide javadoc for non-trivial private and package private code -- Always provide comprehensive tests for new functionality -- Always provide tests for bug fixes - test fails before the fix, passes after the fix -- All code needs to strive to be lean in terms of resources consumption and easy to follow - - do not shy away from factoring out self containing code to shorter functions with explicit name - -### C/C++ Code Style -- **Indentation**: Match the exact indentation style of the surrounding code in each file. Do not introduce inconsistent indentation — reviewers will flag it. -- **Minimal complexity**: Do not split inline logic into separate helper functions unless the helpers are reused or the original is genuinely hard to follow. Unnecessary splits add indirection without value. -- **Comment precision**: Comments explaining "why" must reference concrete mechanisms (e.g., "ASAN's allocator lock is reentrant" not "internal bookkeeping"). Vague comments get challenged in review. Every claim in a comment must be verifiable from the code or documented behavior of the referenced system (ASAN, glibc NPTL, HotSpot, etc.). -- **No speculative comments**: Do not claim a system (HotSpot, glibc, ASAN) uses a specific mechanism unless you are certain. If unsure, describe the observable symptom instead of guessing the cause. diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 6ad80216b..000000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,2 +0,0 @@ -# Changelog -!TODO! diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 8dada3eda..000000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/README.md b/README.md deleted file mode 100644 index 3715202ba..000000000 --- a/README.md +++ /dev/null @@ -1,473 +0,0 @@ -# Datadog Java Profiler Library -_Based on [async-profiler 2.8.3](https://github.com/jvm-profiling-tools/java-profiler/releases/tag/v2.8.3)_ - -## Disclaimer -This is not a fork of [async-profiler](https://github.com/jvm-profiling-tools/async-profiler). This is a work derived from __async-profiler__ but tailored very specifically for Datadog needs. -See [gritty details](#gritty-details) for more info. -If you need a full-fledged Java profiler head back to [async-profiler](https://github.com/jvm-profiling-tools/async-profiler) - -## Build - -### Prerequisites -1. JDK 21 or later (required for building - Gradle 9 requirement) -2. Gradle 9.3.1 (included in wrapper) -3. C++ compiler (clang++ preferred, g++ supported) - - Build system auto-detects clang++ or g++ - - Override with: `./gradlew build -Pnative.forceCompiler=g++` -4. Make (included in XCode on Macos) -5. Google Test (for unit testing) - - On Ubuntu/Debian: `sudo apt install libgtest-dev` - - On macOS: `brew install googletest` - - On Alpine: `apk add gtest-dev` -6. clang-format 11 (for code formatting) - - On Ubuntu/Debian: `sudo apt install clang-format-11` - - On macOS: `brew install clang-format@11` - - On Alpine: `apk add clang-format-11` - -### Building the Project -1. Clone the repository: -```bash -git clone https://github.com/DataDog/java-profiler.git -cd java-profiler -``` - -2. Build a release version of the project: -```bash -./gradlew buildRelease -``` - -The resulting artifact will be in `ddprof-lib/build/libs/ddprof-.jar` - -#### Gritty details -Since the upstream code might not be 100% compatible with the current version of the project, we extend the base classes -with Datadog-specific functionality. These extensions are integrated directly into the base files (e.g., `stackWalker.h`) -with optional parameters and backward-compatible interfaces. - -See [ddprof-lib/src/main/cpp/stackWalker.h](ddprof-lib/src/main/cpp/stackWalker.h) for an example of how we extend the upstream code with additional features like truncation detection. - -## Claude Code Integration - -This project includes Claude Code commands for streamlined development workflows when using [Claude Code](https://claude.ai/code): - -### Available Commands - -#### `/build-and-summarize ` -Automated build execution with intelligent log analysis: -```bash -# Build with automated analysis -/build-and-summarize buildRelease - -# Run tests with summary -/build-and-summarize testDebug - -# Custom gradle tasks -/build-and-summarize clean buildDebug testDebug -``` - -**Features:** -- Executes Gradle builds with appropriate logging (`-i --console=plain`) -- Captures timestamped build logs in `build/logs/` -- Automatically analyzes build results using the gradle-logs-analyst agent -- Generates structured reports: `build/reports/claude/gradle-summary.md` and `build/reports/claude/gradle-summary.json` -- Extracts key information: build status, failed tasks, test results, warnings, and performance metrics - -#### `/compare-and-patch ` -Upstream/local file comparison and patch analysis: -```bash -# Analyze differences between upstream and local versions -/compare-and-patch stackFrame.h -/compare-and-patch symbols.cpp -/compare-and-patch buffers.h -``` - -**Features:** -- Automatically resolves upstream (`ddprof-lib/build/async-profiler/src/`) and local (`ddprof-lib/src/main/cpp/`) file paths -- Intelligently compares files while ignoring newline and copyright-only changes -- Uses the patch-analyst agent to generate comprehensive analysis -- Creates structured patch reports: `build/reports/claude/patches/.patch.json` and `build/reports/claude/patches/.patch.md` -- Identifies required patch operations compatible with the unified patching system -- Handles cases where files are Datadog-specific additions with no upstream equivalent - -### Integration Benefits - -These commands complement the existing patching workflow by providing: -- **Automated Analysis**: Intelligent parsing of build logs and patch requirements -- **Structured Output**: Machine-readable JSON and human-readable Markdown reports -- **Consistency**: Standardized analysis format across all patch operations -- **Efficiency**: Streamlined workflow for patch development and maintenance -- **Documentation**: Automatic generation of patch documentation - -### Usage in Development Workflow - -1. **Build Analysis**: Use `/build-and-summarize` to quickly identify build issues and performance bottlenecks -2. **Patch Development**: Use `/compare-and-patch` to analyze upstream changes and generate patch requirements -3. **Maintenance**: Regular patch analysis helps maintain compatibility with upstream updates - -The generated reports integrate seamlessly with the existing `gradle/patching.gradle` configuration system, making it easier to maintain and update patches as the upstream codebase evolves. - -## Testing - -### Unit Tests -The project includes both Java and C++ unit tests. You can run them using: - -```bash -# Run all tests -./gradlew test - -# Run specific test configurations -./gradlew testRelease # Run release build tests -./gradlew testDebug # Run debug build tests -./gradlew testAsan # Run tests with ASan -./gradlew testTsan # Run tests with TSan - -# Run C++ unit tests only -./gradlew gtestDebug # Run C++ tests in debug mode -./gradlew gtestRelease # Run C++ tests in release mode -``` - -### Test Options -- Skip all tests: `./gradlew build -Pskip-tests` -- Keep JFR recordings: `./gradlew test -PkeepJFRs` -- Skip native build: `./gradlew build -Pskip-native` -- Skip C++ tests: `./gradlew build -Pskip-gtest` - -### Cross-JDK Testing -`JAVA_TEST_HOME= ./gradlew testDebug` - -### Container-Based Testing (musl/glibc) -Run tests in containers to test on different libc implementations. The script defaults to Podman; use `--container=docker` to use Docker instead. Uses two-level container image caching for fast subsequent runs: -1. **Base image** (`java-profiler-base:-`) - OS with all build tools + sanitizers -2. **JDK image** (`java-profiler-test:-jdk-`) - Adds JDK + Gradle - -By default, the script clones the repository at the current commit for clean builds. Use `--mount` to mount the local directory instead (faster but may have stale artifacts). - -```bash -# Run specific test on musl (Alpine) with JDK 21 (clone mode - clean build) -./utils/run-containers-tests.sh --libc=musl --jdk=21 --tests="CTimerGCStressTest" - -# Run all tests on glibc (Ubuntu) with JDK 17 -./utils/run-containers-tests.sh --libc=glibc --jdk=17 - -# Run tests on aarch64 architecture (requires container runtime with multi-arch support) -./utils/run-containers-tests.sh --libc=musl --jdk=21 --arch=aarch64 - -# Mount local repo for faster iteration (may have stale artifacts) -./utils/run-containers-tests.sh --libc=musl --jdk=21 --mount --tests="MyTest" - -# Drop to interactive shell in musl container -./utils/run-containers-tests.sh --libc=musl --jdk=21 --shell - -# Run one C++ gtest binary only -./utils/run-containers-tests.sh --libc=glibc --config=asan --gtest-task=elfparser_ut - -# Use Docker instead of the default Podman runtime -./utils/run-containers-tests.sh --container=docker --libc=glibc --jdk=21 - -# Force rebuild of all cached container images -./utils/run-containers-tests.sh --libc=musl --jdk=21 --rebuild - -# Force rebuild of base image only (useful after Alpine/Ubuntu updates) -./utils/run-containers-tests.sh --libc=musl --rebuild-base - -# Show options -./utils/run-containers-tests.sh --help -``` - -Supported options: -- `--libc=glibc|musl` (default: glibc) -- `--jdk=8|11|17|21|25|8-j9|11-j9|17-j9|21-j9|17-graal|21-graal|25-graal` (default: 21) -- `--arch=x64|aarch64` (default: auto-detect) -- `--config=debug|release|asan|tsan` (default: debug) -- `--container=podman|docker` (default: podman) -- `--tests="TestPattern"` -- `--gtest` (enable C++ gtests, disabled by default for faster runs) -- `--gtest-task=Task` (run one C++ gtest task; accepts `elfparser_ut` or a full task path like `:ddprof-lib:gtestAsan_elfparser_ut`) -- `--shell` (interactive shell instead of running tests) -- `--mount` (mount local repo instead of cloning - faster but may have stale artifacts) -- `--rebuild` (force rebuild of all container images) -- `--rebuild-base` (force rebuild of base image only) - -## Unwinding Validation Tool - -The project includes a comprehensive unwinding validation tool that tests JIT compilation unwinding scenarios to detect stack frame issues. This tool validates the profiler's ability to correctly unwind stack frames during complex JIT compilation scenarios. - -### Running the Unwinding Validator - -```bash -# Run all unwinding validation scenarios (release or debug build required) -./gradlew :ddprof-test:runUnwindingValidator - -# Run specific scenario -./gradlew :ddprof-test:runUnwindingValidator -PvalidatorArgs="--scenario=C2CompilationTriggers" - -# Generate markdown report for CI -./gradlew :ddprof-test:unwindingReport - -# Show available options -./gradlew :ddprof-test:runUnwindingValidator -PvalidatorArgs="--help" -``` - -### Available Scenarios - -The validator includes 13 specialized scenarios targeting different unwinding challenges: - -- **C2CompilationTriggers** - Heavy computational workloads that trigger C2 compilation -- **OSRScenarios** - On-Stack Replacement compilation scenarios -- **ConcurrentC2Compilation** - Concurrent C2 compilation stress testing -- **C2DeoptScenarios** - C2 deoptimization and transition edge cases -- **ExtendedJNIScenarios** - Extended JNI operation patterns -- **MultipleStressRounds** - Multiple concurrent stress rounds -- **ExtendedPLTScenarios** - PLT (Procedure Linkage Table) resolution scenarios -- **ActivePLTResolution** - Intensive PLT resolution during profiling -- **ConcurrentCompilationStress** - Heavy JIT compilation + native activity -- **VeneerHeavyScenarios** - ARM64 veneer/trampoline intensive workloads -- **RapidTierTransitions** - Rapid compilation tier transitions -- **DynamicLibraryOps** - Dynamic library operations during profiling -- **StackBoundaryStress** - Stack boundary stress scenarios - -### Output Formats - -The validator supports multiple output formats: - -```bash -# Text output (default) -./gradlew :ddprof-test:runUnwindingValidator - -# JSON format for programmatic analysis -./gradlew :ddprof-test:runUnwindingValidator -PvalidatorArgs="--output-format=json --output-file=unwinding-report.json" - -# Markdown format for documentation -./gradlew :ddprof-test:runUnwindingValidator -PvalidatorArgs="--output-format=markdown --output-file=unwinding-report.md" -``` - -### CI Integration - -The unwinding validator is automatically integrated into GitHub Actions CI pipeline: - -- Runs only on **debug builds** in CI (provides clean measurements without optimization interference) -- Generates rich markdown reports displayed directly in job summaries -- Creates downloadable report artifacts for historical analysis -- Fails builds when critical unwinding issues are detected - -The validator provides immediate visibility into unwinding quality across all supported platforms and Java versions without requiring artifact downloads. - -### Understanding Results - -The tool analyzes JFR (Java Flight Recorder) data to measure: - -- **Error Rate** - Percentage of samples with unwinding failures (`.unknown()`, `.break_interpreted()`) -- **Native Coverage** - Percentage of samples successfully unwound in native code -- **Sample Count** - Total profiling samples captured during validation -- **Error Types** - Breakdown of specific unwinding failure patterns - -Results are categorized as: -- 🟢 **Excellent** - Error rate < 0.1% -- 🟢 **Good** - Error rate < 1.0% -- 🟡 **Moderate** - Error rate < 5.0% -- 🔴 **Needs Work** - Error rate ≥ 5.0% - -## Release Builds and Debug Information - -### Split Debug Information -Release builds automatically generate split debug information to optimize deployment size while preserving debugging capabilities: - -- **Stripped libraries** (~1.2MB): Production-ready binaries with symbols removed for deployment -- **Debug symbol files** (~6.1MB): Separate `.debug` files containing full debugging information -- **Debug links**: Stripped libraries include `.gnu_debuglink` sections pointing to debug files - -### Build Artifacts Structure -``` -ddprof-lib/build/ -├── lib/main/release/linux/x64/ -│ ├── libjavaProfiler.so # Original library with debug symbols -│ ├── stripped/ -│ │ └── libjavaProfiler.so # Stripped library (83% smaller) -│ └── debug/ -│ └── libjavaProfiler.so.debug # Debug symbols only -├── native/release/ -│ └── META-INF/native-libs/linux-x64/ -│ └── libjavaProfiler.so # Final stripped library (deployed) -└── native/release-debug/ - └── META-INF/native-libs/linux-x64/ - └── libjavaProfiler.so.debug # Debug symbols package -``` - -### Build Options -- **Skip debug extraction**: `./gradlew buildRelease -Pskip-debug-extraction=true` -- **Debug extraction requires**: `objcopy` (Linux) or `dsymutil` (macOS) - - Ubuntu/Debian: `sudo apt-get install binutils` - - Alpine: `apk add binutils` - - macOS: Included with Xcode command line tools - -### Compiler Selection -The build system automatically detects the best available C++ compiler (prefers clang++, falls back to g++). - -```bash -# Auto-detection (default) -./gradlew build - -# Force specific compiler -./gradlew build -Pnative.forceCompiler=clang++ -./gradlew build -Pnative.forceCompiler=g++ -./gradlew build -Pnative.forceCompiler=/usr/bin/g++-13 - -# Test with specific compiler -./gradlew testDebug -Pnative.forceCompiler=g++ -``` - -This is useful for: -- **Reproducibility**: Ensure builds use the same compiler across machines -- **clang-only systems**: macOS with Xcode but no gcc (sanitizer builds work) -- **Testing**: Verify code compiles with both gcc and clang - -## Development - -### Code Quality -The project uses several tools for code quality: - -1. clang-format for C++ code formatting -2. scan-build for static analysis -3. cppcheck for additional C++ checks - -Run code quality checks: -```bash -# Run scan-build (this will use the scan-build binary) -./gradlew scanBuild - -# Run cppcheck (if configured) -./gradlew cppcheck - -# Run spotless (including code formatting) -./gradlew spotlessApply -``` - -### Debugging -**!TODO!** - -### Stress Tests -The project includes JMH-based stress tests: - -```bash -# Run all stress tests -./gradlew :ddprof-stresstest:runStressTests - -### Common Issues -1. If you encounter strange crashes Asan: - ```bash - sudo sysctl vm.mmap_rnd_bits=28 - ``` - -2. For ASan/TSan issues, ensure you have the required libraries installed: - - ASan: `libasan` - - TSan: `libtsan` - -## Architectural Tidbits - -This section documents important architectural decisions and enhancements made to the profiler core. - -### Critical Section Management (2025) - -Introduced race-free critical section management using atomic compare-and-swap operations instead of expensive signal blocking syscalls: - -- **`CriticalSection` class**: Thread-local atomic flag-based protection against signal handler reentrancy -- **Lock-free design**: Uses `compare_exchange_strong` for atomic claiming of critical sections -- **Signal handler safety**: Eliminates race conditions between signal handlers and normal code execution -- **Performance improvement**: Avoids costly `sigprocmask`/`pthread_sigmask` syscalls in hot paths - -**Key files**: `criticalSection.h`, `criticalSection.cpp` - -### Triple-Buffered Call Trace Storage (2025) - -Enhanced the call trace storage system from double-buffered to triple-buffered architecture with hazard pointer-based memory reclamation: - -- **Triple buffering**: Active, standby, and cleanup storage rotation for smoother transitions -- **Hazard pointer system**: Per-instance thread-safe memory reclamation without global locks -- **ABA protection**: Generation counter prevents race conditions during table swaps -- **Instance-based trace IDs**: 64-bit IDs combining instance ID and slot for collision-free trace management -- **Lock-free hot paths**: Atomic operations minimize contention during profiling events - -**Key changes**: -- Replaced `SpinLock` with atomic pointers and hazard pointer system -- Added generation counter for safe table swapping -- Enhanced liveness preservation across storage rotations -- Improved thread safety for high-frequency profiling scenarios - -**Key files**: `callTraceStorage.h`, `callTraceStorage.cpp`, `callTraceHashTable.h`, `callTraceHashTable.cpp` - -### Enhanced Testing Infrastructure (2025) - -Comprehensive testing improvements for better debugging and stress testing: - -- **GTest crash handler**: Detailed crash reporting with backtraces and register state for native code failures -- **Stress testing framework**: Multi-threaded stress tests for call trace storage under high contention -- **Platform-specific debugging**: macOS and Linux register state capture in crash handlers -- **Async-signal-safe reporting**: Crash handlers use only signal-safe functions for reliable diagnostics - -**Key files**: `gtest_crash_handler.h`, `stress_callTraceStorage.cpp` - -### TLS Priming Enhancements (2025) - -Improved thread-local storage initialization to prevent race conditions: - -- **Solid TLS priming**: Enhanced thread-local variable initialization timing -- **Signal handler compatibility**: Ensures TLS is fully initialized before signal handler access -- **Cross-platform consistency**: Unified TLS handling across Linux and macOS platforms - -These architectural improvements focus on eliminating race conditions, improving performance in high-throughput scenarios, and providing better debugging capabilities for the native profiling engine. - -### Remote Symbolication Support (2025) - -Added support for remote symbolication to enable offloading symbol resolution from the agent to backend services: - -- **Build-ID extraction**: Automatically extracts GNU build-id from ELF binaries on Linux -- **Raw addressing information**: Stores build-id and PC offset instead of resolved symbol names -- **Remote symbolication mode**: Enable with `remotesym=true` profiler argument -- **JFR integration**: Remote frames serialized with build-id and offset for backend resolution -- **Zero encoding overhead**: Uses dedicated frame type (FRAME_NATIVE_REMOTE) for efficient serialization - -**Benefits**: -- Reduces agent overhead by eliminating local symbol resolution -- Enables centralized symbol resolution with better caching -- Supports scenarios where debug symbols are not available locally - -**Key files**: `elfBuildId.h`, `elfBuildId.cpp`, `profiler.cpp`, `flightRecorder.cpp` - -For detailed documentation, see [doc/RemoteSymbolication.md](doc/plans/RemoteSymbolication.md). - -## Contributing -1. Fork the repository -2. Create a feature branch -3. Make your changes -4. Run tests: `./gradlew test` -5. Submit a pull request - -## Utility Scripts - -The [`utils/`](utils/) directory contains helper scripts for common workflows. See [`utils/README.md`](utils/README.md) for full documentation. - -| Script | Description | -|--------|-------------| -| `release.sh` | Trigger a validated release (major/minor/patch) via GitHub Actions | -| `backport-pr.sh` | Cherry-pick a merged PR onto a release branch and open a backport PR | -| `patch-dd-java-agent.sh` | Patch `dd-java-agent.jar` with a local ddprof build for quick testing | -| `run-containers-tests.sh` | Run tests in containers (musl/glibc, multiple JDKs) | -| `check_upstream_changes.sh` | Check for upstream async-profiler changes locally | -| `track_upstream_changes.sh` | Track upstream changes and generate reports | -| `generate_tracked_files.sh` | Generate the list of files tracked from upstream | - -## License -This project is licensed under the Apache License 2.0 - see the LICENSE file for details. - -## Usage - -### Example - -Download the latest version of dd-trace-java and add `-Ddd.profiling.ddprof.debug.lib`. Example of a command line: -```bash -DD_SERVICE=your-service DD_TRACE_DEBUG=true java -javaagent:./temp/dd-java-agent.jar -Ddd.profiling.enabled=true -Ddd.profiling.ddprof.enabled=true -Ddd.profiling.ddprof.liveheap.enabled=true -Ddd.profiling.upload.period=10 -Ddd.profiling.start-force-first=true -Ddd.profiling.ddprof.debug.lib=~/dd/java-profiler/ddprof-lib/build/lib/main/debug/linux/x64/libjavaProfiler.so -XX:ErrorFile=${PWD}/hs_err_pid%p.log -XX:OnError='java -jar temp/dd-java-agent.jar uploadCrash hs_err_pid%p.log' -jar ./temp/renaissance-gpl-0.15.0.jar akka-uct -r 5 -``` - -### Consuming the artifact - -For dd-trace-java you just need to set the `ddprof.jar` project property. -Eg. you can run the gradle build like this - ./gradlew clean -Pddprof.jar=file:// :dd-java-agent:shadowJar` - which will result in a custom `dd-java-agent.jar` build containing your test version of Java profiler. diff --git a/_config.yml b/_config.yml new file mode 100644 index 000000000..07b3e5a13 --- /dev/null +++ b/_config.yml @@ -0,0 +1,4 @@ +title: Java Profiler Build - Test Dashboard +description: Test reports for dd-trace-java integration, benchmarks, and reliability +theme: jekyll-theme-minimal +baseurl: /java-profiler diff --git a/_data/integration.json b/_data/integration.json new file mode 100644 index 000000000..526696e5b --- /dev/null +++ b/_data/integration.json @@ -0,0 +1,644 @@ +{ + "runs": [ + { + "id": "121891935", + "timestamp": "2026-06-30T16:29:21Z", + "ddprof_branch": "main", + "ddprof_sha": "984428f01786a79f724db8d962376ec91a941cd1", + "ddprof_pr": null, + "pipeline": { + "id": "121891935", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121891935" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + }, + { + "id": "121891837", + "timestamp": "2026-06-30T15:51:22Z", + "ddprof_branch": "main", + "ddprof_sha": "e4975db27e8c362b053cac5580dd15d6dbd500c4", + "ddprof_pr": null, + "pipeline": { + "id": "121891837", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121891837" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + }, + { + "id": "121890528", + "timestamp": "2026-06-30T15:45:39Z", + "ddprof_branch": "main", + "ddprof_sha": "76b61abaea0cf9ede5d30a16b89e2471589040c5", + "ddprof_pr": null, + "pipeline": { + "id": "121890528", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121890528" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + }, + { + "id": "121876849", + "timestamp": "2026-06-30T15:04:53Z", + "ddprof_branch": "main", + "ddprof_sha": "c7046666bca1d8fcd011d0db1495f9992bb7e6fc", + "ddprof_pr": null, + "pipeline": { + "id": "121876849", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121876849" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + }, + { + "id": "121872420", + "timestamp": "2026-06-30T15:01:09Z", + "ddprof_branch": "main", + "ddprof_sha": "c6718586da6ae5690dbf754310ce6a97d7bef624", + "ddprof_pr": null, + "pipeline": { + "id": "121872420", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121872420" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + }, + { + "id": "121862090", + "timestamp": "2026-06-30T14:25:43Z", + "ddprof_branch": "main", + "ddprof_sha": "13222ae60b16a5593ec99e430471fff7e23548ae", + "ddprof_pr": null, + "pipeline": { + "id": "121862090", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121862090" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + }, + { + "id": "121848831", + "timestamp": "2026-06-30T13:38:56Z", + "ddprof_branch": "main", + "ddprof_sha": "b31f27050ddabe821a34a3bc3a97b4b6f1b42954", + "ddprof_pr": null, + "pipeline": { + "id": "121848831", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121848831" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + }, + { + "id": "121847651", + "timestamp": "2026-06-30T13:38:03Z", + "ddprof_branch": "main", + "ddprof_sha": "dc9612ce4b4be3572f5e5aba2da32b86500cd1f8", + "ddprof_pr": null, + "pipeline": { + "id": "121847651", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121847651" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + }, + { + "id": "121802370", + "timestamp": "2026-06-30T09:50:51Z", + "ddprof_branch": "main", + "ddprof_sha": "6f4cbb0b3f66541b7ccd17ac98b5907bf1cf2f98", + "ddprof_pr": null, + "pipeline": { + "id": "121802370", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121802370" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + }, + { + "id": "121800513", + "timestamp": "2026-06-30T09:45:41Z", + "ddprof_branch": "main", + "ddprof_sha": "1c61f1819b238ac034387f6dbf57647334fe4253", + "ddprof_pr": null, + "pipeline": { + "id": "121800513", + "url": "https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121800513" + }, + "lib_version": "unknown", + "status": "failed", + "summary": { + "total_jobs": 40, + "passed_jobs": 0, + "failed_jobs": 40, + "total_scenarios": 80, + "passed_scenarios": 0, + "failed_scenarios": 0, + "unknown_scenarios": 80, + "failed_configs": [ + "glibc-arm64-hotspot-jdk11", + "glibc-arm64-hotspot-jdk17", + "glibc-arm64-hotspot-jdk21", + "glibc-arm64-hotspot-jdk25", + "glibc-arm64-hotspot-jdk8", + "glibc-arm64-openj9-jdk11", + "glibc-arm64-openj9-jdk17", + "glibc-arm64-openj9-jdk21", + "glibc-arm64-openj9-jdk25", + "glibc-arm64-openj9-jdk8", + "glibc-x64-hotspot-jdk11", + "glibc-x64-hotspot-jdk17", + "glibc-x64-hotspot-jdk21", + "glibc-x64-hotspot-jdk25", + "glibc-x64-hotspot-jdk8", + "glibc-x64-openj9-jdk11", + "glibc-x64-openj9-jdk17", + "glibc-x64-openj9-jdk21", + "glibc-x64-openj9-jdk25", + "glibc-x64-openj9-jdk8", + "musl-arm64-hotspot-jdk11", + "musl-arm64-hotspot-jdk17", + "musl-arm64-hotspot-jdk21", + "musl-arm64-hotspot-jdk25", + "musl-arm64-hotspot-jdk8", + "musl-arm64-openj9-jdk11", + "musl-arm64-openj9-jdk17", + "musl-arm64-openj9-jdk21", + "musl-arm64-openj9-jdk25", + "musl-arm64-openj9-jdk8", + "musl-x64-hotspot-jdk11", + "musl-x64-hotspot-jdk17", + "musl-x64-hotspot-jdk21", + "musl-x64-hotspot-jdk25", + "musl-x64-hotspot-jdk8", + "musl-x64-openj9-jdk11", + "musl-x64-openj9-jdk17", + "musl-x64-openj9-jdk21", + "musl-x64-openj9-jdk25", + "musl-x64-openj9-jdk8" + ] + } + } + ] +} diff --git a/benchmarks/index.md b/benchmarks/index.md new file mode 100644 index 000000000..ccfb0b1ef --- /dev/null +++ b/benchmarks/index.md @@ -0,0 +1,18 @@ +--- +layout: default +title: Benchmark Test History +--- + +# Benchmark Test History + +[← Back to Dashboard](../) + +Performance regression testing using Renaissance benchmark suite. + +## Last 10 Runs + +*No test runs recorded yet.* + +--- + +[← Back to Dashboard](../) | [View git history](https://github.com/DataDog/java-profiler/commits/gh-pages) diff --git a/build-logic/QUICKSTART.md b/build-logic/QUICKSTART.md deleted file mode 100644 index 6cceaf1b2..000000000 --- a/build-logic/QUICKSTART.md +++ /dev/null @@ -1,1196 +0,0 @@ -# Native Build Plugins - Quick Start Guide - -This guide provides practical examples, workflows, and tips for using the Datadog native build plugin suite. For architectural details and reference documentation, see [README.md](README.md). - -## Table of Contents - -- [Getting Started](#getting-started) -- [Common Workflows](#common-workflows) -- [How-To Guides](#how-to-guides) -- [Tips and Tricks](#tips-and-tricks) -- [Troubleshooting](#troubleshooting) - ---- - -## Getting Started - -### Minimal Setup - -**build.gradle.kts:** -```kotlin -plugins { - id("com.datadoghq.native-build") -} - -nativeBuild { - version.set(project.version.toString()) -} -``` - -That's it! The plugin will: -- Auto-detect your compiler (clang++ or g++) -- Create standard configurations (release, debug, asan, tsan, fuzzer) -- Generate compile, link, and assemble tasks -- Use `src/main/cpp` as default source directory - -### Quick Build - -```bash -# Build release configuration -./gradlew assembleRelease - -# Build all active configurations -./gradlew assembleAll - -# Build specific configuration -./gradlew assembleDebug -``` - -### Adding Tests - -**build.gradle.kts:** -```kotlin -plugins { - id("com.datadoghq.native-build") - id("com.datadoghq.gtest") -} - -gtest { - testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) - mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) - - val javaHome = com.datadoghq.native.util.PlatformUtils.javaHome() - includes.from("src/main/cpp", "$javaHome/include") -} -``` - -```bash -# Run all tests -./gradlew gtest - -# Run tests for specific configuration -./gradlew gtestDebug -./gradlew gtestRelease -``` - ---- - -## Common Workflows - -### 1. Development: Fast Iteration with Debug Build - -```bash -# One-time: ensure you have debug config -./gradlew tasks --group=build | grep Debug - -# Edit code, then compile and link -./gradlew assembleDebug - -# Run debug tests -./gradlew gtestDebug -``` - -**Why debug config?** -- No optimization (`-O0`) = faster compilation -- Full debug symbols embedded -- Assertions enabled (no `-DNDEBUG`) -- No symbol stripping - -### 2. Testing: Memory Safety with ASan - -```bash -# Check if ASan is available -./gradlew tasks | grep -i asan - -# Build with AddressSanitizer -./gradlew assembleAsan - -# Run tests with ASan instrumentation -./gradlew gtestAsan -``` - -**ASan detects:** -- Heap buffer overflow/underflow -- Stack buffer overflow -- Use-after-free -- Use-after-return -- Memory leaks -- Double-free - -### 3. Testing: Thread Safety with TSan - -```bash -# Build with ThreadSanitizer -./gradlew assembleTsan - -# Run tests with TSan instrumentation -./gradlew gtestTsan -``` - -**TSan detects:** -- Data races -- Deadlocks -- Thread leaks -- Signal-unsafe functions in signal handlers - -### 4. Release: Production Build with Debug Symbols - -```bash -# Build release configuration -./gradlew assembleRelease - -# Output structure: -# build/lib/main/release/{platform}/{arch}/ -# ├── libjavaProfiler.so # Stripped library (~1.2MB) -# └── debug/ -# └── libjavaProfiler.so.debug # Debug symbols (~6MB) -``` - -**Key features:** -- Fully optimized (`-O3 -DNDEBUG`) -- Debug symbols extracted to separate file -- 69% size reduction in production binary -- Symbols linked via `.gnu_debuglink` - -### 5. Static Analysis: Clang scan-build - -The `scanbuild` plugin integrates Clang's static analyzer to detect bugs without running code. - -**build.gradle.kts:** -```kotlin -plugins { - id("com.datadoghq.scanbuild") -} - -scanBuild { - makefileDir.set(layout.projectDirectory.dir("src/test/make")) - outputDir.set(layout.buildDirectory.dir("reports/scan-build")) - analyzer.set("/usr/bin/clang++") - parallelJobs.set(4) - makeTargets.set(listOf("all", "test")) // Optional: specify make targets -} -``` - -```bash -# Run static analysis -./gradlew scanBuild - -# View HTML report -open build/reports/scan-build/*/index.html - -# Or on Linux -xdg-open build/reports/scan-build/*/index.html -``` - -**What scan-build detects:** -- Null pointer dereferences -- Memory leaks -- Use of uninitialized values -- Dead stores (unused assignments) -- Division by zero -- API misuse -- Logic errors -- Buffer overflows - -**Note:** scan-build is only available on Linux by default. The plugin will skip on macOS unless you have scan-build installed via Homebrew. - ---- - -## How-To Guides - -### Override Compiler - -```bash -# Use specific compiler -./gradlew build -Pnative.forceCompiler=clang++ -./gradlew build -Pnative.forceCompiler=g++-13 -./gradlew build -Pnative.forceCompiler=/usr/local/bin/clang++ - -# The plugin validates the compiler exists and works -``` - -### Customize Source Directories - -```kotlin -nativeBuild { - version.set("1.2.3") - cppSourceDirs.set(listOf( - "src/main/cpp", - "src/vendor/library" - )) - includeDirectories.set(listOf( - "src/main/cpp", - "src/vendor/library/include", - "/usr/local/include" - )) -} -``` - -### Add Custom Configurations - -```kotlin -nativeBuild { - version.set(project.version.toString()) - - // Override standard configs - buildConfigurations { - // Create custom configuration - register("production") { - platform.set(com.datadoghq.native.model.Platform.LINUX) - architecture.set(com.datadoghq.native.model.Architecture.X64) - active.set(true) - - compilerArgs.set(listOf( - "-O3", - "-DNDEBUG", - "-march=native", // Optimize for current CPU - "-flto", // Link-time optimization - "-fPIC" - )) - - linkerArgs.set(listOf( - "-Wl,--gc-sections", - "-flto" - )) - } - } -} -``` - -**Generated tasks:** -- `compileProduction` -- `linkProduction` -- `assembleProduction` - -### Add Common Flags to All Configurations - -```kotlin -nativeBuild { - version.set(project.version.toString()) - - // Apply to all configurations - commonCompilerArgs( - "-Wall", - "-Wextra", - "-Werror" - ) - - commonLinkerArgs( - "-Wl,--as-needed" - ) -} -``` - -### Control Google Test Behavior - -```kotlin -gtest { - testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) - mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) - - // Custom Google Test location (macOS) - googleTestHome.set(file("/usr/local/opt/googletest")) - - // Always enable assertions (remove -DNDEBUG) - enableAssertions.set(true) - - // Keep debug symbols in release test builds - keepSymbols.set(true) - - // Stop on first test failure - failFast.set(true) - - // Always re-run tests (ignore up-to-date checks) - alwaysRun.set(true) - - // Skip building native test support libraries - buildNativeLibs.set(false) - - includes.from( - "src/main/cpp", - "third-party/include" - ) -} -``` - -### Skip Builds Selectively - -```bash -# Skip all tests -./gradlew build -Pskip-tests - -# Skip only gtest (keep Java tests) -./gradlew build -Pskip-gtest - -# Skip all native compilation -./gradlew build -Pskip-native -``` - -### Cross-Platform Configuration - -```kotlin -nativeBuild { - buildConfigurations { - // Linux x64 release - register("linuxRelease") { - platform.set(Platform.LINUX) - architecture.set(Architecture.X64) - active.set(PlatformUtils.currentPlatform == Platform.LINUX) - // ... compiler/linker args - } - - // macOS ARM release - register("macosRelease") { - platform.set(Platform.MACOS) - architecture.set(Architecture.ARM64) - active.set(PlatformUtils.currentPlatform == Platform.MACOS) - // ... compiler/linker args - } - } -} -``` - -### Integrate with Java Resource Packaging - -```kotlin -// Copy native libraries to Java resources -val copyReleaseLibs by tasks.registering(Copy::class) { - from("build/lib/main/release") - into(layout.buildDirectory.dir("resources/main/native")) - - dependsOn(tasks.named("assembleRelease")) -} - -tasks.named("processResources") { - dependsOn(copyReleaseLibs) -} - -// Include in JAR -tasks.named("jar") { - from(layout.buildDirectory.dir("resources/main/native")) { - into("native") - } -} -``` - -### Configure Static Analysis with scan-build - -The scan-build plugin requires a Makefile-based build for analysis. This is typically separate from the Gradle native build. - -**Basic configuration:** -```kotlin -plugins { - id("com.datadoghq.scanbuild") -} - -scanBuild { - // Directory containing Makefile (required) - makefileDir.set(layout.projectDirectory.dir("src/test/make")) - - // Where to output HTML reports - outputDir.set(layout.buildDirectory.dir("reports/scan-build")) - - // Clang analyzer to use - analyzer.set("/usr/bin/clang++") - - // Parallel make jobs - parallelJobs.set(4) - - // Make targets to build (default: ["all"]) - makeTargets.set(listOf("all")) -} -``` - -**Advanced configuration:** -```kotlin -scanBuild { - makefileDir.set(layout.projectDirectory.dir("src/test/make")) - outputDir.set(layout.buildDirectory.dir("reports/scan-build")) - - // Use specific clang version - analyzer.set("/usr/bin/clang++-15") - - // Increase parallelism for faster analysis - parallelJobs.set(Runtime.getRuntime().availableProcessors()) - - // Analyze multiple targets - makeTargets.set(listOf("library", "tests")) -} -``` - -**Example Makefile structure:** - -Create `src/test/make/Makefile`: -```makefile -# Compiler (will be intercepted by scan-build) -CXX = clang++ -CXXFLAGS = -std=c++17 -Wall -Wextra -I../../main/cpp - -# Source files -SOURCES = $(wildcard ../../main/cpp/*.cpp) -OBJECTS = $(SOURCES:.cpp=.o) - -# Targets -all: library - -library: $(OBJECTS) - $(CXX) -shared -o libjavaProfiler.so $(OBJECTS) - -%.o: %.cpp - $(CXX) $(CXXFLAGS) -c $< -o $@ - -clean: - rm -f $(OBJECTS) libjavaProfiler.so - -.PHONY: all clean -``` - -**Integration with CI:** -```kotlin -// Make scanBuild part of verification -tasks.named("check") { - dependsOn("scanBuild") -} -``` - -**Platform-specific configuration:** -```kotlin -import com.datadoghq.native.util.PlatformUtils -import com.datadoghq.native.model.Platform - -if (PlatformUtils.currentPlatform == Platform.LINUX) { - scanBuild { - makefileDir.set(layout.projectDirectory.dir("src/test/make")) - outputDir.set(layout.buildDirectory.dir("reports/scan-build")) - } -} -``` - ---- - -## Tips and Tricks - -### Performance Tips - -#### 1. Parallel Compilation -Gradle automatically parallelizes compilation at the file level. Each `.cpp` file compiles independently. - -```bash -# Use more parallel workers -./gradlew build --parallel --max-workers=8 -``` - -#### 2. Incremental Builds -The plugin tracks: -- Source file changes -- Header file changes (via `-MMD` dependency tracking) -- Compiler flag changes - -Only modified files recompile: -```bash -# First build -./gradlew assembleDebug # Compiles all files - -# Edit one file -vim src/main/cpp/profiler.cpp - -# Second build -./gradlew assembleDebug # Only recompiles profiler.cpp -``` - -#### 3. Build Faster During Development - -```bash -# Use debug config (no optimization) -./gradlew assembleDebug - -# Skip tests -./gradlew assembleDebug -Pskip-tests - -# Use clang++ (generally faster than g++) -./gradlew assembleDebug -Pnative.forceCompiler=clang++ -``` - -### Debugging Tips - -#### 1. Verbose Compiler Output - -```kotlin -tasks.withType { - doFirst { - println("Compiler: ${compiler.get()}") - println("Args: ${compilerArgs.get()}") - println("Sources: ${sources.files}") - } -} -``` - -#### 2. Inspect Generated Build Files - -```bash -# Debug symbols location -ls -lh build/lib/main/release/*/*/debug/ - -# Object files -ls -lh build/obj/main/release/ - -# Task dependency tree -./gradlew assembleRelease --dry-run -``` - -#### 3. Check Active Configurations - -```bash -# View all build tasks -./gradlew tasks --group=build - -# The plugin logs active configurations: -./gradlew assembleAll | grep "Active configurations" -``` - -#### 4. Validate Debug Symbol Extraction - -**Linux:** -```bash -# Check if symbols are stripped -nm build/lib/main/release/linux/x64/libjavaProfiler.so | wc -l - -# Verify debug link -readelf -p .gnu_debuglink build/lib/main/release/linux/x64/libjavaProfiler.so - -# Check debug file -file build/lib/main/release/linux/x64/debug/libjavaProfiler.so.debug -``` - -**macOS:** -```bash -# Check stripped library -nm -gU build/lib/main/release/macos/arm64/libjavaProfiler.dylib - -# Verify dSYM bundle -dwarfdump --uuid build/lib/main/release/macos/arm64/libjavaProfiler.dylib.dSYM -``` - -### Testing Tips - -#### 1. Run Specific Test - -```bash -# Run one test from specific config -./gradlew gtestDebug_test_callTraceStorage -``` - -#### 2. Test with Multiple Configurations - -```bash -# Run tests with sanitizers in parallel -./gradlew gtestDebug gtestAsan gtestTsan --parallel -``` - -#### 3. Investigate Test Failures - -```bash -# Enable detailed test output -./gradlew gtestDebug --info - -# Run specific test binary directly -./build/bin/gtest/debug_test_callTraceStorage/test_callTraceStorage -``` - -#### 4. Test Environment Variables - -ASan and TSan tests automatically set environment variables: -```kotlin -// In BuildConfiguration: -testEnvironment.put("ASAN_OPTIONS", "...") -testEnvironment.put("TSAN_OPTIONS", "...") - -// Add custom variables: -buildConfigurations { - named("debug") { - testEnvironment.put("LOG_LEVEL", "debug") - testEnvironment.put("TEST_DATA_DIR", "$projectDir/testdata") - } -} -``` - -### Static Analysis Tips - -#### 1. Reading scan-build Reports - -```bash -# Run analysis -./gradlew scanBuild - -# Open report in browser -open build/reports/scan-build/*/index.html - -# Reports are organized by bug type: -# - Dead store: Unused assignments -# - Memory leak: Leaked allocations -# - Null dereference: Potential null pointer access -# - Uninitialized value: Use of uninitialized variables -``` - -#### 2. Focus on High-Priority Issues - -scan-build categorizes bugs by severity. Start with: -1. **Logic errors** - Wrong behavior -2. **Memory errors** - Leaks, use-after-free -3. **Null pointer issues** - Crashes -4. **Dead code** - Optimization opportunities - -#### 3. Incremental Analysis - -```bash -# Analyze after significant changes -./gradlew scanBuild - -# Compare with previous run -diff -u old-report/index.html build/reports/scan-build/*/index.html -``` - -#### 4. Customize Analyzer Options - -```kotlin -scanBuild { - makefileDir.set(layout.projectDirectory.dir("src/test/make")) - outputDir.set(layout.buildDirectory.dir("reports/scan-build")) - - // Use latest clang for better analysis - analyzer.set("/usr/bin/clang++-15") - - // Faster analysis with more parallelism - parallelJobs.set(8) -} -``` - -#### 5. Integration with Code Review - -```bash -# Run in CI before merging -./gradlew scanBuild - -# Fail build on new issues (requires custom script) -if grep -q "bugs found" build/reports/scan-build/*/index.html; then - echo "Static analysis found issues" - exit 1 -fi -``` - -#### 6. Suppressing False Positives - -If scan-build reports false positives, add assertions in code: -```cpp -void processData(Data* data) { - // Tell analyzer this can't be null - assert(data != nullptr); - - // Now scan-build knows data is valid - data->process(); -} -``` - -#### 7. Combine with Other Tools - -```bash -# Static analysis + runtime sanitizers = comprehensive coverage -./gradlew scanBuild # Find logic errors -./gradlew gtestAsan # Find memory errors -./gradlew gtestTsan # Find race conditions -``` - -### CI/CD Tips - -#### 1. Minimal CI Build -```bash -# Quick validation build -./gradlew assembleRelease -Pskip-tests -``` - -#### 2. Full CI Build -```bash -# Build all configs and run all tests -./gradlew assembleAll gtest -``` - -#### 3. CI with Sanitizers -```bash -# Test memory safety in CI -./gradlew gtestAsan gtestTsan - -# These are conditionally skipped if libasan/libtsan not available -``` - -#### 4. CI with Static Analysis -```bash -# Run static analysis in CI (Linux only) -./gradlew scanBuild - -# Archive reports as CI artifacts -# GitHub Actions example: -# - name: Upload scan-build reports -# uses: actions/upload-artifact@v3 -# with: -# name: scan-build-reports -# path: build/reports/scan-build/ - -# GitLab CI example: -# artifacts: -# paths: -# - build/reports/scan-build/ -# when: always -``` - -#### 5. Comprehensive CI Pipeline -```bash -# Full verification pipeline -./gradlew clean \ - assembleAll \ - gtest \ - gtestAsan \ - gtestTsan \ - scanBuild - -# This covers: -# - Compilation for all configs -# - Unit tests -# - Memory safety (ASan) -# - Thread safety (TSan) -# - Static analysis (scan-build) -``` - -#### 6. Release Artifact Packaging -```bash -# Build release with extracted debug symbols -./gradlew assembleRelease - -# Package production library (stripped) -tar czf library.tar.gz \ - build/lib/main/release/linux/x64/libjavaProfiler.so - -# Package debug symbols separately -tar czf library-debug.tar.gz \ - build/lib/main/release/linux/x64/debug/ -``` - -### Platform-Specific Tips - -#### macOS: Homebrew Google Test -```bash -# Install Google Test -brew install googletest - -# Plugin auto-detects at: -# /opt/homebrew/opt/googletest (Apple Silicon) -# /usr/local/opt/googletest (Intel) -``` - -#### Linux: System Google Test -```bash -# Ubuntu/Debian -sudo apt-get install libgtest-dev libgmock-dev - -# Fedora/RHEL -sudo dnf install gtest-devel gmock-devel -``` - -#### musl libc Detection -The plugin automatically detects musl libc and adds `-D__musl__`: -```bash -# Check musl detection -./gradlew assembleRelease | grep __musl__ - -# Force musl detection (advanced) -./gradlew -Pnative.musl=true assembleRelease -``` - -#### Linux: scan-build Installation - -scan-build is typically available on Linux but needs separate installation: - -```bash -# Ubuntu/Debian - includes clang static analyzer -sudo apt-get install clang-tools - -# Fedora/RHEL -sudo dnf install clang-tools-extra - -# Arch Linux -sudo pacman -S clang - -# Verify installation -which scan-build -scan-build --version -``` - -**Common scan-build locations:** -- `/usr/bin/scan-build` (most distros) -- `/usr/lib/llvm-*/bin/scan-build` (Ubuntu with specific LLVM versions) - -If you have multiple clang versions: -```bash -# List available scan-build versions -ls /usr/lib/llvm-*/bin/scan-build - -# Use specific version in plugin -scanBuild { - analyzer.set("/usr/lib/llvm-15/bin/clang++") -} -``` - ---- - -## Troubleshooting - -### Compiler Not Found - -**Problem:** -``` -Could not find any suitable C++ compiler -``` - -**Solutions:** -1. Install a compiler: - - macOS: `xcode-select --install` - - Linux: `sudo apt-get install build-essential` or `sudo dnf install gcc-c++` - -2. Force specific compiler: - ```bash - ./gradlew build -Pnative.forceCompiler=/path/to/compiler - ``` - -### Google Test Not Found - -**Problem:** -``` -WARNING: Google Test not found - skipping native tests -``` - -**Solutions:** -1. Install Google Test (see Platform-Specific Tips above) - -2. Set custom location (macOS): - ```kotlin - gtest { - googleTestHome.set(file("/custom/path/googletest")) - } - ``` - -3. Skip tests if not needed: - ```bash - ./gradlew build -Pskip-gtest - ``` - -### Sanitizer Libraries Not Available - -**Problem:** -ASan/TSan configurations are inactive. - -**Check availability:** -```bash -# Check for libasan -find /usr/lib /usr/local/lib -name "libasan.so*" 2>/dev/null - -# Check for libtsan -find /usr/lib /usr/local/lib -name "libtsan.so*" 2>/dev/null -``` - -**Solutions:** -1. Install sanitizer libraries: - - Ubuntu/Debian: `sudo apt-get install libasan6 libtsan0` - - Fedora/RHEL: `sudo dnf install libasan libtsan` - -2. Use clang (includes sanitizers): - ```bash - ./gradlew build -Pnative.forceCompiler=clang++ - ``` - -### Compilation Errors - -**Problem:** -``` -error: 'std::optional' is not a member of 'std' -``` - -**Solution:** -Ensure C++17 standard: -```kotlin -nativeBuild { - commonCompilerArgs("-std=c++17") -} -``` - -### Linking Errors - -**Problem:** -``` -undefined reference to `pthread_create' -``` - -**Solution:** -Add missing linker flags: -```kotlin -buildConfigurations { - named("debug") { - linkerArgs.add("-lpthread") - } -} -``` - -### Test Failures with ASan/TSan - -**Problem:** -Tests pass in debug but fail with ASan/TSan. - -**Analysis:** -This indicates real bugs! ASan/TSan found: -- Memory leaks -- Race conditions -- Use-after-free -- Buffer overflows - -**Solutions:** -1. Review the sanitizer output carefully -2. Fix the underlying bug (don't suppress unless false positive) -3. Add suppressions only for false positives: - ```bash - # ASan suppressions - echo "leak:third_party_library" >> gradle/sanitizers/asan.supp - - # TSan suppressions - echo "race:false_positive_function" >> gradle/sanitizers/tsan.supp - ``` - -### Clean Build Issues - -**Problem:** -Build fails after clean. - -**Solution:** -```bash -# Full clean including native artifacts -./gradlew clean - -# Rebuild -./gradlew assembleAll -``` - -### Task Not Found - -**Problem:** -``` -Task 'assembleAsan' not found -``` - -**Cause:** -Configuration is inactive (sanitizer not available). - -**Check:** -```bash -./gradlew tasks --group=build | grep -i asan -``` - -If not listed, the configuration is skipped on your platform. - -### scan-build Not Found - -**Problem:** -``` -scan-build not found in PATH - scanBuild task will fail if executed -``` - -**Solutions:** -1. Install scan-build on Linux: - ```bash - # Ubuntu/Debian - sudo apt-get install clang-tools - - # Fedora/RHEL - sudo dnf install clang-tools-extra - - # Arch Linux - sudo pacman -S clang - ``` - -2. Install on macOS (optional, not typical): - ```bash - brew install llvm - # Add to PATH: - export PATH="/opt/homebrew/opt/llvm/bin:$PATH" - ``` - -3. Verify installation: - ```bash - which scan-build - scan-build --help - ``` - -### scan-build Fails on macOS - -**Problem:** -Plugin skips scan-build task on macOS. - -**Explanation:** -By design, the plugin only runs scan-build on Linux. This matches typical CI environments. - -**Solution:** -If you need scan-build on macOS: -1. Install via Homebrew (see above) -2. Modify plugin check (advanced): - ```kotlin - // Override platform check - tasks.register("scanBuildMac", Exec::class) { - workingDir = file("src/test/make") - commandLine( - "scan-build", - "-o", "build/reports/scan-build", - "make", "all" - ) - } - ``` - -### scan-build Reports No Issues - -**Problem:** -scan-build runs but reports 0 bugs, even when issues exist. - -**Possible causes:** -1. **Makefile not using compiler correctly:** - ```makefile - # Wrong - hardcoded command - build: - /usr/bin/g++ -o output source.cpp - - # Correct - use $(CXX) variable - build: - $(CXX) -o output source.cpp - ``` - -2. **Precompiled objects:** - ```bash - # Clean before analysis - cd src/test/make - make clean - ./gradlew scanBuild - ``` - -3. **Compiler wrappers not intercepted:** - ```bash - # Verify scan-build is wrapping compiler - ./gradlew scanBuild --info | grep "scan-build" - ``` - -### scan-build Makefile Errors - -**Problem:** -``` -make: *** No rule to make target 'all'. Stop. -``` - -**Solution:** -Verify Makefile exists and has required targets: -```bash -# Check Makefile location -ls -la src/test/make/Makefile - -# Test make directly -cd src/test/make -make all - -# If it works, then try scan-build -./gradlew scanBuild -``` - -### scan-build Reports Inaccessible - -**Problem:** -Can't find or open HTML reports. - -**Solution:** -```bash -# Find the latest report directory -find build/reports/scan-build -name "index.html" - -# Open with browser -open $(find build/reports/scan-build -name "index.html" | head -1) - -# Or view summary in terminal -grep -A 5 "bugs found" build/reports/scan-build/*/index.html -``` - ---- - -## Advanced Topics - -### Custom Task Integration - -Hook into native build lifecycle: - -```kotlin -// Run custom validation after compilation -tasks.named("compileRelease") { - doLast { - println("Compiled ${outputs.files.files.size} object files") - } -} - -// Custom post-link processing -tasks.named("linkRelease") { - doLast { - val library = outputs.files.singleFile - println("Built library: ${library.absolutePath}") - println("Size: ${library.length() / 1024}KB") - } -} -``` - -### Multi-Project Builds - -Share native configurations across projects: - -**root/buildSrc/src/main/kotlin/NativeConventions.kt:** -```kotlin -fun Project.configureNative() { - apply(plugin = "com.datadoghq.native-build") - - configure { - version.set(rootProject.version.toString()) - commonCompilerArgs("-Wall", "-Wextra") - } -} -``` - -**subproject/build.gradle.kts:** -```kotlin -configureNative() - -nativeBuild { - cppSourceDirs.set(listOf("src/cpp")) -} -``` - -### Conditional Platform Builds - -```kotlin -import com.datadoghq.native.util.PlatformUtils -import com.datadoghq.native.model.Platform - -if (PlatformUtils.currentPlatform == Platform.LINUX) { - tasks.register("packageDeb") { - dependsOn("assembleRelease") - doLast { - // Create .deb package - } - } -} -``` - ---- - -## Further Reading - -- [README.md](README.md) - Full reference documentation -- [Plugin source code](conventions/src/main/kotlin/) - Implementation details -- [Gradle documentation](https://docs.gradle.org/) - Gradle build system -- [Google Test documentation](https://google.github.io/googletest/) - Unit testing framework diff --git a/build-logic/README.md b/build-logic/README.md deleted file mode 100644 index 16900afc5..000000000 --- a/build-logic/README.md +++ /dev/null @@ -1,359 +0,0 @@ -# Native Build Plugins - -This directory contains a Gradle composite build that provides plugins for building C++ libraries and tests: - -- **`com.datadoghq.native-build`** - Core C++ compilation and linking -- **`com.datadoghq.gtest`** - Google Test integration for C++ unit tests -- **`com.datadoghq.scanbuild`** - Clang static analyzer integration - -> **📚 New to these plugins?** Check out [QUICKSTART.md](QUICKSTART.md) for practical examples, common workflows, tips and tricks, and troubleshooting guidance. - -## Architecture - -The plugin uses Kotlin DSL for type-safe build configuration and follows modern Gradle conventions: - -- **Composite Build**: Independent Gradle project for build logic versioning -- **Type-Safe DSL**: Kotlin-based configuration with compile-time checking -- **Property API**: Lazy evaluation using Gradle's Property types -- **Automatic Task Generation**: Creates compile, link, and assemble tasks per configuration - -## Plugin Usage - -```kotlin -plugins { - id("com.datadoghq.native-build") -} - -nativeBuild { - version.set(project.version.toString()) - cppSourceDirs.set(listOf("src/main/cpp")) - includeDirectories.set(listOf("src/main/cpp")) -} -``` - -The plugin automatically creates standard configurations (release, debug, asan, tsan, fuzzer) and generates tasks: -- `compile{Config}` - Compiles C++ sources -- `link{Config}` - Links shared library -- `assemble{Config}` - Assembles configuration -- `assembleAll` - Builds all active configurations - -## Standard Configurations - -### Release -- **Optimization**: `-O3 -DNDEBUG` -- **Debug symbols**: Extracted to separate files (69% size reduction) -- **Strip**: Yes (production binaries) -- **Output**: Stripped library + .dSYM bundle (macOS) or .debug file (Linux) - -### Debug -- **Optimization**: `-O0 -g` -- **Debug symbols**: Embedded -- **Strip**: No -- **Output**: Full debug library - -### ASan (AddressSanitizer) -- Conditionally active if libasan is available -- Memory error detection - -### TSan (ThreadSanitizer) -- Conditionally active if libtsan is available -- Thread safety validation - -### Fuzzer -- Fuzzing instrumentation -- Requires libFuzzer - -## Compiler Detection - -The plugin automatically detects and selects the best available C++ compiler: - -### Auto-Detection (Default) -```bash -./gradlew build -# Logs: "Auto-detected compiler: clang++" -# or: "Auto-detected compiler: g++" -``` - -**Detection order:** -1. `clang++` (preferred - better optimization and diagnostics) -2. `g++` (fallback) -3. `c++` (last resort) - -If no compiler is found, the build fails with a clear error message. - -### Force Specific Compiler -Use the `-Pnative.forceCompiler` property to override auto-detection: - -```bash -# Force clang++ -./gradlew build -Pnative.forceCompiler=clang++ - -# Force g++ -./gradlew build -Pnative.forceCompiler=g++ - -# Force specific version (full path) -./gradlew build -Pnative.forceCompiler=/usr/bin/g++-13 -./gradlew build -Pnative.forceCompiler=/opt/homebrew/bin/clang++ -``` - -**Validation:** The specified compiler is validated by running ` --version`. If validation fails, the build errors immediately with an actionable message. - -### Sanitizer Library Detection -ASan and TSan library detection uses the detected/forced compiler instead of hardcoding `gcc`. This enables sanitizer builds on clang-only systems (e.g., macOS with Xcode but no gcc installed). - -## Debug Symbol Extraction - -Release builds automatically extract debug symbols for optimal production deployment: - -### macOS -``` -dsymutil library.dylib -o library.dylib.dSYM -strip -S library.dylib -``` -- **Stripped library**: ~404KB (production) -- **Debug bundle**: ~3.7MB (.dSYM) - -### Linux -``` -objcopy --only-keep-debug library.so library.so.debug -objcopy --strip-debug library.so -objcopy --add-gnu-debuglink=library.so.debug library.so -``` -- **Stripped library**: ~1.2MB (production) -- **Debug file**: ~6MB (.debug) - -## Advanced Features - -### Source Sets - -Source sets allow different parts of the codebase to have different compilation flags. This is useful for: -- Legacy code requiring older C++ standards -- Third-party code with specific compiler warnings -- Platform-specific optimizations - -**Example:** -```kotlin -tasks.register("compileLib", NativeCompileTask::class) { - compiler.set("clang++") - compilerArgs.set(listOf("-std=c++17", "-O3")) // Base flags for all files - includes.from("src/main/cpp") - - // Define source sets with per-set compiler flags - sourceSets { - create("main") { - sources.from(fileTree("src/main/cpp")) - compilerArgs.add("-fPIC") // Additional flags for main code - } - create("legacy") { - sources.from(fileTree("src/legacy")) - compilerArgs.addAll("-Wno-deprecated", "-std=c++11") // Different standard - excludes.add("**/broken/*.cpp") // Exclude specific files - } - } - - objectFileDir.set(file("build/obj")) -} -``` - -**Key features:** -- **Include/exclude patterns**: Ant-style patterns (e.g., `**/*.cpp`, `**/test_*.cpp`) -- **Merged compiler args**: Base args + source-set-specific args -- **Conveniences**: `from()`, `include()`, `exclude()`, `compileWith()` methods - -### Symbol Visibility Control - -Symbol visibility controls which symbols are exported from shared libraries. This is essential for: -- Hiding internal implementation details -- Reducing symbol table size -- Preventing symbol conflicts -- Creating clean JNI interfaces - -**Example:** -```kotlin -tasks.register("linkLib", NativeLinkTask::class) { - linker.set("clang++") - objectFiles.from(fileTree("build/obj")) - outputFile.set(file("build/lib/libjavaProfiler.dylib")) - - // Export only JNI symbols - exportSymbols.set(listOf( - "Java_*", // All JNI methods - "JNI_OnLoad", // JNI initialization - "JNI_OnUnload" // JNI cleanup - )) - - // Hide specific internal symbols (overrides exports) - hideSymbols.set(listOf( - "*_internal*", // Internal functions - "*_test*" // Test utilities - )) -} -``` - -**Platform-specific implementation:** -- **Linux**: Generates version script (`.ver` file) with wildcard pattern support (e.g., `Java_*` matches all JNI methods) -- **macOS**: Generates exported symbols list (`.exp` file) - **Note:** Wildcards are not supported on macOS. Patterns like `Java_*` are treated as literal symbol names. For JNI exports, you must either list individual symbols or use `-fvisibility` compiler flags instead. - -**Generated files** (in `temporaryDir`): -- Linux: `library.ver` → `-Wl,--version-script=library.ver` -- macOS: `library.exp` → `-Wl,-exported_symbols_list,library.exp` - -**Symbol visibility best practices:** -1. Start with `-fvisibility=hidden` compiler flag -2. Mark public API with `__attribute__((visibility("default")))` in source -3. OR use `exportSymbols` linker flag for pattern-based export -4. Verify with: `nm -gU library.dylib` (macOS) or `nm -D library.so` (Linux) - -## Task Dependencies - -``` -compileConfig → linkConfig → assembleConfig - ↓ - extractDebugSymbols (release only) - ↓ - stripSymbols (release only) - ↓ - copyConfigLibs → assembleConfigJar -``` - -## Design Benefits - -The Kotlin-based build system provides: -- ✅ Compile-time type checking via Kotlin DSL -- ✅ Gradle idiomatic design (Property API, composite builds) -- ✅ Automatic debug symbol extraction (69% size reduction) -- ✅ Clean builds work from scratch -- ✅ Centralized configuration definitions - ---- - -# Google Test Plugin - -The `com.datadoghq.gtest` plugin provides Google Test integration for C++ unit testing. - -## Plugin Usage - -```kotlin -plugins { - id("com.datadoghq.native-build") // Required - provides configurations - id("com.datadoghq.gtest") -} - -gtest { - testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) - mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) - - includes.from( - "src/main/cpp", - "${javaHome}/include", - "${javaHome}/include/${platformInclude}" - ) -} -``` - -## Generated Tasks - -For each test file in `testSourceDir`, the plugin creates: - -| Task Pattern | Description | -|--------------|-------------| -| `compileGtest{Config}_{TestName}` | Compile main sources + test file | -| `linkGtest{Config}_{TestName}` | Link test executable with gtest libraries | -| `gtest{Config}_{TestName}` | Execute the test | - -Aggregation tasks: -- `gtest` - Run all tests across all configurations -- `gtest{Config}` - Run all tests for a specific configuration (e.g., `gtestDebug`) - -## Configuration Options - -| Property | Type | Default | Description | -|----------|------|---------|-------------| -| `testSourceDir` | `DirectoryProperty` | Required | Directory containing test `.cpp` files | -| `mainSourceDir` | `DirectoryProperty` | Required | Directory containing main source files | -| `includes` | `ConfigurableFileCollection` | Empty | Include directories for compilation | -| `googleTestHome` | `DirectoryProperty` | Auto-detected | Google Test installation directory (macOS) | -| `enableAssertions` | `Property` | `true` | Remove `-DNDEBUG` to enable assertions | -| `keepSymbols` | `Property` | `true` | Keep debug symbols in release test builds | -| `failFast` | `Property` | `false` | Stop on first test failure | -| `alwaysRun` | `Property` | `true` | Ignore up-to-date checks for tests | -| `buildNativeLibs` | `Property` | `true` | Build native test support libraries (Linux) | - -## Platform Detection - -The plugin automatically detects Google Test installation: - -- **macOS**: `/opt/homebrew/opt/googletest` (Homebrew default) -- **Linux**: System includes (`/usr/include/gtest`) - -Override with `googleTestHome`: -```kotlin -gtest { - googleTestHome.set(file("/custom/path/to/googletest")) -} -``` - -## Integration with NativeBuildPlugin - -GtestPlugin consumes configurations from NativeBuildPlugin: - -1. **Shared configurations**: Uses the same release/debug/asan/tsan/fuzzer configs -2. **Compiler detection**: Uses `PlatformUtils.findCompiler()` with `-Pnative.forceCompiler` support -3. **Consistent flags**: Inherits compiler/linker args from build configurations - -## Example Output - -``` -$ ./gradlew gtestDebug - -> Task :ddprof-lib:compileGtestDebug_test_callTraceStorage -Compiling 45 C++ source files with clang++... - -> Task :ddprof-lib:linkGtestDebug_test_callTraceStorage -Linking executable: test_callTraceStorage - -> Task :ddprof-lib:gtestDebug_test_callTraceStorage -[==========] Running 5 tests from 1 test suite. -... -[ PASSED ] 5 tests. - -BUILD SUCCESSFUL -``` - -## Skip Options - -```bash -# Skip all tests -./gradlew build -Pskip-tests - -# Skip only gtest (keep Java tests) -./gradlew build -Pskip-gtest - -# Skip native compilation entirely -./gradlew build -Pskip-native -``` - ---- - -## Files - -- `settings.gradle` - Composite build configuration -- `conventions/build.gradle.kts` - Plugin module -- `conventions/src/main/kotlin/` - Plugin implementation - - `NativeBuildPlugin.kt` - Native build plugin - - `NativeBuildExtension.kt` - Native build DSL extension - - `gtest/GtestPlugin.kt` - Google Test plugin - - `gtest/GtestExtension.kt` - Google Test DSL extension - - `scanbuild/ScanBuildPlugin.kt` - Static analysis plugin - - `scanbuild/ScanBuildExtension.kt` - Static analysis DSL extension - - `model/` - Type-safe configuration models - - `tasks/` - Compile and link tasks - - `config/` - Configuration presets - - `util/` - Platform utilities - ---- - -## Documentation - -- **[QUICKSTART.md](QUICKSTART.md)** - Quick start guide with practical examples, workflows, tips and troubleshooting -- **[README.md](README.md)** (this file) - Architecture details, API reference, and design documentation diff --git a/build-logic/conventions/build.gradle.kts b/build-logic/conventions/build.gradle.kts deleted file mode 100644 index 2a163b2ce..000000000 --- a/build-logic/conventions/build.gradle.kts +++ /dev/null @@ -1,62 +0,0 @@ -plugins { - `kotlin-dsl` -} - -repositories { - val mavenRepositoryProxy = providers.gradleProperty("mavenRepositoryProxy").orNull - if (mavenRepositoryProxy != null) { - maven { url = uri(mavenRepositoryProxy) } - } - gradlePluginPortal() - mavenCentral() -} - -dependencies { - implementation("org.jetbrains.kotlin:kotlin-stdlib") - implementation("com.diffplug.spotless:spotless-plugin-gradle:8.7.0") -} - -gradlePlugin { - plugins { - create("nativeBuild") { - id = "com.datadoghq.native-build" - implementationClass = "com.datadoghq.native.NativeBuildPlugin" - } - create("nativeRoot") { - id = "com.datadoghq.native-root" - implementationClass = "com.datadoghq.native.RootProjectPlugin" - } - create("gtest") { - id = "com.datadoghq.gtest" - implementationClass = "com.datadoghq.native.gtest.GtestPlugin" - } - create("scanbuild") { - id = "com.datadoghq.scanbuild" - implementationClass = "com.datadoghq.native.scanbuild.ScanBuildPlugin" - } - create("profilerTest") { - id = "com.datadoghq.profiler-test" - implementationClass = "com.datadoghq.profiler.ProfilerTestPlugin" - } - create("spotlessConvention") { - id = "com.datadoghq.spotless-convention" - implementationClass = "com.datadoghq.profiler.SpotlessConventionPlugin" - } - create("javaConventions") { - id = "com.datadoghq.java-conventions" - implementationClass = "com.datadoghq.profiler.JavaConventionsPlugin" - } - create("fuzzTargets") { - id = "com.datadoghq.fuzz-targets" - implementationClass = "com.datadoghq.native.fuzz.FuzzTargetsPlugin" - } - create("simpleNativeLib") { - id = "com.datadoghq.simple-native-lib" - implementationClass = "com.datadoghq.native.SimpleNativeLibPlugin" - } - create("versionedSources") { - id = "com.datadoghq.versioned-sources" - implementationClass = "com.datadoghq.java.versionedsources.VersionedSourcesPlugin" - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourceSet.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourceSet.kt deleted file mode 100644 index a4e075d1c..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourceSet.kt +++ /dev/null @@ -1,54 +0,0 @@ - -package com.datadoghq.java.versionedsources - -import org.gradle.api.Named -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.provider.Property -import javax.inject.Inject - -/** - * Represents a versioned source set for Java version-specific code. - * - * Each instance defines a Java version-specific source directory that contains - * classes compiled with a specific Java release target. These classes are selected - * at runtime via Class.forName() based on the running JVM version. - * - * @property name The source set name (e.g., "java9", "java11") - */ -abstract class VersionedSourceSet @Inject constructor( - private val name: String -) : Named { - - /** - * The Java release version (e.g., 9, 11, 17, 21). - * Used with javac --release flag for compilation. - */ - abstract val release: Property - - /** - * Source directory for version-specific classes. - * Default: src/main/{name} (e.g., src/main/java9) - */ - abstract val sourceDir: DirectoryProperty - - /** - * Minimum toolchain version required to compile this source set. - * Must be >= release version. Defaults to the release version. - * Use when a higher JDK is needed (e.g., compile Java 9 code with JDK 11). - */ - abstract val minToolchainVersion: Property - - override fun getName(): String = name - - /** - * Returns the capitalized name for task naming. - * Example: "Java9" for name "java9" - */ - fun capitalizedName(): String = name.replaceFirstChar { it.titlecase() } - - /** - * The compile task name for this versioned source set. - * Example: "compileJava9Java" for name "java9" - */ - val compileTaskName: String get() = "compile${capitalizedName()}Java" -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesExtension.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesExtension.kt deleted file mode 100644 index 9f55a3462..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesExtension.kt +++ /dev/null @@ -1,109 +0,0 @@ - -package com.datadoghq.java.versionedsources - -import org.gradle.api.Action -import org.gradle.api.JavaVersion -import org.gradle.api.NamedDomainObjectContainer -import org.gradle.api.Project -import org.gradle.api.model.ObjectFactory -import org.gradle.api.tasks.SourceSetContainer -import org.gradle.jvm.tasks.Jar -import javax.inject.Inject - -/** - * Extension for configuring versioned source sets. - * - * This plugin helps manage Java version-specific source directories where code - * requires different Java versions to compile (e.g., VarHandle in Java 9+). - * Classes are compiled separately and merged into the JAR root for runtime - * selection via Class.forName(). - * - * Usage: - * ```kotlin - * versionedSources { - * versions { - * register("java9") { - * release.set(9) - * minToolchainVersion.set(11) - * } - * register("java11") { - * release.set(11) - * } - * } - * } - * ``` - */ -abstract class VersionedSourcesExtension @Inject constructor( - private val project: Project, - objects: ObjectFactory -) { - /** - * Container for versioned source set configurations. - */ - val versions: NamedDomainObjectContainer = - objects.domainObjectContainer(VersionedSourceSet::class.java) - - /** - * Configure versioned source sets using a DSL block. - */ - fun versions(action: Action>) { - action.execute(versions) - } - - /** - * Get all versioned source sets that can be compiled with the current JDK. - */ - fun getCompilableVersions(): List { - val available = JavaVersion.current().majorVersion.toInt() - return versions.filter { version -> - val required = version.minToolchainVersion.orNull ?: version.release.get() - required <= available - } - } - - /** - * Configures a JAR task to include all versioned classes at the JAR root. - * - * This adds the classes compiled from version-specific source sets to the JAR root, - * alongside the main source set classes. This is useful when the application uses - * runtime class loading (Class.forName) to select the appropriate implementation - * based on Java version. - * - * Usage: - * ```kotlin - * tasks.register("myJar", Jar::class) { - * from(sourceSets.main.get().output) - * versionedSources.configureJar(this) - * } - * ``` - */ - fun configureJar(jar: Jar) { - val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) - - versions.forEach { version -> - val sourceSet = sourceSets.findByName(version.name) ?: return@forEach - jar.from(sourceSet.output.classesDirs) - jar.dependsOn(version.compileTaskName) - } - } - - /** - * Configures a source JAR task to include all versioned source files. - * - * Usage: - * ```kotlin - * val sourcesJar by tasks.registering(Jar::class) { - * from(sourceSets.main.get().allJava) - * versionedSources.configureSourceJar(this) - * } - * ``` - */ - fun configureSourceJar(jar: Jar) { - val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) - - versions.forEach { version -> - val sourceSet = sourceSets.findByName(version.name) ?: return@forEach - jar.from(sourceSet.allJava) - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesPlugin.kt deleted file mode 100644 index 57c94418d..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/java/versionedsources/VersionedSourcesPlugin.kt +++ /dev/null @@ -1,131 +0,0 @@ - -package com.datadoghq.java.versionedsources - -import org.gradle.api.JavaVersion -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.plugins.JavaPlugin -import org.gradle.api.tasks.SourceSetContainer -import org.gradle.api.tasks.compile.JavaCompile -import org.gradle.jvm.toolchain.JavaLanguageVersion -import org.gradle.jvm.toolchain.JavaToolchainService - -/** - * Gradle plugin for managing versioned source sets. - * - * This plugin simplifies compiling Java version-specific code by: - * 1. Creating versioned source sets automatically - * 2. Configuring compilation with correct --release flags and toolchains - * 3. Setting up classpath dependencies (version N depends on main) - * 4. Providing utilities for JAR task configuration - * - * Classes are added to the JAR root (not META-INF/versions/) for runtime - * selection via Class.forName() based on Java version. - * - * Usage: - * ```kotlin - * plugins { - * java - * id("com.datadoghq.versioned-sources") - * } - * - * versionedSources { - * versions { - * register("java9") { - * release.set(9) - * minToolchainVersion.set(11) - * } - * } - * } - * ``` - */ -class VersionedSourcesPlugin : Plugin { - - override fun apply(project: Project) { - // Ensure Java plugin is applied - project.pluginManager.apply(JavaPlugin::class.java) - - // Create extension - val extension = project.extensions.create( - "versionedSources", - VersionedSourcesExtension::class.java, - project, - project.objects - ) - - // Configure after project evaluation to access all registered versions - project.afterEvaluate { - configureVersionedSourceSets(project, extension) - configureCompilationTasks(project, extension) - } - } - - private fun configureVersionedSourceSets( - project: Project, - extension: VersionedSourcesExtension - ) { - val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) - - extension.versions.forEach { version -> - // Create source set if it doesn't exist - val sourceSet = sourceSets.maybeCreate(version.name) - - // Configure source directory - val srcDir = if (version.sourceDir.isPresent) { - version.sourceDir.get().asFile.path - } else { - "src/main/${version.name}" - } - - sourceSet.java.srcDirs(srcDir) - - project.logger.info( - "VersionedSources: Created source set '${version.name}' with source dir: $srcDir" - ) - } - } - - private fun configureCompilationTasks( - project: Project, - extension: VersionedSourcesExtension - ) { - val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) - val mainSourceSet = sourceSets.getByName("main") - val javaToolchains = project.extensions.getByType(JavaToolchainService::class.java) - - extension.versions.forEach { version -> - val compileTask = project.tasks.findByName(version.compileTaskName) as? JavaCompile - ?: return@forEach - - val releaseVersion = version.release.get() - val minToolchain = version.minToolchainVersion.orNull ?: releaseVersion - - // Determine actual toolchain version to use (current JDK if >= minToolchain) - val currentJdk = JavaVersion.current().majorVersion.toInt() - val toolchainVersion = if (currentJdk >= minToolchain) currentJdk else minToolchain - - compileTask.apply { - // Set toolchain for compilation - javaCompiler.set( - javaToolchains.compilerFor { - languageVersion.set(JavaLanguageVersion.of(toolchainVersion)) - } - ) - - // Set release flag for target version - options.release.set(releaseVersion) - - // Set classpath to include main source set output + compile dependencies - classpath = mainSourceSet.output + project.configurations.getByName("compileClasspath") - - // Depend on main compilation - dependsOn(project.tasks.named("compileJava")) - } - - project.logger.info( - "VersionedSources: Configured ${version.compileTaskName}: " + - "release=$releaseVersion, toolchain=$toolchainVersion" - ) - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildExtension.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildExtension.kt deleted file mode 100644 index d1a1872c9..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildExtension.kt +++ /dev/null @@ -1,136 +0,0 @@ - -package com.datadoghq.native - -import com.datadoghq.native.model.Architecture -import com.datadoghq.native.model.BuildConfiguration -import com.datadoghq.native.model.Platform -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.Action -import org.gradle.api.NamedDomainObjectContainer -import org.gradle.api.Project -import org.gradle.api.file.Directory -import org.gradle.api.model.ObjectFactory -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.Property -import org.gradle.api.provider.Provider -import javax.inject.Inject - -abstract class NativeBuildExtension @Inject constructor( - private val project: Project, - private val objects: ObjectFactory -) { - /** - * Container for build configurations (release, debug, asan, tsan, etc.) - */ - val buildConfigurations: NamedDomainObjectContainer = - objects.domainObjectContainer(BuildConfiguration::class.java) - - /** - * Project version to embed in binaries - */ - abstract val version: Property - - /** - * Source directories for C++ code - */ - abstract val cppSourceDirs: ListProperty - - /** - * Include directories for compilation - */ - abstract val includeDirectories: ListProperty - - init { - version.convention(project.version.toString()) - cppSourceDirs.convention(listOf("src/main/cpp")) - includeDirectories.convention(emptyList()) - } - - /** - * Configure build configurations using a DSL block - */ - fun buildConfigurations(action: Action>) { - action.execute(buildConfigurations) - } - - /** - * Get all configurations that are active for the current platform and architecture - */ - fun getActiveConfigurations(platform: Platform, architecture: Architecture): List { - return buildConfigurations.filter { it.isActiveFor(platform, architecture) } - } - - /** - * Get configuration names for the current platform/architecture - */ - fun getActiveConfigurationNames(): List { - val platform = PlatformUtils.currentPlatform - val arch = PlatformUtils.currentArchitecture - return getActiveConfigurations(platform, arch).map { it.name } - } - - /** - * Convenience method to define common compiler args for all configurations - */ - fun commonCompilerArgs(vararg args: String) { - buildConfigurations.configureEach { - compilerArgs.addAll(*args) - } - } - - /** - * Convenience method to define common linker args for all configurations - */ - fun commonLinkerArgs(vararg args: String) { - buildConfigurations.configureEach { - linkerArgs.addAll(*args) - } - } - - // ==================== Path Utilities ==================== - // These methods provide consistent path calculations for native library locations - - /** - * Platform identifier for library paths (e.g., "LINUX-x64", "MACOS-arm64-musl") - */ - fun platformIdentifier(): String { - val muslSuffix = if (PlatformUtils.isMusl()) "-musl" else "" - return "${PlatformUtils.currentPlatform}-${PlatformUtils.currentArchitecture}$muslSuffix" - } - - /** - * Directory where native libraries are built for a given configuration. - * Structure: build/lib/main/{config}/{platform}/{arch}/ - */ - fun librarySourceDir(config: String): Provider { - return project.layout.buildDirectory.dir( - "lib/main/$config/${PlatformUtils.currentPlatform}/${PlatformUtils.currentArchitecture}" - ) - } - - /** - * Directory for packaging native libraries into JAR. - * Structure: build/native/{config}/META-INF/native-libs/{platform-arch}/ - */ - fun libraryTargetDir(config: String): Provider { - return project.layout.buildDirectory.dir( - "native/$config/META-INF/native-libs/${platformIdentifier()}" - ) - } - - /** - * Base directory for native library packaging (without platform subdirs). - * Structure: build/native/{config}/ - */ - fun libraryTargetBase(config: String): Provider { - return project.layout.buildDirectory.dir("native/$config") - } - - /** - * Returns the platform-appropriate shared library filename. - * Examples: "libjavaProfiler.so" (Linux), "libjavaProfiler.dylib" (macOS) - */ - fun sharedLibraryName(baseName: String): String { - return "lib$baseName.${PlatformUtils.sharedLibExtension()}" - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildPlugin.kt deleted file mode 100644 index 33628c1f3..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/NativeBuildPlugin.kt +++ /dev/null @@ -1,171 +0,0 @@ - -package com.datadoghq.native - -import com.datadoghq.native.config.ConfigurationPresets -import com.datadoghq.native.model.Architecture -import com.datadoghq.native.model.BuildConfiguration -import com.datadoghq.native.model.Platform -import com.datadoghq.native.tasks.NativeCompileTask -import com.datadoghq.native.tasks.NativeLinkTask -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.Plugin -import org.gradle.api.Project - -/** - * Gradle plugin that provides native C++ build configuration management. - * - * This plugin: - * 1. Creates a `nativeBuild` extension for configuring build configurations - * 2. Automatically generates compile, link, and assemble tasks for each configuration - * 3. Provides standard configuration presets (release, debug, asan, tsan, fuzzer) - * 4. Handles platform-specific compiler and linker flags - * - * Usage example: - * ``` - * plugins { - * id("com.datadoghq.native-build") - * } - * - * nativeBuild { - * buildConfigurations { - * register("release") { - * platform.set(Platform.LINUX) - * architecture.set(Architecture.X64) - * compilerArgs.set(listOf("-O3", "-DNDEBUG")) - * } - * } - * } - * ``` - */ -class NativeBuildPlugin : Plugin { - override fun apply(project: Project) { - // Create the extension - val extension = project.extensions.create( - "nativeBuild", - NativeBuildExtension::class.java, - project, - project.objects - ) - - // Setup standard configurations after project evaluation - project.afterEvaluate { - setupStandardConfigurations(project, extension) - createTasks(project, extension) - } - } - - private fun setupStandardConfigurations(project: Project, extension: NativeBuildExtension) { - ConfigurationPresets.setupStandardConfigurations(extension, project) - } - - private fun createTasks(project: Project, extension: NativeBuildExtension) { - val currentPlatform = PlatformUtils.currentPlatform - val currentArch = PlatformUtils.currentArchitecture - - // Get active configurations for current platform - val activeConfigs = extension.getActiveConfigurations(currentPlatform, currentArch) - - project.logger.lifecycle("Active configurations: ${activeConfigs.map { it.name }.joinToString(", ")}") - - // Create tasks for each active configuration - activeConfigs.forEach { config -> - createConfigurationTasks(project, extension, config) - } - - // Create aggregation tasks - createAggregationTasks(project, activeConfigs) - } - - private fun createConfigurationTasks( - project: Project, - extension: NativeBuildExtension, - config: BuildConfiguration - ) { - val configName = config.capitalizedName() - val platform = config.platform.get() - val arch = config.architecture.get() - - // Define paths - val objDir = project.file("build/obj/main/${config.name}") - val libDir = project.file("build/lib/main/${config.name}/$platform/$arch") - val libName = "libjavaProfiler.${PlatformUtils.sharedLibExtension()}" - val outputLib = project.file("$libDir/$libName") - - // Create compile task - val compileTask = project.tasks.register("compile$configName", NativeCompileTask::class.java) { - group = "build" - description = "Compiles C++ sources for ${config.name} configuration" - - // Find compiler - val compilerPath = findCompiler(project) - compiler.set(compilerPath) - compilerArgs.set(config.compilerArgs.get()) - - // Set sources - default to src/main/cpp - val srcDirs = extension.cppSourceDirs.get() - sources.from(srcDirs.map { dir -> - project.fileTree(dir) { - include("**/*.cpp", "**/*.cc", "**/*.c") - } - }) - - // Set includes - default + JNI - val includeList = extension.includeDirectories.get().toMutableList() - includeList.addAll(PlatformUtils.jniIncludePaths()) - includes.from(includeList) - - objectFileDir.set(objDir) - } - - // Create link task - val linkTask = project.tasks.register("link$configName", NativeLinkTask::class.java) { - group = "build" - description = "Links ${config.name} shared library" - dependsOn(compileTask) - - val compilerPath = findCompiler(project) - linker.set(compilerPath) - linkerArgs.set(config.linkerArgs.get()) - objectFiles.from(project.fileTree(objDir) { - include("*.o") - }) - outputFile.set(outputLib) - - // Enable debug symbol extraction for release builds - if (config.name == "release") { - extractDebugSymbols.set(true) - stripSymbols.set(true) - debugSymbolsDir.set(project.file("$libDir/debug")) - } - } - - // Create assemble task - project.tasks.register("assemble$configName") { - group = "build" - description = "Assembles ${config.name} configuration" - dependsOn(linkTask) - } - - project.logger.debug("Created tasks for configuration: ${config.name}") - } - - private fun findCompiler(project: Project): String = PlatformUtils.findCompiler(project) - - private fun createAggregationTasks( - project: Project, - activeConfigs: List - ) { - // Create assembleAll task that depends on all assemble tasks - project.tasks.register("assembleAll") { - group = "build" - description = "Assembles all active build configurations" - // Depend on all individual assemble tasks - activeConfigs.forEach { config -> - val configName = config.capitalizedName() - dependsOn("assemble$configName") - } - } - - project.logger.lifecycle("Created assembleAll task") - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/RootProjectPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/RootProjectPlugin.kt deleted file mode 100644 index 0808db99a..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/RootProjectPlugin.kt +++ /dev/null @@ -1,53 +0,0 @@ - -package com.datadoghq.native - -import com.datadoghq.native.config.ConfigurationPresets -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.GradleException -import org.gradle.api.Plugin -import org.gradle.api.Project - -/** - * Gradle plugin that provides native build configuration access to the root project. - * - * This plugin creates a `nativeBuild` extension on the root project that exposes - * standard build configurations (release, debug, asan, tsan, fuzzer) to all subprojects. - * This allows subprojects to query and access build configurations in a type-safe manner. - * - * Usage example: - * ``` - * // In root build.gradle.kts - * plugins { - * id("com.datadoghq.native-root") - * } - * - * // In subproject build.gradle.kts - * val nativeBuildExt = rootProject.extensions.getByType(NativeBuildExtension::class.java) - * val activeConfigs = nativeBuildExt.getActiveConfigurations(currentPlatform, currentArch) - * ``` - */ -class RootProjectPlugin : Plugin { - override fun apply(project: Project) { - // Only apply to root project - if (project != project.rootProject) { - throw GradleException("RootProjectPlugin must be applied to root project only") - } - - // Create nativeBuild extension on root - val extension = project.extensions.create( - "nativeBuild", - NativeBuildExtension::class.java, - project, - project.objects - ) - - // Setup standard configurations after project evaluation - project.afterEvaluate { - setupStandardConfigurationsIfNeeded(project, extension) - } - } - - private fun setupStandardConfigurationsIfNeeded(project: Project, extension: NativeBuildExtension) { - ConfigurationPresets.setupStandardConfigurations(extension, project) - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/SimpleNativeLibPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/SimpleNativeLibPlugin.kt deleted file mode 100644 index f39087c02..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/SimpleNativeLibPlugin.kt +++ /dev/null @@ -1,226 +0,0 @@ - -package com.datadoghq.native - -import com.datadoghq.native.model.Platform -import com.datadoghq.native.tasks.NativeCompileTask -import com.datadoghq.native.tasks.NativeLinkTask -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.Property -import javax.inject.Inject - -/** - * Simplified plugin for single-output native library projects. - * - * Creates standard compile and link tasks with sensible defaults: - * - compileLib: Compiles C/C++ sources - * - linkLib: Links shared library - * - Wires linkLib into assemble lifecycle - * - * Usage: - * ```kotlin - * plugins { - * id("com.datadoghq.simple-native-lib") - * } - * - * simpleNativeLib { - * libraryName.set("mylib") - * sourceDir.set(file("src/main/cpp")) - * compilerArgs.set(listOf("-O3", "-fPIC")) - * } - * ``` - */ -class SimpleNativeLibPlugin : Plugin { - override fun apply(project: Project) { - val extension = project.extensions.create( - "simpleNativeLib", - SimpleNativeLibExtension::class.java, - project - ) - - project.afterEvaluate { - if (!extension.enabled.get()) { - return@afterEvaluate - } - - val compiler = extension.compiler.getOrElse(PlatformUtils.findCompiler(project)) - val linker = extension.linker.getOrElse(compiler) - val libraryName = extension.libraryName.get() - val objectDir = extension.objectDir.get().asFile - val outputLib = extension.outputDir.get().asFile.resolve( - "lib$libraryName.${PlatformUtils.sharedLibExtension()}" - ) - - // Compile task - val compileTask = project.tasks.register("compileLib", NativeCompileTask::class.java) { - onlyIf { extension.enabled.get() && !project.hasProperty("skip-native") } - group = "build" - description = "Compile the $libraryName library" - - this.compiler.set(compiler) - this.compilerArgs.set(extension.compilerArgs.get()) - sources.from(extension.sourceDir.map { dir -> - project.fileTree(dir) { - include("**/*.cpp", "**/*.cc", "**/*.c") - } - }) - if (extension.includeJni.get()) { - includes.from(PlatformUtils.jniIncludePaths()) - } - includes.from(extension.includeDirs.get()) - objectFileDir.set(objectDir) - } - - // Link task - val linkTask = project.tasks.register("linkLib", NativeLinkTask::class.java) { - onlyIf { extension.enabled.get() && !project.hasProperty("skip-native") } - dependsOn(compileTask) - group = "build" - description = "Link the $libraryName shared library" - - this.linker.set(linker) - this.linkerArgs.set(extension.linkerArgs.get()) - objectFiles.from(project.fileTree(objectDir) { include("*.o") }) - outputFile.set(outputLib) - } - - // Wire into assemble - project.tasks.named("assemble") { - dependsOn(linkTask) - } - - // Create consumable configurations if requested - if (extension.createConfigurations.get()) { - createConsumableConfigurations(project, extension, compileTask, linkTask) - } - } - } - - private fun createConsumableConfigurations( - project: Project, - extension: SimpleNativeLibExtension, - compileTask: org.gradle.api.tasks.TaskProvider, - linkTask: org.gradle.api.tasks.TaskProvider - ) { - // Runtime library configuration - project.configurations.create("nativeLib") { - isCanBeConsumed = true - isCanBeResolved = false - description = "Native shared library for runtime loading" - outgoing.artifact(linkTask.flatMap { it.outputFile }) { - type = "native-lib" - } - } - - // Object files configuration - project.configurations.create("nativeObjects") { - isCanBeConsumed = true - isCanBeResolved = false - description = "Object files for static linking" - outgoing.artifact(compileTask.flatMap { it.objectFileDir }) { - type = "native-objects" - } - } - - // Library directory configuration - project.configurations.create("nativeLibDir") { - isCanBeConsumed = true - isCanBeResolved = false - description = "Directory containing native library" - outgoing.artifact(extension.outputDir) { - type = "native-lib-dir" - } - } - - // Headers configuration - project.configurations.create("nativeHeaders") { - isCanBeConsumed = true - isCanBeResolved = false - description = "Header files for compilation" - outgoing.artifact(extension.sourceDir) { - type = "native-headers" - } - } - } -} - -/** - * Extension for simple native library configuration. - */ -abstract class SimpleNativeLibExtension @Inject constructor(project: Project) { - /** - * Whether this native build is enabled (e.g., platform-specific builds). - */ - val enabled: Property = project.objects.property(Boolean::class.java) - - /** - * Name of the library (without lib prefix or extension). - */ - val libraryName: Property = project.objects.property(String::class.java) - - /** - * Source directory containing C/C++ files. - */ - val sourceDir: DirectoryProperty = project.objects.directoryProperty() - - /** - * Output directory for compiled library. - */ - val outputDir: DirectoryProperty = project.objects.directoryProperty() - - /** - * Object file directory. - */ - val objectDir: DirectoryProperty = project.objects.directoryProperty() - - /** - * Compiler to use (auto-detected if not set). - */ - val compiler: Property = project.objects.property(String::class.java) - - /** - * Linker to use (defaults to compiler if not set). - */ - val linker: Property = project.objects.property(String::class.java) - - /** - * Compiler arguments. - */ - val compilerArgs: ListProperty = project.objects.listProperty(String::class.java) - - /** - * Linker arguments. - */ - val linkerArgs: ListProperty = project.objects.listProperty(String::class.java) - - /** - * Additional include directories. - */ - val includeDirs: ListProperty = project.objects.listProperty(String::class.java) - - /** - * Whether to include JNI headers. - */ - val includeJni: Property = project.objects.property(Boolean::class.java) - - /** - * Whether to create consumable configurations (nativeLib, nativeObjects, etc.). - */ - val createConfigurations: Property = project.objects.property(Boolean::class.java) - - init { - enabled.convention(true) - libraryName.convention("native") - sourceDir.convention(project.layout.projectDirectory.dir("src/main/cpp")) - outputDir.convention(project.layout.buildDirectory.dir("lib")) - objectDir.convention(project.layout.buildDirectory.dir("obj")) - compilerArgs.convention(listOf("-fPIC", "-O3")) - linkerArgs.convention(emptyList()) - includeDirs.convention(emptyList()) - includeJni.convention(false) - createConfigurations.convention(false) - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/config/ConfigurationPresets.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/config/ConfigurationPresets.kt deleted file mode 100644 index 8f9f9e093..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/config/ConfigurationPresets.kt +++ /dev/null @@ -1,356 +0,0 @@ - -package com.datadoghq.native.config - -import com.datadoghq.native.model.Architecture -import com.datadoghq.native.model.BuildConfiguration -import com.datadoghq.native.model.Platform -import com.datadoghq.native.util.PlatformUtils -import java.io.File - -/** - * Provides factory methods for creating standard build configurations - * (release, debug, asan, tsan, fuzzer) with appropriate compiler and linker flags. - */ -object ConfigurationPresets { - - /** - * Sets up standard build configurations (release, debug, asan, tsan, fuzzer) on the extension. - * This is the shared implementation used by both NativeBuildPlugin and RootProjectPlugin. - * - * @param extension The NativeBuildExtension to configure - * @param project The Gradle project (used for rootDir, logger, and compiler detection) - */ - fun setupStandardConfigurations( - extension: com.datadoghq.native.NativeBuildExtension, - project: org.gradle.api.Project - ) { - if (extension.buildConfigurations.isNotEmpty()) { - return // Don't override explicitly defined configurations - } - - val currentPlatform = PlatformUtils.currentPlatform - val currentArch = PlatformUtils.currentArchitecture - val version = extension.version.get() - val rootDir = project.rootDir - val compiler = PlatformUtils.findCompiler(project) - - project.logger.lifecycle("Setting up standard build configurations for $currentPlatform-$currentArch") - project.logger.lifecycle("Using compiler: $compiler") - - extension.buildConfigurations.apply { - register("release") { - configureRelease(this, currentPlatform, currentArch, version) - } - register("debug") { - configureDebug(this, currentPlatform, currentArch, version) - } - register("asan") { - configureAsan(this, currentPlatform, currentArch, version, rootDir, compiler) - } - register("tsan") { - configureTsan(this, currentPlatform, currentArch, version, rootDir, compiler) - } - register("fuzzer") { - configureFuzzer(this, currentPlatform, currentArch, version, rootDir) - } - } - - val activeConfigs = extension.getActiveConfigurations(currentPlatform, currentArch) - project.logger.lifecycle("Active configurations: ${activeConfigs.map { it.name }.joinToString(", ")}") - } - - private fun commonLinuxCompilerArgs(version: String): List { - val args = mutableListOf( - "-fPIC", - "-fno-omit-frame-pointer", - "-momit-leaf-frame-pointer", - "-fvisibility=hidden", - "-fdata-sections", - "-ffunction-sections", - "-std=c++17", - "-DPROFILER_VERSION=\"$version\"", - "-DCOUNTERS" - ) - // Define __musl__ when building on musl libc (it doesn't define this by default) - if (PlatformUtils.isMusl()) { - args.add("-D__musl__") - } - return args - } - - private fun commonLinuxLinkerArgs(): List = listOf( - "-ldl", - "-Wl,-z,defs", - "--verbose", - "-lpthread", - "-lm", - "-lrt", - "-Wl,--build-id" - ) - - private fun commonMacosCompilerArgs(version: String): List = - commonLinuxCompilerArgs(version) + listOf("-D_XOPEN_SOURCE", "-D_DARWIN_C_SOURCE") - - fun configureRelease( - config: BuildConfiguration, - platform: Platform, - architecture: Architecture, - version: String - ) { - config.platform.set(platform) - config.architecture.set(architecture) - config.active.set(true) - - when (platform) { - Platform.LINUX -> { - config.compilerArgs.set( - listOf("-O3", "-DNDEBUG", "-g") + commonLinuxCompilerArgs(version) - ) - config.linkerArgs.set( - commonLinuxLinkerArgs() + listOf( - "-Wl,-z,nodelete", - "-static-libstdc++", - "-static-libgcc", - "-Wl,--exclude-libs,ALL", - "-Wl,--gc-sections" - ) - ) - } - Platform.MACOS -> { - config.compilerArgs.set( - commonMacosCompilerArgs(version) + listOf("-O3", "-DNDEBUG", "-g") - ) - config.linkerArgs.set(emptyList()) - } - } - } - - fun configureDebug( - config: BuildConfiguration, - platform: Platform, - architecture: Architecture, - version: String - ) { - config.platform.set(platform) - config.architecture.set(architecture) - config.active.set(true) - - when (platform) { - Platform.LINUX -> { - config.compilerArgs.set( - listOf("-O0", "-g", "-DDEBUG") + commonLinuxCompilerArgs(version) - ) - config.linkerArgs.set(commonLinuxLinkerArgs()) - } - Platform.MACOS -> { - config.compilerArgs.set( - commonMacosCompilerArgs(version) + listOf("-O0", "-g", "-DDEBUG") - ) - config.linkerArgs.set(emptyList()) - } - } - } - - fun configureAsan( - config: BuildConfiguration, - platform: Platform, - architecture: Architecture, - version: String, - rootDir: File, - compiler: String = "gcc" - ) { - config.platform.set(platform) - config.architecture.set(architecture) - config.active.set(PlatformUtils.hasAsan(compiler)) - - val asanCompilerArgs = listOf( - "-g", - "-DDEBUG", - "-fPIC", - "-fsanitize=address", - "-fsanitize=undefined", - "-fno-sanitize-recover=all", - "-fsanitize=float-divide-by-zero", - "-fstack-protector-all", - "-fsanitize=leak", - "-fsanitize=pointer-overflow", - "-fsanitize=return", - "-fsanitize=bounds", - "-fsanitize=alignment", - "-fsanitize=object-size", - "-fno-omit-frame-pointer", - "-fno-optimize-sibling-calls" - ) - - when (platform) { - Platform.LINUX -> { - config.compilerArgs.set(asanCompilerArgs + commonLinuxCompilerArgs(version)) - - val libasan = PlatformUtils.locateLibasan(compiler) - // Link against the sanitizer runtime that matches the compiler: - // - clang: locateLibasan returns libclang_rt.asan-.so, which - // includes UBSan symbols; -lclang_rt.asan- satisfies -z defs - // for both __asan_* and __ubsan_* and matches the runtime that - // -fsanitize=address links into executables — one runtime, no conflict. - // - gcc: locateLibasan returns libasan.so; -lasan + -lubsan as before. - val asanLinkerArgs = if (libasan != null) { - val asanLibDir = File(libasan).parent - val asanLibName = File(libasan).nameWithoutExtension.removePrefix("lib") - val ubsanLibs = if (asanLibName.startsWith("clang_rt")) emptyList() - else listOf("-lubsan") - listOf("-L$asanLibDir", "-l$asanLibName", - "-Wl,-rpath,$asanLibDir") + - ubsanLibs + - listOf("-fsanitize=address", "-fsanitize=undefined", "-fno-omit-frame-pointer") - } else { - listOf("-fsanitize=address", "-fsanitize=undefined", "-fno-omit-frame-pointer") - } - - config.linkerArgs.set(commonLinuxLinkerArgs() + asanLinkerArgs) - - if (libasan != null) { - config.testEnvironment.apply { - put("LD_PRELOAD", libasan) - put("ASAN_OPTIONS", "allocator_may_return_null=1:unwind_abort_on_malloc=1:use_sigaltstack=0:detect_stack_use_after_return=0:handle_segv=0:halt_on_error=0:abort_on_error=0:print_stacktrace=1:symbolize=1:log_path=/tmp/asan.log:suppressions=$rootDir/gradle/sanitizers/asan.supp") - put("UBSAN_OPTIONS", "halt_on_error=0:abort_on_error=0:print_stacktrace=1:suppressions=$rootDir/gradle/sanitizers/ubsan.supp") - put("LSAN_OPTIONS", "detect_leaks=0") - } - // G1GC's heap reservation is placed just below 2 GB (0x7fff7000) by ASLR on - // some kernel configurations, which is exactly where ASan needs to mmap its - // shadow bytes [0x7fff7000-0x10007fff7fff]. Force the heap to a very low - // base address so the entire JVM footprint stays below the shadow range. - // HeapBaseMinAddress is not accepted by JDK <= 11 (constraint violation); - // those JDKs rely on the vm.mmap_rnd_bits=8 CI-level mitigation instead. - if (PlatformUtils.testJvmMajorVersion() >= 12) { - config.testJvmArgs.addAll(listOf( - "-XX:HeapBaseMinAddress=0x4000000", - "-Xmx512m", - "-XX:CompressedClassSpaceSize=256m" - )) - } - } - } - Platform.MACOS -> { - // ASAN not typically configured for macOS in this project - config.active.set(false) - } - } - } - - fun configureTsan( - config: BuildConfiguration, - platform: Platform, - architecture: Architecture, - version: String, - rootDir: File, - compiler: String = "gcc" - ) { - config.platform.set(platform) - config.architecture.set(architecture) - config.active.set(PlatformUtils.hasTsan(compiler)) - - val tsanCompilerArgs = listOf( - "-g", - "-DDEBUG", - "-fPIC", - "-fsanitize=thread", - "-fno-omit-frame-pointer", - "-fno-optimize-sibling-calls" - ) - - when (platform) { - Platform.LINUX -> { - config.compilerArgs.set(tsanCompilerArgs + commonLinuxCompilerArgs(version)) - - val libtsan = PlatformUtils.locateLibtsan(compiler) - // Use the library name from the resolved path so that clang's own - // libclang_rt.tsan-.so is linked by name (not as -ltsan). - val tsanLinkerArgs = if (libtsan != null) { - val tsanLibDir = File(libtsan).parent - val tsanLibName = File(libtsan).nameWithoutExtension.removePrefix("lib") - listOf( - "-L$tsanLibDir", - "-l$tsanLibName", - "-Wl,-rpath,$tsanLibDir", - "-fsanitize=thread", - "-fno-omit-frame-pointer" - ) - } else { - listOf("-fsanitize=thread", "-fno-omit-frame-pointer") - } - - config.linkerArgs.set(commonLinuxLinkerArgs() + tsanLinkerArgs) - - config.testEnvironment.apply { - if (libtsan != null) { - put("LD_PRELOAD", libtsan) - // handle_segv=0 / handle_sigbus=0: let the JVM handle these signals - // (SafeFetch, NullPointerException, memory bus errors). - // use_sigaltstack=0: JVM manages its own alternate signal stack. - // halt_on_error=0: report all races; process exits with code 66 at end. - // abort_on_error=0: use exit() not abort() so Java shutdown hooks run. - // io_sync=0: disable TSan's own FD-tracking, which races internally - // when the JVM concurrently closes/reads file descriptors. - put("TSAN_OPTIONS", "handle_segv=0:handle_sigbus=0:use_sigaltstack=0:halt_on_error=0:abort_on_error=0:io_sync=0:suppressions=$rootDir/gradle/sanitizers/tsan.supp") - } - // fork() is unsupported under TSan; threadsafe style uses execve instead. - put("GTEST_DEATH_TEST_STYLE", "threadsafe") - } - } - Platform.MACOS -> { - // TSAN not typically configured for macOS in this project - config.active.set(false) - } - } - } - - fun configureFuzzer( - config: BuildConfiguration, - platform: Platform, - architecture: Architecture, - version: String, - rootDir: File - ) { - config.platform.set(platform) - config.architecture.set(architecture) - config.active.set(PlatformUtils.hasFuzzer()) - - val fuzzerCompilerArgs = listOf( - "-g", - "-DDEBUG", - "-fPIC", - "-fsanitize=address", - "-fsanitize=undefined", - "-fno-sanitize-recover=all", - "-fno-omit-frame-pointer", - "-fno-optimize-sibling-calls" - ) - - val fuzzerLinkerArgs = listOf( - "-fsanitize=address", - "-fsanitize=undefined", - "-fno-omit-frame-pointer" - ) - - when (platform) { - Platform.LINUX -> { - config.compilerArgs.set(fuzzerCompilerArgs + commonLinuxCompilerArgs(version)) - config.linkerArgs.set(commonLinuxLinkerArgs() + fuzzerLinkerArgs) - - config.testEnvironment.apply { - put("ASAN_OPTIONS", "allocator_may_return_null=1:detect_stack_use_after_return=0:handle_segv=0:abort_on_error=1:symbolize=1:suppressions=$rootDir/gradle/sanitizers/asan.supp") - put("UBSAN_OPTIONS", "halt_on_error=1:abort_on_error=1:print_stacktrace=1:suppressions=$rootDir/gradle/sanitizers/ubsan.supp") - } - } - Platform.MACOS -> { - config.compilerArgs.set(fuzzerCompilerArgs + commonMacosCompilerArgs(version)) - config.linkerArgs.set(fuzzerLinkerArgs) - - config.testEnvironment.apply { - put("ASAN_OPTIONS", "allocator_may_return_null=1:detect_stack_use_after_return=0:abort_on_error=1:symbolize=1") - put("UBSAN_OPTIONS", "halt_on_error=1:abort_on_error=1:print_stacktrace=1") - } - } - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/fuzz/FuzzTargetsPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/fuzz/FuzzTargetsPlugin.kt deleted file mode 100644 index c4a547739..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/fuzz/FuzzTargetsPlugin.kt +++ /dev/null @@ -1,330 +0,0 @@ - -package com.datadoghq.native.fuzz - -import com.datadoghq.native.model.Platform -import com.datadoghq.native.tasks.NativeCompileTask -import com.datadoghq.native.tasks.NativeLinkExecutableTask -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.file.ConfigurableFileCollection -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.Property -import org.gradle.api.tasks.Exec -import java.io.File -import javax.inject.Inject - -/** - * Plugin for libFuzzer-based fuzz testing. - * - * Automatically discovers fuzz targets (*.cpp files) in a source directory and generates: - * - compileFuzz_{name} - Compile task - * - linkFuzz_{name} - Link task - * - fuzz_{name} - Execute task - * - fuzz - Aggregate task running all fuzz targets - * - listFuzzTargets - Help task - * - * Usage: - * ```kotlin - * plugins { - * id("com.datadoghq.fuzz-targets") - * } - * - * fuzzTargets { - * fuzzSourceDir.set(file("src/test/fuzz")) - * corpusDir.set(file("src/test/fuzz/corpus")) - * profilerSourceDir.set(file("src/main/cpp")) - * duration.set(60) - * } - * ``` - */ -class FuzzTargetsPlugin : Plugin { - override fun apply(project: Project) { - val extension = project.extensions.create( - "fuzzTargets", - FuzzTargetsExtension::class.java, - project - ) - - project.afterEvaluate { - configureFuzzTargets(project, extension) - } - } - - private fun configureFuzzTargets(project: Project, extension: FuzzTargetsExtension) { - val hasFuzzer = PlatformUtils.hasFuzzer() - - // Master fuzz task - val fuzzAll = project.tasks.register("fuzz") { - onlyIf { - hasFuzzer && - !project.hasProperty("skip-tests") && - !project.hasProperty("skip-native") && - !project.hasProperty("skip-fuzz") - } - group = "verification" - description = "Run all fuzz targets" - - doFirst { - if (!hasFuzzer) { - project.logger.warn("WARNING: libFuzzer not available - skipping fuzz tests (requires clang with -fsanitize=fuzzer)") - } - } - } - - // Build-only aggregate: compiles and links all targets without running them - val buildFuzz = project.tasks.register("buildFuzz") { - onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") } - group = "build" - description = "Build all fuzz targets without running them" - } - - if (!hasFuzzer) { - val msg = if (PlatformUtils.currentPlatform == Platform.MACOS) { - "WARNING: libFuzzer not available on macOS — skipping fuzz targets. " + - "Install LLVM via Homebrew and ensure 'clang' resolves to the Homebrew clang." - } else { - "WARNING: libFuzzer not available — skipping fuzz targets (requires clang with -fsanitize=fuzzer)." - } - project.logger.lifecycle(msg) - createListFuzzTargetsTask(project, extension) - return - } - - val compiler = PlatformUtils.findFuzzerCompiler(project) - val homebrewLLVM = PlatformUtils.findHomebrewLLVM() - val clangResourceDir = PlatformUtils.findClangResourceDir(homebrewLLVM) - - // Build include paths - val includeFiles = buildIncludePaths(project, extension, homebrewLLVM) - - // Build compiler/linker args - val compilerArgs = buildFuzzCompilerArgs(project) - val linkerArgs = buildFuzzLinkerArgs(homebrewLLVM, clangResourceDir, project.logger) - - val fuzzSourceDir = extension.fuzzSourceDir.get().asFile - val corpusBaseDir = extension.corpusDir.get().asFile - val crashDir = extension.crashDir.get().asFile - val duration = extension.duration.get() - - // Discover and create tasks for each fuzz target - if (fuzzSourceDir.exists()) { - fuzzSourceDir.listFiles()?.filter { file -> file.name.endsWith(".cpp") }?.forEach { fuzzFile -> - val fullName = fuzzFile.nameWithoutExtension - val fuzzName = if (fullName.startsWith("fuzz_")) fullName.substring(5) else fullName - - val objDir = project.file("${project.layout.buildDirectory.get()}/obj/fuzz/$fuzzName") - val binDir = project.file("${project.layout.buildDirectory.get()}/bin/fuzz/$fuzzName") - val binary = project.file("$binDir/$fuzzName") - val targetCorpusDir = File(corpusBaseDir, fuzzName) - - // Compile task - val compileTask = project.tasks.register("compileFuzz_$fuzzName", NativeCompileTask::class.java) { - onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") } - group = "build" - description = "Compile the fuzz target $fuzzName" - - this.compiler.set(compiler) - this.compilerArgs.set(compilerArgs) - sources.from( - extension.profilerSourceDir.map { dir -> - project.fileTree(dir) { include("**/*.cpp") } - }, - fuzzFile - ) - includes.from(includeFiles) - objectFileDir.set(objDir) - } - - // Link task - val linkTask = project.tasks.register("linkFuzz_$fuzzName", NativeLinkExecutableTask::class.java) { - onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") } - dependsOn(compileTask) - group = "build" - description = "Link the fuzz target $fuzzName" - - linker.set(compiler) - this.linkerArgs.set(linkerArgs) - objectFiles.from(project.fileTree(objDir) { include("*.o") }) - outputFile.set(binary) - } - - // Execute task - val executeTask = project.tasks.register("fuzz_$fuzzName", Exec::class.java) { - onlyIf { hasFuzzer && !project.hasProperty("skip-tests") && !project.hasProperty("skip-native") && !project.hasProperty("skip-fuzz") } - dependsOn(linkTask) - group = "verification" - description = "Run the fuzz target $fuzzName for $duration seconds" - - doFirst { - crashDir.mkdirs() - targetCorpusDir.mkdirs() - } - - executable = binary.absolutePath - args( - targetCorpusDir.absolutePath, - "-max_total_time=$duration", - "-artifact_prefix=${crashDir.absolutePath}/$fuzzName-", - "-print_final_stats=1" - ) - - inputs.files(binary) - outputs.upToDateWhen { false } - } - - fuzzAll.configure { dependsOn(executeTask) } - buildFuzz.configure { dependsOn(linkTask) } - } - } - - createListFuzzTargetsTask(project, extension) - } - - private fun buildIncludePaths(project: Project, extension: FuzzTargetsExtension, homebrewLLVM: String?): ConfigurableFileCollection { - val javaHome = PlatformUtils.javaHome() - val platformInclude = when (PlatformUtils.currentPlatform) { - Platform.LINUX -> "linux" - Platform.MACOS -> "darwin" - } - - val includes = project.files() - includes.from( - extension.profilerSourceDir, - "$javaHome/include", - "$javaHome/include/$platformInclude" - ) - - // Add additional include directories - extension.additionalIncludes.get().forEach { dir -> - includes.from(dir) - } - - // Add Homebrew LLVM includes on macOS - if (PlatformUtils.currentPlatform == Platform.MACOS && homebrewLLVM != null) { - includes.from("$homebrewLLVM/include") - } - - return includes - } - - private fun buildFuzzCompilerArgs(project: Project): List { - val version = project.version.toString() - val args = mutableListOf( - "-O1", - "-g", - "-fno-omit-frame-pointer", - "-fsanitize=fuzzer,address,undefined", - "-fvisibility=hidden", - "-std=c++17", - "-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION", - "-DPROFILER_VERSION=\"$version\"" - ) - if (PlatformUtils.currentPlatform == Platform.LINUX && PlatformUtils.isMusl()) { - args.add("-D__musl__") - } - return args - } - - private fun buildFuzzLinkerArgs(homebrewLLVM: String?, clangResourceDir: String?, logger: org.gradle.api.logging.Logger): List { - val args = mutableListOf() - - // libFuzzer linking strategy - if (PlatformUtils.currentPlatform == Platform.MACOS && clangResourceDir != null) { - val fuzzerLib = "$clangResourceDir/lib/darwin/libclang_rt.fuzzer_osx.a" - if (File(fuzzerLib).exists()) { - logger.info("Using Homebrew libFuzzer: $fuzzerLib") - args.add(fuzzerLib) - args.add("-L$homebrewLLVM/lib/c++") - args.add("-lc++") - args.add("-Wl,-rpath,$homebrewLLVM/lib/c++") - } else { - logger.warn("Homebrew libFuzzer not found, falling back to -fsanitize=fuzzer") - args.add("-fsanitize=fuzzer,address,undefined") - } - } else { - args.add("-fsanitize=fuzzer,address,undefined") - } - - args.addAll(listOf("-ldl", "-lpthread", "-lm")) - if (PlatformUtils.currentPlatform == Platform.LINUX) { - args.add("-lrt") - } - - return args - } - - private fun createListFuzzTargetsTask(project: Project, extension: FuzzTargetsExtension) { - project.tasks.register("listFuzzTargets") { - group = "help" - description = "List all available fuzz targets" - doLast { - val fuzzSrcDir = extension.fuzzSourceDir.get().asFile - if (fuzzSrcDir.exists()) { - println("Available fuzz targets:") - fuzzSrcDir.listFiles()?.filter { file -> file.name.endsWith(".cpp") }?.forEach { fuzzFile -> - val fullName = fuzzFile.nameWithoutExtension - val fuzzName = if (fullName.startsWith("fuzz_")) fullName.substring(5) else fullName - println(" - fuzz_$fuzzName") - } - println() - println("Run individual targets with: ./gradlew :${project.path}:fuzz_") - println("Run all targets with: ./gradlew :${project.path}:fuzz") - println("Configure duration with: -Pfuzz-duration= (default: ${extension.duration.get()})") - } else { - println("No fuzz targets found. Create .cpp files in ${fuzzSrcDir.path}") - } - } - } - } -} - -/** - * Extension for configuring fuzz targets. - */ -abstract class FuzzTargetsExtension @Inject constructor(project: Project) { - /** - * Directory containing fuzz target source files (*.cpp). - */ - val fuzzSourceDir: DirectoryProperty = project.objects.directoryProperty() - - /** - * Directory for seed corpus files. - */ - val corpusDir: DirectoryProperty = project.objects.directoryProperty() - - /** - * Directory for crash artifacts. - */ - val crashDir: DirectoryProperty = project.objects.directoryProperty() - - /** - * Directory containing the profiler C++ sources to compile with fuzz targets. - */ - val profilerSourceDir: DirectoryProperty = project.objects.directoryProperty() - - /** - * Additional include directories. - */ - val additionalIncludes: ListProperty = project.objects.listProperty(String::class.java) - - /** - * Fuzz duration in seconds. - */ - val duration: Property = project.objects.property(Int::class.java) - - init { - // Set reasonable defaults - these should be overridden by the user - crashDir.convention(project.layout.buildDirectory.dir("fuzz-crashes")) - additionalIncludes.convention(emptyList()) - - // Duration from property or default - val propDuration = if (project.hasProperty("fuzz-duration")) { - project.property("fuzz-duration").toString().toIntOrNull() ?: 60 - } else { - 60 - } - duration.convention(propDuration) - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestExtension.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestExtension.kt deleted file mode 100644 index 0767b576b..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestExtension.kt +++ /dev/null @@ -1,110 +0,0 @@ - -package com.datadoghq.native.gtest - -import org.gradle.api.file.ConfigurableFileCollection -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.model.ObjectFactory -import org.gradle.api.provider.Property -import javax.inject.Inject - -/** - * Extension for configuring Google Test integration in C++ projects. - * - * Provides a declarative DSL for setting up Google Test compilation, linking, and execution - * across multiple build configurations (debug, release, asan, tsan). - * - * Usage example: - * ```kotlin - * gtest { - * testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) - * mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) - * includes.from("src/main/cpp", "${javaHome}/include") - * } - * ``` - */ -abstract class GtestExtension @Inject constructor(objects: ObjectFactory) { - - // === Source Directories === - - /** - * Directory containing test source files (.cpp). - * Required - must be set explicitly. - */ - abstract val testSourceDir: DirectoryProperty - - /** - * Directory containing main source files to compile with tests. - * Required - must be set explicitly. - */ - abstract val mainSourceDir: DirectoryProperty - - /** - * Optional Google Test installation directory. - * Used for include and library paths on macOS. - * Default: /opt/homebrew/opt/googletest on macOS - */ - abstract val googleTestHome: DirectoryProperty - - // === Compiler/Linker Configuration === - - /** - * Include directories for compilation. - * Should include main source, JNI headers, and any dependencies. - */ - abstract val includes: ConfigurableFileCollection - - // === Test Behavior === - - /** - * Enable assertions by removing -DNDEBUG from compiler args. - * Default: true - */ - abstract val enableAssertions: Property - - /** - * Keep symbols in release builds (skip minimizing linker flags). - * Default: true - */ - abstract val keepSymbols: Property - - /** - * Stop on first test failure (fail-fast). - * Default: false (collect all failures) - */ - abstract val failFast: Property - - /** - * Always re-run tests (ignore up-to-date checks). - * Default: true - */ - abstract val alwaysRun: Property - - // === Build Native Libs Task === - - /** - * Enable building native test support libraries (Linux only). - * Default: true - */ - abstract val buildNativeLibs: Property - - /** - * Directory containing native test library sources. - * Default: src/test/resources/native-libs - */ - abstract val nativeLibsSourceDir: DirectoryProperty - - /** - * Output directory for built native test libraries. - * Default: build/test/resources/native-libs - */ - abstract val nativeLibsOutputDir: DirectoryProperty - - init { - // Set default conventions - enableAssertions.convention(true) - keepSymbols.convention(true) - failFast.convention(false) - alwaysRun.convention(true) - buildNativeLibs.convention(true) - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestPlugin.kt deleted file mode 100644 index d2196cf8d..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestPlugin.kt +++ /dev/null @@ -1,271 +0,0 @@ - -package com.datadoghq.native.gtest - -import com.datadoghq.native.NativeBuildExtension -import com.datadoghq.native.model.BuildConfiguration -import com.datadoghq.native.model.Platform -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.Plugin -import org.gradle.api.Project -import java.io.File - -/** - * Gradle plugin for Google Test integration in C++ projects. - * - * This plugin automatically creates compilation, linking, and execution tasks for Google Test - * tests across multiple build configurations. It handles platform-specific differences (macOS, Linux) - * and integrates with the NativeBuildPlugin's BuildConfiguration model. - * - * For each test file in the test source directory, the plugin creates: - * - compileGtest{Config}_{TestName} - Compile all main sources + test file - * - linkGtest{Config}_{TestName} - Link test executable with gtest libraries - * - gtest{Config}_{TestName} - Execute the test - * - * Aggregation tasks: - * - gtest{Config} - Run all tests for a specific configuration (e.g., gtestDebug) - * - gtest - Run all tests across all configurations - * - * Usage: - * ```kotlin - * plugins { - * id("com.datadoghq.native-build") - * id("com.datadoghq.gtest") - * } - * - * gtest { - * testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) - * mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) - * includes.from("src/main/cpp", "${javaHome}/include") - * } - * ``` - */ -class GtestPlugin : Plugin { - - override fun apply(project: Project) { - // Create extension - val extension = project.extensions.create("gtest", GtestExtension::class.java) - - // Register tasks after project evaluation - project.afterEvaluate { - // Check if testSourceDir is set - if (!extension.testSourceDir.isPresent) { - project.logger.warn("WARNING: gtest.testSourceDir not configured - skipping Google Test tasks") - return@afterEvaluate - } - - // Get configurations from NativeBuildExtension - val nativeBuildExtension = project.extensions.findByType(NativeBuildExtension::class.java) - if (nativeBuildExtension == null) { - project.logger.warn("WARNING: NativeBuildExtension not found - apply com.datadoghq.native-build plugin first") - return@afterEvaluate - } - - val activeConfigs = nativeBuildExtension.getActiveConfigurations( - PlatformUtils.currentPlatform, - PlatformUtils.currentArchitecture - ) - - if (activeConfigs.isEmpty()) { - project.logger.warn("WARNING: No active build configurations - skipping Google Test tasks") - return@afterEvaluate - } - - // Check if gtest is available - val hasGtest = checkGtestAvailable() - if (!hasGtest) { - project.logger.warn("WARNING: Google Test not found - skipping native tests") - } - - // Create buildNativeLibs task (Linux only) - if (extension.buildNativeLibs.get()) { - createBuildNativeLibsTask(project, extension, hasGtest) - } - - // Create master aggregation task - val gtestAll = createMasterAggregationTask(project, hasGtest) - - // Create tasks for each active configuration - activeConfigs.forEach { config -> - createConfigTasks(project, extension, config, hasGtest, gtestAll) - } - } - } - - private fun checkGtestAvailable(): Boolean { - // Check common gtest locations - val locations = when (PlatformUtils.currentPlatform) { - Platform.MACOS -> listOf( - "/opt/homebrew/opt/googletest", - "/usr/local/opt/googletest" - ) - Platform.LINUX -> listOf( - "/usr/include/gtest", - "/usr/local/include/gtest" - ) - } - return locations.any { File(it).exists() } - } - - private fun createBuildNativeLibsTask(project: Project, extension: GtestExtension, hasGtest: Boolean) { - project.tasks.register("buildNativeLibs") { - group = "build" - description = "Build the native libs for the Google Tests" - - onlyIf { - hasGtest && - !project.hasProperty("skip-native") && - !project.hasProperty("skip-gtest") && - PlatformUtils.currentPlatform == Platform.LINUX && - extension.nativeLibsSourceDir.isPresent && - extension.nativeLibsOutputDir.isPresent - } - - val srcDir = if (extension.nativeLibsSourceDir.isPresent) { - extension.nativeLibsSourceDir.get().asFile - } else { - project.file("src/test/resources/native-libs") - } - val targetDir = if (extension.nativeLibsOutputDir.isPresent) { - extension.nativeLibsOutputDir.get().asFile - } else { - project.file("build/test/resources/native-libs") - } - - doLast { - if (!srcDir.exists()) { - project.logger.info("Native libs source directory does not exist: $srcDir") - return@doLast - } - - srcDir.listFiles()?.filter { it.isDirectory }?.forEach { dir -> - val libName = dir.name - val libDir = File("$targetDir/$libName") - val libSrcDir = File("$srcDir/$libName") - - // Use ProcessBuilder directly (Gradle 9 removed project.exec in task actions) - val process = ProcessBuilder("sh", "-c", """ - echo "Processing library: $libName @ $libSrcDir" - mkdir -p $libDir - cd $libSrcDir - make TARGET_DIR=$libDir - """.trimIndent()) - .inheritIO() - .start() - val exitCode = process.waitFor() - if (exitCode != 0) { - throw org.gradle.api.GradleException("Failed to build native lib: $libName (exit code: $exitCode)") - } - } - } - - inputs.files(project.fileTree(srcDir)) - outputs.dir(targetDir) - } - } - - private fun createMasterAggregationTask(project: Project, hasGtest: Boolean): org.gradle.api.tasks.TaskProvider<*> { - return project.tasks.register("gtest") { - group = "verification" - description = "Run all Google Tests for all build configurations of the library" - - onlyIf { - hasGtest && - !project.hasProperty("skip-tests") && - !project.hasProperty("skip-native") && - !project.hasProperty("skip-gtest") - } - } - } - - private fun createConfigTasks( - project: Project, - extension: GtestExtension, - config: BuildConfiguration, - hasGtest: Boolean, - gtestAll: org.gradle.api.tasks.TaskProvider<*> - ) { - // Find compiler and build include paths - val compiler = findCompiler(project) - val includeFiles = extension.includes.plus(project.files(getGtestIncludes(extension))) - - // Create per-config aggregation task (compile + link + run) - val gtestConfigTask = project.tasks.register("gtest${config.capitalizedName()}") { - group = "verification" - description = "Run all Google Tests for the ${config.name} build of the library" - } - - // Per-config build-only aggregation task (compile + link, no run). - // Useful in CI where binaries are executed directly without going - // through Gradle's logging infrastructure. - val buildGtestConfigTask = project.tasks.register("buildGtest${config.capitalizedName()}") { - group = "build" - description = "Compile and link all Google Tests for the ${config.name} build (no run)" - } - - // Compile all library sources ONCE for this config. Each test - // binary only compiles its own test file and links against these - // shared objects, reducing compilations from O(n_tests × n_sources) - // to O(n_sources + n_tests). - val sharedBuilder = GtestTaskBuilder(project, extension, config) - .withCompiler(compiler) - .withIncludes(includeFiles) - .onlyIfGtest(hasGtest) - val sharedCompilerArgs = sharedBuilder.sharedCompilerArgs() - val sharedLibCompileTask = project.tasks.register( - "compileGtestLibrary${config.capitalizedName()}", - com.datadoghq.native.tasks.NativeCompileTask::class.java - ) { - onlyIf { hasGtest && !sharedBuilder.skipConditions() } - group = "build" - description = "Compile shared library sources for ${config.name} gtest binaries" - - this.compiler.set(compiler) - this.compilerArgs.set(sharedCompilerArgs) - sources.from(project.fileTree(extension.mainSourceDir.get()) { include("**/*.cpp") }) - includes.from(includeFiles) - objectFileDir.set(project.file( - "${project.layout.buildDirectory.get()}/obj/gtest/${config.name}/lib" - )) - } - - // Discover and create tasks for each test file using builder - val testDir = extension.testSourceDir.get().asFile - if (!testDir.exists()) { - project.logger.info("Test source directory does not exist: $testDir") - return - } - - testDir.listFiles()?.filter { it.name.endsWith(".cpp") }?.forEach { testFile -> - val taskBundle = GtestTaskBuilder(project, extension, config) - .forTest(testFile) - .withCompiler(compiler) - .withIncludes(includeFiles) - .withSharedLibObjects(sharedLibCompileTask) - .onlyIfGtest(hasGtest) - .build() - - gtestConfigTask.configure { dependsOn(taskBundle) } - gtestAll.configure { dependsOn(taskBundle) } - // buildGtest depends on the link task, not the run task - buildGtestConfigTask.configure { - dependsOn("linkGtest${config.capitalizedName()}_${testFile.nameWithoutExtension}") - } - } - } - - private fun findCompiler(project: Project): String = PlatformUtils.findCompiler(project) - - private fun getGtestIncludes(extension: GtestExtension): List { - return when (PlatformUtils.currentPlatform) { - Platform.MACOS -> { - val gtestPath = if (extension.googleTestHome.isPresent) { - extension.googleTestHome.get().asFile.absolutePath - } else { - "/opt/homebrew/opt/googletest" - } - listOf(File("$gtestPath/include")) - } - Platform.LINUX -> emptyList() // System includes - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestTaskBuilder.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestTaskBuilder.kt deleted file mode 100644 index 81d574226..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/gtest/GtestTaskBuilder.kt +++ /dev/null @@ -1,275 +0,0 @@ - -package com.datadoghq.native.gtest - -import com.datadoghq.native.model.BuildConfiguration -import com.datadoghq.native.model.Platform -import com.datadoghq.native.tasks.NativeCompileTask -import com.datadoghq.native.tasks.NativeLinkExecutableTask -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.Project -import org.gradle.api.file.FileCollection -import org.gradle.api.tasks.Exec -import org.gradle.api.tasks.TaskProvider -import java.io.File - -/** - * Builder for creating Google Test compile, link, and execute tasks. - * - * Groups related configuration and provides a fluent API for task creation. - * - * Usage: - * ```kotlin - * GtestTaskBuilder(project, extension, config) - * .forTest(testFile) - * .withCompiler(compiler) - * .withIncludes(includeFiles) - * .onlyIf { hasGtest } - * .build() - * ``` - */ -class GtestTaskBuilder( - private val project: Project, - private val extension: GtestExtension, - private val config: BuildConfiguration -) { - private lateinit var testFile: File - private lateinit var testName: String - private lateinit var compiler: String - private lateinit var includeFiles: FileCollection - private var hasGtest: Boolean = true - private var sharedLibCompileTask: TaskProvider? = null - - private val configName: String get() = config.capitalizedName() - - /** - * Set the test file to build tasks for. - */ - fun forTest(file: File): GtestTaskBuilder { - testFile = file - testName = file.nameWithoutExtension - return this - } - - /** - * Set the compiler to use. - */ - fun withCompiler(comp: String): GtestTaskBuilder { - compiler = comp - return this - } - - /** - * Set include directories. - */ - fun withIncludes(includes: FileCollection): GtestTaskBuilder { - includeFiles = includes - return this - } - - /** - * Set whether gtest is available. - */ - fun onlyIfGtest(available: Boolean): GtestTaskBuilder { - hasGtest = available - return this - } - - /** - * Provide the shared library compile task whose objects are linked into - * every test binary. Allows the 59 library sources to be compiled once - * instead of once per test file. - */ - fun withSharedLibObjects(task: TaskProvider): GtestTaskBuilder { - sharedLibCompileTask = task - return this - } - - /** - * Returns the compiler args used for compiling library and test sources. - * Exposed so GtestPlugin can configure the shared library compile task - * with identical flags without duplicating the adjustment logic. - */ - fun sharedCompilerArgs(): List = adjustCompilerArgs() - - /** - * Build all tasks (compile, link, execute) and return the execute task provider. - */ - fun build(): TaskProvider { - val compileTask = buildCompileTask() - val linkTask = buildLinkTask(compileTask) - return buildExecuteTask(linkTask) - } - - private fun buildCompileTask(): TaskProvider { - val compilerArgs = adjustCompilerArgs() - val objDir = project.file("${project.layout.buildDirectory.get()}/obj/gtest/${config.name}/$testName") - - return project.tasks.register("compileGtest${configName}_$testName", NativeCompileTask::class.java) { - onlyIf { hasGtest && !skipConditions() } - group = "build" - description = "Compile the Google Test $testName for the ${config.name} build" - - this.compiler.set(this@GtestTaskBuilder.compiler) - this.compilerArgs.set(compilerArgs) - - // When a shared library compile task is provided, library sources are - // compiled once there. Only compile the test file itself here. - if (sharedLibCompileTask != null) { - sources.from(testFile) - } else { - sources.from( - project.fileTree(extension.mainSourceDir.get()) { include("**/*.cpp") }, - testFile - ) - } - includes.from(includeFiles) - objectFileDir.set(objDir) - } - } - - private fun buildLinkTask(compileTask: TaskProvider): TaskProvider { - // Strip explicit sanitizer -l/-L/-rpath flags so the executable relies solely - // on clang's automatic static embedding of the sanitizer runtime. Mixing - // clang's statically-embedded runtime with an explicit GCC libtsan/libasan - // causes "incompatible runtimes" at startup (two __tsan_init/__asan_init calls). - val sanitizerLibPattern = Regex("^(-lasan|-lubsan|-ltsan|-lclang_rt\\.asan.*|-lclang_rt\\.ubsan.*|-lclang_rt\\.tsan.*|-L.*/clang.*/|-Wl,-rpath,.*/clang/.*)") - val linkerArgs = config.linkerArgs.get().filter { !sanitizerLibPattern.containsMatchIn(it) } - val objDir = project.file("${project.layout.buildDirectory.get()}/obj/gtest/${config.name}/$testName") - val binary = project.file("${project.layout.buildDirectory.get()}/bin/gtest/${config.name}_$testName/$testName") - - return project.tasks.register("linkGtest${configName}_$testName", NativeLinkExecutableTask::class.java) { - onlyIf { hasGtest && !skipConditions() } - dependsOn(compileTask) - group = "build" - description = "Link the Google Test $testName for the ${config.name} build" - - linker.set(compiler) - this.linkerArgs.set(linkerArgs) - objectFiles.from(project.fileTree(objDir) { include("*.o") }) - sharedLibCompileTask?.let { sharedTask -> - dependsOn(sharedTask) - objectFiles.from(sharedTask.map { it.objectFileDir.get().asFileTree.matching { include("*.o") } }) - } - outputFile.set(binary) - - // Add gtest library paths - when (PlatformUtils.currentPlatform) { - Platform.MACOS -> { - val gtestPath = gtestHomePath() - libPath("$gtestPath/lib") - } - Platform.LINUX -> { /* System paths */ } - } - - // Add gtest libraries - lib("gtest", "gtest_main", "gmock", "gmock_main", "dl", "pthread", "m") - if (PlatformUtils.currentPlatform == Platform.LINUX) { - lib("rt") - } - } - } - - private fun buildExecuteTask(linkTask: TaskProvider): TaskProvider { - val binary = project.file("${project.layout.buildDirectory.get()}/bin/gtest/${config.name}_$testName/$testName") - - return project.tasks.register("gtest${configName}_$testName", Exec::class.java) { - onlyIf { hasGtest && !skipConditions() } - dependsOn(linkTask) - - // Add dependency on buildNativeLibs if it exists (Linux only) - if (PlatformUtils.currentPlatform == Platform.LINUX && extension.buildNativeLibs.get()) { - project.tasks.findByName("buildNativeLibs")?.let { dependsOn(it) } - } - - group = "verification" - description = "Run the Google Test $testName for the ${config.name} build" - - executable = binary.absolutePath - - // Set test environment variables from configuration. - // LD_PRELOAD is excluded: gtest binaries are compiled with -fsanitize=address - // and already have libasan.so in their NEEDED entries. Preloading it again - // causes "incompatible ASan runtimes" → immediate abort before any test runs. - config.testEnvironment.get() - .filter { (key, _) -> key != "LD_PRELOAD" } - .forEach { (key, value) -> - environment(key, hardenSanitizerOptions(key, value)) - } - - inputs.files(binary) - - // Gradle's default Exec task buffers child output and discards it on - // failure. /dev/std* bypass the logging infrastructure and stream - // bytes directly to fd 1/2 of the Gradle JVM so sanitizer reports - // are always visible in CI. - if (PlatformUtils.currentPlatform == Platform.LINUX) { - val devStdout = java.io.FileOutputStream("/dev/stdout") - val devStderr = java.io.FileOutputStream("/dev/stderr") - standardOutput = devStdout - errorOutput = devStderr - doLast { - devStdout.flush(); devStdout.close() - devStderr.flush(); devStderr.close() - } - } - - if (extension.alwaysRun.get()) { - outputs.upToDateWhen { false } - } - - isIgnoreExitValue = !extension.failFast.get() - } - } - - fun skipConditions(): Boolean { - return project.hasProperty("skip-tests") || - project.hasProperty("skip-native") || - project.hasProperty("skip-gtest") - } - - private fun gtestHomePath(): String { - return if (extension.googleTestHome.isPresent) { - extension.googleTestHome.get().asFile.absolutePath - } else { - "/opt/homebrew/opt/googletest" - } - } - - private fun adjustCompilerArgs(): List { - val args = config.compilerArgs.get().toMutableList() - - // Remove -std= so we can re-add it consistently - args.removeAll { it.startsWith("-std=") } - - // Remove -DNDEBUG if assertions are enabled - if (extension.enableAssertions.get()) { - args.removeAll { it == "-DNDEBUG" } - } - - // Re-add C++17 standard - args.add("-std=c++17") - - // Add musl define if needed - if (PlatformUtils.currentPlatform == Platform.LINUX && PlatformUtils.isMusl()) { - args.add("-D__musl__") - } - - // Mark unit-test builds so test-only production APIs are compiled in. - args.add("-DUNIT_TEST") - - return args - } - - // Gtest binaries have no JVM, so halt_on_error=0 / abort_on_error=0 (which exists - // to avoid conflicts with JVM signal handlers in Java integration tests) is wrong: - // it silently swallows ASan/TSan findings and lets the test exit 0 despite errors. - // Promote both flags to =1 so any sanitizer finding fails the gtest task immediately. - private fun hardenSanitizerOptions(key: String, value: String): String { - if (key != "ASAN_OPTIONS" && key != "TSAN_OPTIONS" && key != "UBSAN_OPTIONS") { - return value - } - return value - .replace("halt_on_error=0", "halt_on_error=1") - .replace("abort_on_error=0", "abort_on_error=1") - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Architecture.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Architecture.kt deleted file mode 100644 index 1bab5b103..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Architecture.kt +++ /dev/null @@ -1,29 +0,0 @@ - -package com.datadoghq.native.model - -enum class Architecture { - X64, - ARM64, - X86, - ARM; - - override fun toString(): String = when (this) { - X64 -> "x64" - ARM64 -> "arm64" - X86 -> "x86" - ARM -> "arm" - } - - companion object { - fun current(): Architecture { - val osArch = System.getProperty("os.arch").lowercase() - return when { - osArch.contains("amd64") || osArch.contains("x86_64") -> X64 - osArch.contains("aarch64") || osArch.contains("arm64") -> ARM64 - osArch.contains("x86") || osArch.contains("i386") -> X86 - osArch.contains("arm") -> ARM - else -> throw IllegalStateException("Unsupported architecture: $osArch") - } - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/BuildConfiguration.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/BuildConfiguration.kt deleted file mode 100644 index 1224abd67..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/BuildConfiguration.kt +++ /dev/null @@ -1,58 +0,0 @@ - -package com.datadoghq.native.model - -import org.gradle.api.Named -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.MapProperty -import org.gradle.api.provider.Property -import javax.inject.Inject - -abstract class BuildConfiguration @Inject constructor( - private val configName: String -) : Named { - abstract val platform: Property - abstract val architecture: Property - abstract val compilerArgs: ListProperty - abstract val linkerArgs: ListProperty - abstract val testEnvironment: MapProperty - abstract val testJvmArgs: ListProperty - abstract val active: Property - - override fun getName(): String = configName - - init { - // Default to active unless overridden - active.convention(true) - testEnvironment.convention(emptyMap()) - testJvmArgs.convention(emptyList()) - } - - fun isActiveFor(targetPlatform: Platform, targetArch: Architecture): Boolean { - return active.get() && - platform.get() == targetPlatform && - architecture.get() == targetArch - } - - /** - * Returns a unique identifier for this configuration combining name, platform, and architecture. - * Example: "releaseLinuxX64" - */ - fun identifier(): String { - val platformStr = platform.get().toString() - val archStr = architecture.get().toString() - return "$configName${platformStr.replaceFirstChar { it.titlecase() }}${archStr.replaceFirstChar { it.titlecase() }}" - } - - /** - * Returns the capitalized name for task generation. - * Example: "Release" for name "release" - */ - fun capitalizedName(): String = configName.replaceFirstChar { it.titlecase() } - - // Task name helpers for consistent naming across plugins - val compileTaskName: String get() = "compile${capitalizedName()}" - val linkTaskName: String get() = "link${capitalizedName()}" - val assembleTaskName: String get() = "assemble${capitalizedName()}" - val testTaskName: String get() = "test${capitalizedName()}" - val gtestTaskName: String get() = "gtest${capitalizedName()}" -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/ErrorHandlingMode.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/ErrorHandlingMode.kt deleted file mode 100644 index 04a0cad05..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/ErrorHandlingMode.kt +++ /dev/null @@ -1,13 +0,0 @@ - -package com.datadoghq.native.model - -/** - * Error handling strategy for compilation. - */ -enum class ErrorHandlingMode { - /** Stop on first error (default) */ - FAIL_FAST, - - /** Compile all files, collect all errors, report at end */ - COLLECT_ALL -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/LogLevel.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/LogLevel.kt deleted file mode 100644 index 3797cbc7d..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/LogLevel.kt +++ /dev/null @@ -1,19 +0,0 @@ - -package com.datadoghq.native.model - -/** - * Logging verbosity level for native build tasks. - */ -enum class LogLevel { - /** Only errors */ - QUIET, - - /** Standard lifecycle messages (default) */ - NORMAL, - - /** Detailed progress information */ - VERBOSE, - - /** Full command lines and output */ - DEBUG -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Platform.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Platform.kt deleted file mode 100644 index 861e9637b..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/Platform.kt +++ /dev/null @@ -1,20 +0,0 @@ - -package com.datadoghq.native.model - -enum class Platform { - LINUX, - MACOS; - - override fun toString(): String = name.lowercase() - - companion object { - fun current(): Platform { - val osName = System.getProperty("os.name").lowercase() - return when { - osName.contains("mac") || osName.contains("darwin") -> MACOS - osName.contains("linux") -> LINUX - else -> throw IllegalStateException("Unsupported OS: $osName") - } - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/SourceSet.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/SourceSet.kt deleted file mode 100644 index 4325265ef..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/model/SourceSet.kt +++ /dev/null @@ -1,93 +0,0 @@ - -package com.datadoghq.native.model - -import org.gradle.api.Named -import org.gradle.api.file.ConfigurableFileCollection -import org.gradle.api.provider.ListProperty -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.InputFiles -import org.gradle.api.tasks.Internal -import org.gradle.api.tasks.Optional -import javax.inject.Inject - -/** - * Represents a named set of source files with optional per-set compiler flags. - * Allows different parts of the codebase to have different compilation settings. - * - * Example usage: - * sourceSets { main { sources.from(fileTree("src/main/cpp")) } } - */ -abstract class SourceSet @Inject constructor( - private val name: String -) : Named { - - /** - * Source files for this source set. - */ - @get:InputFiles - abstract val sources: ConfigurableFileCollection - - /** - * Include patterns for filtering source files (Ant-style). - * Default: all C++ source files (.cpp, .c, .cc) - */ - @get:Input - @get:Optional - abstract val includes: ListProperty - - /** - * Exclude patterns for filtering source files (Ant-style). - */ - @get:Input - @get:Optional - abstract val excludes: ListProperty - - /** - * Additional compiler arguments specific to this source set. - * These are added to the base compiler arguments. - */ - @get:Input - @get:Optional - abstract val compilerArgs: ListProperty - - init { - includes.convention(listOf("**/*.cpp", "**/*.c", "**/*.cc")) - excludes.convention(emptyList()) - compilerArgs.convention(emptyList()) - } - - @Internal - override fun getName(): String = name - - /** - * Convenience method to set source directory. - */ - fun from(vararg sources: Any) { - this.sources.from(*sources) - } - - /** - * Convenience method to add include patterns. - */ - fun include(vararg patterns: String) { - includes.addAll(*patterns) - } - - /** - * Convenience method to add exclude patterns. - */ - fun exclude(vararg patterns: String) { - excludes.addAll(*patterns) - } - - /** - * Convenience method to add compiler args. - */ - fun compileWith(vararg args: String) { - compilerArgs.addAll(*args) - } - - override fun toString(): String { - return "SourceSet[name=$name, sources=${sources.files.size} files]" - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildExtension.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildExtension.kt deleted file mode 100644 index a7808165f..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildExtension.kt +++ /dev/null @@ -1,51 +0,0 @@ - -package com.datadoghq.native.scanbuild - -import org.gradle.api.Project -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.Property -import javax.inject.Inject - -/** - * Extension for configuring the scan-build static analysis task. - */ -abstract class ScanBuildExtension @Inject constructor(project: Project) { - /** - * Directory containing the Makefile for scan-build. - * Default: src/test/make - */ - abstract val makefileDir: DirectoryProperty - - /** - * Output directory for scan-build reports. - * Default: build/reports/scan-build - */ - abstract val outputDir: DirectoryProperty - - /** - * Path to the clang analyzer to use. - * Default: /usr/bin/clang++ - */ - abstract val analyzer: Property - - /** - * Number of parallel jobs for make. - * Default: 4 - */ - abstract val parallelJobs: Property - - /** - * Make targets to build. - * Default: ["all"] - */ - abstract val makeTargets: ListProperty - - init { - makefileDir.convention(project.layout.projectDirectory.dir("src/test/make")) - outputDir.convention(project.layout.buildDirectory.dir("reports/scan-build")) - analyzer.convention("/usr/bin/clang++") - parallelJobs.convention(4) - makeTargets.convention(listOf("all")) - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildPlugin.kt deleted file mode 100644 index 30151b8e4..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/scanbuild/ScanBuildPlugin.kt +++ /dev/null @@ -1,103 +0,0 @@ - -package com.datadoghq.native.scanbuild - -import com.datadoghq.native.model.Platform -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.tasks.Exec - -/** - * Gradle plugin that provides clang static analysis via scan-build. - * - * This plugin creates a `scanBuild` task that runs the clang static analyzer - * on the C++ codebase using a Makefile-based build. - * - * Usage: - * ```kotlin - * plugins { - * id("com.datadoghq.scanbuild") - * } - * - * scanBuild { - * makefileDir.set(layout.projectDirectory.dir("src/test/make")) - * outputDir.set(layout.buildDirectory.dir("reports/scan-build")) - * analyzer.set("/usr/bin/clang++") - * parallelJobs.set(4) - * } - * ``` - */ -class ScanBuildPlugin : Plugin { - override fun apply(project: Project) { - // Create the extension - val extension = project.extensions.create( - "scanBuild", - ScanBuildExtension::class.java, - project - ) - - // Create the task after project evaluation - project.afterEvaluate { - createScanBuildTask(project, extension) - } - } - - private fun createScanBuildTask(project: Project, extension: ScanBuildExtension) { - // Only create the task on Linux (scan-build is typically Linux-only in CI) - if (PlatformUtils.currentPlatform != Platform.LINUX) { - project.logger.info("Skipping scanBuild task - only available on Linux") - return - } - - // Check if scan-build is available - if (!isScanBuildAvailable()) { - project.logger.warn("scan-build not found in PATH - scanBuild task will fail if executed") - } - - val makefileDir = extension.makefileDir.get().asFile - val outputDir = extension.outputDir.get().asFile - val analyzer = extension.analyzer.get() - val parallelJobs = extension.parallelJobs.get() - val makeTargets = extension.makeTargets.get() - - val scanBuildTask = project.tasks.register("scanBuild", Exec::class.java) - scanBuildTask.configure { - group = "verification" - description = "Run clang static analyzer via scan-build" - - workingDir(makefileDir) - - // Build command line as a single list to avoid vararg ambiguity - val command = mutableListOf( - "scan-build", - "-o", outputDir.absolutePath, - "--force-analyze-debug-code", - // core.StackAddressEscape fires on the intentional setjmp/longjmp pattern in - // StackWalker::walkVM: the jmp_buf address is stored in vm_thread->exception() - // for the duration of the stack walk and is always restored before the function - // returns. The analyzer cannot prove the lifetime is safe, but we can. - "-disable-checker", "core.StackAddressEscape", - "--use-analyzer", analyzer, - "make", "-j$parallelJobs" - ) - command.addAll(makeTargets) - commandLine(command) - - // Ensure output directory exists - doFirst { - outputDir.mkdirs() - } - } - } - - private fun isScanBuildAvailable(): Boolean { - return try { - val process = ProcessBuilder("which", "scan-build") - .redirectErrorStream(true) - .start() - process.waitFor() == 0 - } catch (e: Exception) { - false - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeCompileTask.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeCompileTask.kt deleted file mode 100644 index 4a9ec0e75..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeCompileTask.kt +++ /dev/null @@ -1,473 +0,0 @@ - -package com.datadoghq.native.tasks - -import com.datadoghq.native.model.ErrorHandlingMode -import com.datadoghq.native.model.LogLevel -import com.datadoghq.native.model.SourceSet -import org.gradle.api.DefaultTask -import org.gradle.api.NamedDomainObjectContainer -import org.gradle.api.file.ConfigurableFileCollection -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.model.ObjectFactory -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.Property -import org.gradle.api.tasks.* -import org.gradle.process.ExecOperations -import java.io.ByteArrayOutputStream -import java.io.File -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.concurrent.atomic.AtomicInteger -import javax.inject.Inject - -/** - * Kotlin-based C++ compilation task that directly invokes gcc/clang. - * - * Supports both simple mode (single sources collection) and source sets mode - * (multiple source collections with per-set compiler flags). - */ -abstract class NativeCompileTask @Inject constructor( - private val execOperations: ExecOperations, - private val objects: ObjectFactory -) : DefaultTask() { - - /** - * The C++ compiler executable (e.g., 'g++', 'clang++', or full path). - */ - @get:Input - abstract val compiler: Property - - /** - * Compiler arguments (flags) to pass to the compiler. - */ - @get:Input - abstract val compilerArgs: ListProperty - - /** - * The C++ source files to compile. - */ - @get:InputFiles - @get:SkipWhenEmpty - abstract val sources: ConfigurableFileCollection - - /** - * Include directories for header file lookup. - */ - @get:InputFiles - @get:Optional - abstract val includes: ConfigurableFileCollection - - /** - * Output directory for object files. - */ - @get:OutputDirectory - abstract val objectFileDir: DirectoryProperty - - /** - * Number of parallel compilation jobs. - */ - @get:Input - @get:Optional - abstract val parallelJobs: Property - - /** - * Show detailed compilation output. - */ - @get:Input - @get:Optional - abstract val verbose: Property - - /** - * Source sets for per-directory compiler flags. - * When used, the simple 'sources' property is ignored. - */ - @get:Nested - @get:Optional - val sourceSets: NamedDomainObjectContainer = objects.domainObjectContainer(SourceSet::class.java) - - // === Logging and Verbosity === - - /** - * Logging verbosity level. - * Default: NORMAL - */ - @get:Input - @get:Optional - abstract val logLevel: Property - - /** - * Progress reporting interval (log every N files during compilation). - * Default: 10 - */ - @get:Input - @get:Optional - abstract val progressReportInterval: Property - - /** - * Show full command line for each file compilation. - * Default: false (only shown at DEBUG level) - */ - @get:Input - @get:Optional - abstract val showCommandLines: Property - - /** - * Enable ANSI color codes in output. - * Default: true - */ - @get:Input - @get:Optional - abstract val colorOutput: Property - - // === Error Handling === - - /** - * Error handling mode. - * FAIL_FAST: Stop on first compilation error (default) - * COLLECT_ALL: Compile all files, collect errors, report at end - */ - @get:Input - @get:Optional - abstract val errorHandling: Property - - /** - * Maximum number of errors to show when using COLLECT_ALL mode. - * Default: 10 - */ - @get:Input - @get:Optional - abstract val maxErrorsToShow: Property - - /** - * Treat compiler warnings as errors (-Werror). - * Default: false - */ - @get:Input - @get:Optional - abstract val treatWarningsAsErrors: Property - - // === Convenience Properties === - - /** - * Compiler defines (-D flags). - * Use define() method to add: define("DEBUG", "VERSION=\"1.0\"") - */ - @get:Input - @get:Optional - abstract val defines: ListProperty - - /** - * Compiler undefines (-U flags). - * Use undefine() method to add: undefine("NDEBUG") - */ - @get:Input - @get:Optional - abstract val undefines: ListProperty - - /** - * C++ standard version (e.g., "c++17", "c++20"). - * Use standard() method to set: standard("c++20") - */ - @get:Input - @get:Optional - abstract val standardVersion: Property - - init { - parallelJobs.convention(Runtime.getRuntime().availableProcessors()) - verbose.convention(false) - logLevel.convention(LogLevel.NORMAL) - progressReportInterval.convention(10) - showCommandLines.convention(false) - colorOutput.convention(true) - errorHandling.convention(ErrorHandlingMode.FAIL_FAST) - maxErrorsToShow.convention(10) - treatWarningsAsErrors.convention(false) - defines.convention(emptyList()) - undefines.convention(emptyList()) - group = "build" - description = "Compiles C++ source files" - } - - /** - * Configure source sets using a DSL block. - */ - fun sourceSets(action: org.gradle.api.Action>) { - action.execute(sourceSets) - } - - // === Convenience Methods === - - /** - * Add compiler defines (-D flags). - * Example: define("DEBUG", "VERSION=\"1.0\"") - */ - fun define(vararg defs: String) { - defines.addAll(*defs) - } - - /** - * Add compiler undefines (-U flags). - * Example: undefine("NDEBUG", "DEBUG") - */ - fun undefine(vararg undefs: String) { - undefines.addAll(*undefs) - } - - /** - * Set C++ standard version. - * Example: standard("c++20") generates -std=c++20 - */ - fun standard(version: String) { - standardVersion.set(version) - } - - // === Logging Helpers === - - private fun logNormal(message: String) { - if (logLevel.get() >= LogLevel.NORMAL) { - logger.lifecycle(message) - } - } - - private fun logVerbose(message: String) { - if (logLevel.get() >= LogLevel.VERBOSE) { - logger.lifecycle(message) - } - } - - private fun logDebug(message: String) { - if (logLevel.get() == LogLevel.DEBUG) { - logger.lifecycle(message) - } - } - - private fun shouldShowCommandLine(): Boolean { - return showCommandLines.get() || logLevel.get() == LogLevel.DEBUG - } - - @TaskAction - fun compile() { - val objDir = objectFileDir.get().asFile - objDir.mkdirs() - - // Build base compiler arguments with convenience properties - val baseArgs = compilerArgs.get().toMutableList() - - // Add C++ standard if specified - if (standardVersion.isPresent) { - baseArgs.add("-std=${standardVersion.get()}") - } - - // Add defines (-D) - defines.get().forEach { define -> - baseArgs.add("-D$define") - } - - // Add undefines (-U) - undefines.get().forEach { undefine -> - baseArgs.add("-U$undefine") - } - - // Add -Werror if warnings should be treated as errors - if (treatWarningsAsErrors.get()) { - baseArgs.add("-Werror") - } - - // Build include arguments - val includeArgs = mutableListOf() - includes.files.forEach { dir -> - if (dir.exists()) { - includeArgs.add("-I") - includeArgs.add(dir.absolutePath) - } - } - - val errors = ConcurrentLinkedQueue() - val compiled = AtomicInteger(0) - - // Choose compilation mode: source sets or simple sources - if (sourceSets.isEmpty()) { - // Simple mode: compile all sources with base args - compileSimpleMode(objDir, baseArgs, includeArgs, compiled, errors) - } else { - // Source sets mode: compile each set with merged args - compileSourceSetsMode(objDir, baseArgs, includeArgs, compiled, errors) - } - - // Report errors if any - if (errors.isNotEmpty()) { - val maxErrors = maxErrorsToShow.get() - val errorMsg = buildString { - appendLine("Compilation failed with ${errors.size} error(s):") - errors.take(maxErrors).forEach { error -> - appendLine(" - $error") - } - if (errors.size > maxErrors) { - appendLine(" ... and ${errors.size - maxErrors} more error(s)") - } - } - throw RuntimeException(errorMsg) - } - - logNormal("Successfully compiled ${compiled.get()} file${if (compiled.get() == 1) "" else "s"}") - } - - private fun compileSimpleMode( - objDir: File, - baseArgs: List, - includeArgs: List, - compiled: AtomicInteger, - errors: ConcurrentLinkedQueue - ) { - val sourceFiles = sources.files.toList() - if (sourceFiles.isEmpty()) { - logNormal("No source files to compile") - return - } - - val total = sourceFiles.size - logNormal("Compiling $total C++ source file${if (total == 1) "" else "s"} with ${compiler.get()}...") - - // Compile files in parallel (or sequentially for FAIL_FAST) - if (errorHandling.get() == ErrorHandlingMode.FAIL_FAST) { - // Use sequential stream for FAIL_FAST to ensure immediate termination on error - sourceFiles.stream().forEach { sourceFile -> - try { - compileFile(sourceFile, objDir, baseArgs, includeArgs, compiled, total, errors) - } catch (e: Exception) { - errors.add("Exception compiling ${sourceFile.name}: ${e.message}") - throw e // Re-throw to stop compilation immediately in FAIL_FAST mode - } - } - } else { - // Use parallel stream for COLLECT_ALL mode - sourceFiles.parallelStream().forEach { sourceFile -> - try { - compileFile(sourceFile, objDir, baseArgs, includeArgs, compiled, total, errors) - } catch (e: Exception) { - errors.add("Exception compiling ${sourceFile.name}: ${e.message}") - } - } - } - } - - private fun compileSourceSetsMode( - objDir: File, - baseArgs: List, - includeArgs: List, - compiled: AtomicInteger, - errors: ConcurrentLinkedQueue - ) { - // Collect all files from all source sets - val allFiles = mutableListOf>>() - - sourceSets.forEach { sourceSet -> - val setFiles = sourceSet.sources.asFileTree - .matching { - sourceSet.includes.get().forEach { pattern -> include(pattern) } - sourceSet.excludes.get().forEach { pattern -> exclude(pattern) } - } - .files - .toList() - - // Merge base args with source-set-specific args - val mergedArgs = baseArgs + sourceSet.compilerArgs.get() - - setFiles.forEach { file -> - allFiles.add(file to mergedArgs) - } - } - - if (allFiles.isEmpty()) { - logNormal("No source files to compile in source sets") - return - } - - val total = allFiles.size - logNormal("Compiling $total C++ source file${if (total == 1) "" else "s"} from ${sourceSets.size} source set${if (sourceSets.size == 1) "" else "s"} with ${compiler.get()}...") - - // Compile files in parallel (or sequentially for FAIL_FAST) with their specific args - if (errorHandling.get() == ErrorHandlingMode.FAIL_FAST) { - // Use sequential stream for FAIL_FAST to ensure immediate termination on error - allFiles.stream().forEach { (sourceFile, specificArgs) -> - try { - compileFile(sourceFile, objDir, specificArgs, includeArgs, compiled, total, errors) - } catch (e: Exception) { - errors.add("Exception compiling ${sourceFile.name}: ${e.message}") - throw e // Re-throw to stop compilation immediately in FAIL_FAST mode - } - } - } else { - // Use parallel stream for COLLECT_ALL mode - allFiles.parallelStream().forEach { (sourceFile, specificArgs) -> - try { - compileFile(sourceFile, objDir, specificArgs, includeArgs, compiled, total, errors) - } catch (e: Exception) { - errors.add("Exception compiling ${sourceFile.name}: ${e.message}") - } - } - } - } - - private fun compileFile( - sourceFile: File, - objDir: File, - baseArgs: List, - includeArgs: List, - compiled: AtomicInteger, - total: Int, - errors: ConcurrentLinkedQueue - ) { - // Determine object file name - val baseName = sourceFile.nameWithoutExtension - val objectFile = File(objDir, "$baseName.o") - - // Build full command line - val cmdLine = mutableListOf().apply { - add(compiler.get()) - addAll(baseArgs) - addAll(includeArgs) - add("-c") - add(sourceFile.absolutePath) - add("-o") - add(objectFile.absolutePath) - } - - if (shouldShowCommandLine()) { - logDebug(" ${cmdLine.joinToString(" ")}") - } - - // Execute compilation - val stdout = ByteArrayOutputStream() - val stderr = ByteArrayOutputStream() - - val result = execOperations.exec { - commandLine(cmdLine) - standardOutput = stdout - errorOutput = stderr - isIgnoreExitValue = true - } - - if (result.exitValue != 0) { - val allOutput = (stdout.toString() + stderr.toString()).trim() - val errorMsg = buildString { - append("Failed to compile ${sourceFile.name}: exit code ${result.exitValue}") - if (allOutput.isNotEmpty()) { - appendLine() - append(allOutput) - } - } - errors.add(errorMsg) - - // FAIL_FAST: throw immediately on first error - if (errorHandling.get() == ErrorHandlingMode.FAIL_FAST) { - throw RuntimeException(errorMsg) - } - } else { - val count = compiled.incrementAndGet() - val interval = progressReportInterval.get() - if (logLevel.get() >= LogLevel.VERBOSE && (count % interval == 0 || count == total)) { - logVerbose(" Compiled $count/$total files...") - } - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkExecutableTask.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkExecutableTask.kt deleted file mode 100644 index d2d8d5e08..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkExecutableTask.kt +++ /dev/null @@ -1,196 +0,0 @@ - -package com.datadoghq.native.tasks - -import com.datadoghq.native.model.LogLevel -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.DefaultTask -import org.gradle.api.file.ConfigurableFileCollection -import org.gradle.api.file.RegularFileProperty -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.Property -import org.gradle.api.tasks.* -import org.gradle.process.ExecOperations -import java.io.ByteArrayOutputStream -import javax.inject.Inject - -/** - * Kotlin-based executable linking task that directly invokes the linker. - * - * Used for linking test executables (gtest) and other standalone binaries. - */ -abstract class NativeLinkExecutableTask @Inject constructor( - private val execOperations: ExecOperations -) : DefaultTask() { - - /** - * The linker executable (usually same as compiler: 'g++', 'clang++'). - */ - @get:Input - abstract val linker: Property - - /** - * Linker arguments (flags). - */ - @get:Input - abstract val linkerArgs: ListProperty - - /** - * The object files to link. - */ - @get:InputFiles - @get:SkipWhenEmpty - abstract val objectFiles: ConfigurableFileCollection - - /** - * The output executable file. - */ - @get:OutputFile - abstract val outputFile: RegularFileProperty - - /** - * Library search paths (-L). - */ - @get:Input - @get:Optional - abstract val libraryPaths: ListProperty - - /** - * Libraries to link against (-l). - */ - @get:Input - @get:Optional - abstract val libraries: ListProperty - - /** - * Runtime library search paths (-Wl,-rpath). - */ - @get:Input - @get:Optional - abstract val runtimePaths: ListProperty - - /** - * Logging verbosity level. - */ - @get:Input - @get:Optional - abstract val logLevel: Property - - /** - * Show full command line. - */ - @get:Input - @get:Optional - abstract val showCommandLine: Property - - init { - libraryPaths.convention(emptyList()) - libraries.convention(emptyList()) - runtimePaths.convention(emptyList()) - logLevel.convention(LogLevel.NORMAL) - showCommandLine.convention(false) - group = "build" - description = "Links object files into an executable" - } - - /** - * Add libraries to link against. - */ - fun lib(vararg libs: String) { - libraries.addAll(*libs) - } - - /** - * Add library search paths. - */ - fun libPath(vararg paths: String) { - libraryPaths.addAll(*paths) - } - - /** - * Add runtime library search paths. - */ - fun runtimePath(vararg paths: String) { - runtimePaths.addAll(*paths) - } - - private fun logNormal(message: String) { - if (logLevel.get() >= LogLevel.NORMAL) { - logger.lifecycle(message) - } - } - - private fun logDebug(message: String) { - if (logLevel.get() == LogLevel.DEBUG) { - logger.lifecycle(message) - } - } - - @TaskAction - fun link() { - val outFile = outputFile.get().asFile - outFile.parentFile.mkdirs() - - val objectPaths = objectFiles.files.map { it.absolutePath } - - // Build command line - val cmdLine = mutableListOf().apply { - add(linker.get()) - addAll(objectPaths) - addAll(linkerArgs.get()) - - // Add library search paths (-L) - libraryPaths.get().forEach { path -> - add("-L$path") - } - - // Add libraries (-l) - libraries.get().forEach { lib -> - add("-l$lib") - } - - // Add runtime search paths (-rpath) - runtimePaths.get().forEach { path -> - add("-Wl,-rpath,$path") - } - - // Add output file - add("-o") - add(outFile.absolutePath) - } - - logNormal("Linking executable: ${outFile.name}") - - if (showCommandLine.get() || logLevel.get() == LogLevel.DEBUG) { - logDebug(" ${cmdLine.joinToString(" ")}") - } - - // Execute linking - val stdout = ByteArrayOutputStream() - val stderr = ByteArrayOutputStream() - - val result = execOperations.exec { - commandLine(cmdLine) - standardOutput = stdout - errorOutput = stderr - isIgnoreExitValue = true - } - - if (result.exitValue != 0) { - val allOutput = (stdout.toString() + stderr.toString()).trim() - val errorMsg = buildString { - append("Failed to link executable: exit code ${result.exitValue}") - if (allOutput.isNotEmpty()) { - appendLine() - append(allOutput) - } - } - throw RuntimeException(errorMsg) - } - - // Make executable - outFile.setExecutable(true) - - val sizeKB = outFile.length() / 1024 - logNormal("Successfully linked ${outFile.name} (${sizeKB}KB)") - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkTask.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkTask.kt deleted file mode 100644 index 59185c678..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/tasks/NativeLinkTask.kt +++ /dev/null @@ -1,543 +0,0 @@ - -package com.datadoghq.native.tasks - -import com.datadoghq.native.model.LogLevel -import com.datadoghq.native.model.Platform -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.DefaultTask -import org.gradle.api.file.ConfigurableFileCollection -import org.gradle.api.file.DirectoryProperty -import org.gradle.api.file.RegularFileProperty -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.Property -import org.gradle.api.tasks.* -import org.gradle.process.ExecOperations -import java.io.ByteArrayOutputStream -import javax.inject.Inject - -/** - * Kotlin-based shared library linking task that directly invokes the linker. - * - * Simplified from the Groovy SimpleLinkShared to focus on core functionality: - * - Linking object files into shared libraries - * - Library path and library flag management - * - Platform-specific flag handling (soname vs install_name) - * - Symbol stripping (optional) - */ -abstract class NativeLinkTask @Inject constructor( - private val execOperations: ExecOperations -) : DefaultTask() { - - /** - * The linker executable (usually same as compiler: 'g++', 'clang++'). - */ - @get:Input - abstract val linker: Property - - /** - * Linker arguments (flags and libraries). - */ - @get:Input - abstract val linkerArgs: ListProperty - - /** - * The object files to link. - */ - @get:InputFiles - @get:SkipWhenEmpty - abstract val objectFiles: ConfigurableFileCollection - - /** - * The output shared library file. - */ - @get:OutputFile - abstract val outputFile: RegularFileProperty - - /** - * Library search paths (-L). - */ - @get:Input - @get:Optional - abstract val libraryPaths: ListProperty - - /** - * Libraries to link against (-l). - */ - @get:Input - @get:Optional - abstract val libraries: ListProperty - - /** - * SO name for Linux (-Wl,-soname). - */ - @get:Input - @get:Optional - abstract val soname: Property - - /** - * Install name for macOS (-Wl,-install_name). - */ - @get:Input - @get:Optional - abstract val installName: Property - - /** - * Strip symbols after linking. - */ - @get:Input - @get:Optional - abstract val stripSymbols: Property - - /** - * Extract debug symbols to separate file before stripping. - */ - @get:Input - @get:Optional - abstract val extractDebugSymbols: Property - - /** - * Output directory for extracted debug symbols. - */ - @get:OutputDirectory - @get:Optional - abstract val debugSymbolsDir: DirectoryProperty - - /** - * Show detailed linking output. - */ - @get:Input - @get:Optional - abstract val verbose: Property - - // === Logging and Verbosity === - - /** - * Logging verbosity level. - * Default: NORMAL - */ - @get:Input - @get:Optional - abstract val logLevel: Property - - /** - * Show full command line for the link operation. - * Default: false (only shown at DEBUG level) - */ - @get:Input - @get:Optional - abstract val showCommandLine: Property - - /** - * Show linker map (symbol resolution details). - * Default: false - */ - @get:Input - @get:Optional - abstract val showLinkerMap: Property - - /** - * Linker map output file (when showLinkerMap is true). - * Default: null (stdout/stderr) - */ - @get:OutputFile - @get:Optional - abstract val linkerMapFile: RegularFileProperty - - /** - * Enable ANSI color codes in output. - * Default: true - */ - @get:Input - @get:Optional - abstract val colorOutput: Property - - // === Symbol Management === - - /** - * Symbol patterns to export (make visible). - * For example: ["Java_*", "JNI_OnLoad", "JNI_OnUnload"] - */ - @get:Input - @get:Optional - abstract val exportSymbols: ListProperty - - /** - * Symbol patterns to hide (make not visible). - * Applied after exportSymbols. - */ - @get:Input - @get:Optional - abstract val hideSymbols: ListProperty - - // === Library Path Management === - - /** - * Runtime library search paths (-Wl,-rpath). - * Use runtimePath() method to add. - */ - @get:Input - @get:Optional - abstract val runtimePaths: ListProperty - - // === Verification === - - /** - * Check for undefined symbols after linking. - * Default: false - */ - @get:Input - @get:Optional - abstract val checkUndefinedSymbols: Property - - /** - * Verify the shared library after linking (ldd/otool -L). - * Default: false - */ - @get:Input - @get:Optional - abstract val verifySharedLib: Property - - init { - libraryPaths.convention(emptyList()) - libraries.convention(emptyList()) - runtimePaths.convention(emptyList()) - stripSymbols.convention(false) - extractDebugSymbols.convention(false) - verbose.convention(false) - logLevel.convention(LogLevel.NORMAL) - showCommandLine.convention(false) - showLinkerMap.convention(false) - colorOutput.convention(true) - exportSymbols.convention(emptyList()) - hideSymbols.convention(emptyList()) - checkUndefinedSymbols.convention(false) - verifySharedLib.convention(false) - group = "build" - description = "Links object files into a shared library" - } - - fun lib(vararg libs: String) { - libraries.addAll(*libs) - } - - fun libPath(vararg paths: String) { - libraryPaths.addAll(*paths) - } - - fun runtimePath(vararg paths: String) { - runtimePaths.addAll(*paths) - } - - // === Logging Helpers === - - private fun logNormal(message: String) { - if (logLevel.get() >= LogLevel.NORMAL) { - logger.lifecycle(message) - } - } - - private fun logVerbose(message: String) { - if (logLevel.get() >= LogLevel.VERBOSE) { - logger.lifecycle(message) - } - } - - private fun logDebug(message: String) { - if (logLevel.get() == LogLevel.DEBUG) { - logger.lifecycle(message) - } - } - - private fun shouldShowCommandLine(): Boolean { - return showCommandLine.get() || logLevel.get() == LogLevel.DEBUG - } - - @TaskAction - fun link() { - val outFile = outputFile.get().asFile - outFile.parentFile.mkdirs() - - val objectPaths = objectFiles.files.map { it.absolutePath } - - // Determine shared library flag based on platform - val sharedFlag = when (PlatformUtils.currentPlatform) { - Platform.MACOS -> "-dynamiclib" - Platform.LINUX -> "-shared" - } - - // Build command line - val cmdLine = mutableListOf().apply { - add(linker.get()) - add(sharedFlag) - addAll(objectPaths) - addAll(linkerArgs.get()) - - // Add library search paths (-L) - libraryPaths.get().forEach { path -> - add("-L$path") - } - - // Add libraries (-l) - libraries.get().forEach { lib -> - add("-l$lib") - } - - // Add runtime search paths (-rpath) - runtimePaths.get().forEach { path -> - add("-Wl,-rpath,$path") - } - - // Add soname/install_name based on platform - when (PlatformUtils.currentPlatform) { - Platform.LINUX -> { - if (soname.isPresent) { - add("-Wl,-soname,${soname.get()}") - } - } - Platform.MACOS -> { - if (installName.isPresent) { - add("-Wl,-install_name,${installName.get()}") - } - } - } - - // Add symbol visibility control if specified - if (exportSymbols.get().isNotEmpty() || hideSymbols.get().isNotEmpty()) { - addAll(generateSymbolVisibilityFlags(outFile)) - } - - // Add output file - add("-o") - add(outFile.absolutePath) - } - - logNormal("Linking shared library: ${outFile.name}") - - if (shouldShowCommandLine()) { - logDebug(" ${cmdLine.joinToString(" ")}") - } - - // Execute linking - val stdout = ByteArrayOutputStream() - val stderr = ByteArrayOutputStream() - - val result = execOperations.exec { - commandLine(cmdLine) - standardOutput = stdout - errorOutput = stderr - isIgnoreExitValue = true - } - - if (result.exitValue != 0) { - val allOutput = (stdout.toString() + stderr.toString()).trim() - val errorMsg = buildString { - append("Failed to link shared library: exit code ${result.exitValue}") - if (allOutput.isNotEmpty()) { - appendLine() - append(allOutput) - } - } - throw RuntimeException(errorMsg) - } - - // Extract debug symbols before stripping if requested - if (extractDebugSymbols.get()) { - extractDebugInfo(outFile) - } - - // Strip symbols if requested - if (stripSymbols.get()) { - stripLibrary(outFile) - } - - val sizeKB = outFile.length() / 1024 - logNormal("Successfully linked ${outFile.name} (${sizeKB}KB)") - } - - /** - * Generate platform-specific symbol visibility flags. - * Returns linker flags to control symbol export/hiding. - */ - private fun generateSymbolVisibilityFlags(outFile: java.io.File): List { - return when (PlatformUtils.currentPlatform) { - Platform.LINUX -> { - generateLinuxVersionScript(outFile) - } - Platform.MACOS -> { - generateMacOSExportList(outFile) - } - } - } - - /** - * Generate Linux version script for symbol visibility control. - */ - private fun generateLinuxVersionScript(outFile: java.io.File): List { - val versionScript = java.io.File(temporaryDir, "${outFile.nameWithoutExtension}.ver") - - val scriptContent = buildString { - appendLine("{") - appendLine(" global:") - - // Export specified symbols - exportSymbols.get().forEach { pattern -> - appendLine(" $pattern;") - } - - // Consolidate all hidden symbols in a single local section - appendLine(" local:") - - // Explicitly hide specified symbols (override exports) - hideSymbols.get().forEach { pattern -> - appendLine(" $pattern;") - } - - // Hide everything else unless it was explicitly exported - if (exportSymbols.get().isNotEmpty() || hideSymbols.get().isNotEmpty()) { - appendLine(" *;") - } - - appendLine("};") - } - - versionScript.writeText(scriptContent) - logVerbose("Generated version script: ${versionScript.name}") - - return listOf("-Wl,--version-script=${versionScript.absolutePath}") - } - - /** - * Generate macOS exported symbols list for symbol visibility control. - */ - private fun generateMacOSExportList(outFile: java.io.File): List { - val exportList = java.io.File(temporaryDir, "${outFile.nameWithoutExtension}.exp") - - // Warn if wildcards are used - macOS doesn't support them - exportSymbols.get().forEach { pattern -> - if (pattern.contains('*') || pattern.contains('?')) { - logger.warn("Symbol pattern '$pattern' contains wildcards which are not supported on macOS. " + - "Pattern will be treated as a literal symbol name. " + - "Consider using -fvisibility compiler flags instead, or list symbols explicitly.") - } - } - - val listContent = buildString { - // Export specified symbols (macOS needs leading underscore for C symbols) - exportSymbols.get().forEach { pattern -> - // Convert glob patterns to exact names or keep as-is - // macOS export list doesn't support wildcards like Linux version scripts - // For wildcards, we'd need to use -exported_symbols_list with all matching symbols - // For now, treat patterns as literal symbol names - val symbol = if (pattern.startsWith("_")) pattern else "_$pattern" - appendLine(symbol) - } - } - - exportList.writeText(listContent) - logVerbose("Generated export list: ${exportList.name}") - - val flags = mutableListOf() - - // Add export list - if (exportSymbols.get().isNotEmpty()) { - flags.add("-Wl,-exported_symbols_list,${exportList.absolutePath}") - } - - // For hiding, use -unexported_symbols_list if needed - if (hideSymbols.get().isNotEmpty()) { - val hideList = java.io.File(temporaryDir, "${outFile.nameWithoutExtension}.hide") - val hideContent = buildString { - hideSymbols.get().forEach { pattern -> - val symbol = if (pattern.startsWith("_")) pattern else "_$pattern" - appendLine(symbol) - } - } - hideList.writeText(hideContent) - flags.add("-Wl,-unexported_symbols_list,${hideList.absolutePath}") - } - - return flags - } - - private fun extractDebugInfo(libFile: java.io.File) { - val debugDir = if (debugSymbolsDir.isPresent) { - debugSymbolsDir.get().asFile - } else { - libFile.parentFile - } - debugDir.mkdirs() - - when (PlatformUtils.currentPlatform) { - Platform.LINUX -> { - extractDebugInfoLinux(libFile, debugDir) - } - Platform.MACOS -> { - extractDebugInfoMacOS(libFile, debugDir) - } - } - } - - private fun extractDebugInfoLinux(libFile: java.io.File, debugDir: java.io.File) { - val debugFile = java.io.File(debugDir, "${libFile.name}.debug") - - logNormal("Extracting debug symbols to ${debugFile.name}...") - - // Extract debug symbols - val extractResult = execOperations.exec { - commandLine("objcopy", "--only-keep-debug", libFile.absolutePath, debugFile.absolutePath) - isIgnoreExitValue = true - } - - if (extractResult.exitValue != 0) { - logger.warn("Failed to extract debug symbols (exit code ${extractResult.exitValue})") - return - } - - // Add GNU debuglink to stripped library - val debuglinkResult = execOperations.exec { - commandLine("objcopy", "--add-gnu-debuglink=${debugFile.absolutePath}", libFile.absolutePath) - isIgnoreExitValue = true - } - - if (debuglinkResult.exitValue != 0) { - logger.warn("Failed to add debuglink (exit code ${debuglinkResult.exitValue})") - } else { - logNormal("Created debug file: ${debugFile.name} (${debugFile.length() / 1024}KB)") - } - } - - private fun extractDebugInfoMacOS(libFile: java.io.File, debugDir: java.io.File) { - val dsymBundle = java.io.File(debugDir, "${libFile.name}.dSYM") - - logNormal("Creating dSYM bundle...") - - val result = execOperations.exec { - commandLine("dsymutil", libFile.absolutePath, "-o", dsymBundle.absolutePath) - isIgnoreExitValue = true - } - - if (result.exitValue != 0) { - logger.warn("Failed to create dSYM bundle (exit code ${result.exitValue})") - } else { - logNormal("Created dSYM bundle: ${dsymBundle.name}") - } - } - - private fun stripLibrary(libFile: java.io.File) { - logNormal("Stripping symbols from ${libFile.name}...") - - val stripCmd = when (PlatformUtils.currentPlatform) { - Platform.LINUX -> listOf("strip", "--strip-debug", libFile.absolutePath) - Platform.MACOS -> listOf("strip", "-S", libFile.absolutePath) - } - - val result = execOperations.exec { - commandLine(stripCmd) - isIgnoreExitValue = true - } - - if (result.exitValue != 0) { - logger.warn("Failed to strip symbols (exit code ${result.exitValue}), continuing...") - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt deleted file mode 100644 index 52719c376..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/native/util/PlatformUtils.kt +++ /dev/null @@ -1,390 +0,0 @@ - -package com.datadoghq.native.util - -import com.datadoghq.native.model.Architecture -import com.datadoghq.native.model.Platform -import org.gradle.api.GradleException -import org.gradle.api.Project -import java.io.File -import kotlin.io.path.createTempFile -import kotlin.io.path.deleteIfExists -import kotlin.io.path.writeText -import java.util.concurrent.TimeUnit - -object PlatformUtils { - val currentPlatform: Platform by lazy { Platform.current() } - val currentArchitecture: Architecture by lazy { Architecture.current() } - - fun isMusl(): Boolean { - if (currentPlatform != Platform.LINUX) { - return false - } - - // Check if running on musl libc by scanning /lib/ld-musl-*.so.1 - return File("/lib").listFiles()?.any { - it.name.startsWith("ld-musl-") && it.name.endsWith(".so.1") - } ?: false - } - - fun javaHome(): String { - return System.getenv("JAVA_HOME") - ?: System.getProperty("java.home") - ?: throw IllegalStateException("Neither JAVA_HOME environment variable nor java.home system property is set") - } - - /** - * Resolve JAVA_HOME for test execution, preferring JAVA_TEST_HOME if set. - * This allows running tests with a different JDK than the build JDK. - */ - fun testJavaHome(): String { - return System.getenv("JAVA_TEST_HOME") - ?: System.getenv("JAVA_HOME") - ?: System.getProperty("java.home") - ?: throw IllegalStateException("Neither JAVA_TEST_HOME, JAVA_HOME, nor java.home is set") - } - - /** - * Get the java executable path for test execution. - */ - fun testJavaExecutable(): String { - return "${testJavaHome()}/bin/java" - } - - fun jniIncludePaths(): List { - val javaHome = javaHome() - val platform = when (currentPlatform) { - Platform.LINUX -> "linux" - Platform.MACOS -> "darwin" - } - return listOf( - "$javaHome/include", - "$javaHome/include/$platform" - ) - } - - /** - * Check if a compiler is available and functional - */ - fun isCompilerAvailable(compiler: String): Boolean { - return try { - val process = ProcessBuilder(compiler, "--version") - .redirectErrorStream(true) - .start() - process.waitFor(5, TimeUnit.SECONDS) - process.exitValue() == 0 - } catch (e: Exception) { - false - } - } - - /** - * Locate a library using compiler's -print-file-name. - * Uses the specified compiler, falling back to gcc if not available. - */ - fun locateLibrary(libName: String, compiler: String = "gcc"): String? { - if (currentPlatform != Platform.LINUX) { - return null - } - - // Try the specified compiler first - if (isCompilerAvailable(compiler)) { - try { - val process = ProcessBuilder(compiler, "-print-file-name=$libName.so") - .redirectErrorStream(true) - .start() - - val output = process.inputStream.bufferedReader().readText().trim() - process.waitFor() - - if (process.exitValue() == 0 && output != "$libName.so") { - return output - } - } catch (e: Exception) { - // Fall through to try gcc - } - } - - // If the specified compiler didn't find it, try gcc as fallback - if (compiler != "gcc" && isCompilerAvailable("gcc")) { - try { - val process = ProcessBuilder("gcc", "-print-file-name=$libName.so") - .redirectErrorStream(true) - .start() - - val output = process.inputStream.bufferedReader().readText().trim() - process.waitFor() - - if (process.exitValue() == 0 && output != "$libName.so") { - return output - } - } catch (e: Exception) { - // Fall through to return null - } - } - - return null - } - - fun locateLibasan(compiler: String = "gcc"): String? { - if (currentPlatform != Platform.LINUX) return null - // For clang, prefer the architecture-specific clang_rt.asan library over - // GCC's libasan. Using GCC's runtime alongside clang's libclang_rt.asan - // (which -fsanitize=address links for executables) causes "incompatible - // ASan runtimes" at startup. The clang runtime also includes UBSan symbols, - // so no separate -lubsan is needed. - if (compiler.contains("clang")) { - val archSuffix = when (currentArchitecture) { - Architecture.X64 -> "x86_64" - Architecture.ARM64 -> "aarch64" - Architecture.X86 -> "i386" - Architecture.ARM -> "arm" - } - val clangAsan = locateLibrary("libclang_rt.asan-$archSuffix", compiler) - if (clangAsan != null) return clangAsan - } - return locateLibrary("libasan", compiler) - } - - fun locateLibtsan(compiler: String = "gcc"): String? { - if (currentPlatform != Platform.LINUX) return null - // For clang, prefer the architecture-specific clang_rt.tsan library over - // GCC's libtsan. GCC 11's libtsan only handles 39-bit VMA on aarch64; - // clang's compiler-rt tsan handles both 39-bit and 48-bit VMA. - if (compiler.contains("clang")) { - val archSuffix = when (currentArchitecture) { - Architecture.X64 -> "x86_64" - Architecture.ARM64 -> "aarch64" - Architecture.X86 -> "i386" - Architecture.ARM -> "arm" - } - val clangTsan = locateLibrary("libclang_rt.tsan-$archSuffix", compiler) - if (clangTsan != null) return clangTsan - } - return locateLibrary("libtsan", compiler) - } - - fun checkFuzzerSupport(): Boolean { - return try { - val testFile = createTempFile("fuzzer_check", ".cpp") - val outFile = createTempFile("fuzzer_check", "") - // Remove the pre-created output file so the compiler writes its own binary - outFile.deleteIfExists() - try { - testFile.writeText("extern \"C\" int LLVMFuzzerTestOneInput(const char*, long) { return 0; }") - - // Link (not just compile) to catch missing libclang_rt.fuzzer_osx.a on Apple clang - val process = ProcessBuilder( - "clang++", - "-fsanitize=fuzzer", - testFile.toAbsolutePath().toString(), - "-o", - outFile.toAbsolutePath().toString() - ).redirectErrorStream(true).start() - - if (!process.waitFor(30, TimeUnit.SECONDS)) { - process.destroyForcibly() - process.waitFor(5, TimeUnit.SECONDS) - return false - } - process.exitValue() == 0 - } finally { - testFile.deleteIfExists() - outFile.deleteIfExists() - } - } catch (e: InterruptedException) { - Thread.currentThread().interrupt() - false - } catch (e: Exception) { - false - } - } - - fun hasAsan(compiler: String = "gcc"): Boolean { - return !isMusl() && locateLibasan(compiler) != null - } - - fun hasTsan(compiler: String = "gcc"): Boolean { - return !isMusl() && locateLibtsan(compiler) != null - } - - fun hasFuzzer(): Boolean { - return !isMusl() && checkFuzzerSupport() - } - - fun sharedLibExtension(): String = when (currentPlatform) { - Platform.LINUX -> "so" - Platform.MACOS -> "dylib" - } - - /** - * Find Homebrew LLVM installation on macOS. - * Returns the LLVM installation path or null if not found. - */ - fun findHomebrewLLVM(): String? { - if (currentPlatform != Platform.MACOS) { - return null - } - - val possiblePaths = listOf( - "/opt/homebrew/opt/llvm", // Apple Silicon - "/usr/local/opt/llvm" // Intel Mac - ) - - for (path in possiblePaths) { - if (File(path).exists() && File("$path/bin/clang++").exists()) { - return path - } - } - - // Try using brew command - return try { - val process = ProcessBuilder("brew", "--prefix", "llvm") - .redirectErrorStream(true) - .start() - process.waitFor(5, TimeUnit.SECONDS) - if (process.exitValue() == 0) { - val brewPath = process.inputStream.bufferedReader().readText().trim() - if (File("$brewPath/bin/clang++").exists()) { - brewPath - } else { - null - } - } else { - null - } - } catch (e: Exception) { - null - } - } - - /** - * Find the clang resource directory within an LLVM installation. - * This is needed for locating libFuzzer on macOS with Homebrew LLVM. - */ - fun findClangResourceDir(llvmPath: String?): String? { - if (llvmPath == null) { - return null - } - - val clangLibDir = File("$llvmPath/lib/clang") - if (!clangLibDir.exists()) { - return null - } - - // Find the version directory (e.g., 18.1.8 or 19) - val versions = clangLibDir.listFiles() - ?.filter { it.isDirectory } - ?.sortedByDescending { it.name } - - return if (versions != null && versions.isNotEmpty()) { - "$llvmPath/lib/clang/${versions[0].name}" - } else { - null - } - } - - /** - * Find a compiler suitable for fuzzing. - * On macOS, prefers Homebrew LLVM's clang++ for libFuzzer support. - */ - fun findFuzzerCompiler(project: Project): String { - if (currentPlatform == Platform.MACOS) { - val homebrewLLVM = findHomebrewLLVM() - if (homebrewLLVM != null) { - return "$homebrewLLVM/bin/clang++" - } - } - return findCompiler(project) - } - - /** - * Detect the installed clang-format version. - * Returns null if clang-format is not available. - */ - fun clangFormatVersion(): String? { - return try { - val process = ProcessBuilder("clang-format", "--version").start() - process.waitFor(5, TimeUnit.SECONDS) - if (process.exitValue() == 0) { - val output = process.inputStream.bufferedReader().readText().trim() - val match = Regex("""clang-format version (\d+\.\d+\.\d+)""").find(output) - match?.groupValues?.get(1) - } else { - null - } - } catch (e: Exception) { - null - } - } - - /** - * Find a C++ compiler, respecting -Pnative.forceCompiler property. - * Auto-detects clang++ or g++ if not specified. - */ - fun findCompiler(project: Project): String { - // Check for forced compiler override - val forcedCompiler = project.findProperty("native.forceCompiler") as? String - if (forcedCompiler != null) { - if (isCompilerAvailable(forcedCompiler)) { - project.logger.info("Using forced compiler: $forcedCompiler") - return forcedCompiler - } - throw GradleException( - "Forced compiler '$forcedCompiler' is not available. " + - "Verify the path or remove -Pnative.forceCompiler to auto-detect." - ) - } - - // Auto-detect: prefer clang++, then g++, then c++ - val compilers = listOf("clang++", "g++", "c++") - for (compiler in compilers) { - if (isCompilerAvailable(compiler)) { - project.logger.info("Auto-detected compiler: $compiler") - return compiler - } - } - - throw GradleException( - "No C++ compiler found. Please install clang++ or g++, " + - "or specify one with -Pnative.forceCompiler=/path/to/compiler" - ) - } - - /** - * Returns the major version of the test JVM (e.g. 8, 11, 17, 21, 25). - * Returns 0 if the version cannot be determined. - */ - fun testJvmMajorVersion(): Int { - val javaHome = testJavaHome() - return try { - val process = ProcessBuilder("$javaHome/bin/java", "-version") - .redirectErrorStream(true) - .start() - val output = process.inputStream.bufferedReader().readText() - process.waitFor(10, TimeUnit.SECONDS) - val match = Regex("""version "(?:1\.)?(\d+)""").find(output) - match?.groupValues?.get(1)?.toIntOrNull() ?: 0 - } catch (_: Exception) { - 0 - } - } - - /** - * Returns true if the test JVM (from JAVA_TEST_HOME or JAVA_HOME) is an OpenJ9/J9 JVM. - * Probes `java -version` stderr output for "J9" or "OpenJ9". - */ - fun isTestJvmJ9(): Boolean { - val javaHome = testJavaHome() - return try { - val process = ProcessBuilder("$javaHome/bin/java", "-version") - .redirectErrorStream(true) - .start() - val output = process.inputStream.bufferedReader().readText() - process.waitFor(10, TimeUnit.SECONDS) - output.contains("J9") || output.contains("OpenJ9") - } catch (_: Exception) { - false - } - } - -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/JavaConventionsPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/JavaConventionsPlugin.kt deleted file mode 100644 index d2c46b2c9..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/JavaConventionsPlugin.kt +++ /dev/null @@ -1,46 +0,0 @@ - -package com.datadoghq.profiler - -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.artifacts.VersionCatalogsExtension -import org.gradle.api.tasks.compile.JavaCompile - -/** - * Convention plugin for Java compilation settings. - * - * Applies standard Java compilation options across all subprojects: - * - Java 8 release target for broad JVM compatibility - * - Suppresses JDK 21+ deprecation warnings for --release 8 - * - Redirects org.lz4:lz4-java to at.yawk.lz4:lz4-java (community fork) - * - * Requires JDK 21+ for building (Gradle 9 requirement). - * The compiled bytecode targets Java 8 runtime. - * - * Usage: - * ```kotlin - * plugins { - * id("com.datadoghq.java-conventions") - * } - * ``` - */ -class JavaConventionsPlugin : Plugin { - override fun apply(project: Project) { - project.tasks.withType(JavaCompile::class.java).configureEach { - // JDK 21+ deprecated --release 8 with warnings; suppress with -Xlint:-options - // The deprecation is informational - Java 8 targeting still works - options.compilerArgs.addAll(listOf("--release", "8", "-Xlint:-options")) - } - - // Redirect org.lz4:lz4-java → at.yawk.lz4:lz4-java (community fork) - // JMC flightrecorder transitively depends on the old coordinates - val libs = project.extensions.getByType(VersionCatalogsExtension::class.java).named("libs") - val lz4Version = libs.findVersion("lz4").orElseThrow().requiredVersion - project.configurations.all { - resolutionStrategy.dependencySubstitution { - substitute(module("org.lz4:lz4-java")) - .using(module("at.yawk.lz4:lz4-java:$lz4Version")) - } - } - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt deleted file mode 100644 index 4dfa580d7..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/ProfilerTestPlugin.kt +++ /dev/null @@ -1,679 +0,0 @@ - -package com.datadoghq.profiler - -import com.datadoghq.native.NativeBuildExtension -import com.datadoghq.native.model.BuildConfiguration -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.DefaultTask -import org.gradle.api.GradleException -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.artifacts.Configuration -import org.gradle.api.file.FileCollection -import org.gradle.api.provider.ListProperty -import org.gradle.api.provider.Property -import org.gradle.api.tasks.Exec -import org.gradle.api.tasks.SourceSetContainer -import org.gradle.api.tasks.testing.Test -import java.time.Duration -import javax.inject.Inject - -/** - * Convention plugin for profiler test configuration. - * - * Provides: - * - Standard JVM arguments for profiler testing (attach self, error files, etc.) - * - Java executable selection (JAVA_TEST_HOME or JAVA_HOME) on ALL platforms - * - Common environment variables (CI, rate limiting) - * - Unified -Ptests flag support across all platforms - * - Automatic multi-config test task generation from NativeBuildExtension - * - * Implementation: - * - glibc/macOS: Uses native Test tasks with Gradle's JUnit Platform integration - * - musl (Alpine): Uses Exec tasks with custom ProfilerTestRunner (bypasses toolchain probe) - * - Unified interface: -Ptests property works identically on all platforms - * - Supports multi-JDK testing via JAVA_TEST_HOME on all platforms - * - Same task names everywhere (testdebug, testrelease, unwindingReportRelease) - * - * Platform Detection: - * - Uses PlatformUtils.isMusl() at configuration time to select task implementation - * - musl systems: Exec task with ProfilerTestRunner (uses JUnit Platform Launcher API directly) - * - glibc/macOS: Normal Test task with native JUnit integration - * - * Custom Test Runner: - * - ProfilerTestRunner uses JUnit Platform Launcher API directly - * - Avoids Console Launcher issues (assertions, JVM args, NoSuchMethodError on musl + JDK 11) - * - Same API used by IDEs and Gradle's Test task internally - * - Supports test filtering via -Dtest.filter system property - * - * Usage: - * ```kotlin - * plugins { - * id("com.datadoghq.profiler-test") - * } - * - * profilerTest { - * // Required: the project providing the profiler library - * profilerLibProject.set(":ddprof-lib") - * - * // Optional: override native library directory - * nativeLibDir.set(layout.buildDirectory.dir("libs/native")) - * - * // Optional: add extra JVM args - * extraJvmArgs.add("-Xms256m") - * - * // Optional: specify which configs get application tasks (default: all active configs) - * applicationConfigs.set(listOf("release", "debug")) - * - * // Optional: main class for application tasks - * applicationMainClass.set("com.datadoghq.profiler.unwinding.UnwindingValidator") - * } - * - * // Run tests (all platforms use same syntax): - * ./gradlew :ddprof-test:testdebug -Ptests=ClassName.methodName - * ./gradlew :ddprof-test:testdebug -Ptests=ClassName - * ./gradlew :ddprof-test:testdebug -Ptests="*.Pattern*" - * ``` - */ -class ProfilerTestPlugin : Plugin { - override fun apply(project: Project) { - val extension = project.extensions.create( - "profilerTest", - ProfilerTestExtension::class.java, - project - ) - - // Create base configurations eagerly so they can be extended by build scripts - // without needing afterEvaluate - project.configurations.maybeCreate("testCommon").apply { - isCanBeConsumed = false - isCanBeResolved = true - } - project.configurations.maybeCreate("mainCommon").apply { - isCanBeConsumed = false - isCanBeResolved = true - } - - // After evaluation, generate multi-config tasks if profilerLibProject is set - project.afterEvaluate { - if (extension.profilerLibProject.isPresent) { - generateMultiConfigTasks(project, extension) - } - } - } - - /** - * Shared test task configuration extracted for reuse between Test and Exec paths. - */ - private data class TestTaskConfiguration( - val configName: String, - val isActive: Boolean, - val testClasspath: FileCollection, - val standardJvmArgs: List, - val extraJvmArgs: List, - val configJvmArgs: List, - val systemProperties: Map, - val environmentVariables: Map - ) - - /** - * Build shared test configuration used by both Test and Exec task creation. - */ - private fun buildTestConfiguration( - project: Project, - extension: ProfilerTestExtension, - config: BuildConfiguration, - testCfg: Configuration, - sourceSets: SourceSetContainer - ): TestTaskConfiguration { - val configName = config.name - val testEnv = config.testEnvironment.get() - - // Build classpath - val testClasspath = sourceSets.getByName("test").runtimeClasspath.filter { file -> - !file.name.contains("ddprof-") || file.name.contains("test-tracer") - } + testCfg - - // System properties - val keepRecordings = project.hasProperty("keepJFRs") || - System.getenv("KEEP_JFRS")?.toBoolean() ?: false - val systemPropsBase = mapOf( - "ddprof_test.keep_jfrs" to keepRecordings.toString(), - "ddprof_test.config" to configName, - "ddprof_test.ci" to (project.hasProperty("CI")).toString(), - "DDPROF_TEST_DISABLE_RATE_LIMIT" to "1", - "CI" to (project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false).toString() - ) - val systemProps = systemPropsBase + testEnv - - // Environment variables (explicit for consistency across both paths) - val envVars = buildMap { - putAll(testEnv) - put("DDPROF_TEST_DISABLE_RATE_LIMIT", "1") - put("CI", (project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false).toString()) - // Pass through CI vars (needed for Exec, optional for Test) - System.getenv("LIBC")?.let { put("LIBC", it) } - System.getenv("KEEP_JFRS")?.let { put("KEEP_JFRS", it) } - System.getenv("TEST_COMMIT")?.let { put("TEST_COMMIT", it) } - System.getenv("TEST_CONFIGURATION")?.let { put("TEST_CONFIGURATION", it) } - System.getenv("SANITIZER")?.let { put("SANITIZER", it) } - } - - return TestTaskConfiguration( - configName = configName, - isActive = config.active.get(), - testClasspath = testClasspath, - standardJvmArgs = extension.standardJvmArgs.get(), - extraJvmArgs = extension.extraJvmArgs.get(), - configJvmArgs = config.testJvmArgs.get(), - systemProperties = systemProps, - environmentVariables = envVars - ) - } - - /** - * Create native Test task for glibc/macOS (normal path). - * Uses Gradle's Test task with -Ptests property support. - */ - private fun createTestTask( - project: Project, - extension: ProfilerTestExtension, - testConfig: TestTaskConfiguration, - testCfg: Configuration, - sourceSets: SourceSetContainer - ) { - project.tasks.register("test${testConfig.configName.replaceFirstChar { it.uppercase() }}", Test::class.java) { - val testTask = this - testTask.description = "Runs unit tests with the ${testConfig.configName} library variant" - testTask.group = "verification" - testTask.onlyIf { testConfig.isActive && !project.hasProperty("skip-tests") } - - // Dependencies - testTask.dependsOn(project.tasks.named("compileTestJava")) - testTask.dependsOn(testCfg) - testTask.dependsOn(sourceSets.getByName("test").output) - - // Test class directories and classpath - testTask.testClassesDirs = sourceSets.getByName("test").output.classesDirs - testTask.classpath = testConfig.testClasspath - - // Use JUnit Platform - testTask.useJUnitPlatform() - - // Configure Java executable - bypasses toolchain system - testTask.setExecutable(PlatformUtils.testJavaExecutable()) - - // Environment variables (from testConfig which already includes DDPROF_TEST_DISABLE_RATE_LIMIT and CI) - testConfig.environmentVariables.forEach { (key, value) -> - testTask.environment(key, value) - } - - // Test output - testTask.testLogging { - val logging = this - logging.events("started", "passed", "skipped", "failed") - logging.showStandardStreams = true - } - - // UNIFIED INTERFACE: Support -Ptests property (same as musl) - val testsFilter = project.findProperty("tests") as String? - if (testsFilter != null) { - // Forward -Ptests to Test task's filter - testTask.filter.includeTestsMatching(testsFilter) - } - - // Warn if --tests flag was used instead of -Ptests - testTask.doFirst { - val filterPatterns = testTask.filter.includePatterns - if (filterPatterns.isNotEmpty() && testsFilter == null) { - project.logger.warn("") - project.logger.warn("WARNING: --tests flag detected. While it works on glibc/macOS, it will FAIL on musl systems.") - project.logger.warn("For consistent behavior across all platforms, please use -Ptests instead:") - project.logger.warn(" ./gradlew :ddprof-test:${testTask.name} -Ptests=${filterPatterns.first()}") - project.logger.warn("") - } - } - - // JVM arguments and system properties - configure in doFirst like main does - testTask.doFirst { - val allArgs = mutableListOf() - allArgs.addAll(testConfig.standardJvmArgs) - - if (extension.nativeLibDir.isPresent) { - allArgs.add("-Djava.library.path=${extension.nativeLibDir.get().asFile.absolutePath}") - } - - // System properties as JVM args - testConfig.systemProperties.forEach { (key, value) -> - allArgs.add("-D$key=$value") - } - - allArgs.addAll(testConfig.extraJvmArgs) - allArgs.addAll(testConfig.configJvmArgs) - testTask.jvmArgs(allArgs) - } - - // Sanitizer conditions - when (testConfig.configName) { - "asan" -> testTask.onlyIf { - PlatformUtils.locateLibasan() != null && - // Skip J9+ASAN: OpenJ9 has known GC stack-scanning and defineClass - // race bugs exposed by ASAN timing - // https://github.com/eclipse-openj9/openj9/issues/23514 - !PlatformUtils.isTestJvmJ9() - } - // TSan + JVM integration tests are incompatible: the profiler's signal - // handlers (SIGPROF at 1ms) are TSan-instrumented; when a signal fires - // while TSan is updating its shadow memory it causes re-entrance and a - // SIGSEGV. TSan coverage is provided by the C++ gtest suite (gtestTsan). - "tsan" -> testTask.onlyIf { false } - } - } - } - - /** - * Create Exec task with custom test runner for musl platforms. - * Uses ProfilerTestRunner with JUnit Platform Launcher API directly. - * Supports unified -Ptests property interface for test filtering. - */ - private fun createExecTestTask( - project: Project, - extension: ProfilerTestExtension, - testConfig: TestTaskConfiguration, - testCfg: Configuration, - sourceSets: SourceSetContainer - ) { - project.tasks.register("test${testConfig.configName.replaceFirstChar { it.uppercase() }}", Exec::class.java) { - val execTask = this - execTask.description = "Runs unit tests with the ${testConfig.configName} library variant (musl workaround)" - execTask.group = "verification" - execTask.onlyIf { testConfig.isActive && !project.hasProperty("skip-tests") } - - // Dependencies - execTask.dependsOn(project.tasks.named("compileTestJava")) - execTask.dependsOn(testCfg) - execTask.dependsOn(sourceSets.getByName("test").output) - - // Configure at execution time to capture -Ptests filter - execTask.doFirst { - execTask.executable = PlatformUtils.testJavaExecutable() - - val allArgs = mutableListOf() - - // JVM args - allArgs.addAll(testConfig.standardJvmArgs) - if (extension.nativeLibDir.isPresent) { - allArgs.add("-Djava.library.path=${extension.nativeLibDir.get().asFile.absolutePath}") - } - allArgs.addAll(testConfig.extraJvmArgs) - allArgs.addAll(testConfig.configJvmArgs) - - // System properties - testConfig.systemProperties.forEach { (key, value) -> - allArgs.add("-D$key=$value") - } - - // UNIFIED INTERFACE: Test filter from -Ptests property - val testsFilter = project.findProperty("tests") as String? - if (testsFilter != null) { - allArgs.add("-Dtest.filter=$testsFilter") - } - - // Classpath (includes custom test runner) - allArgs.add("-cp") - allArgs.add(testConfig.testClasspath.asPath) - - // Use custom test runner (NOT ConsoleLauncher) - allArgs.add("com.datadoghq.profiler.test.ProfilerTestRunner") - - execTask.args = allArgs - } - - // Environment variables - testConfig.environmentVariables.forEach { (key, value) -> - execTask.environment(key, value) - } - - // CRITICAL FIX: Remove LD_LIBRARY_PATH to let RPATH work correctly - // The test JDK's launcher has RPATH set to find its own libraries ($ORIGIN/../lib/jli) - // But LD_LIBRARY_PATH overrides RPATH and causes it to load the wrong libjli.so - // Solution: Unset LD_LIBRARY_PATH entirely to let RPATH take precedence - execTask.doFirst { - val currentLdLibPath = (execTask.environment["LD_LIBRARY_PATH"] as? String) ?: System.getenv("LD_LIBRARY_PATH") - if (!currentLdLibPath.isNullOrEmpty()) { - project.logger.info("Removing LD_LIBRARY_PATH to prevent cross-JDK library conflicts (was: $currentLdLibPath)") - execTask.environment.remove("LD_LIBRARY_PATH") - } - } - - // Sanitizer conditions - when (testConfig.configName) { - "asan" -> execTask.onlyIf { - PlatformUtils.locateLibasan() != null && - // Skip J9+ASAN: OpenJ9 has known GC stack-scanning and defineClass - // race bugs exposed by ASAN timing - // https://github.com/eclipse-openj9/openj9/issues/23514 - !PlatformUtils.isTestJvmJ9() - } - "tsan" -> execTask.onlyIf { false } // same reason as testTask above - } - } - } - - private fun generateMultiConfigTasks(project: Project, extension: ProfilerTestExtension) { - val nativeBuildExt = project.rootProject.extensions.findByType(NativeBuildExtension::class.java) - ?: return // No native build extension, nothing to generate - - val currentPlatform = PlatformUtils.currentPlatform - val currentArchitecture = PlatformUtils.currentArchitecture - val activeConfigurations = nativeBuildExt.getActiveConfigurations(currentPlatform, currentArchitecture) - - if (activeConfigurations.isEmpty()) { - return - } - - val profilerLibProjectPath = extension.profilerLibProject.get() - val tracerProjectPath = extension.tracerProject.getOrElse(":ddprof-test-tracer") - // Default to all active configs if not explicitly specified - val explicitAppConfigs = extension.applicationConfigs.get() - val applicationConfigs = if (explicitAppConfigs.isEmpty()) { - activeConfigurations.map { it.name } - } else { - explicitAppConfigs - } - val appMainClass = extension.applicationMainClass.getOrElse("") - - val sourceSets = project.extensions.getByType(SourceSetContainer::class.java) - - // Get the base configurations (created eagerly in apply()) - val testCommon = project.configurations.getByName("testCommon") - val mainCommon = project.configurations.getByName("mainCommon") - - // Add common dependencies to base configurations - addCommonTestDependencies(project, testCommon, tracerProjectPath) - addCommonMainDependencies(project, mainCommon, tracerProjectPath) - - val configNames = mutableListOf() - - // Generate tasks for each active configuration - activeConfigurations.forEach { config -> - val configName = config.name - val isActive = config.active.get() - val testEnv = config.testEnvironment.get() - - configNames.add(configName) - - // Create test configuration - val testCfg = project.configurations.maybeCreate("test${configName.replaceFirstChar { it.uppercaseChar() }}Implementation").apply { - isCanBeConsumed = false - isCanBeResolved = true - extendsFrom(testCommon) - } - testCfg.dependencies.add( - project.dependencies.project(mapOf("path" to profilerLibProjectPath, "configuration" to configName)) - ) - - // Build shared configuration - val testConfig = buildTestConfiguration(project, extension, config, testCfg, sourceSets) - - // Platform-conditional task creation - // Check both PlatformUtils.isMusl() and LIBC environment variable (set by Docker) - val isMuslSystem = PlatformUtils.isMusl() || System.getenv("LIBC") == "musl" - if (isMuslSystem) { - project.logger.info("Creating Exec task for $configName (musl workaround, LIBC=${System.getenv("LIBC")})") - createExecTestTask(project, extension, testConfig, testCfg, sourceSets) - } else { - project.logger.info("Creating Test task for $configName (glibc/macOS, LIBC=${System.getenv("LIBC")})") - createTestTask(project, extension, testConfig, testCfg, sourceSets) - } - - // Create application tasks for specified configs - if (configName in applicationConfigs && appMainClass.isNotEmpty()) { - // Create main configuration - val mainCfg = project.configurations.maybeCreate("${configName}Implementation").apply { - isCanBeConsumed = false - isCanBeResolved = true - extendsFrom(mainCommon) - } - mainCfg.dependencies.add( - project.dependencies.project(mapOf("path" to profilerLibProjectPath, "configuration" to configName)) - ) - - // Create run task using Exec to bypass Gradle's toolchain system - project.tasks.register("runUnwindingValidator${configName.replaceFirstChar { it.uppercaseChar() }}", Exec::class.java) { - val runTask = this - runTask.onlyIf { isActive } - runTask.description = "Run the unwinding validator application ($configName config)" - runTask.group = "application" - - runTask.dependsOn(project.tasks.named("compileJava")) - runTask.dependsOn(mainCfg) - - val mainClasspath = sourceSets.getByName("main").runtimeClasspath + mainCfg - - runTask.doFirst { - // Set executable at execution time so environment variables are read correctly - runTask.executable = PlatformUtils.testJavaExecutable() - - val allArgs = mutableListOf() - allArgs.addAll(extension.standardJvmArgs.get()) - allArgs.addAll(extension.extraJvmArgs.get()) - allArgs.add("-cp") - allArgs.add(mainClasspath.asPath) - allArgs.add(appMainClass) - - // Handle validatorArgs property - if (project.hasProperty("validatorArgs")) { - allArgs.addAll((project.property("validatorArgs") as String).split(" ")) - } - - runTask.args = allArgs - } - - if (testEnv.isNotEmpty()) { - testEnv.forEach { (key, value) -> - runTask.environment(key, value) - } - } - - // CRITICAL FIX: Remove LD_LIBRARY_PATH to let RPATH work correctly - runTask.doFirst { - val currentLdLibPath = (runTask.environment["LD_LIBRARY_PATH"] as? String) ?: System.getenv("LD_LIBRARY_PATH") - if (!currentLdLibPath.isNullOrEmpty()) { - project.logger.info("Removing LD_LIBRARY_PATH to prevent cross-JDK library conflicts (was: $currentLdLibPath)") - runTask.environment.remove("LD_LIBRARY_PATH") - } - } - } - - // Create report task using Exec to bypass Gradle's toolchain system - project.tasks.register("unwindingReport${configName.replaceFirstChar { it.uppercaseChar() }}", Exec::class.java) { - val reportTask = this - reportTask.onlyIf { isActive } - reportTask.description = "Generate unwinding report for CI ($configName config)" - reportTask.group = "verification" - - reportTask.dependsOn(project.tasks.named("compileJava")) - reportTask.dependsOn(mainCfg) - - val mainClasspath = sourceSets.getByName("main").runtimeClasspath + mainCfg - - reportTask.doFirst { - // Set executable at execution time so environment variables are read correctly - reportTask.executable = PlatformUtils.testJavaExecutable() - - project.file("${project.layout.buildDirectory.get()}/reports").mkdirs() - - val allArgs = mutableListOf() - allArgs.addAll(extension.standardJvmArgs.get()) - allArgs.addAll(extension.extraJvmArgs.get()) - allArgs.add("-cp") - allArgs.add(mainClasspath.asPath) - allArgs.add(appMainClass) - allArgs.add("--output-format=markdown") - allArgs.add("--output-file=build/reports/unwinding-summary.md") - - reportTask.args = allArgs - } - - if (testEnv.isNotEmpty()) { - testEnv.forEach { (key, value) -> - reportTask.environment(key, value) - } - } - reportTask.environment("CI", project.hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false) - - // CRITICAL FIX: Remove LD_LIBRARY_PATH to let RPATH work correctly - reportTask.doFirst { - val currentLdLibPath = (reportTask.environment["LD_LIBRARY_PATH"] as? String) ?: System.getenv("LD_LIBRARY_PATH") - if (!currentLdLibPath.isNullOrEmpty()) { - project.logger.info("Removing LD_LIBRARY_PATH to prevent cross-JDK library conflicts (was: $currentLdLibPath)") - reportTask.environment.remove("LD_LIBRARY_PATH") - } - } - } - } - } - - // Create convenience delegate tasks - if (applicationConfigs.isNotEmpty()) { - project.tasks.register("runUnwindingValidator", DefaultTask::class.java) { - val delegateTask = this - delegateTask.description = "Run unwinding validator (delegates to release if available, otherwise debug)" - delegateTask.group = "application" - delegateTask.dependsOn( - project.provider { - when { - project.tasks.findByName("runUnwindingValidatorRelease") != null -> listOf("runUnwindingValidatorRelease") - project.tasks.findByName("runUnwindingValidatorDebug") != null -> listOf("runUnwindingValidatorDebug") - else -> throw GradleException("No suitable build configuration available for unwinding validator") - } - } - ) - } - - project.tasks.register("unwindingReport", DefaultTask::class.java) { - val delegateTask = this - delegateTask.description = "Generate unwinding report (delegates to release if available, otherwise debug)" - delegateTask.group = "verification" - delegateTask.dependsOn( - project.provider { - when { - project.tasks.findByName("unwindingReportRelease") != null -> listOf("unwindingReportRelease") - project.tasks.findByName("unwindingReportDebug") != null -> listOf("unwindingReportDebug") - else -> throw GradleException("No suitable build configuration available for unwinding report") - } - } - ) - } - } - - // Wire up gtest -> test dependencies (C++ tests run before Java tests) - project.gradle.projectsEvaluated { - configNames.forEach { cfgName -> - val capitalizedCfgName = cfgName.replaceFirstChar { it.uppercaseChar() } - val testTaskName = "test$capitalizedCfgName" - val testTask = project.tasks.findByName(testTaskName) - val profilerLibProject = project.rootProject.findProject(profilerLibProjectPath) - - if (profilerLibProject != null && testTask != null) { - // gtest runs before test (C++ unit tests run before Java integration tests) - val gtestTaskName = "gtest${capitalizedCfgName}" - try { - val gtestTask = profilerLibProject.tasks.named(gtestTaskName) - testTask.dependsOn(gtestTask) - } catch (e: org.gradle.api.UnknownTaskException) { - project.logger.info("Task $gtestTaskName not found in $profilerLibProjectPath - gtest may not be available") - } - } - } - } - } - - private fun addCommonTestDependencies(project: Project, configuration: Configuration, tracerProjectPath: String) { - // Add tracer project dependency - other deps added via version catalog in build file - configuration.dependencies.add( - project.dependencies.project(mapOf("path" to tracerProjectPath)) - ) - } - - private fun addCommonMainDependencies(project: Project, configuration: Configuration, tracerProjectPath: String) { - // Add tracer project dependency - other deps added via version catalog in build file - configuration.dependencies.add( - project.dependencies.project(mapOf("path" to tracerProjectPath)) - ) - } -} - -/** - * Extension for profiler test configuration. - */ -abstract class ProfilerTestExtension @Inject constructor( - project: Project, - objects: org.gradle.api.model.ObjectFactory -) { - - /** - * Standard JVM arguments applied to all Exec-based test and application tasks. - * These are the common profiler testing requirements. - */ - abstract val standardJvmArgs: ListProperty - - /** - * Additional JVM arguments to add beyond the standard set. - */ - abstract val extraJvmArgs: ListProperty - - /** - * Directory containing native test libraries. - * When set, adds -Djava.library.path to test Exec tasks. - */ - val nativeLibDir: org.gradle.api.file.DirectoryProperty = objects.directoryProperty() - - /** - * Whether to skip tests when -Pskip-tests is set. - * Default: true - */ - abstract val respectSkipTests: Property - - /** - * The project path providing the profiler library (e.g., ":ddprof-lib"). - * When set, enables automatic multi-config task generation. - */ - abstract val profilerLibProject: Property - - /** - * The project path providing the test tracer (default: ":ddprof-test-tracer"). - */ - abstract val tracerProject: Property - - /** - * Configurations that should have application tasks (runUnwindingValidator*, unwindingReport*). - * Default: empty list means all active configurations get application tasks. - * Set explicitly to restrict which configs get app tasks. - */ - abstract val applicationConfigs: ListProperty - - /** - * Main class for application tasks. - * Default: "com.datadoghq.profiler.unwinding.UnwindingValidator" - */ - abstract val applicationMainClass: Property - - init { - // Standard JVM arguments for profiler testing - standardJvmArgs.convention(listOf( - "-Djdk.attach.allowAttachSelf", // Allow profiler to attach to self - "-Djol.tryWithSudo=true", // JOL memory layout analysis - "-XX:ErrorFile=build/hs_err_pid%p.log", // HotSpot error file location - "-XX:+ResizeTLAB", // Allow TLAB resizing for allocation profiling - "-Xmx512m" // Default heap size for tests - )) - - extraJvmArgs.convention(emptyList()) - respectSkipTests.convention(true) - tracerProject.convention(":ddprof-test-tracer") - applicationConfigs.convention(emptyList()) // Empty = all active configs get app tasks - applicationMainClass.convention("com.datadoghq.profiler.unwinding.UnwindingValidator") - } -} diff --git a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/SpotlessConventionPlugin.kt b/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/SpotlessConventionPlugin.kt deleted file mode 100644 index 1d9e5fc69..000000000 --- a/build-logic/conventions/src/main/kotlin/com/datadoghq/profiler/SpotlessConventionPlugin.kt +++ /dev/null @@ -1,105 +0,0 @@ - -package com.datadoghq.profiler - -import com.diffplug.gradle.spotless.SpotlessExtension -import org.gradle.api.Plugin -import org.gradle.api.Project - -/** - * Convention plugin that configures Spotless code formatting for all project types. - * Supports Java, Groovy, Kotlin, Scala, C++, and miscellaneous file formatting. - */ -class SpotlessConventionPlugin : Plugin { - override fun apply(project: Project) { - project.pluginManager.apply("com.diffplug.spotless") - - val configPath = project.rootProject.file("gradle").absolutePath - val spotless = project.extensions.getByType(SpotlessExtension::class.java) - - // Java formatting (Google Java Format 1.7 - last version supporting Java 8) - if (project.plugins.hasPlugin("java")) { - spotless.java { - toggleOffOn() - target("src/**/*.java") - targetExclude("src/test/resources/**") - googleJavaFormat("1.7") - } - } - - // Groovy Gradle files - spotless.groovyGradle { - toggleOffOn() - target("*.gradle", "gradle/**/*.gradle") - greclipse().configFile("$configPath/enforcement/spotless-groovy.properties") - } - - // Kotlin Gradle files - spotless.kotlinGradle { - toggleOffOn() - target("*.gradle.kts") - // ktlint 1.5.0 is compatible with Kotlin 2.x and Spotless 7.x - ktlint("1.5.0").editorConfigOverride( - mapOf( - "indent_size" to "2", - "continuation_indent_size" to "2" - ) - ) - } - - // Groovy source files - if (project.plugins.hasPlugin("groovy")) { - val skipJavaExclude = project.findProperty("groovySkipJavaExclude") as? Boolean ?: false - spotless.groovy { - toggleOffOn() - if (!skipJavaExclude) { - excludeJava() - } - greclipse().configFile("$configPath/enforcement/spotless-groovy.properties") - } - } - - // Scala source files - if (project.plugins.hasPlugin("scala")) { - spotless.scala { - toggleOffOn() - scalafmt("2.7.5").configFile("$configPath/enforcement/spotless-scalafmt.conf") - } - } - - // TODO: Enable C++ formatting in a follow-up PR - // This requires reformatting all C/C++ source files which would pollute this PR. - // - // project.pluginManager.withPlugin("com.datadoghq.native-build") { - // val clangVersion = PlatformUtils.clangFormatVersion() - // if (clangVersion != null) { - // spotless.cpp { - // target("src/main/cpp/**") - // clangFormat(clangVersion).style("file") - // } - // } - // } - - // Miscellaneous files (markdown, shell scripts, etc.) - spotless.format("misc") { - toggleOffOn() - target( - ".gitignore", - "*.md", - ".github/**/*.md", - "src/**/*.md", - "application/**/*.md", - "*.sh", - "tooling/*.sh", - ".circleci/*.sh" - ) - leadingTabsToSpaces() - trimTrailingWhitespace() - endWithNewline() - } - - // Wire spotlessCheck into the check task - project.tasks.named("check") { - dependsOn("spotlessCheck") - } - } -} diff --git a/build-logic/settings.gradle b/build-logic/settings.gradle deleted file mode 100644 index 2e69f0cf8..000000000 --- a/build-logic/settings.gradle +++ /dev/null @@ -1,14 +0,0 @@ -pluginManagement { - def mavenRepositoryProxy = providers.gradleProperty('mavenRepositoryProxy').orNull - repositories { - if (mavenRepositoryProxy != null) { - maven { url = uri(mavenRepositoryProxy) } - } - gradlePluginPortal() - mavenCentral() - } -} - -rootProject.name = 'build-logic' - -include 'conventions' diff --git a/build.gradle.kts b/build.gradle.kts deleted file mode 100644 index b811fcd96..000000000 --- a/build.gradle.kts +++ /dev/null @@ -1,98 +0,0 @@ -import java.net.URI - -buildscript { - dependencies { - classpath("com.dipien:semantic-version-gradle-plugin:2.0.0") - } - repositories { - val mavenRepositoryProxy = providers.gradleProperty("mavenRepositoryProxy").orNull - mavenLocal() - if (mavenRepositoryProxy != null) { - maven { url = uri(mavenRepositoryProxy) } - } - mavenCentral() - gradlePluginPortal() - } -} - -plugins { - id("io.github.gradle-nexus.publish-plugin") version "2.0.0" - id("com.datadoghq.native-root") -} - -version = "1.45.0" - -apply(plugin = "com.dipien.semantic-version") -version = findProperty("ddprof_version") as? String ?: version - -allprojects { - repositories { - mavenCentral() - gradlePluginPortal() - } -} - -repositories { - mavenLocal() - mavenCentral() - gradlePluginPortal() - - maven { - content { - includeGroup("com.datadoghq") - } - mavenContent { - snapshotsOnly() - } - // see https://central.sonatype.org/publish/publish-portal-snapshots/#consuming-via-gradle - url = URI("https://central.sonatype.com/repository/maven-snapshots/") - } -} - -allprojects { - group = "com.datadoghq" - - // Apply spotless formatting convention to all projects - apply(plugin = "com.datadoghq.spotless-convention") -} - -subprojects { - version = rootProject.version -} - -val isCI = hasProperty("CI") || System.getenv("CI")?.toBoolean() ?: false - -nexusPublishing { - repositories { - val forceLocal = hasProperty("forceLocal") - - if (forceLocal && !isCI) { - create("local") { - // For testing use with https://hub.docker.com/r/sonatype/nexus - // docker run --rm -d -p 8081:8081 --name nexus sonatype/nexus - // Doesn't work for testing releases though... (due to staging) - nexusUrl.set(URI("http://localhost:8081/nexus/content/repositories/releases/")) - snapshotRepositoryUrl.set(URI("http://localhost:8081/nexus/content/repositories/snapshots/")) - username.set("admin") - password.set("admin123") - } - } else { - // see https://github.com/gradle-nexus/publish-plugin#publishing-to-maven-central-via-sonatype-central - // For official documentation: - // staging repo publishing https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration - // snapshot publishing https://central.sonatype.org/publish/publish-portal-snapshots/#publishing-via-other-methods - create("sonatype") { - // see https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration - // see https://github.com/gradle-nexus/publish-plugin#publishing-to-maven-central-via-sonatype-central - // Also for official doc - // staging repo publishing https://central.sonatype.org/publish/publish-portal-ossrh-staging-api/#configuration - // snapshot publishing https://central.sonatype.org/publish/publish-portal-snapshots/#publishing-via-other-methods - nexusUrl.set(URI("https://ossrh-staging-api.central.sonatype.com/service/local/")) - snapshotRepositoryUrl.set(URI("https://central.sonatype.com/repository/maven-snapshots/")) - - username.set(System.getenv("SONATYPE_USERNAME")) - password.set(System.getenv("SONATYPE_PASSWORD")) - } - } - } -} diff --git a/ddprof-lib/benchmarks/build.gradle.kts b/ddprof-lib/benchmarks/build.gradle.kts deleted file mode 100644 index d3f0d2f3a..000000000 --- a/ddprof-lib/benchmarks/build.gradle.kts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Benchmark for testing unwinding failures. - * Uses NativeCompileTask/NativeLinkExecutableTask from build-logic. - */ - -import com.datadoghq.native.model.Platform -import com.datadoghq.native.tasks.NativeCompileTask -import com.datadoghq.native.tasks.NativeLinkExecutableTask -import com.datadoghq.native.util.PlatformUtils - -plugins { - base - // Note: Does NOT use native-build plugin - tasks are created manually below - // because this project has a single benchmark executable, not the standard - // multi-config library structure -} - -val benchmarkName = "unwind_failures_benchmark" - -// Determine if we should build for this platform -val shouldBuild = PlatformUtils.currentPlatform == Platform.MACOS || - PlatformUtils.currentPlatform == Platform.LINUX - -if (shouldBuild) { - val compiler = PlatformUtils.findCompiler(project) - - // Compile task - val compileTask = tasks.register("compileBenchmark") { - onlyIf { shouldBuild && !project.hasProperty("skip-native") } - group = "build" - description = "Compile the unwinding failures benchmark" - - this.compiler.set(compiler) - compilerArgs.set(listOf("-O2", "-g", "-std=c++17")) - sources.from(file("src/unwindFailuresBenchmark.cpp")) - includes.from(project(":ddprof-lib").file("src/main/cpp")) - objectFileDir.set(file("${layout.buildDirectory.get()}/obj/benchmark")) - } - - // Link task - val binary = file("${layout.buildDirectory.get()}/bin/$benchmarkName") - val linkTask = tasks.register("linkBenchmark") { - onlyIf { shouldBuild && !project.hasProperty("skip-native") } - dependsOn(compileTask) - group = "build" - description = "Link the unwinding failures benchmark" - - linker.set(compiler) - val args = mutableListOf("-ldl", "-lpthread") - if (PlatformUtils.currentPlatform == Platform.LINUX) { - args.add("-lrt") - } - linkerArgs.set(args) - objectFiles.from(fileTree("${layout.buildDirectory.get()}/obj/benchmark") { include("*.o") }) - outputFile.set(binary) - } - - // Wire linkBenchmark into the standard assemble lifecycle - tasks.named("assemble") { - dependsOn(linkTask) - } - - // Add a task to run the benchmark - tasks.register("runBenchmark") { - dependsOn(linkTask) - group = "verification" - description = "Run the unwinding failures benchmark" - - executable = binary.absolutePath - - // Add any additional arguments passed to the Gradle task - doFirst { - if (project.hasProperty("args")) { - args(project.property("args").toString().split(" ")) - } - println("Running benchmark: ${binary.absolutePath}") - } - - doLast { - println("Benchmark completed.") - } - } -} diff --git a/ddprof-lib/benchmarks/build_run.sh b/ddprof-lib/benchmarks/build_run.sh deleted file mode 100755 index 868d844ad..000000000 --- a/ddprof-lib/benchmarks/build_run.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -cd "${HERE}/.." - -# Build and run the benchmark using Gradle -./gradlew :ddprof-lib:benchmarks:runBenchmark diff --git a/ddprof-lib/benchmarks/src/benchmarkConfig.h b/ddprof-lib/benchmarks/src/benchmarkConfig.h deleted file mode 100644 index de47b8655..000000000 --- a/ddprof-lib/benchmarks/src/benchmarkConfig.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include -#include - -struct BenchmarkResult { - std::string name; - long long total_time_ns; - int iterations; - double avg_time_ns; -}; - -struct BenchmarkConfig { - int warmup_iterations; - int measurement_iterations; - std::string csv_file; - std::string json_file; - bool debug; - - BenchmarkConfig() : warmup_iterations(100000), measurement_iterations(1000000), debug(false) { - } -}; diff --git a/ddprof-lib/benchmarks/src/unwindFailuresBenchmark.cpp b/ddprof-lib/benchmarks/src/unwindFailuresBenchmark.cpp deleted file mode 100644 index 833da9a21..000000000 --- a/ddprof-lib/benchmarks/src/unwindFailuresBenchmark.cpp +++ /dev/null @@ -1,268 +0,0 @@ -#include "benchmarkConfig.h" -#include "unwindStats.h" -#include -#include -#include -#include -#include -#include -#include - -// Test data - using JVM stub function names -const char *TEST_NAMES[] = {"Java_java_lang_String_toString", "Java_java_lang_Object_hashCode", - "Java_java_lang_System_arraycopy", - "Java_java_lang_Thread_currentThread", "Java_java_lang_Class_getName"}; -const int NUM_NAMES = sizeof(TEST_NAMES) / sizeof(TEST_NAMES[0]); - -// Global variables -std::vector results; -BenchmarkConfig config; - -// Pre-generated random values for benchmarking -struct RandomValues { - std::vector names; - std::vector kinds; - std::vector name_indices; - - void generate(int count) { - std::mt19937 rng(42); // Fixed seed for reproducibility - names.resize(count); - kinds.resize(count); - name_indices.resize(count); - - for (int i = 0; i < count; i++) { - name_indices[i] = rng() % NUM_NAMES; - names[i] = TEST_NAMES[name_indices[i]]; - kinds[i] = static_cast(rng() % 3); - } - } -}; - -RandomValues random_values; - -void exportResultsToCSV(const std::string &filename) { - std::ofstream file(filename); - if (!file.is_open()) { - std::cerr << "Failed to open file: " << filename << std::endl; - return; - } - - // Write header - file << "Benchmark,Total Time (ns),Iterations,Average Time (ns)\n"; - - // Write data - for (const auto &result : results) { - file << result.name << "," << result.total_time_ns << "," << result.iterations << "," - << result.avg_time_ns << "\n"; - } - - file.close(); - std::cout << "Results exported to CSV: " << filename << std::endl; -} - -void exportResultsToJSON(const std::string &filename) { - std::ofstream file(filename); - if (!file.is_open()) { - std::cerr << "Failed to open file: " << filename << std::endl; - return; - } - - file << "{\n \"benchmarks\": [\n"; - for (size_t i = 0; i < results.size(); ++i) { - const auto &result = results[i]; - file << " {\n" - << " \"name\": \"" << result.name << "\",\n" - << " \"total_time_ns\": " << result.total_time_ns << ",\n" - << " \"iterations\": " << result.iterations << ",\n" - << " \"avg_time_ns\": " << result.avg_time_ns << "\n" - << " }" << (i < results.size() - 1 ? "," : "") << "\n"; - } - file << " ]\n}\n"; - - file.close(); - std::cout << "Results exported to JSON: " << filename << std::endl; -} - -// Helper function to run a benchmark with warmup -template -BenchmarkResult runBenchmark(const std::string &name, F &&func, double rng_overhead = 0.0) { - std::cout << "\n--- Benchmark: " << name << " ---" << std::endl; - - // Warmup phase - if (config.warmup_iterations > 0) { - std::cout << "Warming up with " << config.warmup_iterations << " iterations..." - << std::endl; - for (int i = 0; i < config.warmup_iterations; i++) { - func(i); - } - } - - // Measurement phase - std::cout << "Running " << config.measurement_iterations << " iterations..." << std::endl; - auto start = std::chrono::high_resolution_clock::now(); - - for (int i = 0; i < config.measurement_iterations; i++) { - func(i); - if (config.debug && i % 100000 == 0) { - std::cout << "Progress: " << (i * 100 / config.measurement_iterations) << "%" - << std::endl; - } - } - - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - - double avg_time = (double)duration.count() / config.measurement_iterations; - if (rng_overhead > 0) { - avg_time -= rng_overhead; - } - - std::cout << "Total time: " << duration.count() << " ns" << std::endl; - std::cout << "Average time per operation: " << avg_time << " ns" << std::endl; - if (rng_overhead > 0) { - std::cout << " (RNG overhead of " << rng_overhead << " ns has been subtracted)" - << std::endl; - } - - return {name, static_cast(avg_time * config.measurement_iterations), - config.measurement_iterations, avg_time}; -} - -// Benchmark just the RNG overhead -BenchmarkResult measureRNGOverhead() { - std::mt19937 rng(42); - std::vector names(config.measurement_iterations); - std::vector kinds(config.measurement_iterations); - std::vector indices(config.measurement_iterations); - - return runBenchmark("RNG Overhead", [&](int i) { - indices[i] = rng() % NUM_NAMES; - names[i] = TEST_NAMES[indices[i]]; - kinds[i] = static_cast(rng() % 3); - }); -} - -// Main benchmark function -void benchmarkUnwindFailures() { - UnwindFailures failures; - results.clear(); // Clear any previous results - - std::cout << "=== Benchmarking UnwindFailures ===" << std::endl; - std::cout << "Configuration:" << std::endl; - std::cout << " Warmup iterations: " << config.warmup_iterations << std::endl; - std::cout << " Measurement iterations: " << config.measurement_iterations << std::endl; - std::cout << " Number of test names: " << NUM_NAMES << std::endl; - std::cout << " Debug mode: " << (config.debug ? "enabled" : "disabled") << std::endl; - - // First measure RNG overhead - std::cout << "\nMeasuring RNG overhead..." << std::endl; - auto rng_overhead = measureRNGOverhead(); - double overhead_per_op = rng_overhead.avg_time_ns; - std::cout << "RNG overhead per operation: " << overhead_per_op << " ns" << std::endl; - - // Create RNG for actual benchmarks - std::mt19937 rng(42); - - // Run actual benchmarks with RNG inline and overhead subtracted internally - results.push_back(runBenchmark( - "Record Single Failure Kind", - [&](int) { - int idx = rng() % NUM_NAMES; - auto kind = static_cast(rng() % 3); - failures.record(UNWIND_FAILURE_STUB, TEST_NAMES[idx]); - }, - overhead_per_op)); - - results.push_back(runBenchmark( - "Record Mixed Failures", - [&](int) { - int idx = rng() % NUM_NAMES; - auto kind = static_cast(rng() % 3); - failures.record(kind, TEST_NAMES[idx]); - }, - overhead_per_op)); - - results.push_back(runBenchmark( - "Find Name", - [&](int) { - int idx = rng() % NUM_NAMES; - failures.findName(TEST_NAMES[idx]); - }, - overhead_per_op)); - - results.push_back(runBenchmark( - "Count Failures with Mixed Kinds", - [&](int) { - int idx = rng() % NUM_NAMES; - auto kind = static_cast(rng() % 3); - failures.count(TEST_NAMES[idx], kind); - }, - overhead_per_op)); - - // For merge benchmark, we'll pre-populate the collections since that's not part of what we're - // measuring - UnwindFailures failures1; - UnwindFailures failures2; - // Use a smaller number of items for pre-population to avoid overflow - const int prePopulateCount = std::min(1000, config.measurement_iterations / 2); - for (int i = 0; i < prePopulateCount; i++) { - int idx = rng() % NUM_NAMES; - auto kind = static_cast(rng() % 3); - failures1.record(kind, TEST_NAMES[idx]); - failures2.record(kind, TEST_NAMES[idx]); - } - - results.push_back(runBenchmark("Merge Failures", [&](int) { - failures1.merge(failures2); - })); - - std::cout << "\n=== Benchmark Complete ===" << std::endl; -} - -void printUsage(const char *programName) { - std::cout << "Usage: " << programName << " [options]\n" - << "Options:\n" - << " --csv Export results to CSV file\n" - << " --json Export results to JSON file\n" - << " --warmup Number of warmup iterations (default: 100000)\n" - << " --iterations Number of measurement iterations (default: 1000000)\n" - << " --debug Enable debug output\n" - << " -h, --help Show this help message\n"; -} - -int main(int argc, char *argv[]) { - // Parse command line arguments - for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--csv") == 0 && i + 1 < argc) { - config.csv_file = argv[++i]; - } else if (strcmp(argv[i], "--json") == 0 && i + 1 < argc) { - config.json_file = argv[++i]; - } else if (strcmp(argv[i], "--warmup") == 0 && i + 1 < argc) { - config.warmup_iterations = std::atoi(argv[++i]); - } else if (strcmp(argv[i], "--iterations") == 0 && i + 1 < argc) { - config.measurement_iterations = std::atoi(argv[++i]); - } else if (strcmp(argv[i], "--debug") == 0) { - config.debug = true; - } else if (strcmp(argv[i], "-h") == 0 || strcmp(argv[i], "--help") == 0) { - printUsage(argv[0]); - return 0; - } else { - std::cerr << "Unknown option: " << argv[i] << std::endl; - printUsage(argv[0]); - return 1; - } - } - - std::cout << "Running UnwindFailures benchmark..." << std::endl; - benchmarkUnwindFailures(); - - // Export results if requested - if (!config.csv_file.empty()) { - exportResultsToCSV(config.csv_file); - } - if (!config.json_file.empty()) { - exportResultsToJSON(config.json_file); - } - - return 0; -} diff --git a/ddprof-lib/build.gradle.kts b/ddprof-lib/build.gradle.kts deleted file mode 100644 index b39dbfc24..000000000 --- a/ddprof-lib/build.gradle.kts +++ /dev/null @@ -1,291 +0,0 @@ -import com.datadoghq.native.model.Platform -import com.datadoghq.native.util.PlatformUtils -import org.gradle.api.publish.maven.tasks.AbstractPublishToMaven -import org.gradle.api.tasks.VerificationTask - -plugins { - java - `maven-publish` - signing - id("com.github.ben-manes.versions") version "0.54.0" - id("de.undercouch.download") version "5.7.0" - id("com.datadoghq.native-build") - id("com.datadoghq.gtest") - id("com.datadoghq.scanbuild") - id("com.datadoghq.versioned-sources") -} - -val libraryName = "ddprof" -description = "Datadog Java Profiler Library" - -val componentVersion = findProperty("ddprof_version") as? String ?: version.toString() - -// Configure native build with the new plugin -nativeBuild { - version.set(componentVersion) - cppSourceDirs.set(listOf("src/main/cpp")) - includeDirectories.set( - listOf( - "src/main/cpp", - "${project(":malloc-shim").file("src/main/public")}", - ), - ) -} - -// Configure Google Test -gtest { - testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) - mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) - - // Include paths for compilation - val javaHome = PlatformUtils.javaHome() - val platformInclude = when (PlatformUtils.currentPlatform) { - Platform.LINUX -> "linux" - Platform.MACOS -> "darwin" - } - - includes.from( - "src/main/cpp", - "$javaHome/include", - "$javaHome/include/$platformInclude", - project(":malloc-shim").file("src/main/public"), - ) - - failFast.set(true) -} - -// Java configuration - using sourceCompatibility (not --release 8) -// because BufferWriter8 needs access to internal sun.nio.ch package -java { - sourceCompatibility = JavaVersion.VERSION_1_8 - targetCompatibility = JavaVersion.VERSION_1_8 -} - -// Configure versioned sources for runtime version-specific implementations -versionedSources { - versions { - register("java9") { - release.set(9) - minToolchainVersion.set(11) // Compile Java 9 code with JDK 11+ - } - } -} - -// Test configuration -tasks.test { - onlyIf { - !project.hasProperty("skip-tests") - } - useJUnitPlatform() -} - -// Copy external libs task -val copyExternalLibs by tasks.registering(Copy::class) { - if (project.hasProperty("with-libs")) { - from(project.property("with-libs") as String) { - include("**/*.so", "**/*.dylib", "**/*.debug", "**/*.dSYM/**") - } - into("$projectDir/build/classes/java/main/META-INF/native-libs") - } -} - -// Gradle 9 requires explicit dependency: compileJava9Java uses mainSourceSet.output -// which includes the copyExternalLibs destination directory -afterEvaluate { - tasks.named("compileJava9Java") { - dependsOn(copyExternalLibs) - } -} - -// Create JAR tasks for each build configuration using nativeBuild extension utilities -// Uses afterEvaluate to discover configurations dynamically from NativeBuildExtension -afterEvaluate { - nativeBuild.buildConfigurations.names.forEach { name -> - val capitalizedName = name.replaceFirstChar { it.uppercase() } - - val copyTask = tasks.register("copy${capitalizedName}Libs", Copy::class) { - from(nativeBuild.librarySourceDir(name)) { - // Exclude debug symbols from production JAR - exclude("debug/**", "*.debug", "*.dSYM/**") - } - into(nativeBuild.libraryTargetDir(name)) - - // Ensure library is built before copying (link task created by NativeBuildPlugin) - val linkTaskName = "link$capitalizedName" - if (tasks.names.contains(linkTaskName)) { - dependsOn(linkTaskName) - } - } - - val assembleJarTask = tasks.register("assemble${capitalizedName}Jar", Jar::class) { - group = "build" - description = "Assemble the $name build of the library" - dependsOn(copyExternalLibs) - - if (!project.hasProperty("skip-native")) { - dependsOn(copyTask) - } - - if (name == "debug") { - manifest { - attributes("Premain-Class" to "com.datadoghq.profiler.Main") - } - } - - from(sourceSets.main.get().output.classesDirs) - versionedSources.configureJar(this) - from(nativeBuild.libraryTargetBase(name)) { - include("**/*") - // Exclude debug symbols from production JAR - exclude("**/debug/**", "**/*.debug", "**/*.dSYM/**") - } - archiveBaseName.set(libraryName) - archiveClassifier.set(if (name == "release") "" else name) - archiveVersion.set(componentVersion) - } - - // Create consumable configuration for inter-project dependencies - // This allows other projects to depend on specific build configurations - configurations.create(name) { - isCanBeConsumed = true - isCanBeResolved = false - outgoing.artifact(assembleJarTask) - } - } -} - -// Add runBenchmarks task -tasks.register("runBenchmarks") { - dependsOn(":ddprof-lib:benchmarks:runBenchmark") - group = "verification" - description = "Run all benchmarks" -} - -// Standard JAR task -tasks.jar { - dependsOn(copyExternalLibs) -} - -// Source JAR -val sourcesJar by tasks.registering(Jar::class) { - from(sourceSets.main.get().allJava) - versionedSources.configureSourceJar(this) - archiveBaseName.set(libraryName) - archiveClassifier.set("sources") - archiveVersion.set(componentVersion) -} - -// Javadoc configuration -tasks.withType().configureEach { - dependsOn(copyExternalLibs) - // Allow javadoc to access internal sun.nio.ch package used by BufferWriter8 - (options as StandardJavadocDocletOptions).addStringOption("-add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") -} - -// Javadoc JAR -val javadocJar by tasks.registering(Jar::class) { - dependsOn(tasks.javadoc) - archiveBaseName.set(libraryName) - archiveClassifier.set("javadoc") - archiveVersion.set(componentVersion) - from( - tasks.javadoc.map { - it.destinationDir ?: throw GradleException("Javadoc task destinationDir is null - task may not have been configured properly") - }, - ) -} - -// Publishing configuration -val isGitlabCI = System.getenv("GITLAB_CI") != null -val isCI = System.getenv("CI") != null - -publishing { - publications { - create("assembled") { - groupId = "com.datadoghq" - artifactId = "ddprof" - - // Add artifacts from each build configuration - afterEvaluate { - nativeBuild.buildConfigurations.names.forEach { name -> - val capitalizedName = name.replaceFirstChar { it.uppercase() } - artifact(tasks.named("assemble${capitalizedName}Jar")) - } - } - artifact(sourcesJar) - artifact(javadocJar) - - pom { - name.set(project.name) - description.set("${project.description} ($componentVersion)") - packaging = "jar" - url.set("https://github.com/datadog/java-profiler") - - licenses { - license { - name.set("The Apache Software License, Version 2.0") - url.set("http://www.apache.org/licenses/LICENSE-2.0.txt") - distribution.set("repo") - } - } - - scm { - connection.set("scm:https://datadog@github.com/datadog/java-profiler") - developerConnection.set("scm:git@github.com:datadog/java-profiler") - url.set("https://github.com/datadog/java-profiler") - } - - developers { - developer { - id.set("datadog") - name.set("Datadog") - } - } - } - } - } -} - -signing { - useInMemoryPgpKeys(System.getenv("GPG_PRIVATE_KEY"), System.getenv("GPG_PASSWORD")) - sign(publishing.publications["assembled"]) -} - -tasks.withType().configureEach { - onlyIf { - isGitlabCI || (System.getenv("GPG_PRIVATE_KEY") != null && System.getenv("GPG_PASSWORD") != null) - } -} - -// Publication assertions -gradle.taskGraph.whenReady { - if (hasTask(":ddprof-lib:publish") || hasTask(":publishToSonatype")) { - check(project.findProperty("removeJarVersionNumbers") != true) { - "Cannot publish with removeJarVersionNumbers=true" - } - if (hasTask(":publishToSonatype")) { - checkNotNull(System.getenv("SONATYPE_USERNAME")) { "SONATYPE_USERNAME must be set" } - checkNotNull(System.getenv("SONATYPE_PASSWORD")) { "SONATYPE_PASSWORD must be set" } - if (isCI) { - checkNotNull(System.getenv("GPG_PRIVATE_KEY")) { "GPG_PRIVATE_KEY must be set in CI" } - checkNotNull(System.getenv("GPG_PASSWORD")) { "GPG_PASSWORD must be set in CI" } - } - } - } -} - -// Verify project has description (required for published projects) -afterEvaluate { - requireNotNull(description) { "Project ${project.path} is published, must have a description" } -} - -// Ensure published artifacts depend on release JAR -// Note: assembleReleaseJar is registered in afterEvaluate, so use matching instead of named -tasks.withType().configureEach { - if (name.contains("AssembledPublication")) { - dependsOn(tasks.matching { it.name == "assembleReleaseJar" }) - } - rootProject.subprojects.forEach { subproject -> - mustRunAfter(subproject.tasks.matching { it is VerificationTask }) - } -} diff --git a/ddprof-lib/fuzz/build.gradle.kts b/ddprof-lib/fuzz/build.gradle.kts deleted file mode 100644 index db87efffc..000000000 --- a/ddprof-lib/fuzz/build.gradle.kts +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Gradle build file for libFuzzer-based fuzz testing. - * This module compiles and runs fuzz targets against the profiler's C++ code - * to discover bugs through automated input generation. - */ - -plugins { - base - id("com.datadoghq.fuzz-targets") -} - -fuzzTargets { - // Source directory containing fuzz target files (fuzz_*.cpp) - fuzzSourceDir.set(project(":ddprof-lib").file("src/test/fuzz")) - - // Seed corpus directory for each target (subdirectories named by target) - corpusDir.set(project(":ddprof-lib").file("src/test/fuzz/corpus")) - - // Main profiler sources to compile with fuzz targets - profilerSourceDir.set(project(":ddprof-lib").file("src/main/cpp")) - - // Additional include directories - additionalIncludes.set( - listOf( - project(":malloc-shim").file("src/main/public").absolutePath, - ), - ) -} diff --git a/ddprof-lib/settings.gradle b/ddprof-lib/settings.gradle deleted file mode 100644 index 74d033079..000000000 --- a/ddprof-lib/settings.gradle +++ /dev/null @@ -1,3 +0,0 @@ -rootProject.name = "JavaProfiler" - -include ':ddprof-lib:benchmarks' diff --git a/ddprof-lib/src/main/cpp/arch.h b/ddprof-lib/src/main/cpp/arch.h deleted file mode 100644 index 9f03ab30c..000000000 --- a/ddprof-lib/src/main/cpp/arch.h +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _ARCH_H -#define _ARCH_H - - -#ifndef likely -# define likely(x) (__builtin_expect(!!(x), 1)) -#endif - -#ifndef unlikely -# define unlikely(x) (__builtin_expect(!!(x), 0)) -#endif - -#ifdef _LP64 -# define LP64_ONLY(code) code -#else // !_LP64 -# define LP64_ONLY(code) -#endif // _LP64 - -#define COMMA , - -typedef unsigned char u8; -typedef unsigned short u16; -typedef unsigned int u32; -typedef unsigned long long u64; - -constexpr int DEFAULT_CACHE_LINE_SIZE = 64; - -static inline u64 atomicInc(volatile u64& var, u64 increment = 1) { - return __sync_fetch_and_add(&var, increment); -} - -static inline int atomicInc(volatile u32& var, int increment = 1) { - return __sync_fetch_and_add(&var, increment); -} - -static inline int atomicInc(volatile int& var, int increment = 1) { - return __sync_fetch_and_add(&var, increment); -} - -static inline long long atomicInc(volatile long long &var, - long long increment = 1) { - return __sync_fetch_and_add(&var, increment); -} - -template -static inline long long atomicIncRelaxed(volatile T &var, - T increment = 1) { - return __atomic_fetch_add(&var, increment, __ATOMIC_RELAXED); -} - -// Atomic load/store (unordered) -template -static inline T load(volatile T& var) { - return __atomic_load_n(&var, __ATOMIC_RELAXED); -} - -static inline u64 loadAcquire(u64& var) { - return __atomic_load_n(&var, __ATOMIC_ACQUIRE); -} - -// Atomic load-acquire/release-store -template -static inline T loadAcquire(volatile T& var) { - return __atomic_load_n(&var, __ATOMIC_ACQUIRE); -} - -template -static inline void store(volatile T& var, T value) { - return __atomic_store_n(&var, value, __ATOMIC_RELAXED); -} - -static inline void storeRelease(u64& var, u64 value) { - return __atomic_store_n(&var, value, __ATOMIC_RELEASE); -} - -template -static inline void storeRelease(volatile T& var, T value) { - return __atomic_store_n(&var, value, __ATOMIC_RELEASE); -} - -#if defined(__x86_64__) || defined(__i386__) - -typedef unsigned char instruction_t; -const instruction_t BREAKPOINT = 0xcc; -const int BREAKPOINT_OFFSET = 0; - -const int SYSCALL_SIZE = 2; -const int FRAME_PC_SLOT = 1; -const int PROBE_SP_LIMIT = 4; -const int PLT_HEADER_SIZE = 16; -const int PLT_ENTRY_SIZE = 16; -const int PERF_REG_PC = 8; // PERF_REG_X86_IP - -#define spinPause() asm volatile("pause") -#define rmb() asm volatile("lfence" : : : "memory") -#define flushCache(addr) asm volatile("mfence; clflush (%0); mfence" : : "r" (addr) : "memory") - -#define callerPC() __builtin_return_address(0) -#define callerFP() __builtin_frame_address(1) -#define callerSP() ((void**)__builtin_frame_address(0) + 2) - -#elif defined(__arm__) || defined(__thumb__) - -typedef unsigned int instruction_t; -const instruction_t BREAKPOINT = 0xe7f001f0; -const instruction_t BREAKPOINT_THUMB = 0xde01de01; -const int BREAKPOINT_OFFSET = 0; - -const int SYSCALL_SIZE = sizeof(instruction_t); -const int FRAME_PC_SLOT = 1; -const int PROBE_SP_LIMIT = 0; -const int PLT_HEADER_SIZE = 20; -const int PLT_ENTRY_SIZE = 12; -const int PERF_REG_PC = 15; // PERF_REG_ARM_PC - -#define spinPause() asm volatile("yield") -#define rmb() asm volatile("dmb ish" : : : "memory") -#define flushCache(addr) __builtin___clear_cache((char*)(addr), (char*)(addr) + sizeof(instruction_t)) - -#define callerPC() __builtin_return_address(0) -#define callerFP() __builtin_frame_address(1) -#define callerSP() __builtin_frame_address(1) - -#elif defined(__aarch64__) - -typedef unsigned int instruction_t; -const instruction_t BREAKPOINT = 0xd4200000; -const int BREAKPOINT_OFFSET = 0; - -const int SYSCALL_SIZE = sizeof(instruction_t); -const int FRAME_PC_SLOT = 1; -const int PROBE_SP_LIMIT = 0; -const int PLT_HEADER_SIZE = 32; -const int PLT_ENTRY_SIZE = 16; -const int PERF_REG_PC = 32; // PERF_REG_ARM64_PC - -#define spinPause() asm volatile("isb") -#define rmb() asm volatile("dmb ish" : : : "memory") -#define flushCache(addr) __builtin___clear_cache((char*)(addr), (char*)(addr) + sizeof(instruction_t)) - -#define callerPC() ({ void* pc; asm volatile("adr %0, ." : "=r"(pc)); pc; }) -#define callerFP() ({ void* fp; asm volatile("mov %0, fp" : "=r"(fp)); fp; }) -#define callerSP() ({ void* sp; asm volatile("mov %0, sp" : "=r"(sp)); sp; }) - -#elif defined(__PPC64__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) - -typedef unsigned int instruction_t; -const instruction_t BREAKPOINT = 0x7fe00008; -// We place the break point in the third instruction slot on PPCLE as the first two are skipped if -// the call comes from within the same compilation unit according to the LE ABI. -const int BREAKPOINT_OFFSET = 8; - -const int SYSCALL_SIZE = sizeof(instruction_t); -const int FRAME_PC_SLOT = 2; -const int PROBE_SP_LIMIT = 0; -const int PLT_HEADER_SIZE = 24; -const int PLT_ENTRY_SIZE = 24; -const int PERF_REG_PC = 32; // PERF_REG_POWERPC_NIP - -#define spinPause() asm volatile("yield") // does nothing, but using or 1,1,1 would lead to other problems -#define rmb() asm volatile ("sync" : : : "memory") // lwsync would do but better safe than sorry -#define flushCache(addr) __builtin___clear_cache((char*)(addr), (char*)(addr) + sizeof(instruction_t)) - -#define callerPC() __builtin_return_address(0) -#define callerFP() __builtin_frame_address(1) -#define callerSP() __builtin_frame_address(0) - -#elif defined(__riscv) && (__riscv_xlen == 64) - -typedef unsigned int instruction_t; -#if defined(__riscv_compressed) -const instruction_t BREAKPOINT = 0x9002; // EBREAK (compressed form) -#else -const instruction_t BREAKPOINT = 0x00100073; // EBREAK -#endif -const int BREAKPOINT_OFFSET = 0; - -const int SYSCALL_SIZE = sizeof(instruction_t); -const int FRAME_PC_SLOT = 1; // return address is at -1 from FP -const int PROBE_SP_LIMIT = 0; -const int PLT_HEADER_SIZE = 24; // Best guess from examining readelf -const int PLT_ENTRY_SIZE = 24; // ...same... -const int PERF_REG_PC = 0; // PERF_REG_RISCV_PC - -#define spinPause() // No architecture support -#define rmb() asm volatile ("fence" : : : "memory") -#define flushCache(addr) __builtin___clear_cache((char*)(addr), (char*)(addr) + sizeof(instruction_t)) - -#define callerPC() __builtin_return_address(0) -#define callerFP() __builtin_frame_address(1) -#define callerSP() __builtin_frame_address(0) - -#elif defined(__loongarch_lp64) - -typedef unsigned int instruction_t; -const instruction_t BREAKPOINT = 0x002a0005; // EBREAK -const int BREAKPOINT_OFFSET = 0; - -const int SYSCALL_SIZE = sizeof(instruction_t); -const int FRAME_PC_SLOT = 1; -const int PROBE_SP_LIMIT = 0; -const int PLT_HEADER_SIZE = 32; -const int PLT_ENTRY_SIZE = 16; -const int PERF_REG_PC = 0; // PERF_REG_LOONGARCH_PC - -#define spinPause() asm volatile("ibar 0x0") -#define rmb() asm volatile("dbar 0x0" : : : "memory") -#define flushCache(addr) __builtin___clear_cache((char*)(addr), (char*)(addr) + sizeof(instruction_t)) - -#define callerPC() __builtin_return_address(0) -#define callerFP() __builtin_frame_address(1) -#define callerSP() __builtin_frame_address(0) - -#else - -#error "Compiling on unsupported arch" - -#endif - - -// On Apple M1 and later processors, memory is either writable or executable (W^X) -#if defined(__aarch64__) && defined(__APPLE__) -# define WX_MEMORY true -#else -# define WX_MEMORY false -#endif - -// Pointer authentication (PAC) support. -// Only 48-bit virtual addresses are currently supported. -#ifdef __aarch64__ -const unsigned long PAC_MASK = WX_MEMORY ? 0x7fffffffffffUL : 0xffffffffffffUL; - -static inline const void* stripPointer(const void* p) { - return (const void*) ((unsigned long)p & PAC_MASK); -} -#else -# define stripPointer(p) (p) -#endif - - -#endif // _ARCH_H diff --git a/ddprof-lib/src/main/cpp/arguments.cpp b/ddprof-lib/src/main/cpp/arguments.cpp deleted file mode 100644 index e76352cd5..000000000 --- a/ddprof-lib/src/main/cpp/arguments.cpp +++ /dev/null @@ -1,540 +0,0 @@ -/* - * Copyright 2017 Andrei Pangin - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "arguments.h" -#include "vmEntry.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -// Predefined value that denotes successful operation -const Error Error::OK(NULL); - -// Extra buffer space for expanding file pattern -const size_t EXTRA_BUF_SIZE = 512; - -static const Multiplier NANOS[] = { - {'n', 1}, {'u', 1000}, {'m', 1000000}, {'s', 1000000000}, {0, 0}}; -static const Multiplier BYTES[] = { - {'b', 1}, {'k', 1024}, {'m', 1048576}, {'g', 1073741824}, {0, 0}}; -static const Multiplier SECONDS[] = { - {'s', 1}, {'m', 60}, {'h', 3600}, {'d', 86400}, {0, 0}}; -static const Multiplier UNIVERSAL[] = { - {'n', 1}, {'u', 1000}, {'m', 1000000}, {'s', 1000000000}, - {'b', 1}, {'k', 1024}, {'g', 1073741824}, {0, 0}}; - -// Simulate switch statement over string hashes -#define SWITCH(arg) \ - if (0) - -#define CASE(s) \ - } \ - else if (strcasecmp(arg, s) == 0) { - -#define DEFAULT() \ - } \ - else { - -// Parses agent arguments. -// The format of the string is: -// arg[,arg...] -// where arg is one of the following options: -// start - start profiling -// resume - start or resume profiling without resetting -// collected data stop - stop profiling check - -// check if the specified profiling event is available status - -// print profiling status (inactive / running for X seconds) list - show the -// list of available profiling events version[=full] - display the agent -// version event=EVENT - which event to trace (cpu, wall, -// cache-misses, etc.) cpu[=INTERVAL] - enable CPU profiling with the -// specified sampling interval wall[=INTERVAL] - enable wall-clock -// profiling with the specified sampling interval walltpt=THREADS - -// sample THREADS threads per tick in wall-clock profiling -// memory[=BYTES[:[a|[l|L[:RATIO]]]]] -// - memory profiling with suggested interval in bytes -// - 'a'=record allocation,'l'=record liveness,'L'=record -// liveness and heap usage -// - RATIO is a floating point number between 0.01 -// and 1.0 to subsample live samples -// - eg. 'memory=1048576:a:L:0.1' to sample every 1MB -// with allocation, liveness and heap usage tracking, -// and keep the liveness track of 10% of the allocation -// samples -// generations - track surviving generations -// lightweight[=BOOL] - enable lightweight profiling - events without -// stacktraces (default: true) -// remotesym[=BOOL] - enable remote symbolication for native frames -// (stores build-id and PC offset instead of symbol names) -// jfr - dump events in Java -// Flight Recorder format interval=N - sampling interval in ns -// (default: 10'000'000, i.e. 10 ms) jstackdepth=N - maximum Java stack -// depth (default: 2048) safemode=BITS - disable stack recovery -// techniques (default: 0, i.e. everything enabled) file=FILENAME - -// output file name for dumping log=FILENAME - log warnings and errors -// to the given dedicated stream loglevel=LEVEL - logging level: TRACE, -// DEBUG, INFO, WARN, ERROR, or NONE filter=FILTER - thread filter -// cstack=MODE - how to collect C stack frames in addition to Java -// stack -// MODE is 'fp', 'dwarf', 'lbr', 'vm' or 'no' -// allkernel - include only kernel-mode events -// alluser - include only user-mode events -// - -Error Arguments::parse(const char *args) { - if (args == NULL) { - return Error::OK; - } - - size_t len = strlen(args); - if (_buf != NULL) { - free(_buf); - } - _buf = (char *)malloc(len + EXTRA_BUF_SIZE + 1); - if (_buf == NULL) { - return Error("Not enough memory to parse arguments"); - } - char *args_copy = strcpy(_buf + EXTRA_BUF_SIZE, args); - - const char *msg = NULL; - - for (char *arg = strtok(args_copy, ","); arg != NULL; - arg = strtok(NULL, ",")) { - char *value = strchr(arg, '='); - if (value != NULL) - *value++ = 0; - - SWITCH(arg) { - // Actions - CASE("start") - _action = ACTION_START; - - CASE("resume") - _action = ACTION_RESUME; - - CASE("stop") - _action = ACTION_STOP; - - CASE("check") - _action = ACTION_CHECK; - - CASE("status") - _action = ACTION_STATUS; - - CASE("list") - _action = ACTION_LIST; - - CASE("version") - _action = ACTION_VERSION; - - CASE("jfr") - if (value != NULL) { - _jfr_options = (int)strtol(value, NULL, 0); - } - - CASE("cpu") - _cpu = value == NULL ? 0 : parseUnits(value, NANOS); - if (_cpu < 0) { - msg = "cpu must be >= 0"; - } - // vtable_target: resolve vtable/itable stub receiver classes in CPU traces. - // Signal handler stores the raw receiver VMSymbol* in a BCI_VTABLE_RECEIVER - // frame (no lock, no map lookup, no allocation). Resolution happens at dump - // time via SafeAccess-protected reads in Lookup::resolveVTableReceiver, - // which is crash-safe against concurrent class unloading. _class_map only - // grows with classes actually sampled during the chunk. - _features.vtable_target = 1; - - CASE("wall") - if (value == NULL) { - _wall = 0; - } else { - if (value[0] == '~') { - _wall_collapsing = true; - _wall = parseUnits(value + 1, NANOS); - } else { - _wall = parseUnits(value, NANOS); - } - } - if (_wall < 0) { - msg = "wall must be >= 0"; - } - - CASE("walltpt") - if (value == NULL || (_wall_threads_per_tick = atoi(value)) <= 0) { - msg = "walltpt must be > 0"; - } - - CASE("event") - if (value == NULL || value[0] == 0) { - msg = "event must not be empty"; - } else if (strcmp(value, EVENT_ALLOC) == 0) { - if (_memory < 0) - _memory = 0; - } else if (_event != NULL) { - msg = "Duplicate event argument"; - } else { - _event = value; - } - - CASE("memory") - char *config = value ? strchr(value, ':') : nullptr; - char *ratio = nullptr; - if (config) { - *(config++) = 0; // terminate the 'value' string and update the pointer - // to the 'config' section - ratio = strchr(config, ':'); - if (ratio) { - *(ratio++) = 0; // terminate the preceding 'config' string and update - // the pointer to the 'ratio' section - } - } - _memory = - value == nullptr ? DEFAULT_ALLOC_INTERVAL : parseUnits(value, BYTES); - if (_memory >= 0) { - if (config) { - if (strchr(config, 'a')) { - _record_allocations = true; - } - if (strchr(config, 'l')) { - _record_liveness = true; - } else if (strchr(config, 'L')) { - _record_liveness = true; - _record_heap_usage = true; - } - if (_record_liveness && ratio) { // live-sample ratio is only - // applicable when tracking liveness - char *endptr; - _live_samples_ratio = - std::max(std::min(strtod(ratio, &endptr), 1.0), - 0.01); // subsample at least 1% but not more than 100% - } - } else { - // enable both allocations and liveness tracking - _record_allocations = true; - _record_liveness = true; - } - } - - CASE("generations") - _gc_generations = value != NULL && strcmp(value, "true") == 0; - if (_gc_generations && _memory <= 0) { - _memory = - 4 * 1024 * - 1024; // very conservative sampling interval to reduce overhead - } - - CASE("interval") - if (value == NULL || (_interval = parseUnits(value, UNIVERSAL)) <= 0) { - msg = "Invalid interval"; - } - - CASE("jstackdepth") - if (value == NULL || (_jstackdepth = atoi(value)) <= 0) { - msg = "jstackdepth must be > 0"; - } - - CASE("safemode") - _safe_mode = value == NULL ? INT_MAX : (int)strtol(value, NULL, 0); - - CASE("file") - if (value == NULL || value[0] == 0) { - msg = "file must not be empty"; - } - _file = value; - - CASE("log") - _log = value == NULL || value[0] == 0 ? NULL : value; - - CASE("loglevel") - if (value == NULL || value[0] == 0) { - msg = "loglevel must not be empty"; - } - _loglevel = value; - - // Filters - CASE("filter") - _filter = value == NULL ? "" : value; - - CASE("allkernel") - _ring = RING_KERNEL; - - CASE("alluser") - _ring = RING_USER; - - CASE("cstack") - if (value != NULL) { - if (strcmp(value, "fp") == 0) { - _cstack = CSTACK_FP; - } else if (strcmp(value, "dwarf") == 0) { - _cstack = CSTACK_DWARF; - } else if (strcmp(value, "lbr") == 0) { - _cstack = CSTACK_LBR; - } else if (strcmp(value, "vm") == 0) { - _cstack = CSTACK_VM; - } else if (strcmp(value, "vmx") == 0) { - // cstack=vmx is a shorthand for cstack=vm,features=mixed; carrier-frame - // unwinding is enabled automatically since vmx already traverses entry frames - _cstack = CSTACK_VM; - _features.mixed = 1; - _features.carrier_frames = 1; - } else { - _cstack = CSTACK_NO; - } - } - - CASE("attributes") - if (value != NULL) { - std::string input(value); - std::size_t start = 0; - std::size_t end; - while ((end = input.find(";", start)) != std::string::npos) { - _context_attributes.push_back(input.substr(start, end - start)); - start = end + 1; - } - _context_attributes.push_back(input.substr(start)); - } - - CASE("lightweight") - if (value != NULL) { - switch (value[0]) { - case 'y': // yes - case 't': // true - _lightweight = true; - break; - default: - _lightweight = false; - } - } - - CASE("mcleanup") - if (value != NULL) { - switch (value[0]) { - case 'n': // no - case 'f': // false - case '0': // 0 - _enable_method_cleanup = false; - break; - case 'y': // yes - case 't': // true - case '1': // 1 - default: - _enable_method_cleanup = true; - } - } else { - // No value means enable - _enable_method_cleanup = true; - } - - CASE("remotesym") - if (value != NULL) { - switch (value[0]) { - case 'n': // no - case 'f': // false - case '0': // 0 - _remote_symbolication = false; - break; - case 'y': // yes - case 't': // true - case '1': // 1 - default: - _remote_symbolication = true; - } - } else { - // No value means enable - _remote_symbolication = true; - } - - CASE("jvmtistacks") - if (value != NULL) { - switch (value[0]) { - case 'y': // yes - case 't': // true - case '1': // 1 - _jvmtistacks = true; - break; - default: - _jvmtistacks = false; - } - } else { - _jvmtistacks = true; - } - - CASE("wallprecheck") - if (value != NULL) { - _wall_precheck = strcmp(value, "false") != 0 && strcmp(value, "0") != 0; - } else { - // No value means enable - _wall_precheck = true; - } - - CASE("wallsampler") - if (value != NULL) { - switch (value[0]) { - case 'j': - _wallclock_sampler = JVMTI; - break; - case 'a': - default: - _wallclock_sampler = ASGCT; - } - } - - CASE("nativemem") - _nativemem = value == NULL ? 0 : parseUnits(value, BYTES); - if (_nativemem < 0) { - msg = "nativemem must be >= 0"; - } - - CASE("natsock") - if (value != NULL) { - _nativesocket_interval = parseUnits(value, NANOS); - if (_nativesocket_interval < 0) { - msg = "natsock interval must be >= 0"; - } else { - _nativesocket = true; - } - } else { - _nativesocket = true; - } - - DEFAULT() - if (_unknown_arg == NULL) - _unknown_arg = arg; - } - } - - // Return error only after parsing all arguments, when 'log' is already set - if (msg != NULL) { - return Error(msg); - } - - if (_event == NULL && _cpu < 0 && _wall < 0 && _memory < 0 && _nativemem < 0) { - _event = EVENT_CPU; - } - - if (VM::isOpenJ9()) { - if (_cstack == CSTACK_FP) { - // J9 is compiled without FP - // switch to DWARF for better results - _cstack = CSTACK_DWARF; - } - } - - return Error::OK; -} - -const char *Arguments::file() { - if (_file != NULL && strchr(_file, '%') != NULL) { - return expandFilePattern(_file); - } - return _file; -} - -// Expands the following patterns: -// %p process id -// %t timestamp (yyyyMMdd-hhmmss) -// %n{MAX} sequence number -// %{ENV} environment variable -const char *Arguments::expandFilePattern(const char *pattern) { - char *ptr = _buf; - char *end = _buf + EXTRA_BUF_SIZE - 1; - - while (ptr < end && *pattern != 0) { - char c = *pattern++; - if (c == '%') { - c = *pattern++; - if (c == 0) { - break; - } else if (c == 'p') { - ptr += snprintf(ptr, end - ptr, "%d", getpid()); - continue; - } else if (c == 't') { - time_t timestamp = time(NULL); - struct tm t; - localtime_r(×tamp, &t); - ptr += snprintf(ptr, end - ptr, "%d%02d%02d-%02d%02d%02d", - t.tm_year + 1900, t.tm_mon + 1, t.tm_mday, t.tm_hour, - t.tm_min, t.tm_sec); - continue; - } else if (c == '{') { - char env_key[128]; - const char *p = strchr(pattern, '}'); - if (p != NULL && p - pattern < static_cast(sizeof(env_key))) { - memcpy(env_key, pattern, p - pattern); - env_key[p - pattern] = 0; - const char *env_value = getenv(env_key); - if (env_value != NULL) { - ptr += snprintf(ptr, end - ptr, "%s", env_value); - pattern = p + 1; - continue; - } - } - } - } - *ptr++ = c; - } - - *(ptr < end ? ptr : end) = 0; - return _buf; -} - -long Arguments::parseUnits(const char *str, const Multiplier *multipliers) { - char *end; - errno = 0; - long result = strtol(str, &end, 0); - if (errno == ERANGE) { - return -1; - } - - char c = *end; - if (c == 0) { - return result; - } - if (c >= 'A' && c <= 'Z') { - c += 'a' - 'A'; - } - - for (const Multiplier *m = multipliers; m->symbol; m++) { - if (c == m->symbol) { - if (m->multiplier != 1 && (result > LONG_MAX / m->multiplier || result < LONG_MIN / m->multiplier)) { - return -1; - } - return result * m->multiplier; - } - } - - return -1; -} - -Arguments::~Arguments() { - if (_buf != NULL) { - free(_buf); - _buf = NULL; - } -} - -void Arguments::save(Arguments &other) { - other = *this; - other._buf = NULL; - other._shared = true; -} diff --git a/ddprof-lib/src/main/cpp/arguments.h b/ddprof-lib/src/main/cpp/arguments.h deleted file mode 100644 index bcf83e317..000000000 --- a/ddprof-lib/src/main/cpp/arguments.h +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Copyright 2017 Andrei Pangin - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _ARGUMENTS_H -#define _ARGUMENTS_H - -#include -#include -#include -#include - -const long DEFAULT_CPU_INTERVAL = 10 * 1000 * 1000; // 10 ms -const long DEFAULT_WALL_INTERVAL = 50 * 1000 * 1000; // 50 ms -const long DEFAULT_ALLOC_INTERVAL = 524287; // 512 KiB -const int DEFAULT_WALL_THREADS_PER_TICK = 16; -const int DEFAULT_JSTACKDEPTH = 2048; - -const char *const EVENT_NOOP = "noop"; -const char *const EVENT_CPU = "cpu"; -const char *const EVENT_ALLOC = "alloc"; -const char *const EVENT_WALL = "wall"; -const char *const EVENT_ITIMER = "itimer"; -const char *const EVENT_CTIMER = "ctimer"; - -enum Action { - ACTION_NONE, - ACTION_START, - ACTION_RESUME, - ACTION_STOP, - ACTION_CHECK, - ACTION_STATUS, - ACTION_LIST, - ACTION_VERSION -}; - -enum Ring { - RING_KERNEL = 1, - RING_USER = 1 << 1, - RING_ANY = RING_KERNEL | RING_USER, -}; - -enum Style { - STYLE_SIMPLE = 1, - STYLE_DOTTED = 2, - STYLE_SIGNATURES = 4, - STYLE_ANNOTATE = 8, - STYLE_LIB_NAMES = 16 -}; - -enum CStack { - CSTACK_DEFAULT, // use perf_event_open stack if available or Frame Pointer links otherwise - CSTACK_NO, // do not collect native frames - CSTACK_FP, // walk stack using Frame Pointer links - CSTACK_DWARF, // use DWARF unwinding info from .eh_frame section - CSTACK_LBR, // Last Branch Record hardware capability - CSTACK_VM // unwind using HotSpot VMStructs (vmx mode uses CSTACK_VM with _features.mixed=1) -}; - -enum Output { OUTPUT_NONE, OUTPUT_COLLAPSED, OUTPUT_JFR }; - -enum JfrOption { - NO_SYSTEM_INFO = 0x1, - NO_SYSTEM_PROPS = 0x2, - NO_NATIVE_LIBS = 0x4, - NO_CPU_LOAD = 0x8, - - JFR_SYNC_OPTS = - NO_SYSTEM_INFO | NO_SYSTEM_PROPS | NO_NATIVE_LIBS | NO_CPU_LOAD -}; - -enum WallclockSampler { - ASGCT, - JVMTI -}; - -enum Clock { - CLK_DEFAULT, - CLK_TSC, - CLK_MONOTONIC -}; - -// Keep this in sync with JfrSync.java -enum EventMask { - EM_CPU = 1, - EM_ALLOC = 2, - EM_LOCK = 4, - EM_WALL = 8, - EM_NATIVEMEM = 16, - EM_METHOD_TRACE = 32, - EM_NATIVESOCKET = 64 -}; -constexpr int EVENT_MASK_SIZE = 7; - -struct StackWalkFeatures { - // Deprecated stack recovery techniques used to workaround AsyncGetCallTrace flaws - unsigned short unknown_java : 1; - unsigned short unwind_stub : 1; - unsigned short unwind_comp : 1; - unsigned short unwind_native : 1; - unsigned short java_anchor : 1; - unsigned short gc_traces : 1; - - // Common features - unsigned short stats : 1; // collect stack walking duration statistics - - // Additional HotSpot-specific features - unsigned short jnienv : 1; // verify JNIEnv* obtained using VMStructs - unsigned short probe_sp : 1; // when AsyncGetCallTrace fails, adjust SP and retry - unsigned short mixed : 1; // mixed stack traces with Java and native frames interleaved - unsigned short vtable_target : 1; // show receiver classes of vtable/itable stubs - unsigned short comp_task : 1; // display current compilation task for JIT threads - unsigned short pc_addr : 1; // record exact PC address for each sample - unsigned short carrier_frames: 1; // walk through VT continuation boundary to carrier frames (enabled automatically with cstack=vmx) - unsigned short _padding : 2; // pad structure to 16 bits -}; - -struct Multiplier { - char symbol; - long multiplier; -}; - -class Error { -private: - const char *_message; - -public: - static const Error OK; - - explicit Error(const char *message) : _message(message) {} - - const char *message() { return _message; } - - operator bool() { return _message != NULL; } -}; - -class Arguments { -private: - char *_buf; - bool _shared; - bool _persistent; - const char *expandFilePattern(const char *pattern); - static long parseUnits(const char *str, const Multiplier *multipliers); - static bool isCpuEvent(const char *event) { - // event == NULL will default to EVENT_CPU - return event == NULL || strcmp(event, EVENT_CPU) == 0 || - strcmp(event, EVENT_ITIMER) == 0 || strcmp(event, EVENT_CTIMER) == 0; - } - -public: - Action _action; - Ring _ring; - const char *_event; - long _interval; - long _cpu; - long _wall; - bool _wall_collapsing; - bool _wall_precheck; - int _wall_threads_per_tick; - WallclockSampler _wallclock_sampler; - long _memory; - bool _record_allocations; - bool _record_liveness; - double _live_samples_ratio; - bool _record_heap_usage; - bool _gc_generations; - long _nativemem; - int _jstackdepth; - int _safe_mode; - StackWalkFeatures _features; - const char* _file; - const char* _log; - const char* _loglevel; - const char* _unknown_arg; - const char* _filter; - CStack _cstack; - Clock _clock; - int _jfr_options; - std::vector _context_attributes; - bool _lightweight; - bool _enable_method_cleanup; - bool _remote_symbolication; // Enable remote symbolication for native frames - bool _jvmtistacks; // Delegate CPU/wall stack walks to HotSpot JFR RequestStackTrace extension - bool _nativesocket; - long _nativesocket_interval; // initial sampling period in nanoseconds; 0 = engine default - - Arguments(bool persistent = false) - : _buf(NULL), - _shared(false), - _persistent(persistent), - _action(ACTION_NONE), - _ring(RING_ANY), - _event(NULL), - _interval(0), - _cpu(-1), - _wall(-1), - _wall_collapsing(false), - _wall_precheck(false), - _wall_threads_per_tick(DEFAULT_WALL_THREADS_PER_TICK), - _wallclock_sampler(ASGCT), - _memory(-1), - _record_allocations(false), - _record_liveness(false), - _live_samples_ratio(0.1), // default to liveness-tracking 10% of the allocation samples - _record_heap_usage(false), - _gc_generations(false), - _nativemem(-1), - _jstackdepth(DEFAULT_JSTACKDEPTH), - _safe_mode(0), - _features{1, 1, 1, 1, 1, 1}, - _file(NULL), - _log(NULL), - _loglevel(NULL), - _unknown_arg(NULL), - _filter(NULL), - _cstack(CSTACK_DEFAULT), - _clock(CLK_DEFAULT), - _jfr_options(0), - _context_attributes({}), - _lightweight(false), - _enable_method_cleanup(true), - _remote_symbolication(false), - _jvmtistacks(false), - _nativesocket(false), - _nativesocket_interval(0) {} - - ~Arguments(); - - void save(Arguments &other); - - Error parse(const char *args); - - const char *file(); - - bool hasOption(JfrOption option) const { - return (_jfr_options & option) != 0; - } - - long cpuSamplerInterval() const { - return isCpuEvent(_event) ? (_cpu > 0 ? _cpu - : _interval > 0 ? _interval - : DEFAULT_CPU_INTERVAL) - : 0; - } - - friend class FrameName; - friend class Recording; -}; - -#endif // _ARGUMENTS_H diff --git a/ddprof-lib/src/main/cpp/asprof.h b/ddprof-lib/src/main/cpp/asprof.h deleted file mode 100644 index 3f6cbfdcd..000000000 --- a/ddprof-lib/src/main/cpp/asprof.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _ASPROF_H -#define _ASPROF_H - -#include -#include - -#ifdef __clang__ -# define DLLEXPORT __attribute__((visibility("default"))) -#else -# define DLLEXPORT __attribute__((visibility("default"),externally_visible)) -#endif - -#define WEAK __attribute__((weak)) - -#ifdef __cplusplus -extern "C" { -#endif - - -typedef const char* asprof_error_t; -typedef void (*asprof_writer_t)(const char* buf, size_t size); - -// Should be called once prior to any other API functions -DLLEXPORT void asprof_init(); -typedef void (*asprof_init_t)(); - -// Returns an error message for the given error code or NULL if there is no error -DLLEXPORT const char* asprof_error_str(asprof_error_t err); -typedef const char* (*asprof_error_str_t)(asprof_error_t err); - -// Executes async-profiler command using output_callback as an optional sink -// for the profiler output. Returning an error code or NULL on success. -DLLEXPORT asprof_error_t asprof_execute(const char* command, asprof_writer_t output_callback); -typedef asprof_error_t (*asprof_execute_t)(const char* command, asprof_writer_t output_callback); - -// This API is UNSTABLE and might change or be removed in the next version of async-profiler. -typedef struct { - // A thread-local sample counter, which increments (not necessarily by 1) every time a - // stack profiling sample is taken using a profiling signal. - // - // The counter might be initialized lazily, only starting counting from 0 the first time - // `asprof_get_thread_local_data` is called on a given thread. Further calls to - // `asprof_get_thread_local_data` on a given thread will of course not reset the counter. - volatile uint64_t sample_counter; -} asprof_thread_local_data; - -// This API is UNSTABLE and might change or be removed in the next version of async-profiler. -// -// Gets a pointer to asprof's thread-local data structure, see `asprof_thread_local_data`'s -// documentation for the details of each field. This function might lazily initialize that -// structure. -// -// This function can return NULL either if the profiler is not yet initializer, or in -// case of an allocation failure. -// -// This function is *not* async-signal-safe. However, it is safe to call concurrently -// with async-profiler operations, including initialization. -DLLEXPORT asprof_thread_local_data* asprof_get_thread_local_data(void); -typedef asprof_thread_local_data* (*asprof_get_thread_local_data_t)(void); - - -typedef int asprof_jfr_event_key; - -// This API is UNSTABLE and might change or be removed in the next version of async-profiler. -// -// Return a asprof_jfr_event_key identifier for a user-defined JFR key. -// That identifier can then be used in `asprof_emit_jfr_event` -// -// The name is required to be valid (since it's a C string, NUL-free) UTF-8. -// -// Returns -1 on failure. -DLLEXPORT asprof_jfr_event_key asprof_register_jfr_event(const char* name); -typedef asprof_jfr_event_key (*asprof_register_jfr_event_t)(const char* name); - - -#define ASPROF_MAX_JFR_EVENT_LENGTH 2048 - -// This API is UNSTABLE and might change or be removed in the next version of async-profiler. -// -// Emits a custom, user-defined JFR event. The key should be created via `asprof_register_jfr_event`. -// The data can be arbitrary binary data, with size <= ASPROF_MAX_JFR_EVENT_LENGTH. -// -// User-defined events are included in the JFR under a `profiler.UserEvent` event type. That type will contain -// (at least) the following fields: -// 1. `startTime` [Long] - the emitted event's time in ticks. -// 2. `eventThread` [java.lang.Thread] - the thread that emitted the events. -// 3. `type` [profiler.types.UserEventType] - the event's type, -// where `profiler.types.UserEventType` is an indexed string from the JFR constant pool. -// 4. `data` [String] - the event data. This is the Latin-1 encoded version of the inputted data. -// The Latin-1 encoding is used as a way to stuff the arbitrary byte input into something -// that JFR supports (JFR technically supports byte arrays, but `jfr print` doesn't). -// -// Returns an error code or NULL on success. -DLLEXPORT asprof_error_t asprof_emit_jfr_event(asprof_jfr_event_key type, const uint8_t* data, size_t len); -typedef asprof_error_t (*asprof_emit_jfr_event_t)(asprof_jfr_event_key type, const uint8_t* data, size_t len); - -#ifdef __cplusplus -} -#endif - -#endif // _ASPROF_H diff --git a/ddprof-lib/src/main/cpp/asyncSampleMutex.h b/ddprof-lib/src/main/cpp/asyncSampleMutex.h deleted file mode 100644 index aa949ebdf..000000000 --- a/ddprof-lib/src/main/cpp/asyncSampleMutex.h +++ /dev/null @@ -1,37 +0,0 @@ -#ifndef ASYNCSAMPLEMUTEX_H -#define ASYNCSAMPLEMUTEX_H - -#include "threadLocalData.h" - -// controls access to AGCT -class AsyncSampleMutex { -private: - ThreadLocalData *_threadLocalData; - bool _acquired; - - bool try_acquire() { - if (_threadLocalData != nullptr && !_threadLocalData->is_unwinding_Java()) { - _threadLocalData->set_unwinding_Java(true); - return true; - } - return false; - } - -public: - AsyncSampleMutex(ThreadLocalData *threadLocalData) - : _threadLocalData(threadLocalData) { - _acquired = try_acquire(); - } - - AsyncSampleMutex(AsyncSampleMutex &other) = delete; - - ~AsyncSampleMutex() { - if (_acquired) { - _threadLocalData->set_unwinding_Java(false); - } - } - - bool acquired() { return _acquired; } -}; - -#endif // ASYNCSAMPLEMUTEX_H diff --git a/ddprof-lib/src/main/cpp/buffers.h b/ddprof-lib/src/main/cpp/buffers.h deleted file mode 100644 index 700a802bc..000000000 --- a/ddprof-lib/src/main/cpp/buffers.h +++ /dev/null @@ -1,216 +0,0 @@ -#ifndef _BUFFERS_H -#define _BUFFERS_H - -#include -#include -#include -#include - -#include - -#include "os.h" - -const int BUFFER_SIZE = 1024; -const int BUFFER_LIMIT = BUFFER_SIZE - 128; -// we use this space, which is larger than the longest string we will store, -// as a temporary defence against overflow. If we ever write into this space -// we may produce a corrupt recording, which is sad, but we can't overwrite -// offsets in the adjacent buffer, which has been a frequent cause of hard to -// diagnose crashes. Ultimately, the entire FlightRecorder implementation needs -// to be reworked for the sake of safety, but this small change derisks the -// legacy implementation at a runtime cost of 16 * 8KiB = 128KiB -const int RECORDING_BUFFER_OVERFLOW = 8192; -const int RECORDING_BUFFER_SIZE = 65536; -const int RECORDING_BUFFER_LIMIT = RECORDING_BUFFER_SIZE - 4096; -const int MAX_STRING_LENGTH = 8191; - -typedef ssize_t (*FlushCallback)(char *data, int len); - -class Buffer { -private: - int _offset; - static const int _limit = BUFFER_SIZE - sizeof(int); -protected: - // this array is 'extended' by the RecordingBuffer - // this will confuse sanitizers and most of the sane people but it seems to - // work - char _data[_limit]; - -public: - Buffer() : _offset(0) { memset(_data, 0, _limit); } - - virtual int limit() const { return _limit; } - - bool flushIfNeeded(FlushCallback callback, int limit = BUFFER_LIMIT) { - if (_offset > limit) { - if (callback(_data, _offset) == _offset) { - reset(); - return true; - } - } - return false; - } - - const char *data() const { return _data; } - - int offset() const { return _offset; } - - // ! This method returns the position *before* skipping ! - int skip(int delta) { - assert(_offset + delta < limit()); - int here = _offset; - _offset = here + delta; - return here; - } - - void reset() { _offset = 0; } - - __attribute__((no_sanitize("bounds"))) - void put(const char *v, u32 len) { - assert(static_cast(_offset + len) < limit()); - memcpy(_data + _offset, v, len); - _offset += (int)len; - } - - // RecordingBuffer extends _data[] beyond its declared size via struct layout; - // suppress bounds sanitizer on all architectures to avoid false positives. - __attribute__((no_sanitize("bounds"))) - void put8(char v) { - assert(_offset < limit()); - _data[_offset++] = v; - } - - __attribute__((no_sanitize("undefined"))) - __attribute__((no_sanitize("bounds"))) - void put16(short v) { - assert(_offset + 2 < limit()); - *(short *)(_data + _offset) = htons(v); - _offset += 2; - } - - // java-profiler/ddprof-lib/src/main/cpp/buffers.h:92:34: runtime error: - // store to misaligned address 0x7f3c446ec81e for type 'int', which - // requires 4 byte alignment 0x7f3c446ec81e: note: pointer points here - __attribute__((no_sanitize("undefined"))) - __attribute__((no_sanitize("bounds"))) - void put32(int v) { - assert(_offset + 4 < limit()); - *(int *)(_data + _offset) = htonl(v); - _offset += 4; - } - - // alignment issue to be looked at - // runtime error: store to misaligned address 0x766bb5a1e814 for type - // 'u64', which requires 8 byte alignment - // 0x766bb5a1e814: note: pointer points here - __attribute__((no_sanitize("undefined"))) - __attribute__((no_sanitize("bounds"))) - void put64(u64 v) { - assert(_offset + 8 < limit()); - *(u64 *)(_data + _offset) = OS::hton64(v); - _offset += 8; - } - - // the trickery of RecordingBuffer extending Buffer::_data array may trip off asan - __attribute__((no_sanitize("bounds"))) - void putFloat(float v) { - union { - float f; - int i; - } u; - - u.f = v; - put32(u.i); - } - - __attribute__((no_sanitize("bounds"))) - void putVar32(u32 v) { - assert(_offset + 5 < limit()); - while (v > 0x7f) { - _data[_offset++] = (char)v | 0x80; - v >>= 7; - } - _data[_offset++] = (char)v; - assert(_offset < limit()); - } - - __attribute__((no_sanitize("bounds"))) - void putVar64(u64 v) { - assert(_offset + 9 < limit()); - int iter = 0; - while (v > 0x1fffff) { - _data[_offset++] = (char)v | 0x80; - v >>= 7; - _data[_offset++] = (char)v | 0x80; - v >>= 7; - _data[_offset++] = (char)v | 0x80; - v >>= 7; - if (++iter == 3) - return; - } - while (v > 0x7f) { - _data[_offset++] = (char)v | 0x80; - v >>= 7; - } - _data[_offset++] = (char)v; - assert(_offset < limit()); - } - - // the trickery of RecordingBuffer extending Buffer::_data array may trip off asan - __attribute__((no_sanitize("bounds"))) - void putUtf8(const char *v) { - if (v == NULL) { - put8(0); - } else { - size_t len = strlen(v); - putUtf8(v, len); - } - } - - __attribute__((no_sanitize("bounds"))) - void putUtf8(const char *v, u32 len) { - len = len < MAX_STRING_LENGTH ? len : MAX_STRING_LENGTH; - put8(3); - putVar32(len); - put(v, len); - } - - __attribute__((no_sanitize("bounds"))) - void put8(int offset, char v) { _data[offset] = v; } - - __attribute__((no_sanitize("bounds"))) - void putVar32(int offset, u32 v) { - assert(offset + 4 < limit()); - _data[offset] = v | 0x80; - _data[offset + 1] = (v >> 7) | 0x80; - _data[offset + 2] = (v >> 14) | 0x80; - _data[offset + 3] = (v >> 21) | 0x80; - _data[offset + 4] = (v >> 28); - } -}; - -class RecordingBuffer : public Buffer { -private: - static const int _limit = RECORDING_BUFFER_SIZE - sizeof(Buffer); - // we reserve 8KiB to overflow in to in case event serialisers in - // the flight recorder are buggy. If we ever use the overflow, - // which is sized to accommodate the largest possible string, we - // will truncate and may produce a corrupt recording, but we will - // not write into arbitrary memory. - char _buf[_limit + RECORDING_BUFFER_OVERFLOW]; - -public: - RecordingBuffer() : Buffer() { - new (_data + sizeof(_data)) char[sizeof(_buf)]; - memset(_buf, 0, _limit); - } - - int limit() const override { return _limit; } - - bool flushIfNeeded(FlushCallback callback, - int limit = RECORDING_BUFFER_LIMIT) { - return Buffer::flushIfNeeded(callback, limit); - } -}; - -#endif // _BUFFERS_H diff --git a/ddprof-lib/src/main/cpp/callTraceHashTable.cpp b/ddprof-lib/src/main/cpp/callTraceHashTable.cpp deleted file mode 100644 index cb0d7e608..000000000 --- a/ddprof-lib/src/main/cpp/callTraceHashTable.cpp +++ /dev/null @@ -1,518 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "callTraceHashTable.h" -#include "callTraceStorage.h" -#include "counters.h" -#include "os.h" -#include "arch.h" -#include "common.h" -#include "primeProbing.h" -#include -#include -#include - -static const u32 INITIAL_CAPACITY = 65536; // 64K initial table size (matches upstream) -static const u32 CALL_TRACE_CHUNK = 8 * 1024 * 1024; -static const u64 OVERFLOW_TRACE_ID = 0x7fffffffffffffffULL; // Max 64-bit signed value - -// Define the sentinel value for CallTraceSample -CallTrace* const CallTraceSample::PREPARING = reinterpret_cast(1); - -class LongHashTable { -private: - LongHashTable *_prev; - void *_padding0; - u32 _capacity; - u32 _padding1[15]; - volatile u32 _size; - u32 _padding2[15]; - - static size_t getSize(u32 capacity) { - size_t size = sizeof(LongHashTable) + - (sizeof(u64) + sizeof(CallTraceSample)) * capacity; - return (size + OS::page_mask) & ~OS::page_mask; - } - -public: - LongHashTable(LongHashTable *prev = nullptr, u32 capacity = 0, bool should_clean = true) - : _prev(prev), _padding0(nullptr), _capacity(capacity), _size(0) { - memset(_padding1, 0, sizeof(_padding1)); - memset(_padding2, 0, sizeof(_padding2)); - if (should_clean) { - clear(); - } - } - - static LongHashTable *allocate(LongHashTable *prev, u32 capacity, LinearAllocator* allocator) { - void *memory = allocator->alloc(getSize(capacity)); - if (memory != nullptr) { - // Use placement new to invoke constructor in-place with parameters - // LinearAllocator doesn't zero memory like OS::safeAlloc with anon mmap - // so we need to explicitly clear the keys and values (should_clean = true) - LongHashTable *table = new (memory) LongHashTable(prev, capacity, true); - return table; - } - return nullptr; - } - - LongHashTable *prev() { return _prev; } - void setPrev(LongHashTable* prev) { _prev = prev; } - - u32 capacity() { return _capacity; } - - u32 size() { return _size; } - - u32 incSize() { return __sync_add_and_fetch(&_size, 1); } - - u64 *keys() { return (u64 *)(this + 1); } - - CallTraceSample *values() { return (CallTraceSample *)(keys() + _capacity); } - - void clear() { - memset(keys(), 0, (sizeof(u64) + sizeof(CallTraceSample)) * _capacity); - _size = 0; - } -}; - -CallTrace CallTraceHashTable::_overflow_trace(false, 1, OVERFLOW_TRACE_ID); - -// Static initializer for overflow trace frame -__attribute__((constructor)) -static void init_overflow_trace() { - CallTraceHashTable::_overflow_trace.frames[0] = {BCI_ERROR, LP64_ONLY(0 COMMA) (jmethodID)"storage_overflow"}; -} - -CallTraceHashTable::CallTraceHashTable() : _instance_id(0), _parent_storage(nullptr), _allocator(CALL_TRACE_CHUNK) { - // Instance ID will be set externally via setInstanceId() - - // Start with initial capacity, allowing expansion as needed - _table = LongHashTable::allocate(nullptr, INITIAL_CAPACITY, &_allocator); - _overflow = 0; -} - -CallTraceHashTable::~CallTraceHashTable() { - // LinearAllocator handles all memory cleanup automatically - // No need to explicitly destroy tables since they're allocated from LinearAllocator - // Note: No synchronization needed here because CallTraceStorage ensures - // no new operations can start by nullifying storage pointers first - _table = nullptr; -} - - -void CallTraceHashTable::decrementCounters() { -#ifdef COUNTERS - // Compute and decrement the global counters for everything in this table. - // Safe to call when (a) this is a standby/scratch table (never _active_storage, - // so no signal-handler put() can target it), or (b) the active-table path is - // guarded by lockAll() — both conditions are enforced by the only caller, - // clearTableOnly(). The _prev traversal is safe because waitForRefCountToClear(this) - // in clearTableOnly() has already drained any in-flight put() operations. - // Use a set to deduplicate: put() may store the same CallTrace* pointer in - // both a newer and an older table (when findCallTrace finds it in prev()), - // but the counter was only incremented once, so we must only count it once. - const size_t header_size = sizeof(CallTrace) - sizeof(ASGCT_CallFrame); - long long freed_bytes = 0; - long long freed_traces = 0; - size_t estimated_entries = 0; - for (LongHashTable *t = _table; t != nullptr; t = t->prev()) { - estimated_entries += t->size(); - } - std::unordered_set seen; - seen.reserve(estimated_entries); - for (LongHashTable *t = _table; t != nullptr; t = t->prev()) { - u64 *keys = t->keys(); - CallTraceSample *values = t->values(); - u32 capacity = t->capacity(); - for (u32 slot = 0; slot < capacity; slot++) { - if (keys[slot] != 0) { - CallTrace *trace = values[slot].acquireTrace(); - if (trace != nullptr && trace != CallTraceSample::PREPARING) { - if (seen.insert(trace).second) { - freed_bytes += header_size + trace->num_frames * sizeof(ASGCT_CallFrame); - freed_traces++; - } - } - } - } - } - Counters::increment(CALLTRACE_STORAGE_BYTES, -freed_bytes); - Counters::increment(CALLTRACE_STORAGE_TRACES, -freed_traces); -#endif // COUNTERS -} - -ChunkList CallTraceHashTable::clearTableOnly() { - // Wait only for in-flight put() operations that hold a RefCountGuard on THIS - // table. Waiting globally (waitForAllRefCountsToClear) would block on - // unrelated puts to the currently-active table, causing 500 ms timeouts under - // sustained wall-clock profiling and leaving collect() racing with a still- - // running put(). Since standby and scratch tables never appear as the - // _active_storage, this wait returns instantly for them; for the active table - // (called from clear() -> clearTableOnly()) the protection comes from the caller - // holding lockAll() (which blocks signal-handler puts) and from this in-function - // targeted wait — there is no prior caller-side drain. - RefCountGuard::waitForRefCountToClear(this); - decrementCounters(); - - // Disconnect the full _prev chain before freeing chunks. The advance step - // must use a pre-saved pointer because setPrev(nullptr) clears the link that - // the original loop used for advancement, causing early termination after only - // the first node on an expanded (multi-node) table. - for (LongHashTable *table = __atomic_load_n(&_table, __ATOMIC_ACQUIRE); - table != nullptr; ) { - LongHashTable *next = table->prev(); - table->setPrev(nullptr); - table = next; - } - - // Detach chunks for deferred deallocation - keeps trace memory alive - ChunkList detached_chunks = _allocator.detachChunks(); - - // Reinitialize with fresh table (using the new chunk from detachChunks) - // Note: If detachChunks() failed to allocate a fresh chunk, the allocator's - // _tail will be nullptr. LongHashTable::allocate will try to allocate, - // which will call LinearAllocator::alloc(), which needs to handle nullptr _tail. - // This is already handled in alloc() by checking _tail before use. - // RELEASE: pairs with ACQUIRE loads in collect() and put() to ensure the - // freshly-initialised table is visible on weakly-ordered architectures (aarch64). - __atomic_store_n(&_table, - LongHashTable::allocate(nullptr, INITIAL_CAPACITY, &_allocator), - __ATOMIC_RELEASE); - _overflow = 0; - - return detached_chunks; -} - -void CallTraceHashTable::clear() { - // Clear table and immediately free chunks (original behavior) - ChunkList chunks = clearTableOnly(); - LinearAllocator::freeChunks(chunks); -} - -// Adaptation of MurmurHash64A by Austin Appleby -u64 CallTraceHashTable::calcHash(int num_frames, ASGCT_CallFrame *frames, - bool truncated) { - const u64 M = 0xc6a4a7935bd1e995ULL; - const int R = 47; - - int len = num_frames * sizeof(ASGCT_CallFrame); - u64 h = len * M * (truncated ? 1 : 2); - - const u64 *data = (const u64 *)frames; - const u64 *end = data + len / sizeof(u64); - - while (data != end) { - u64 k = *data++; - k *= M; - k ^= k >> R; - k *= M; - h ^= k; - h *= M; - } - - if (len & 4) { - h ^= *(u32 *)data; - h *= M; - } - - h ^= h >> R; - h *= M; - h ^= h >> R; - - return h; -} - -CallTrace *CallTraceHashTable::storeCallTrace(int num_frames, - ASGCT_CallFrame *frames, - bool truncated, u64 trace_id) { - const size_t header_size = sizeof(CallTrace) - sizeof(ASGCT_CallFrame); - const size_t total_size = header_size + num_frames * sizeof(ASGCT_CallFrame); - void *memory = _allocator.alloc(total_size); - CallTrace *buf = nullptr; - if (memory != nullptr) { - // Use placement new to invoke constructor in-place - buf = new (memory) CallTrace(truncated, num_frames, trace_id); - // Do not use memcpy inside signal handler - for (int i = 0; i < num_frames; i++) { - buf->frames[i] = frames[i]; - } - Counters::increment(CALLTRACE_STORAGE_BYTES, total_size); - Counters::increment(CALLTRACE_STORAGE_TRACES); - } - return buf; -} - -CallTrace *CallTraceHashTable::findCallTrace(LongHashTable *table, u64 hash) { - u64 *keys = table->keys(); - HashProbe probe(hash, table->capacity()); - - u32 slot = probe.slot(); - while (true) { - // Use atomic load: keys[] can be written concurrently via CAS in put() - // when a table is promoted to prev but still has in-flight insertions. - u64 key = __atomic_load_n(&keys[slot], __ATOMIC_ACQUIRE); - if (key == hash) { - // Use acquireTrace() to pair with the RELEASE store in setTrace(). - // If still PREPARING, treat as not found: callers will create a new entry. - CallTrace *trace = table->values()[slot].acquireTrace(); - if (trace == CallTraceSample::PREPARING) { - return nullptr; - } - return trace; - } - if (key == 0) { - return nullptr; - } - if (!probe.hasNext()) { - break; - } - slot = probe.next(); - }; - - return nullptr; -} - -void CallTraceHashTable::expandTableIfNeeded(LongHashTable* table, u32 size) { - u32 capacity = table->capacity(); - - // EXPANSION LOGIC: Check if load ratio reached after incrementing size - if (size >= (u32) (capacity * LOAD_RATIO) && - table == __atomic_load_n(&_table, __ATOMIC_RELAXED)) { // quick check, if other thread already expanded the table - // Allocate new table with double capacity using LinearAllocator - LongHashTable* new_table = LongHashTable::allocate(table, capacity * 2, &_allocator); - if (new_table != nullptr) { - // Atomic table swap - only one thread succeeds - __atomic_compare_exchange_n(&_table, &table, new_table, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); - } - } -} - -u64 CallTraceHashTable::put(int num_frames, ASGCT_CallFrame *frames, - bool truncated, u64 weight) { - u64 hash = calcHash(num_frames, frames, truncated); - - // ACQUIRE pairs with the ACQ_REL CAS in the expansion path below, ensuring - // that if another thread published a new (expanded) table we see its fully - // initialised contents. - LongHashTable *table = __atomic_load_n(&_table, __ATOMIC_ACQUIRE); - if (table == nullptr) { - // Table allocation failed or was cleared - drop sample - Counters::increment(CALLTRACE_STORAGE_DROPPED); - return CallTraceStorage::DROPPED_TRACE_ID; - } - - u64 *keys = table->keys(); - HashProbe probe(hash, table->capacity()); - - u32 slot = probe.slot(); - while (true) { - u64 key_value = __atomic_load_n(&keys[slot], __ATOMIC_RELAXED); - if (key_value == hash) { - // Hash matches - wait for the preparing thread to complete - CallTrace* current_trace = table->values()[slot].acquireTrace(); - - // If another thread is preparing this slot, wait for completion - if (current_trace == CallTraceSample::PREPARING) { - // Wait for the preparing thread to complete, with timeout - int wait_cycles = 0; - const int MAX_WAIT_CYCLES = 1000; // ~1000 cycles should be enough for allocation - - do { - // Brief spin-wait to allow preparing thread to complete - for (volatile int i = 0; i < 10; i++) { - spinPause(); // Architecture-specific pause instruction - } - - current_trace = table->values()[slot].acquireTrace(); - wait_cycles++; - - // Check if key was cleared (preparation failed) - if (__atomic_load_n(&keys[slot], __ATOMIC_RELAXED) != hash) { - break; // Key cleared, restart search - } - - } while (current_trace == CallTraceSample::PREPARING && wait_cycles < MAX_WAIT_CYCLES); - - // If still preparing after timeout, something is wrong - continue search - if (current_trace == CallTraceSample::PREPARING) { - continue; - } - } - - // Check final state after waiting - if (current_trace != nullptr && current_trace != CallTraceSample::PREPARING) { - // Trace is ready, use it - return current_trace->trace_id; - } else { - // Trace is nullptr but hash exists - preparation failed - u64 recheck_key = __atomic_load_n(&keys[slot], __ATOMIC_ACQUIRE); - if (recheck_key != hash) { - continue; // Key was cleared, retry - } - Counters::increment(CALLTRACE_STORAGE_DROPPED); - return CallTraceStorage::DROPPED_TRACE_ID; - } - } - - if (key_value == 0) { - u64 expected = 0; - if (!__atomic_compare_exchange_n(&keys[slot], &expected, hash, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { - continue; // another thread claimed it, go to next slot - } - - // Mark the slot as being prepared so other threads know to wait - if (!table->values()[slot].markPreparing()) { - // Failed to mark as preparing (shouldn't happen), clear key with full barrier and retry - __atomic_thread_fence(__ATOMIC_SEQ_CST); - __atomic_store_n(&keys[slot], 0, __ATOMIC_RELEASE); - continue; - } - - // Increment size counter for statistics and check for expansion - u32 new_size = table->incSize(); - - probe.updateCapacity(new_size); - - expandTableIfNeeded(table, new_size); - - // Check if trace exists in previous tables to avoid duplication - CallTrace *trace = nullptr; - if (table->prev() != nullptr) { - trace = findCallTrace(table->prev(), hash); - } - - if (trace == nullptr) { - // Generate unique trace ID: upper 32 bits = instance_id, lower 32 bits = slot - // ACQUIRE ordering synchronizes with RELEASE store in setInstanceId() to ensure - // visibility of new instance_id on weakly-ordered architectures (aarch64, POWER) - u64 instance_id = _instance_id.load(std::memory_order_acquire); - u64 trace_id = (instance_id << 32) | slot; - trace = storeCallTrace(num_frames, frames, truncated, trace_id); - if (trace == nullptr) { - // Allocation failure - reset trace first, then clear key - table->values()[slot].setTrace(nullptr); - __atomic_thread_fence(__ATOMIC_SEQ_CST); - __atomic_store_n(&keys[slot], 0, __ATOMIC_RELEASE); - Counters::increment(CALLTRACE_STORAGE_DROPPED); - return CallTraceStorage::DROPPED_TRACE_ID; - } - } - - // Set the actual trace (this changes state from PREPARING to ready) - table->values()[slot].setTrace(trace); - return trace->trace_id; - } - - if (!probe.hasNext()) { - // Table overflow - very unlikely with expansion logic - atomicIncRelaxed(_overflow); - return OVERFLOW_TRACE_ID; - } - // Prime probing for better distribution - slot = probe.next(); - } -} - -void CallTraceHashTable::collect(std::unordered_set &traces, std::function trace_hook) { - // Lock-free collection for read-only tables. - // Use ACQUIRE to pair with the ACQ_REL CAS in put()'s expansion path and the - // RELEASE store in clearTableOnly(); ensures we see the fully-initialised table - // on weakly-ordered architectures (aarch64). - for (LongHashTable *table = __atomic_load_n(&_table, __ATOMIC_ACQUIRE); - table != nullptr; table = table->prev()) { - u64 *keys = table->keys(); - CallTraceSample *values = table->values(); - u32 capacity = table->capacity(); - for (u32 slot = 0; slot < capacity; slot++) { - if (keys[slot] != 0) { - CallTrace *trace = values[slot].acquireTrace(); - if (trace != nullptr && trace != CallTraceSample::PREPARING) { - if (trace_hook) { - trace_hook(trace); // Call hook first if provided - } - traces.insert(trace); - } - } - } - } - - // Handle overflow trace - if (_overflow > 0) { - if (trace_hook) { - trace_hook(&_overflow_trace); // Call hook for overflow trace too - } - traces.insert(&_overflow_trace); - } -} - - -void CallTraceHashTable::putWithExistingId(CallTrace* source_trace, u64 weight) { - // Trace preservation for standby tables (no contention with new puts) - // This is safe because new put() operations go to the new active table - - u64 hash = calcHash(source_trace->num_frames, source_trace->frames, source_trace->truncated); - - // First check if trace already exists in any table in the chain. - // Use ACQUIRE to match the RELEASE store in clearTableOnly(); putWithExistingId() - // is only called on scratch/standby tables with no concurrent writers, so the - // load is safe, but consistent ordering prevents latent issues if callers change. - for (LongHashTable *search_table = __atomic_load_n(&_table, __ATOMIC_ACQUIRE); - search_table != nullptr; search_table = search_table->prev()) { - CallTrace *existing_trace = findCallTrace(search_table, hash); - if (existing_trace != nullptr) { - return; - } - } - - LongHashTable *table = __atomic_load_n(&_table, __ATOMIC_ACQUIRE); - if (table == nullptr) { - return; // Table allocation failed - } - - u64 *keys = table->keys(); - u32 capacity = table->capacity(); - - HashProbe probe(hash, capacity); - u32 slot = probe.slot(); - - // Look for existing entry or empty slot - no locking needed - while (true) { - u64 key_value = __atomic_load_n(&keys[slot], __ATOMIC_RELAXED); - if (key_value == 0) { - // Found empty slot - claim it atomically - u64 expected = 0; - if (__atomic_compare_exchange_n(&keys[slot], &expected, hash, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { - // Successfully claimed the slot - // Create a copy of the source trace preserving its exact ID - const size_t header_size = sizeof(CallTrace) - sizeof(ASGCT_CallFrame); - const size_t total_size = header_size + source_trace->num_frames * sizeof(ASGCT_CallFrame); - void *memory = _allocator.alloc(total_size); - if (memory != nullptr) { - // Use placement new to invoke constructor in-place - CallTrace* copied_trace = new (memory) CallTrace(source_trace->truncated, source_trace->num_frames, source_trace->trace_id); - // memcpy safe since not in signal handler - memcpy(copied_trace->frames, source_trace->frames, source_trace->num_frames * sizeof(ASGCT_CallFrame)); - table->values()[slot].setTrace(copied_trace); - Counters::increment(CALLTRACE_STORAGE_BYTES, total_size); - Counters::increment(CALLTRACE_STORAGE_TRACES); - - // Increment table size - table->incSize(); - } else { - // Allocation failure - clear the key we claimed - __atomic_store_n(&keys[slot], 0, __ATOMIC_RELEASE); - } - break; - } - } - if (probe.hasNext()) { - slot = probe.next(); - } else { - // No more slots. The sample is dropped - Counters::increment(CALLTRACE_STORAGE_DROPPED); - break; - } - } -} diff --git a/ddprof-lib/src/main/cpp/callTraceHashTable.h b/ddprof-lib/src/main/cpp/callTraceHashTable.h deleted file mode 100644 index 351369015..000000000 --- a/ddprof-lib/src/main/cpp/callTraceHashTable.h +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _CALLTRACEHASHTABLE_H -#define _CALLTRACEHASHTABLE_H - -#include "arch.h" -#include "linearAllocator.h" -#include "vmEntry.h" -#include -#include -#include - -class LongHashTable; - -struct CallTrace { - bool truncated; - int num_frames; - u64 trace_id; // 64-bit for JFR constant pool compatibility - ASGCT_CallFrame frames[1]; - - CallTrace(bool truncated, int num_frames, u64 trace_id) - : truncated(truncated), num_frames(num_frames), trace_id(trace_id) { - } -}; - -struct CallTraceSample { - CallTrace *trace; - - // Sentinel value to indicate slot is being prepared - static CallTrace* const PREPARING; - - CallTrace *acquireTrace() { - return __atomic_load_n(&trace, __ATOMIC_ACQUIRE); - } - - void setTrace(CallTrace *value) { - __atomic_store_n(&trace, value, __ATOMIC_RELEASE); - } - - bool markPreparing() { - CallTrace* expected = nullptr; - return __atomic_compare_exchange_n(&trace, &expected, PREPARING, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); - } - - bool isPreparing() { - return acquireTrace() == PREPARING; - } -}; - -// Forward declaration for circular dependency -class CallTraceStorage; - -class CallTraceHashTable { - static constexpr double LOAD_RATIO = 3.0 / 4.0; - -public: - static CallTrace _overflow_trace; - -private: - std::atomic _instance_id; // 64-bit instance ID for this hash table - atomic for thread-safe access - CallTraceStorage* _parent_storage; // Parent storage for RefCountGuard access - - LinearAllocator _allocator; - - // Expandable hash table; put() doubles capacity when fill reaches 75%. - // Memory-ordering protocol: - // - ACQ_REL CAS in put() when installing the expanded table - // - RELEASE store in clearTableOnly() when resetting to a fresh table - // - ACQUIRE loads in collect(), put(), and putWithExistingId() - // Required for correct visibility on weakly-ordered architectures (aarch64). - LongHashTable* _table; - - volatile u64 _overflow; - - u64 calcHash(int num_frames, ASGCT_CallFrame *frames, bool truncated); - CallTrace *storeCallTrace(int num_frames, ASGCT_CallFrame *frames, - bool truncated, u64 trace_id); - CallTrace *findCallTrace(LongHashTable *table, u64 hash); - void decrementCounters(); - - void expandTableIfNeeded(LongHashTable* table, u32 size); - -public: - CallTraceHashTable(); - ~CallTraceHashTable(); - - void clear(); - - /** - * Resets the hash table structure but defers memory deallocation. - * Returns a ChunkList containing the detached memory chunks. - * The caller must call LinearAllocator::freeChunks() on the returned - * ChunkList after processing is complete. - * - * This is used to fix use-after-free in processTraces(): the table - * structure is reset immediately (allowing rotation), but trace memory - * remains valid until the processor finishes accessing it. - */ - ChunkList clearTableOnly(); - - void collect(std::unordered_set &traces, std::function trace_hook = nullptr); - - u64 put(int num_frames, ASGCT_CallFrame *frames, bool truncated, u64 weight); - void putWithExistingId(CallTrace* trace, u64 weight); // For standby tables with no contention - void setInstanceId(u64 instance_id) { - // Use atomic store with RELEASE ordering to ensure visibility across threads - _instance_id.store(instance_id, std::memory_order_release); - } - void setParentStorage(CallTraceStorage* storage) { _parent_storage = storage; } -}; - -#endif // _CALLTRACEHASHTABLE_H diff --git a/ddprof-lib/src/main/cpp/callTraceStorage.cpp b/ddprof-lib/src/main/cpp/callTraceStorage.cpp deleted file mode 100644 index 919cdd503..000000000 --- a/ddprof-lib/src/main/cpp/callTraceStorage.cpp +++ /dev/null @@ -1,281 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "callTraceStorage.h" -#include "counters.h" -#include "log.h" -#include "os.h" -#include "common.h" -#include "thread.h" -#include "vmEntry.h" // For BCI_ERROR constant -#include "arch.h" // For LP64_ONLY macro and COMMA macro -#include "guards.h" // For table swap critical sections -#include "thread.h" -#include -#include - -static const u64 OVERFLOW_TRACE_ID = 0x7fffffffffffffffULL; // Max 64-bit signed value - -// Static atomic for instance ID generation - explicit initialization avoids function-local static issues -u64 CallTraceStorage::_next_instance_id = 1; // Start from 1, 0 is reserved - - -// Lazy initialization helper to avoid global constructor race conditions -u64 CallTraceStorage::getNextInstanceId() { - // Uses GCC atomic builtin (no malloc, async-signal-safe) - u64 instance_id = __atomic_fetch_add(&_next_instance_id, 1, __ATOMIC_ACQ_REL); - return instance_id; -} - -CallTraceStorage::CallTraceStorage() : _active_storage(nullptr), _standby_storage(nullptr), _scratch_storage(nullptr), _generation_counter(1), _liveness_lock(0) { - - // Pre-allocate and pre-size collections with conservative load factor - _traces_buffer.max_load_factor(0.75f); - _traces_buffer.rehash(static_cast(2048 / 0.75f)); - - - - _preserve_set_buffer.max_load_factor(0.75f); - _preserve_set_buffer.rehash(static_cast(1024 / 0.75f)); - - // Initialize triple-buffered storage - auto active_table = std::make_unique(); - active_table->setInstanceId(getNextInstanceId()); - active_table->setParentStorage(this); - __atomic_store_n(&_active_storage, active_table.release(), __ATOMIC_RELEASE); - - auto standby_table = std::make_unique(); - standby_table->setParentStorage(this); - standby_table->setInstanceId(getNextInstanceId()); - __atomic_store_n(&_standby_storage, standby_table.release(), __ATOMIC_RELEASE); - - auto scratch_table = std::make_unique(); - scratch_table->setParentStorage(this); - scratch_table->setInstanceId(getNextInstanceId()); - __atomic_store_n(&_scratch_storage, scratch_table.release(), __ATOMIC_RELEASE); - - // Pre-allocate containers to avoid malloc() during hot path operations - _liveness_checkers.reserve(4); // Typical max: 1-2 checkers, avoid growth - - // Initialize counters - Counters::set(CALLTRACE_STORAGE_BYTES, 0); - Counters::set(CALLTRACE_STORAGE_TRACES, 0); -} - -CallTraceStorage::~CallTraceStorage() { - // Atomically invalidate storage pointers to prevent new put() operations - // ACQ_REL ordering: RELEASE ensures nullptr is visible to put()'s ACQUIRE load, - // ACQUIRE ensures we see the latest pointer value for subsequent deletion - CallTraceHashTable* active = const_cast(__atomic_exchange_n(&_active_storage, nullptr, __ATOMIC_ACQ_REL)); - CallTraceHashTable* standby = const_cast(__atomic_exchange_n(&_standby_storage, nullptr, __ATOMIC_ACQ_REL)); - CallTraceHashTable* scratch = const_cast(__atomic_exchange_n(&_scratch_storage, nullptr, __ATOMIC_ACQ_REL)); - - // Wait for any ongoing refcount usage to complete and delete each unique table - // Note: In triple-buffering, all three pointers should be unique, but check anyway - RefCountGuard::waitForRefCountToClear(active); - delete active; - - if (standby != active) { - RefCountGuard::waitForRefCountToClear(standby); - delete standby; - } - if (scratch != active && scratch != standby) { - RefCountGuard::waitForRefCountToClear(scratch); - delete scratch; - } - -} - - -CallTrace* CallTraceStorage::getDroppedTrace() { - // Static dropped trace object - created once and reused - // Use same pattern as storage_overflow trace for consistent platform handling - static CallTrace dropped_trace(false, 1, DROPPED_TRACE_ID); - // Initialize frame data only once - static bool initialized = false; - if (!initialized) { - dropped_trace.frames[0] = {BCI_ERROR, LP64_ONLY(0 COMMA) (jmethodID)""}; - initialized = true; - } - - return &dropped_trace; -} - -void CallTraceStorage::registerLivenessChecker(LivenessChecker checker) { - ExclusiveLockGuard lock(&_liveness_lock); - _liveness_checkers.push_back(checker); -} - -void CallTraceStorage::clearLivenessCheckers() { - ExclusiveLockGuard lock(&_liveness_lock); - _liveness_checkers.clear(); -} - - -u64 CallTraceStorage::put(int num_frames, ASGCT_CallFrame* frames, bool truncated, u64 weight) { - // Signal handlers can run concurrently with destructor - // MEMORY_ORDER_ACQUIRE: Critical - synchronizes with release stores in processTraces() - CallTraceHashTable* active = const_cast(__atomic_load_n(&_active_storage, __ATOMIC_ACQUIRE)); - - // Safety check - if null, system is shutting down - if (active == nullptr) { - Counters::increment(CALLTRACE_STORAGE_DROPPED); - return DROPPED_TRACE_ID; - } - - // RAII refcount guard automatically manages reference count lifecycle - // Uses pointer-first protocol for race-free operation - RefCountGuard guard(active); - - // Check if refcount guard allocation failed (slot exhaustion) - if (!guard.isActive()) { - // No refcount protection available - return dropped trace ID - Counters::increment(CALLTRACE_STORAGE_DROPPED); - return DROPPED_TRACE_ID; - } - - // CRITICAL REVALIDATION CHECK: Prevents use-after-free during guard construction race - // - // Race scenario: Scanner can see count=0 during guard construction window: - // 1. We load active table pointer above - // 2. RefCountGuard constructor stores pointer but count still 0 (preemption) - // 3. Scanner sees count=0, treats slot as inactive, deletes table - // 4. RefCountGuard constructor completes (count=1), but table already deleted - // 5. This check detects the race: _active_storage was nullified by scanner - // 6. We return DROPPED_TRACE_ID, never touching the deleted table - // - // Memory ordering: ACQUIRE load ensures we see scanner's ACQ_REL exchange to nullptr - CallTraceHashTable* original_active = const_cast(__atomic_load_n(&_active_storage, __ATOMIC_ACQUIRE)); - if (original_active != active || original_active == nullptr) { - // Storage was swapped or nullified during guard construction - // SAFE: We detected the race, drop this trace, never use the table pointer - Counters::increment(CALLTRACE_STORAGE_DROPPED); - return DROPPED_TRACE_ID; - } - - // Refcount guard prevents deletion - u64 result = active->put(num_frames, frames, truncated, weight); - - return result; -} - -/* - * Trace processing with signal blocking for simplified concurrency. - * This function is safe to call concurrently with put() operations. - * It is not designed to be called concurrently with itself. - */ -void CallTraceStorage::processTraces(std::function&)> processor) { - // PHASE 1: Collect liveness information with simple lock (rare operation) - { - SharedLockGuard lock(&_liveness_lock); - - // Use pre-allocated containers to avoid malloc() - _preserve_set_buffer.clear(); - - for (const auto& checker : _liveness_checkers) { - checker(_preserve_set_buffer); - } - } - - // PHASE 2: 10-Step Triple-Buffer Rotation - - // Load storage pointers - ACQUIRE ordering synchronizes with RELEASE stores from - // previous processTraces() calls and constructor initialization - CallTraceHashTable* original_active = const_cast(__atomic_load_n(&_active_storage, __ATOMIC_ACQUIRE)); - CallTraceHashTable* original_standby = const_cast(__atomic_load_n(&_standby_storage, __ATOMIC_ACQUIRE)); - CallTraceHashTable* original_scratch = const_cast(__atomic_load_n(&_scratch_storage, __ATOMIC_ACQUIRE)); - - // Clear process collection for reuse (no malloc/free) - _traces_buffer.clear(); - - // Step 1: Collect from current standby directly to _traces_buffer with hook for immediate preservation - // Only 'original_active' can be used from multiple threads at this moment. - // Both 'original_standby' and 'original_scratch' can be used only from the single thread executing 'processTraces' - // so we can iterate the first one and at the same time add new elements to the second one safely - original_standby->collect(_traces_buffer, [&](CallTrace* trace) { - if (_preserve_set_buffer.find(trace->trace_id) != _preserve_set_buffer.end()) { - original_scratch->putWithExistingId(trace, 1); - } - }); - - // Step 3: Clear standby table structure but DEFER memory deallocation - // The standby traces are now in _traces_buffer as raw pointers. - // We must keep the underlying memory alive until processor() finishes. - // clearTableOnly() resets the table for reuse but returns the chunks for later freeing. - ChunkList standby_chunks = original_standby->clearTableOnly(); - - { - // Critical section for table swap operations - disallow signals to interrupt - // Minimize the critical section by using the lexical scope around the critical code - CriticalSection cs; - - // Step 4: standby (now empty) becomes new active - // SYNCHRONIZATION POINT: After this store, new put() operations will target original_standby - // but ongoing put() operations may still be accessing original_active via RefCountGuards - // MEMORY_ORDER_RELEASE: Critical - synchronizes with acquire loads in put() method - __atomic_store_n(&_active_storage, original_standby, __ATOMIC_RELEASE); - - // Step 5: Complete the rotation: active→scratch, scratch→standby - // MEMORY_ORDER_RELEASE: Ensures visibility of storage state changes to RefCountGuard system - __atomic_exchange_n(&_scratch_storage, original_active, __ATOMIC_RELEASE); - __atomic_store_n(&_standby_storage, original_scratch, __ATOMIC_RELEASE); - } - - // Just make sure all puts to the original_active are done before proceeding - // Do this outside of the critical section not to block the new active area needlessly - RefCountGuard::waitForRefCountToClear(original_active); - - // Step 6: Collect from old active directly to _traces_buffer with hook for immediate preservation - original_active->collect(_traces_buffer, [&](CallTrace* trace) { - if (_preserve_set_buffer.find(trace->trace_id) != _preserve_set_buffer.end()) { - original_scratch->putWithExistingId(trace, 1); - } - }); - - // Step 7: Update the instance id for the recently retired active buffer - original_active->setInstanceId(getNextInstanceId()); - - // Step 8: Add dropped trace and call processor - _traces_buffer.insert(getDroppedTrace()); - - processor(_traces_buffer); - - // Step 9: NOW safe to free standby chunks - processor is done accessing those traces - // This completes the deferred deallocation that prevents use-after-free - LinearAllocator::freeChunks(standby_chunks); - - // Step 10: Clear the original active area (now scratch) - original_active->clear(); - - // Triple-buffer rotation maintains trace continuity with thread-safe malloc-free operations: - // - Pre-allocated collections prevent malloc/free during processTraces - // - Standby traces collected first (safe - no signal handler writes to standby) - // - New active (old standby, now empty) receives new traces from signal handlers - // - Old active (now scratch) safely collected after rotation, then cleared - // - New standby (old scratch) stores preserved traces for next cycle -} - -void CallTraceStorage::clear() { - // Mark critical section during clear operation for consistency - CriticalSection cs; - - // Load current table pointers - ACQUIRE ordering synchronizes with RELEASE stores - // from processTraces() rotation and constructor initialization - CallTraceHashTable* active = const_cast(__atomic_load_n(&_active_storage, __ATOMIC_ACQUIRE)); - CallTraceHashTable* standby = const_cast(__atomic_load_n(&_standby_storage, __ATOMIC_ACQUIRE)); - - // Direct clear operations with critical section protection - if (active) { - active->clear(); - } - if (standby) { - standby->clear(); - } - - // Reset counters when clearing all storage - Counters::set(CALLTRACE_STORAGE_BYTES, 0); - Counters::set(CALLTRACE_STORAGE_TRACES, 0); -} diff --git a/ddprof-lib/src/main/cpp/callTraceStorage.h b/ddprof-lib/src/main/cpp/callTraceStorage.h deleted file mode 100644 index 9be8cb038..000000000 --- a/ddprof-lib/src/main/cpp/callTraceStorage.h +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _CALLTRACESTORAGE_H -#define _CALLTRACESTORAGE_H - -#include "callTraceHashTable.h" -#include "refCountGuard.h" -#include "spinLock.h" -#include "os.h" -#include -#include -#include -#include -#include -#include -#include -#include - -// Forward declarations -class CallTraceHashTable; - -// Liveness checker function type -// Fills the provided set with 64-bit call_trace_id values that should be preserved -// Using reference parameter avoids malloc() for vector creation and copying -typedef std::function&)> LivenessChecker; - -class CallTraceStorage { -public: - // Reserved trace ID for dropped samples due to contention - // Real trace IDs are generated as (instance_id << 32) | slot, where instance_id starts from 1 - // Any ID with 0 in upper 32 bits is guaranteed to not clash with real trace IDs - static constexpr u64 DROPPED_TRACE_ID = 1ULL; - - // Static dropped trace object that appears in JFR constant pool - static CallTrace* getDroppedTrace(); - -private: - // Triple-buffered storage with atomic pointers - // Rotation: tmp=scratch, scratch=active, active=standby, standby=tmp - // New active inherits preserved traces for continuity - volatile CallTraceHashTable* _active_storage; - volatile CallTraceHashTable* _standby_storage; - volatile CallTraceHashTable* _scratch_storage; - - // Generation counter for ABA protection during table swaps - u32 _generation_counter; - - // Liveness checkers - protected by simple spinlock during registration/clear - // Using vector instead of unordered_set since std::function cannot be hashed - std::vector _liveness_checkers; - SpinLock _liveness_lock; // Simple atomic lock for rare liveness operations - - // Static atomic for instance ID generation - avoids function-local static initialization issues - - static u64 _next_instance_id; - - // Lazy initialization helper to avoid global constructor - static u64 getNextInstanceId(); - - // Pre-allocated collections for processTraces (single-threaded operation) - // These collections are reused to eliminate malloc/free cycles - std::unordered_set _traces_buffer; // All traces for JFR processing - std::unordered_set _preserve_set_buffer; // Preserve set for current cycle - -public: - CallTraceStorage(); - ~CallTraceStorage(); - - // Register a liveness checker (rare operation - uses simple lock) - void registerLivenessChecker(LivenessChecker checker); - - // Clear liveness checkers (rare operation - uses simple lock) - void clearLivenessCheckers(); - - // Lock-free put operation for signal handler safety - // Uses RefCountGuard and generation counter for ABA protection - u64 put(int num_frames, ASGCT_CallFrame* frames, bool truncated, u64 weight); - - // Lock-free trace processing with RefCountGuard protection - // The callback receives traces that are guaranteed to be valid during execution - // Uses atomic table swapping with grace period for safe memory reclamation - void processTraces(std::function&)> processor); - - // Enhanced clear with liveness preservation (rarely called - uses atomic operations) - void clear(); -}; - -#endif // _CALLTRACESTORAGE_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/codeCache.cpp b/ddprof-lib/src/main/cpp/codeCache.cpp deleted file mode 100644 index 5203d5c5d..000000000 --- a/ddprof-lib/src/main/cpp/codeCache.cpp +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "codeCache.h" -#include "dwarf.h" -#include "os.h" -#include "safeAccess.h" - -#include -#include -#include -#include - -char *NativeFunc::create(const char *name, short lib_index) { - size_t size = align_up(sizeof(NativeFunc) + 1 + strlen(name), sizeof(NativeFunc*)); - NativeFunc *f = (NativeFunc *)aligned_alloc(sizeof(NativeFunc*), size); - f->_lib_index = lib_index; - f->_mark = 0; - // cppcheck-suppress memleak - return strcpy(f->_name, name); -} - -void NativeFunc::destroy(char *name) { free(from(name)); } - -char NativeFunc::read_mark(const char* name) { - if (name == nullptr) { - return 0; - } - NativeFunc* func = from(name); - if (!is_aligned(func, sizeof(func))) { - return 0; - } - // Use SafeAccess to read the mark field in signal handler context - // Read the first 4 bytes (lib_index + mark + reserved) and extract the mark byte - int32_t prefix = SafeAccess::safeFetch32((int32_t*)func, 0); - // Extract mark byte: shift right by 16 bits to skip lib_index (2 bytes), mask to 1 byte - return (char)((prefix >> 16) & 0xFF); -} - -CodeCache::CodeCache(const char *name, short lib_index, - const void *min_address, const void *max_address, - const char* image_base, bool imports_patchable) { - _name = NativeFunc::create(name, -1); - _lib_index = lib_index; - _min_address = min_address; - _max_address = max_address; - _text_base = NULL; - _image_base = image_base; - - _plt_offset = 0; - _plt_size = 0; - _debug_symbols = false; - - // Initialize build-id fields - _build_id = nullptr; - _build_id_len = 0; - _load_bias = 0; - - memset(_imports, 0, sizeof(_imports)); - _imports_patchable = imports_patchable; - - _dwarf_table = NULL; - _dwarf_table_length = 0; - _default_frame = &FrameDesc::default_frame; - - _capacity = INITIAL_CODE_CACHE_CAPACITY; - _count = 0; - _blobs = new CodeBlob[_capacity]; -} - -void CodeCache::copyFrom(const CodeCache& other) { - _name = NativeFunc::create(other._name, -1); - _lib_index = other._lib_index; - _min_address = other._min_address; - _max_address = other._max_address; - _text_base = other._text_base; - _image_base = other._image_base; - - _plt_offset = other._plt_offset; - _plt_size = other._plt_size; - _debug_symbols = other._debug_symbols; - - // Copy build-id information - _build_id_len = other._build_id_len; - if (other._build_id != nullptr && other._build_id_len > 0) { - size_t hex_str_len = strlen(other._build_id); - _build_id = static_cast(malloc(hex_str_len + 1)); - if (_build_id != nullptr) { - strcpy(_build_id, other._build_id); - } - } else { - _build_id = nullptr; - } - _load_bias = other._load_bias; - - memset(_imports, 0, sizeof(_imports)); - _imports_patchable = other._imports_patchable; - - _dwarf_table_length = other._dwarf_table_length; - if (_dwarf_table_length > 0) { - _dwarf_table = (FrameDesc*)malloc(_dwarf_table_length * sizeof(FrameDesc)); - memcpy(_dwarf_table, other._dwarf_table, - _dwarf_table_length * sizeof(FrameDesc)); - } else { - _dwarf_table = nullptr; - } - _default_frame = other._default_frame; - - _capacity = other._capacity; - _count = other._count; - _blobs = new CodeBlob[_capacity]; - memcpy(_blobs, other._blobs, _count * sizeof(CodeBlob)); -} - -CodeCache::CodeCache(const CodeCache &other) { - copyFrom(other); -} - -CodeCache &CodeCache::operator=(const CodeCache &other) { - if (&other == this) { - return *this; - } - - NativeFunc::destroy(_name); - free(_dwarf_table); - delete[] _blobs; - free(_build_id); - - copyFrom(other); - - return *this; -} - -CodeCache::~CodeCache() { - for (int i = 0; i < _count; i++) { - NativeFunc::destroy(_blobs[i]._name); - } - NativeFunc::destroy(_name); - delete[] _blobs; - free(_dwarf_table); - free(_build_id); // Free build-id memory -} - -void CodeCache::expand() { - CodeBlob *old_blobs = _blobs; - CodeBlob *new_blobs = new CodeBlob[_capacity * 2]; - - memcpy(new_blobs, old_blobs, _count * sizeof(CodeBlob)); - - _capacity *= 2; - _blobs = new_blobs; - delete[] old_blobs; -} - -void CodeCache::add(const void *start, int length, const char *name, - bool update_bounds) { - char *name_copy = NativeFunc::create(name, _lib_index); - // Replace non-printable characters - for (char *s = name_copy; *s != 0; s++) { - if (*s < ' ') - *s = '?'; - } - - if (_count >= _capacity) { - expand(); - } - - const void *end = (const char *)start + length; - _blobs[_count]._start = start; - _blobs[_count]._end = end; - _blobs[_count]._name = name_copy; - _count++; - - if (update_bounds) { - updateBounds(start, end); - } -} - -void CodeCache::updateBounds(const void *start, const void *end) { - if (start < _min_address) - _min_address = start; - if (end > _max_address) - _max_address = end; -} - -void CodeCache::sort() { - if (_count == 0) - return; - - qsort(_blobs, _count, sizeof(CodeBlob), CodeBlob::comparator); - - if (_min_address == NO_MIN_ADDRESS) - _min_address = _blobs[0]._start; - if (_max_address == NO_MAX_ADDRESS) - _max_address = _blobs[_count - 1]._end; -} - -CodeBlob *CodeCache::findBlob(const char *name) { - for (int i = 0; i < _count; i++) { - const char *blob_name = _blobs[i]._name; - if (blob_name != NULL && strcmp(blob_name, name) == 0) { - return &_blobs[i]; - } - } - return NULL; -} - -CodeBlob *CodeCache::findBlobByAddress(const void *address) { - for (int i = 0; i < _count; i++) { - if (address >= _blobs[i]._start && address < _blobs[i]._end) { - return &_blobs[i]; - } - } - return NULL; -} - -const void *CodeCache::binarySearch(const void *address, const char **name) { - int low = 0; - int high = _count - 1; - - while (low <= high) { - int mid = (unsigned int)(low + high) >> 1; - if (_blobs[mid]._end <= address) { - low = mid + 1; - } else if (_blobs[mid]._start > address) { - high = mid - 1; - } else { - if (name != NULL) { - *name = _blobs[mid]._name; - } - return _blobs[mid]._start; - } - } - - // Symbols with zero size can be valid functions: e.g. ASM entry points or - // kernel code. Also, in some cases (endless loop) the return address may - // point beyond the function. - if (low > 0 && (_blobs[low - 1]._start == _blobs[low - 1]._end || - _blobs[low - 1]._end == address)) { - - if (name != NULL) { - *name = _blobs[low - 1]._name; - } - - return _blobs[low - 1]._start; - } - return _name; -} - -void CodeCache::dump() { -#ifdef TRACE - fprintf(stdout, "Dumping symbols for %s:\n+-\n", _name); - for (int i = 0; i < _count; i++) { - fprintf(stdout, "%d. %s\n", i, _blobs[i]._name); - } - fprintf(stdout, "+-\n"); -#endif // TRACE -} - -const void *CodeCache::findSymbol(const char *name) { - CodeBlob *blob = findBlob(name); - return blob == NULL ? NULL : blob->_start; -} - -const void *CodeCache::findSymbolByPrefix(const char *prefix) { - return findSymbolByPrefix(prefix, strlen(prefix)); -} - -const void *CodeCache::findSymbolByPrefix(const char *prefix, int prefix_len) { - for (int i = 0; i < _count; i++) { - const char *blob_name = _blobs[i]._name; - if (blob_name != NULL && strncmp(blob_name, prefix, prefix_len) == 0) { - return _blobs[i]._start; - } - } - return NULL; -} - -void CodeCache::findSymbolsByPrefix(std::vector &prefixes, - std::vector &symbols) { - std::vector prefix_lengths; - prefix_lengths.reserve(prefixes.size()); - for (const char *prefix : prefixes) { - prefix_lengths.push_back(strlen(prefix)); - } - for (int i = 0; i < _count; i++) { - const char *blob_name = _blobs[i]._name; - if (blob_name != NULL) { - for (size_t i = 0; i < prefixes.size(); i++) { - if (strncmp(blob_name, prefixes[i], prefix_lengths[i]) == 0) { - symbols.push_back(_blobs[i]._start); - } - } - } - } -} - -void CodeCache::saveImport(ImportId id, void** entry) { - for (int ty = 0; ty < NUM_IMPORT_TYPES; ty++) { - if (_imports[id][ty] == nullptr) { - _imports[id][ty] = entry; - return; - } - } -} - -void CodeCache::addImport(void **entry, const char *name) { - switch (name[0]) { - case 'a': - if (strcmp(name, "aligned_alloc") == 0) { - saveImport(im_aligned_alloc, entry); - } - break; - case 'c': - if (strcmp(name, "calloc") == 0) { - saveImport(im_calloc, entry); - } - break; - case 'd': - if (strcmp(name, "dlopen") == 0) { - saveImport(im_dlopen, entry); - } - break; - case 'f': - if (strcmp(name, "free") == 0) { - saveImport(im_free, entry); - } - break; - case 'm': - if (strcmp(name, "malloc") == 0) { - saveImport(im_malloc, entry); - } - break; - case 'p': - if (strcmp(name, "pthread_create") == 0) { - saveImport(im_pthread_create, entry); - } else if (strcmp(name, "pthread_exit") == 0) { - saveImport(im_pthread_exit, entry); - } else if (strcmp(name, "pthread_setspecific") == 0) { - saveImport(im_pthread_setspecific, entry); - } else if (strcmp(name, "poll") == 0) { - saveImport(im_poll, entry); - } else if (strcmp(name, "posix_memalign") == 0) { - saveImport(im_posix_memalign, entry); - } - break; - case 'r': - if (strcmp(name, "realloc") == 0) { - saveImport(im_realloc, entry); - } else if (strcmp(name, "recv") == 0) { - saveImport(im_recv, entry); - } else if (strcmp(name, "read") == 0) { - saveImport(im_read, entry); - } - break; - case 's': - if (strcmp(name, "send") == 0) { - saveImport(im_send, entry); - } else if (strcmp(name, "sigaction") == 0) { - saveImport(im_sigaction, entry); - } - break; - case 'w': - if (strcmp(name, "write") == 0) { - saveImport(im_write, entry); - } - break; - } -} - -void **CodeCache::findImport(ImportId id) { - if (!_imports_patchable) { - makeImportsPatchable(); - _imports_patchable = true; - } - return _imports[id][PRIMARY]; -} - -void CodeCache::patchImport(ImportId id, void *hook_func) { - if (!_imports_patchable) { - makeImportsPatchable(); - _imports_patchable = true; - } - - for (int ty = 0; ty < NUM_IMPORT_TYPES; ty++) {void **entry = _imports[id][ty]; - if (entry != NULL) { - *entry = hook_func; - }} -} - -void CodeCache::makeImportsPatchable() { - void **min_import = (void **)-1; - void **max_import = NULL; - for (int i = 0; i < NUM_IMPORTS; i++) { - for (int j = 0; j < NUM_IMPORT_TYPES; j++) { - void** entry = _imports[i][j]; - if (entry == NULL) continue; - if (entry < min_import) - min_import = entry; - if (entry > max_import) - max_import = entry; - } - } - - if (max_import != NULL) { - uintptr_t patch_start = (uintptr_t)min_import & ~OS::page_mask; - uintptr_t patch_end = (uintptr_t)max_import & ~OS::page_mask; - mprotect((void *)patch_start, patch_end - patch_start + OS::page_size, - PROT_READ | PROT_WRITE); - } -} - -void CodeCache::setDwarfTable(FrameDesc *table, int length, const FrameDesc &default_frame) { - _dwarf_table = table; - _dwarf_table_length = length; - _default_frame = &default_frame; -} - -FrameDesc CodeCache::findFrameDesc(const void *pc) { - if (_dwarf_table == NULL || _dwarf_table_length == 0) { - return *_default_frame; - } - - u32 target_loc = (const char *)pc - _text_base; - int low = 0; - int high = _dwarf_table_length - 1; - - while (low <= high) { - int mid = (unsigned int)(low + high) >> 1; - if (_dwarf_table[mid].loc < target_loc) { - low = mid + 1; - } else if (_dwarf_table[mid].loc > target_loc) { - high = mid - 1; - } else { - return _dwarf_table[mid]; - } - } - - if (low > 0) { - return _dwarf_table[low - 1]; - } else if (target_loc - _plt_offset < _plt_size) { - return FrameDesc::empty_frame; - } else { - return *_default_frame; - } -} - -void CodeCache::setBuildId(const char* build_id, size_t build_id_len) { - // Free existing build-id if any - free(_build_id); - _build_id = nullptr; - _build_id_len = 0; - - if (build_id != nullptr && build_id_len > 0) { - // build_id is a hex string, allocate based on actual string length - size_t hex_str_len = strlen(build_id); - _build_id = static_cast(malloc(hex_str_len + 1)); - - if (_build_id != nullptr) { - // Copy the hex string - strcpy(_build_id, build_id); - // Store the original byte length (not hex string length) - _build_id_len = build_id_len; - } - } -} - diff --git a/ddprof-lib/src/main/cpp/codeCache.h b/ddprof-lib/src/main/cpp/codeCache.h deleted file mode 100644 index 92b45bf47..000000000 --- a/ddprof-lib/src/main/cpp/codeCache.h +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _CODECACHE_H -#define _CODECACHE_H - -#include "common.h" -#include "counters.h" -#include "dwarf.h" -#include "utils.h" - -#include -#include -#include -#include - -#define NO_MIN_ADDRESS ((const void *)-1) -#define NO_MAX_ADDRESS ((const void *)0) - -typedef bool (*NamePredicate)(const char *name); - -const int INITIAL_CODE_CACHE_CAPACITY = 1000; -const int MAX_NATIVE_LIBS = 2048; - -enum ImportId { - im_dlopen, - im_pthread_create, - im_pthread_exit, - im_pthread_setspecific, - im_poll, - im_malloc, - im_calloc, - im_realloc, - im_free, - im_posix_memalign, - im_aligned_alloc, - im_sigaction, - im_send, - im_recv, - im_write, - im_read, - NUM_IMPORTS -}; - -enum ImportType { - PRIMARY, - SECONDARY, - NUM_IMPORT_TYPES -}; - -enum Mark { - MARK_VM_RUNTIME = 1, - MARK_INTERPRETER = 2, - MARK_COMPILER_ENTRY = 3, - MARK_ASYNC_PROFILER = 4, // async-profiler internals such as native hooks. - MARK_THREAD_ENTRY = 5, // Thread entry points (thread_native_entry, JavaThread::, etc.) -}; - -class NativeFunc { -private: - short _lib_index; - char _mark; - char _reserved; - char _name[0]; - - static NativeFunc *from(const char *name) { - return (NativeFunc *)(name - sizeof(NativeFunc)); - } - -public: - static char *create(const char *name, short lib_index); - static void destroy(char *name); - - static short libIndex(const char *name) { - if (name == nullptr) { - return -1; - } - NativeFunc* func = from(name); - if (!is_aligned(func, sizeof(func))) { - return -1; - } - return func->_lib_index; - } - - static bool is_marked(const char *name) { - return read_mark(name) != 0; - } - - static char read_mark(const char* name); - - static void set_mark(const char* name, char value) { - if (name == nullptr) { - return; - } - NativeFunc* func = from(name); - if (!is_aligned(func, sizeof(func))) { - return; - } - func->_mark = value; - } -}; - -class CodeBlob { -public: - const void *_start; - const void *_end; - char *_name; - - static int comparator(const void *c1, const void *c2) { - CodeBlob *cb1 = (CodeBlob *)c1; - CodeBlob *cb2 = (CodeBlob *)c2; - if (cb1->_start < cb2->_start) { - return -1; - } else if (cb1->_start > cb2->_start) { - return 1; - } else if (cb1->_end == cb2->_end) { - return 0; - } else { - return cb1->_end > cb2->_end ? -1 : 1; - } - } -}; - -class CodeCache { -private: - char *_name; - short _lib_index; - const void *_min_address; - const void *_max_address; - const char *_text_base; - const char* _image_base; - - unsigned int _plt_offset; - unsigned int _plt_size; - - // Build-ID and load bias for remote symbolication - char *_build_id; // GNU build-id (hex string, null if not available) - size_t _build_id_len; // Build-id length in bytes (raw, not hex string length) - uintptr_t _load_bias; // Load bias (image_base - file_base address) - - void **_imports[NUM_IMPORTS][NUM_IMPORT_TYPES]; - bool _imports_patchable; - bool _debug_symbols; - - FrameDesc *_dwarf_table; - int _dwarf_table_length; - const FrameDesc *_default_frame; - - int _capacity; - int _count; - CodeBlob *_blobs; - - void expand(); - void makeImportsPatchable(); - void saveImport(ImportId id, void** entry); - void copyFrom(const CodeCache& other); - -public: - explicit CodeCache(const char *name, short lib_index = -1, - const void *min_address = NO_MIN_ADDRESS, - const void *max_address = NO_MAX_ADDRESS, - const char* image_base = NULL, - bool imports_patchable = false); - // Copy constructor - CodeCache(const CodeCache &other); - // Copy assignment operator - CodeCache &operator=(const CodeCache &other); - - ~CodeCache(); - - void dump(); - - const char *name() const { return _name; } - - const void *minAddress() const { return _min_address; } - - const void *maxAddress() const { return _max_address; } - - const char* imageBase() const { return _image_base; } - - bool contains(const void *address) const { - return address >= _min_address && address < _max_address; - } - - void setTextBase(const char *text_base) { _text_base = text_base; } - - void setPlt(unsigned int plt_offset, unsigned int plt_size) { - _plt_offset = plt_offset; - _plt_size = plt_size; - } - - bool hasDebugSymbols() const { return _debug_symbols; } - - void setDebugSymbols(bool debug_symbols) { _debug_symbols = debug_symbols; } - - // Build-ID and remote symbolication support - const char* buildId() const { return _build_id; } - size_t buildIdLen() const { return _build_id_len; } - bool hasBuildId() const { return _build_id != nullptr; } - uintptr_t loadBias() const { return _load_bias; } - short libIndex() const { return _lib_index; } - - // Sets the build-id (hex string) and stores the original byte length - // build_id: null-terminated hex string (e.g., "abc123..." for 40-char string) - // build_id_len: original byte length before hex conversion (e.g., 20 bytes) - void setBuildId(const char* build_id, size_t build_id_len); - void setLoadBias(uintptr_t load_bias) { _load_bias = load_bias; } - - void add(const void *start, int length, const char *name, - bool update_bounds = false); - void updateBounds(const void *start, const void *end); - void sort(); - - /** - * Mark symbols matching the predicate with the given mark value. - * - * This is called during profiler initialization to mark JVM internal functions - * (MARK_VM_RUNTIME, MARK_INTERPRETER, MARK_COMPILER_ENTRY, MARK_ASYNC_PROFILER). - */ - template - inline void mark(NamePredicate predicate, char value) { - for (int i = 0; i < _count; i++) { - const char* blob_name = _blobs[i]._name; - if (blob_name != NULL && predicate(blob_name)) { - NativeFunc::set_mark(blob_name, value); - } - } - - if (value == MARK_VM_RUNTIME && _name != NULL) { - // In case a library has no debug symbols - NativeFunc::set_mark(_name, value); - } - } - - void addImport(void **entry, const char *name); - void **findImport(ImportId id); - void patchImport(ImportId, void *hook_func); - - CodeBlob *findBlob(const char *name); - CodeBlob *findBlobByAddress(const void *address); - const void *binarySearch(const void *address, const char **name); - const void *findSymbol(const char *name); - const void *findSymbolByPrefix(const char *prefix); - const void *findSymbolByPrefix(const char *prefix, int prefix_len); - void findSymbolsByPrefix(std::vector &prefixes, - std::vector &symbols); - - void setDwarfTable(FrameDesc *table, int length, const FrameDesc &default_frame = FrameDesc::default_frame); - FrameDesc findFrameDesc(const void *pc); - - long long memoryUsage() { - return _capacity * sizeof(CodeBlob *) + _count * sizeof(NativeFunc); - } - - int count() { return _count; } - CodeBlob* blob(int idx) { - return &_blobs[idx]; - } -}; - -class CodeCacheArray { -private: - CodeCache *_libs[MAX_NATIVE_LIBS]; - volatile int _reserved; // next slot to reserve (CAS by writers) - volatile int _count; // published count (all indices < _count have non-NULL pointers) - volatile size_t _used_memory; - bool _overflow_reported; - -public: - CodeCacheArray() : _reserved(0), _count(0), _used_memory(0), _overflow_reported(false) { - memset(_libs, 0, MAX_NATIVE_LIBS * sizeof(CodeCache *)); - } - - CodeCache *operator[](int index) const { return __atomic_load_n(&_libs[index], __ATOMIC_ACQUIRE); } - - // All indices < count() are guaranteed to have a non-NULL pointer. - int count() const { return __atomic_load_n(&_count, __ATOMIC_ACQUIRE); } - - // Pointer-first add: reserve a slot via CAS on _reserved, store the - // pointer with RELEASE, then advance _count. Readers see count() grow - // only after the pointer is visible, so indices < count() never yield NULL. - bool add(CodeCache *lib) { - int slot = __atomic_load_n(&_reserved, __ATOMIC_RELAXED); - do { - if (slot >= MAX_NATIVE_LIBS) { - Counters::increment(NATIVE_LIBS_DROPPED); - if (!_overflow_reported) { - _overflow_reported = true; - LOG_WARN("Native library limit reached (%d). Additional libraries will not be tracked.", MAX_NATIVE_LIBS); - } - return false; - } - } while (!__atomic_compare_exchange_n(&_reserved, &slot, slot + 1, - true, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); - assert(__atomic_load_n(&_libs[slot], __ATOMIC_RELAXED) == nullptr); - __atomic_fetch_add(&_used_memory, lib->memoryUsage(), __ATOMIC_RELAXED); - // Store pointer before publishing count. The RELEASE here pairs with - // the ACQUIRE load in operator[]/at() and count(). - __atomic_store_n(&_libs[slot], lib, __ATOMIC_RELEASE); - // Advance _count to publish the new slot. Spin until our slot is next - // in line, preserving contiguous ordering when multiple adds race. - while (__atomic_load_n(&_count, __ATOMIC_RELAXED) != slot) { - // wait for preceding slots to publish - } - __atomic_store_n(&_count, slot + 1, __ATOMIC_RELEASE); - return true; - } - - CodeCache* at(int index) const { - if (index >= MAX_NATIVE_LIBS) { - return nullptr; - } - return __atomic_load_n(&_libs[index], __ATOMIC_ACQUIRE); - } - - size_t memoryUsage() const { - return __atomic_load_n(&_used_memory, __ATOMIC_RELAXED); - } -}; - -#endif // _CODECACHE_H diff --git a/ddprof-lib/src/main/cpp/common.h b/ddprof-lib/src/main/cpp/common.h deleted file mode 100644 index 13b8ae9ca..000000000 --- a/ddprof-lib/src/main/cpp/common.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef _COMMON_H -#define _COMMON_H - -#include -#include - -// Sanitizer detection: define ASAN_ENABLED / TSAN_ENABLED uniformly across -// toolchains. clang exposes sanitizers only via __has_feature(...), while gcc -// defines __SANITIZE_ADDRESS__ / __SANITIZE_THREAD__. Guarding code on the gcc -// macros alone silently drops every sanitizer annotation under clang (the CI -// sanitizer compiler), so always go through these macros instead. -#if defined(__has_feature) -# if __has_feature(address_sanitizer) -# ifndef ASAN_ENABLED -# define ASAN_ENABLED 1 -# endif -# endif -# if __has_feature(thread_sanitizer) -# ifndef TSAN_ENABLED -# define TSAN_ENABLED 1 -# endif -# endif -#endif -#ifdef __SANITIZE_ADDRESS__ -# ifndef ASAN_ENABLED -# define ASAN_ENABLED 1 -# endif -#endif -#ifdef __SANITIZE_THREAD__ -# ifndef TSAN_ENABLED -# define TSAN_ENABLED 1 -# endif -#endif - -// Knuth's multiplicative constant (golden ratio * 2^64 for 64-bit) -// Used for hash distribution in various components -constexpr size_t KNUTH_MULTIPLICATIVE_CONSTANT = 0x9e3779b97f4a7c15ULL; - -#ifdef DEBUG -#define TEST_LOG(fmt, ...) do { \ - fprintf(stdout, "[TEST::INFO] " fmt "\n", ##__VA_ARGS__); \ - fflush(stdout); \ -} while (0) -#else -#define TEST_LOG(fmt, ...) // No-op in non-debug mode -#endif - -// Lightweight stderr warning that does not depend on the Log subsystem. -// Safe to call from low-level code where Log may not be initialized. -#define LOG_WARN(fmt, ...) do { \ - fprintf(stderr, "[ddprof] [WARN] " fmt "\n", ##__VA_ARGS__); \ -} while (0) - -#endif // _COMMON_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/context.h b/ddprof-lib/src/main/cpp/context.h deleted file mode 100644 index ca4e3d5d8..000000000 --- a/ddprof-lib/src/main/cpp/context.h +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2021, 2022 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _CONTEXT_H -#define _CONTEXT_H - -#include "arch.h" - -static const u32 DD_TAGS_CAPACITY = 10; - -typedef struct { - u32 value; -} Tag; - -class alignas(DEFAULT_CACHE_LINE_SIZE) Context { -public: - u64 spanId; - u64 rootSpanId; - Tag tags[DD_TAGS_CAPACITY]; - - Tag get_tag(int i) { return tags[i]; } -}; - -#endif /* _CONTEXT_H */ diff --git a/ddprof-lib/src/main/cpp/context_api.cpp b/ddprof-lib/src/main/cpp/context_api.cpp deleted file mode 100644 index 53c989fa1..000000000 --- a/ddprof-lib/src/main/cpp/context_api.cpp +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "context_api.h" -#include "context.h" -#include "guards.h" -#include "otel_context.h" -#include "profiler.h" -#include "thread.h" -#include - -/** - * Initialize context TLS for the current thread on first use. - * Must be called with signals blocked to prevent musl TLS deadlock: - * on musl, the first write to a TLS variable triggers lazy slot allocation, - * which acquires an internal lock that is also held during signal delivery, - * causing deadlock if a signal fires mid-init. - * The OtelThreadContextRecord is already zero-initialized by the ProfiledThread ctor. - */ -void ContextApi::initializeContextTLS(ProfiledThread* thrd) { - SignalBlocker blocker; - // Set the TLS pointer permanently to this thread's record. - // This first write triggers musl's TLS slot initialization (see above). - // The pointer remains stable for the thread's lifetime; external profilers - // rely solely on the valid flag for consistency, not pointer nullness. - otel_thread_ctx_v1 = thrd->getOtelContextRecord(); - thrd->markContextInitialized(); -} - -bool ContextApi::get(u64& span_id, u64& root_span_id) { - ProfiledThread* thrd = ProfiledThread::currentSignalSafe(); - if (thrd == nullptr || !thrd->isContextInitialized()) { - return false; - } - - OtelThreadContextRecord* record = thrd->getOtelContextRecord(); - if (__atomic_load_n(&record->valid, __ATOMIC_ACQUIRE) != 1) { - return false; - } - u64 val = 0; - for (int i = 0; i < 8; i++) { val = (val << 8) | record->span_id[i]; } - span_id = val; - - root_span_id = thrd->getOtelLocalRootSpanId(); - return true; -} - -Context ContextApi::snapshot() { - ProfiledThread* thrd = ProfiledThread::currentSignalSafe(); - if (thrd == nullptr) { - return {}; - } - size_t numAttrs = Profiler::instance()->numContextAttributes(); - return thrd->snapshotContext(numAttrs); -} diff --git a/ddprof-lib/src/main/cpp/context_api.h b/ddprof-lib/src/main/cpp/context_api.h deleted file mode 100644 index de3249de4..000000000 --- a/ddprof-lib/src/main/cpp/context_api.h +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _CONTEXT_API_H -#define _CONTEXT_API_H - -#include "arch.h" -#include "context.h" -#include - -class ProfiledThread; - -/** - * Unified context API for trace/span context storage. - * - * Uses OTEP #4947 TLS pointer (otel_thread_ctx_v1) for all - * context reads and writes. The OTEP record is embedded in ProfiledThread - * and discovered by external profilers via ELF dynsym. - */ -class ContextApi { -public: - /** - * Initialize context TLS for the given thread on first use. - * Must be called with signals blocked (SignalBlocker). - */ - static void initializeContextTLS(ProfiledThread* thrd); - - /** - * Read span ID and local root span ID for the current thread. - * - * Used by signal handlers to get the current trace context. - * Returns false if the OTEP valid flag is not set (record being mutated - * or thread not yet initialized). Does not modify span_id or root_span_id - * on failure; callers must pre-initialize output parameters to their - * desired default. Does not detect torn reads (the valid flag guards that). - * - * Unlike snapshot(), this reads only spanId and rootSpanId — use - * snapshot() when tag encodings are also needed. - * - * @param span_id Output: the span ID - * @param root_span_id Output: the root span ID (from sidecar) - * @return true if the valid flag was set and context was read - */ - static bool get(u64& span_id, u64& root_span_id); - - /** - * Snapshot the current thread's full context into a Context struct. - * - * Populates a Context with spanId, rootSpanId (from sidecar) - * and tag encodings (from sidecar) so that writeContextSnapshot() - * works for both live and deferred event paths. Unlike get(), this - * also captures custom attribute tag encodings. - * - * @return A Context struct representing the current thread's context - */ - static Context snapshot(); -}; - -#endif /* _CONTEXT_API_H */ diff --git a/ddprof-lib/src/main/cpp/counters.cpp b/ddprof-lib/src/main/cpp/counters.cpp deleted file mode 100644 index 5b21e4eaf..000000000 --- a/ddprof-lib/src/main/cpp/counters.cpp +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright 2023 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include "counters.h" -#include - -long long *Counters::init() { - u32 alignment = sizeof(long long) * ALIGNMENT; - long long *counters = - (long long *)aligned_alloc(alignment, DD_NUM_COUNTERS * alignment); - memset(counters, 0, DD_NUM_COUNTERS * alignment); - return counters; -} diff --git a/ddprof-lib/src/main/cpp/counters.h b/ddprof-lib/src/main/cpp/counters.h deleted file mode 100644 index c6b606b0e..000000000 --- a/ddprof-lib/src/main/cpp/counters.h +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright 2023 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef JAVA_PROFILER_LIBRARY_COUNTERS_H -#define JAVA_PROFILER_LIBRARY_COUNTERS_H - -#include "arch.h" -#include -#include - -#define DD_COUNTER_TABLE(X) \ - X(DICTIONARY_BYTES, "dictionary_bytes") \ - X(DICTIONARY_CLASSES_BYTES, "dictionary_classes_bytes") \ - X(DICTIONARY_ENDPOINTS_BYTES, "dictionary_endpoints_bytes") \ - X(DICTIONARY_CONTEXT_BYTES, "dictionary_context_bytes") \ - X(DICTIONARY_PAGES, "dictionary_pages") \ - X(DICTIONARY_CLASSES_PAGES, "dictionary_classes_pages") \ - X(DICTIONARY_ENDPOINTS_PAGES, "dictionary_endpoints_pages") \ - X(DICTIONARY_CONTEXT_PAGES, "dictionary_context_pages") \ - X(DICTIONARY_KEYS, "dictionary_keys") \ - X(DICTIONARY_CLASSES_KEYS, "dictionary_classes_keys") \ - X(DICTIONARY_ENDPOINTS_KEYS, "dictionary_endpoints_keys") \ - X(DICTIONARY_CONTEXT_KEYS, "dictionary_context_keys") \ - X(DICTIONARY_KEYS_BYTES, "dictionary_keys_bytes") \ - X(DICTIONARY_CLASSES_KEYS_BYTES, "dictionary_classes_keys_bytes") \ - X(DICTIONARY_ENDPOINTS_KEYS_BYTES, "dictionary_endpoints_keys_bytes") \ - X(DICTIONARY_CONTEXT_KEYS_BYTES, "dictionary_context_keys_bytes") \ - X(DICTIONARY_ARENA_WASTE_BYTES, "dictionary_arena_waste_bytes") \ - X(DICTIONARY_CLASSES_ARENA_WASTE_BYTES, "dictionary_classes_arena_waste_bytes") \ - X(DICTIONARY_ENDPOINTS_ARENA_WASTE_BYTES, "dictionary_endpoints_arena_waste_bytes") \ - X(DICTIONARY_CONTEXT_ARENA_WASTE_BYTES, "dictionary_context_arena_waste_bytes") \ - X(DICTIONARY_DRAIN_TIMEOUTS, "dictionary_drain_timeouts") \ - X(CONTEXT_STORAGE_BYTES, "context_storage_bytes") \ - X(CONTEXT_STORAGE_PAGES, "context_storage_pages") \ - X(CONTEXT_BOUNDS_MISS_INITS, "context_bounds_miss_inits") \ - X(CONTEXT_BOUNDS_MISS_GETS, "context_bounds_miss_gets") \ - X(CONTEXT_NULL_PAGE_GETS, "context_null_page_gets") \ - X(CONTEXT_ALLOC_FAILS, "context_alloc_fails") \ - X(CALLTRACE_STORAGE_BYTES, "calltrace_storage_bytes") \ - X(CALLTRACE_STORAGE_TRACES, "calltrace_storage_traces") \ - X(LINEAR_ALLOCATOR_BYTES, "linear_allocator_bytes") \ - X(LINEAR_ALLOCATOR_CHUNKS, "linear_allocator_chunks") \ - X(THREAD_IDS_COUNT, "thread_ids_count") \ - X(THREAD_NAMES_COUNT, "thread_names_count") \ - X(THREAD_FILTER_PAGES, "thread_filter_pages") \ - X(THREAD_FILTER_BYTES, "thread_filter_bytes") \ - X(JMETHODID_SKIPPED, "jmethodid_skipped_count") \ - X(CODECACHE_NATIVE_SIZE_BYTES, "codecache_native_size_bytes") \ - X(CODECACHE_NATIVE_COUNT, "native_codecache_count") \ - X(CODECACHE_RUNTIME_STUBS_SIZE_BYTES, "codecache_runtime_stubs_size_bytes") \ - X(AGCT_NOT_REGISTERED_IN_TLS, "agct_not_registered_in_tls") \ - X(AGCT_NOT_JAVA, "agct_not_java") \ - X(AGCT_NATIVE_NO_JAVA_CONTEXT, "agct_native_no_java_context") \ - X(AGCT_BLOCKED_IN_VM, "agct_blocked_in_vm") \ - X(SKIPPED_WALLCLOCK_UNWINDS, "skipped_wallclock_unwinds") \ - X(WC_SIGNAL_SUPPRESSED_SAMPLED_RUN, "wc_signals_suppressed_sampled_run") \ - X(WC_UNOWNED_BLOCKED_SUPPRESSED, "wc_unowned_blocked_suppressed") \ - X(WC_UNOWNED_BLOCKED_RECORDED, "wc_unowned_blocked_recorded") \ - X(WC_SIGNAL_QUEUE_FULL, "wc_signals_queue_full") \ - X(UNWINDING_TIME_ASYNC, "unwinding_ticks_async") \ - X(UNWINDING_TIME_JVMTI, "unwinding_ticks_jvmti") \ - X(CALLTRACE_STORAGE_DROPPED, "calltrace_storage_dropped_traces") \ - X(LINE_NUMBER_TABLES, "line_number_tables") \ - X(REMOTE_SYMBOLICATION_FRAMES, "remote_symbolication_frames") \ - X(REMOTE_SYMBOLICATION_LIBS_WITH_BUILD_ID, "remote_symbolication_libs_with_build_id") \ - X(REMOTE_SYMBOLICATION_BUILD_ID_CACHE_HITS, "remote_symbolication_build_id_cache_hits") \ - X(VTABLE_RECEIVER_RESOLVE_FAILED, "vtable_receiver_resolve_failed") \ - X(THREAD_ENTRY_MARK_DETECTIONS, "thread_entry_mark_detections") \ - X(WALKVM_THREAD_INACCESSIBLE, "walkvm_thread_inaccessible") \ - X(WALKVM_ANCHOR_NULL, "walkvm_anchor_null") \ - X(WALKVM_CACHED_NOT_JAVA, "walkvm_cached_not_java") \ - X(WALKVM_NO_VMTHREAD, "walkvm_no_vmthread") \ - X(WALKVM_VMTHREAD_OK, "walkvm_vmthread_ok") \ - X(WALKVM_ANCHOR_USED_INLINE, "walkvm_anchor_used_inline") \ - X(WALKVM_ANCHOR_FALLBACK, "walkvm_anchor_fallback") \ - X(WALKVM_CODEH_NO_VM, "walkvm_codeh_no_vm") \ - X(WALKVM_DEPTH_ZERO, "walkvm_depth_zero") \ - X(WALKVM_HIT_CODEHEAP, "walkvm_hit_codeheap") \ - X(WALKVM_ANCHOR_FALLBACK_FAIL, "walkvm_anchor_fallback_fail") \ - X(WALKVM_ANCHOR_CONSUMED, "walkvm_anchor_consumed") \ - X(WALKVM_BREAK_INTERPRETED, "walkvm_break_interpreted") \ - X(WALKVM_BREAK_COMPILED, "walkvm_break_compiled") \ - X(WALKVM_JAVA_FRAME_OK, "walkvm_java_frame_ok") \ - X(WALKVM_ANCHOR_INLINE_NO_ANCHOR, "walkvm_anchor_inline_no_anchor") \ - X(WALKVM_ANCHOR_INLINE_NO_SP, "walkvm_anchor_inline_no_sp") \ - X(WALKVM_ANCHOR_INLINE_BAD_SP, "walkvm_anchor_inline_bad_sp") \ - X(WALKVM_SAVED_ANCHOR_USED, "walkvm_saved_anchor_used") \ - X(WALKVM_STUB_GENERIC_UNWIND, "walkvm_stub_generic_unwind") \ - X(WALKVM_STUB_FRAMESIZE_FALLBACK, "walkvm_stub_framesize_fallback") \ - X(WALKVM_FP_CHAIN_ATTEMPT, "walkvm_fp_chain_attempt") \ - X(WALKVM_FP_CHAIN_REACHED_CODEHEAP, "walkvm_fp_chain_reached_codeheap") \ - X(WALKVM_ANCHOR_NOT_IN_JAVA, "walkvm_anchor_not_in_java") \ - X(WALKVM_CONT_BARRIER_HIT, "walkvm_cont_barrier_hit") \ - X(WALKVM_ENTER_SPECIAL_HIT, "walkvm_enter_special_hit") \ - X(WALKVM_CONT_SPECULATIVE_HIT,"walkvm_cont_speculative_hit") \ - X(WALKVM_CONT_ENTRY_NULL, "walkvm_cont_entry_null") \ - X(NATIVE_LIBS_DROPPED, "native_libs_dropped") \ - X(SIGACTION_PATCHED_LIBS, "sigaction_patched_libs") \ - X(SIGACTION_INTERCEPTED, "sigaction_intercepted") \ - X(CTIMER_SIGNAL_OWN, "ctimer_signal_own") \ - X(CTIMER_SIGNAL_FOREIGN, "ctimer_signal_foreign") \ - X(WALLCLOCK_SIGNAL_OWN, "wallclock_signal_own") \ - X(WALLCLOCK_SIGNAL_FOREIGN, "wallclock_signal_foreign") \ - X(JVMTI_STACKS_INIT_OK, "jvmti_stacks_init_ok") \ - X(JVMTI_STACKS_INIT_FAILED, "jvmti_stacks_init_failed") \ - X(JVMTI_STACKS_REQUESTED, "jvmti_stacks_requested") \ - X(JVMTI_STACKS_FAILED_WRONG_PHASE, "jvmti_stacks_failed_wrong_phase") \ - X(JVMTI_STACKS_FAILED_OTHER, "jvmti_stacks_failed_other") \ - /* Delegated stacks dropped at slot-lock. Rec-lock drops from all recording \ - * paths (delegated and direct) go into SAMPLES_DROPPED_REC_LOCK. */ \ - X(JVMTI_STACKS_DROPPED_LOCK, "jvmti_stacks_dropped_lock") \ - X(SAMPLES_DROPPED_REC_LOCK, "samples_dropped_rec_lock") -#define X_ENUM(a, b) a, -typedef enum CounterId : int { - DD_COUNTER_TABLE(X_ENUM) DD_NUM_COUNTERS -} CounterId; -#undef X_ENUM - -class Counters { -private: - static const u32 ALIGNMENT = 16; - volatile long long *_counters; - static long long *init(); - Counters() : _counters() { -#ifdef COUNTERS - _counters = Counters::init(); -#endif // COUNTERS - } - -public: - static Counters &instance() { - static Counters instance; - return instance; - } - - Counters(Counters const &) = delete; - void operator=(Counters const &) = delete; - - static constexpr int address(int index) { return index * ALIGNMENT; } - - static constexpr int size() { - return address(DD_NUM_COUNTERS * sizeof(long long)); - } - - static long long *getCounters() { -#ifdef COUNTERS - return const_cast(Counters::instance()._counters); -#else - return nullptr; -#endif // COUNTERS - } - - static long long getCounter(CounterId counter, int offset = 0) { -#ifdef COUNTERS - return Counters::instance() - ._counters[address(static_cast(counter) + offset)]; -#else - return 0; -#endif // COUNTERS - } - - static void set(CounterId counter, long long value, int offset = 0) { -#ifdef COUNTERS - storeRelease(Counters::instance() - ._counters[address(static_cast(counter) + offset)], - value); -#endif // COUNTERS - } - - static void increment(CounterId counter, long long delta = 1, - int offset = 0) { -#ifdef COUNTERS - atomicIncRelaxed(Counters::instance() - ._counters[address(static_cast(counter) + offset)], - delta); -#endif // COUNTERS - } - - static void decrement(CounterId counter, long long delta = 1, - int offset = 0) { -#ifdef COUNTERS - increment(counter, -delta, offset); -#endif // COUNTERS - } - - static std::vector describeCounters() { -#ifdef COUNTERS -#define X_NAME(a, b) b, - return {DD_COUNTER_TABLE(X_NAME)}; -#undef X_NAME -#else - return {}; -#endif // COUNTERS - } - - static void reset() { -#ifdef COUNTERS - memset((void *)Counters::instance()._counters, 0, size()); -#endif // COUNTERS - } -}; - -#endif // JAVA_PROFILER_LIBRARY_COUNTERS_H diff --git a/ddprof-lib/src/main/cpp/cpuEngine.h b/ddprof-lib/src/main/cpp/cpuEngine.h deleted file mode 100644 index da8becbd9..000000000 --- a/ddprof-lib/src/main/cpp/cpuEngine.h +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _CPUENGINE_H -#define _CPUENGINE_H - -#include -#include "engine.h" - - -// Base class for CPU sampling engines: PerfEvents, CTimer, ITimer -class CpuEngine : public Engine { - protected: - static void** _pthread_entry; - static CpuEngine* _current; - - static long _interval; - static CStack _cstack; - static int _signal; - static bool _count_overrun; - - static void signalHandler(int signo, siginfo_t* siginfo, void* ucontext); - static void signalHandlerJ9(int signo, siginfo_t* siginfo, void* ucontext); - - static bool setupThreadHook(); - - void enableThreadHook(); - void disableThreadHook(); - - bool isResourceLimit(int err); - - int createForAllThreads(); - - virtual int createForThread(int tid) { return -1; } - virtual void destroyForThread(int tid) {} - - public: - const char* title() { - return "CPU profile"; - } - - const char* units() { - return "ns"; - } - - static void onThreadStart(); - static void onThreadEnd(); -}; - -#endif // _CPUENGINE_H diff --git a/ddprof-lib/src/main/cpp/ctimer.h b/ddprof-lib/src/main/cpp/ctimer.h deleted file mode 100644 index c99d2cd0e..000000000 --- a/ddprof-lib/src/main/cpp/ctimer.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2017 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _CTIMER_H -#define _CTIMER_H - -#include "engine.h" -#include -#ifdef __linux__ - -#include "arch.h" -#include - -class CTimer : public Engine { -protected: - // This is accessed from signal handlers, so must be async-signal-safe - static bool _enabled; - static long _interval; - static CStack _cstack; - static int _signal; - - static int _max_timers; - static int *_timers; - - int registerThread(int tid); - void unregisterThread(int tid); - -private: - // cppcheck-suppress unusedPrivateFunction - static void signalHandler(int signo, siginfo_t *siginfo, void *ucontext); - -public: - const char *units() { return "ns"; } - - const char *name() { return "CTimer"; } - - long interval() const { return _interval; } - - Error check(Arguments &args); - Error start(Arguments &args); - void stop(); - - inline void enableEvents(bool enabled) { - __atomic_store_n(&_enabled, enabled, __ATOMIC_RELEASE); - } - - // Get the signal number used by CTimer (0 if not initialized) - static int getSignal() { return _signal; } -}; - -// A CPU-time engine that reuses CTimer's per-thread timer_create / SIGPROF -// dispatch, but instead of walking the stack in the signal handler delegates -// the walk to HotSpot's JFR RequestStackTrace JVMTI extension. The sampled -// event is emitted on our side with only a correlation ID; the JVM writes -// the stack trace (and its own JFR stack-trace id) into the concurrent JFR -// recording as jdk.StackTraceRequest. See VM::canRequestStackTrace(). -class CTimerJvmti : public CTimer { -private: - // cppcheck-suppress unusedPrivateFunction - static void signalHandler(int signo, siginfo_t *siginfo, void *ucontext); - -public: - const char *name() { return "CTimerJvmti"; } - - Error check(Arguments &args); - Error start(Arguments &args); -}; - -#else - -class CTimer : public Engine { -public: - Error check(Arguments &args) { - return Error("CTimer is not supported on this platform"); - } - - Error start(Arguments &args) { - return Error("CTimer is not supported on this platform"); - } - - static bool supported() { return false; } -}; - -class CTimerJvmti : public Engine { -public: - const char *name() { return "CTimerJvmti"; } - - Error check(Arguments &args) { - return Error("CTimerJvmti is not supported on this platform"); - } - - Error start(Arguments &args) { - return Error("CTimerJvmti is not supported on this platform"); - } -}; - -#endif // __linux__ - -#endif // _CTIMER_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/ctimer_linux.cpp b/ddprof-lib/src/main/cpp/ctimer_linux.cpp deleted file mode 100644 index e0d2b8b0c..000000000 --- a/ddprof-lib/src/main/cpp/ctimer_linux.cpp +++ /dev/null @@ -1,305 +0,0 @@ -/* - * Copyright 2023 Andrei Pangin - * Copyright 2025, 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ - -#include "counters.h" -#include "guards.h" -#include "ctimer.h" -#include "debugSupport.h" -#include "jvmThread.h" -#include "libraries.h" -#include "log.h" -#include "profiler.h" -#include "signalCookie.h" -#include "threadState.inline.h" -#include -#include -#include -#include -#include -#include -#include -#include - -#ifndef SIGEV_THREAD_ID -#define SIGEV_THREAD_ID 4 -#endif - -static inline clockid_t thread_cpu_clock(unsigned int tid) { - return ((~tid) << 3) | 6; // CPUCLOCK_SCHED | CPUCLOCK_PERTHREAD_MASK -} - -long CTimer::_interval; -int CTimer::_max_timers = 0; -int *CTimer::_timers = NULL; -CStack CTimer::_cstack; -bool CTimer::_enabled = false; -int CTimer::_signal; - -int CTimer::registerThread(int tid) { - if (tid >= _max_timers) { - Log::warn("tid[%d] > pid_max[%d]. Restart profiler after changing pid_max", - tid, _max_timers); - return -1; - } - - struct sigevent sev; - // Zero the whole struct first so any padding / future fields the kernel - // inspects (sigev_notify_function, sigev_notify_attributes on glibc) are - // not populated from stack garbage. - memset(&sev, 0, sizeof(sev)); - // Cookie identifying this timer as ddprof-owned. When the signal is delivered - // the handler checks siginfo->si_value.sival_ptr against SignalCookie::cpu() - // and drops/forwards any SIGPROF that does not carry it (e.g. from a Go - // runtime's setitimer(ITIMER_PROF) or a foreign library's raise()). - sev.sigev_value.sival_ptr = SignalCookie::cpu(); - sev.sigev_signo = _signal; - sev.sigev_notify = SIGEV_THREAD_ID; - // glibc/musl layout convention: sigev_notify_thread_id sits immediately - // after sigev_notify inside the union — the tid is written as the *second* - // int starting at &sev.sigev_notify, so bytes [sizeof(int), 2*sizeof(int)) - // of that int-pointer must be in-bounds of struct sigevent. Guard against - // a future libc change by statically asserting that both ints fit. - static_assert(offsetof(struct sigevent, sigev_notify) + 2 * sizeof(int) - <= sizeof(struct sigevent), - "sigevent layout assumption broken: tid write would overflow"); - ((int *)&sev.sigev_notify)[1] = tid; - - // Use raw syscalls, since libc wrapper allows only predefined clocks - clockid_t clock = thread_cpu_clock(tid); - int timer; - if (syscall(__NR_timer_create, clock, &sev, &timer) < 0) { - return -1; - } - - // Kernel timer ID may start with zero, but we use zero as an empty slot - if (!__sync_bool_compare_and_swap(&_timers[tid], 0, timer + 1)) { - // Lost race - syscall(__NR_timer_delete, timer); - return -1; - } - - struct itimerspec ts; - ts.it_interval.tv_sec = (time_t)(_interval / 1000000000); - ts.it_interval.tv_nsec = _interval % 1000000000; - ts.it_value = ts.it_interval; - if (syscall(__NR_timer_settime, timer, 0, &ts, NULL) < 0) { - // Arming failed after publishing the timer in _timers[tid]. Reclaim the - // slot only if it still contains this timer; otherwise a concurrent - // unregisterThread(tid) has already claimed responsibility for cleanup - // (avoids a double timer_delete). - int settime_errno = errno; - char errbuf[64]; - strerror_r(settime_errno, errbuf, sizeof(errbuf)); - Log::warn("timer_settime failed for tid=%d: %s", tid, errbuf); - errno = settime_errno; - if (__sync_bool_compare_and_swap(&_timers[tid], timer + 1, 0)) { - syscall(__NR_timer_delete, timer); - } - return -1; - } - return 0; -} - -void CTimer::unregisterThread(int tid) { - if (tid >= _max_timers) { - return; - } - // Atomic acquire to avoid possible leak when unregistering - // This was raised by tsan, with registers and unregisters done in separate - // threads. - int timer = __atomic_load_n(&_timers[tid], __ATOMIC_ACQUIRE); - if (timer != 0 && __sync_bool_compare_and_swap(&_timers[tid], timer--, 0)) { - syscall(__NR_timer_delete, timer); - } -} - -Error CTimer::check(Arguments &args) { - timer_t timer; - if (timer_create(CLOCK_THREAD_CPUTIME_ID, NULL, &timer) < 0) { - return Error("Failed to create CPU timer"); - } - timer_delete(timer); - - return Error::OK; -} - -Error CTimer::start(Arguments &args) { - if (args._interval < 0) { - return Error("interval must be positive"); - } - - _interval = args.cpuSamplerInterval(); - _cstack = args._cstack; - _signal = SIGPROF; - - int max_timers = OS::getMaxThreadId(); - if (max_timers != _max_timers) { - free(_timers); - _timers = (int *)calloc(max_timers, sizeof(int)); - _max_timers = max_timers; - } - - // Prime the origin-check cache from this non-signal context before any - // SIGPROF can fire — reading the env var lazily from the handler itself - // would go through a C++ function-local-static guard, which is not - // async-signal-safe. - OS::primeSignalOriginCheck(); - - OS::installSignalHandler(_signal, signalHandler); - - // Register all existing threads. Individual failures are benign — a thread - // may exit between listThreads() and registerThread(), and new threads - // will register themselves on creation. check() already validated that the - // timer mechanism works on this system. - ThreadList *thread_list = OS::listThreads(); - while (thread_list->hasNext()) { - registerThread(thread_list->next()); - } - delete thread_list; - - return Error::OK; -} - -void CTimer::stop() { - for (int i = 0; i < _max_timers; i++) { - unregisterThread(i); - } -} - -Error CTimerJvmti::check(Arguments &args) { - if (!VM::canRequestStackTrace()) { - return Error("HotSpot RequestStackTrace JVMTI extension not available"); - } - return CTimer::check(args); -} - -Error CTimerJvmti::start(Arguments &args) { - if (!VM::canRequestStackTrace()) { - return Error("HotSpot RequestStackTrace JVMTI extension not available"); - } - Error result = CTimer::start(args); - if (result) return result; - // Override the signal handler installed by CTimer::start with our own, - // which delegates stack walking to the HotSpot JFR extension. - OS::installSignalHandler(_signal, CTimerJvmti::signalHandler); - return Error::OK; -} - -void CTimerJvmti::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { - SIGNAL_HANDLER_GUARD(); - if (!OS::shouldProcessSignal(siginfo, SI_TIMER, SignalCookie::cpu())) { - Counters::increment(CTIMER_SIGNAL_FOREIGN); - OS::forwardForeignSignal(signo, siginfo, ucontext); - return; - } - Counters::increment(CTIMER_SIGNAL_OWN); - - CriticalSection cs; - if (!cs.entered()) { - return; - } - int saved_errno = errno; - if (!__atomic_load_n(&_enabled, __ATOMIC_ACQUIRE)) { - errno = saved_errno; - return; - } - int tid = 0; - ProfiledThread *current = ProfiledThread::currentSignalSafe(); - assert(current == nullptr || !current->isDeepCrashHandler()); - if (current != nullptr && JVMThread::isInitialized() && JVMThread::current() == nullptr - && current->inInitWindow()) { - current->tickInitWindow(); - errno = saved_errno; - return; - } - if (current != NULL) { - current->noteCPUSample(Profiler::instance()->recordingEpoch()); - tid = current->tid(); - } else { - tid = OS::threadId(); - } - Shims::instance().setSighandlerTid(tid); - - ExecutionEvent event; - event._execution_mode = getThreadExecutionMode(); - // Opted into JVMTI delegation; drop the sample if the JVM rejects the - // request (WRONG_PHASE if JFR is not recording, NOT_AVAILABLE if - // jdk.StackTraceRequest is disabled). recordSampleDelegated() bumps the - // failure counters; there is no fallback to ASGCT in this engine. - Profiler::instance()->recordSampleDelegated(ucontext, _interval, tid, - BCI_CPU, &event); - Shims::instance().setSighandlerTid(-1); - errno = saved_errno; -} - -void CTimer::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { - SIGNAL_HANDLER_GUARD(); - // Reject signals that did not originate from our timer_create timers. - // This guards against Go's process-wide setitimer(ITIMER_PROF) and other - // foreign SIGPROF sources that would otherwise drive our handler onto - // threads we never registered — see doc/plans/SignalOriginValidation.md. - if (!OS::shouldProcessSignal(siginfo, SI_TIMER, SignalCookie::cpu())) { - Counters::increment(CTIMER_SIGNAL_FOREIGN); - OS::forwardForeignSignal(signo, siginfo, ucontext); - return; - } - Counters::increment(CTIMER_SIGNAL_OWN); - - // Atomically try to enter critical section - prevents all reentrancy races - CriticalSection cs; - if (!cs.entered()) { - return; // Another critical section is active, defer profiling - } - // Save the current errno value - int saved_errno = errno; - // we want to ensure memory order because of the possibility the instance gets - // cleared - if (!__atomic_load_n(&_enabled, __ATOMIC_ACQUIRE)) - return; - int tid = 0; - ProfiledThread *current = ProfiledThread::currentSignalSafe(); - assert(current == nullptr || !current->isDeepCrashHandler()); - // Guard against the race window between Profiler::registerThread() and - // thread_native_entry setting JVM TLS (PROF-13072): skip at most one signal - // per thread. Pure native threads (where JVMThread::current() is always null) - // are allowed through once the one-shot window expires. - if (current != nullptr && JVMThread::isInitialized() && JVMThread::current() == nullptr - && current->inInitWindow()) { - current->tickInitWindow(); - errno = saved_errno; - return; - } - if (current != NULL) { - current->noteCPUSample(Profiler::instance()->recordingEpoch()); - tid = current->tid(); - } else { - tid = OS::threadId(); - } - Shims::instance().setSighandlerTid(tid); - - ExecutionEvent event; - event._execution_mode = getThreadExecutionMode(); - Profiler::instance()->recordSample(ucontext, _interval, tid, BCI_CPU, 0, - &event); - Shims::instance().setSighandlerTid(-1); - // we need to avoid spoiling the value of errno (tsan report) - errno = saved_errno; -} - -#endif // __linux__ diff --git a/ddprof-lib/src/main/cpp/debugSupport.cpp b/ddprof-lib/src/main/cpp/debugSupport.cpp deleted file mode 100644 index e0cbd71f7..000000000 --- a/ddprof-lib/src/main/cpp/debugSupport.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include -#include - -#include "debug.h" -#include "debugSupport.h" - -Shims Shims::_instance; - -Shims::Shims() : _tid_setter_ref(NULL) { -#ifdef DEBUG - if (_tid_setter_ref == NULL) { - void *sym_handle = dlsym(RTLD_DEFAULT, "set_sighandler_tid"); - __atomic_compare_exchange_n(&_tid_setter_ref, - (SetSigHandlerTidRef *)(&sym_handle), NULL, - false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED); - } -#endif -} - -void Shims::setSighandlerTid(int tid) { -#ifdef DEBUG - SetSigHandlerTidRef ref = __atomic_load_n(&_tid_setter_ref, __ATOMIC_ACQUIRE); - if (ref != NULL) { - ref(tid); - } -#endif -} \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/debugSupport.h b/ddprof-lib/src/main/cpp/debugSupport.h deleted file mode 100644 index 512fb86d7..000000000 --- a/ddprof-lib/src/main/cpp/debugSupport.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef _DEBUGSUPPORT_H -#define _DEBUGSUPPORT_H - -typedef void (*SetSigHandlerTidRef)(int tid); - -class Shims { -private: - static Shims _instance; - volatile SetSigHandlerTidRef _tid_setter_ref; - Shims(); - -public: - void setSighandlerTid(int tid); - inline static Shims instance() { return _instance; } -}; - -#endif //_DEBUGSUPPORT_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/dictionary.cpp b/ddprof-lib/src/main/cpp/dictionary.cpp deleted file mode 100644 index 29db2eef5..000000000 --- a/ddprof-lib/src/main/cpp/dictionary.cpp +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2020 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "dictionary.h" -#include "arch.h" -#include "counters.h" -#include "signalSafety.h" -#include -#include -#include - -static inline char *allocateKey(const char *key, size_t length) { - char *result = (char *)malloc(length + 1); - memcpy(result, key, length); - result[length] = 0; - return result; -} - -static inline bool keyEquals(const char *candidate, const char *key, - size_t length) { - return strncmp(candidate, key, length) == 0 && candidate[length] == 0; -} - -Dictionary::~Dictionary() { - clear(_table, _id); - free(_table); - Counters::set(DICTIONARY_BYTES, 0, _id); - Counters::set(DICTIONARY_PAGES, 0, _id); -} - -void Dictionary::clear() { - DEBUG_ASSERT_NOT_IN_SIGNAL(); - clear(_table, _id); - memset(_table, 0, sizeof(DictTable)); - _table->base_index = _base_index = 1; - Counters::set(DICTIONARY_KEYS, 0, _id); - Counters::set(DICTIONARY_KEYS_BYTES, 0, _id); - Counters::set(DICTIONARY_BYTES, sizeof(DictTable), _id); - Counters::set(DICTIONARY_PAGES, 1, _id); - _size = 0; -} - -void Dictionary::clear(DictTable *table, int id) { - for (int i = 0; i < ROWS; i++) { - DictRow *row = &table->rows[i]; - for (int j = 0; j < CELLS; j++) { - if (row->keys[j]) { - free(row->keys[j]); // content is zeroed en-mass in the clear() function - } - } - if (row->next != NULL) { - clear(row->next, id); - DictTable *tmp = row->next; - row->next = NULL; - free(tmp); - } - } -} - -// Many popular symbols are quite short, e.g. "[B", "()V" etc. -// FNV-1a is reasonably fast and sufficiently random. -unsigned int Dictionary::hash(const char *key, size_t length) { - unsigned int h = 2166136261U; - for (size_t i = 0; i < length; i++) { - h = (h ^ key[i]) * 16777619; - } - return h; -} - -unsigned int Dictionary::lookup(const char *key) { - return lookup(key, strlen(key)); -} - -unsigned int Dictionary::lookup(const char *key, size_t length) { - return lookup(key, length, true, 0); -} - -unsigned int Dictionary::lookup(const char *key, size_t length, bool for_insert, - unsigned int sentinel) { - // The insert path mallocs (allocateKey) and may calloc a DictTable — - // both AS-unsafe. Read-only lookups (for_insert == false, used by - // check() and bounded_lookup at capacity) only touch already-allocated - // memory and are AS-safe. Assert here rather than in the overloads - // so bounded_lookup's runtime-decided for_insert is also covered. - if (for_insert) { - DEBUG_ASSERT_NOT_IN_SIGNAL(); - } - - DictTable *table = _table; - unsigned int h = hash(key, length); - - while (true) { - DictRow *row = &table->rows[h % ROWS]; - for (int c = 0; c < CELLS; c++) { - if (for_insert && row->keys[c] == NULL) { - char *new_key = allocateKey(key, length); - if (__sync_bool_compare_and_swap(&row->keys[c], NULL, new_key)) { - Counters::increment(DICTIONARY_KEYS, 1, _id); - Counters::increment(DICTIONARY_KEYS_BYTES, length + 1, _id); - atomicInc(_size); - return table->index(h % ROWS, c); - } - free(new_key); - } - if (row->keys[c] && keyEquals(row->keys[c], key, length)) { - return table->index(h % ROWS, c); - } - } - - if (row->next == NULL) { - if (for_insert) { - DictTable *new_table = (DictTable *)calloc(1, sizeof(DictTable)); - new_table->base_index = - __sync_add_and_fetch(&_base_index, TABLE_CAPACITY); - if (!__sync_bool_compare_and_swap(&row->next, NULL, new_table)) { - free(new_table); - } else { - Counters::increment(DICTIONARY_PAGES, 1, _id); - Counters::increment(DICTIONARY_BYTES, sizeof(DictTable), _id); - } - } else { - return sentinel; - } - } - - table = row->next; - h = (h >> ROW_BITS) | (h << (32 - ROW_BITS)); - } -} - -bool Dictionary::check(const char* key) { - return lookup(key, strlen(key), false, 0) != 0; -} - -unsigned int Dictionary::bounded_lookup(const char *key, size_t length, - int size_limit) { - // bounded lookup will find the encoding if the key is already mapped, - // but will only grow the dictionary if the current size is below the limit - return lookup(key, length, _size < size_limit, INT_MAX); -} - -void Dictionary::collect(std::map &map) { - collect(map, _table); -} - -void Dictionary::collect(std::map &map, - DictTable *table) { - for (int i = 0; i < ROWS; i++) { - DictRow *row = &table->rows[i]; - for (int j = 0; j < CELLS; j++) { - if (row->keys[j] != NULL) { - map[table->index(i, j)] = row->keys[j]; - } - } - if (row->next != NULL) { - collect(map, row->next); - } - } -} diff --git a/ddprof-lib/src/main/cpp/dictionary.h b/ddprof-lib/src/main/cpp/dictionary.h deleted file mode 100644 index e5df18f82..000000000 --- a/ddprof-lib/src/main/cpp/dictionary.h +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2020 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _DICTIONARY_H -#define _DICTIONARY_H - -#include "counters.h" -#include -#include -#include - -#define ROW_BITS 7 -#define ROWS (1 << ROW_BITS) -#define CELLS 3 -#define TABLE_CAPACITY (ROWS * CELLS) - -struct DictTable; - -struct DictRow { - char *keys[CELLS]; - DictTable *next; -}; - -struct DictTable { - DictRow rows[ROWS]; - unsigned int base_index; - - unsigned int index(int row, int col) { - return base_index + (col << ROW_BITS) + row; - } -}; - -// Append-only concurrent hash table based on multi-level arrays -class Dictionary { -private: - DictTable *_table; - const int _id; - volatile unsigned int _base_index; - volatile int _size; - - static void clear(DictTable *table, int id); - - static unsigned int hash(const char *key, size_t length); - - static void collect(std::map &map, - DictTable *table); - - unsigned int lookup(const char *key, size_t length, bool for_insert, - unsigned int sentinel); - -public: - Dictionary() : Dictionary(0) {} - Dictionary(int id) : _id(id) { - _table = (DictTable *)calloc(1, sizeof(DictTable)); - Counters::set(DICTIONARY_PAGES, 1, id); - Counters::set(DICTIONARY_BYTES, sizeof(DictTable), id); - _table->base_index = _base_index = 1; - _size = 0; - } - ~Dictionary(); - - void clear(); - - bool check(const char* key); - // NOT signal-safe: the inserting lookup overloads call malloc/calloc on miss - // (see allocateKey and the calloc in dictionary.cpp). Signal handlers must use - // bounded_lookup(key, length, 0) instead, which never inserts and returns - // INT_MAX on miss. - unsigned int lookup(const char *key); - unsigned int lookup(const char *key, size_t length); - unsigned int bounded_lookup(const char *key, size_t length, int size_limit); - - void collect(std::map &map); -}; - -#endif // _DICTIONARY_H diff --git a/ddprof-lib/src/main/cpp/dwarf.cpp b/ddprof-lib/src/main/cpp/dwarf.cpp deleted file mode 100644 index b4dee8877..000000000 --- a/ddprof-lib/src/main/cpp/dwarf.cpp +++ /dev/null @@ -1,609 +0,0 @@ -/* - * Copyright 2021 Andrei Pangin - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "dwarf.h" -#include "common.h" -#include "log.h" -#include -#include - -enum { - DW_CFA_nop = 0x0, - DW_CFA_set_loc = 0x1, - DW_CFA_advance_loc1 = 0x2, - DW_CFA_advance_loc2 = 0x3, - DW_CFA_advance_loc4 = 0x4, - DW_CFA_offset_extended = 0x5, - DW_CFA_restore_extended = 0x6, - DW_CFA_undefined = 0x7, - DW_CFA_same_value = 0x8, - DW_CFA_register = 0x9, - DW_CFA_remember_state = 0xa, - DW_CFA_restore_state = 0xb, - DW_CFA_def_cfa = 0xc, - DW_CFA_def_cfa_register = 0xd, - DW_CFA_def_cfa_offset = 0xe, - DW_CFA_def_cfa_expression = 0xf, - DW_CFA_expression = 0x10, - DW_CFA_offset_extended_sf = 0x11, - DW_CFA_def_cfa_sf = 0x12, - DW_CFA_def_cfa_offset_sf = 0x13, - DW_CFA_val_offset = 0x14, - DW_CFA_val_offset_sf = 0x15, - DW_CFA_val_expression = 0x16, - DW_CFA_AARCH64_negate_ra_state = 0x2d, - DW_CFA_GNU_args_size = 0x2e, - - DW_CFA_advance_loc = 0x1, - DW_CFA_offset = 0x2, - DW_CFA_restore = 0x3, -}; - -enum { - DW_OP_breg_pc = 0x70 + DW_REG_PC, - DW_OP_const1u = 0x08, - DW_OP_const1s = 0x09, - DW_OP_const2u = 0x0a, - DW_OP_const2s = 0x0b, - DW_OP_const4u = 0x0c, - DW_OP_const4s = 0x0d, - DW_OP_constu = 0x10, - DW_OP_consts = 0x11, - DW_OP_minus = 0x1c, - DW_OP_plus = 0x22, -}; - -enum { - // DWARF Exception Header value format - DW_EH_PE_uleb128 = 0x01, - DW_EH_PE_udata2 = 0x02, - DW_EH_PE_udata4 = 0x03, - DW_EH_PE_udata8 = 0x04, - DW_EH_PE_sleb128 = 0x09, - DW_EH_PE_sdata2 = 0x0a, - DW_EH_PE_sdata4 = 0x0b, - DW_EH_PE_sdata8 = 0x0c, - // DWARF Exception Header application - DW_EH_PE_absptr = 0x00, - DW_EH_PE_pcrel = 0x10, - DW_EH_PE_datarel = 0x30, - // valid in both - DW_EH_PE_omit = 0xff, -}; - -FrameDesc FrameDesc::empty_frame = {0, DW_REG_SP | EMPTY_FRAME_SIZE << 8, - DW_SAME_FP, -EMPTY_FRAME_SIZE}; -FrameDesc FrameDesc::default_frame = {0, DW_REG_FP | LINKED_FRAME_SIZE << 8, - -LINKED_FRAME_SIZE, - -LINKED_FRAME_SIZE + DW_STACK_SLOT}; -FrameDesc FrameDesc::default_clang_frame = {0, DW_REG_FP | LINKED_FRAME_CLANG_SIZE << 8, -LINKED_FRAME_CLANG_SIZE, -LINKED_FRAME_CLANG_SIZE + DW_STACK_SLOT}; -FrameDesc FrameDesc::no_dwarf_frame = {0, DW_REG_INVALID, DW_REG_INVALID, DW_REG_INVALID}; - -void DwarfParser::init(const char *name, const char *image_base, const char *image_end) { - _name = name; - _image_base = image_base; - _section_start = NULL; - _section_end = reinterpret_cast(~(size_t)0); - _image_end = image_end; - - _capacity = 128; - _count = 0; - _table = (FrameDesc *)malloc(_capacity * sizeof(FrameDesc)); - _prev = NULL; - - _code_align = sizeof(instruction_t); - _data_align = -(int)sizeof(void *); - _linked_frame_size = -1; - _has_z_augmentation = false; -} - -DwarfParser::DwarfParser(const char *name, const char *image_base, - const char *eh_frame_hdr, size_t eh_frame_hdr_size, - EhFrameHdrTag, const char *image_end) { - init(name, image_base, image_end); - parse(eh_frame_hdr, eh_frame_hdr_size, image_end); -} - -DwarfParser::DwarfParser(const char *name, const char *image_base, - const char *eh_frame, size_t eh_frame_size) { - init(name, image_base, eh_frame + eh_frame_size); - parseEhFrame(eh_frame, eh_frame_size); -} - -static constexpr u8 omit_sign_bit(u8 value) { - // each signed flag = unsigned equivalent | 0x80 - return value & 0xf7; -} - -static constexpr u8 omit_sign_bit_mask_low(u8 value) { - // each signed flag = unsigned equivalent | 0x80 - return value & 0x7; -} - -void DwarfParser::parse(const char *eh_frame_hdr, size_t size, const char *image_end) { - // Fixed .eh_frame_hdr header: version (1) + 3 encoding bytes + eh_frame_ptr (4) - // + fde_count at offset 8 (4), binary-search table starting at offset 12. - // Refuse anything too small. - if (eh_frame_hdr == NULL || size < 16) { - return; - } - // The version/encoding bytes [0..3] and fde_count [8..11] are read directly - // (not via canRead), so reject if image_end does not cover the 12-byte prefix. - if (image_end < eh_frame_hdr + 12) { - return; - } - // Bound FDE reads to the full ELF image [image_base, image_end) so that - // pointers into the adjacent .eh_frame section are validated against mapped - // memory, not left unbounded. - _section_start = _image_base; - _section_end = image_end; - - - u8 version = eh_frame_hdr[0]; - u8 eh_frame_ptr_enc = eh_frame_hdr[1]; - u8 fde_count_enc = eh_frame_hdr[2]; - u8 table_enc = eh_frame_hdr[3]; - - if (version != 1 || - omit_sign_bit_mask_low(eh_frame_ptr_enc) != DW_EH_PE_udata4 || - omit_sign_bit_mask_low(fde_count_enc) != DW_EH_PE_udata4 - // note that DW_EH_PE_pcrel is not supported, it remains to be seen - // whether support should be added - || omit_sign_bit(table_enc) != (DW_EH_PE_datarel | DW_EH_PE_udata4)) { - Log::warn("Unsupported .eh_frame_hdr [%02x%02x%02x%02x] in %s", version, - eh_frame_ptr_enc, fde_count_enc, table_enc, _name); - return; - } - - u32 fde_count = *(u32 *)(eh_frame_hdr + 8); - // Table starts at offset 12 (4-byte header + 4-byte eh_frame_ptr + 4-byte fde_count). - // Each entry is a (initial_loc, fde_ptr) pair of 4-byte section-relative - // offsets (DW_EH_PE_datarel | DW_EH_PE_udata4). Reject a count that would - // make the table walk read past the section. - u32 *table = (u32 *)(eh_frame_hdr + 12); - if (fde_count > (size - 12) / 8) { - Log::warn("Truncated or invalid .eh_frame_hdr (fde_count=%u, size=%lu) in %s", - fde_count, (unsigned long)size, _name); - return; - } - for (u32 i = 0; i < fde_count; i++) { - // table[i*2] is initial_loc; table[i*2+1] is the FDE pointer (datarel sdata4). - // Cast to int to correctly handle negative offsets (FDE before the header). - _ptr = eh_frame_hdr + (int)table[i * 2 + 1]; - parseFde(); - } -} - -// Parse raw .eh_frame (or __eh_frame on macOS) without a binary-search index. -// Records are CIE/FDE sequences laid out linearly; terminated by a 4-byte zero or EOF. -void DwarfParser::parseEhFrame(const char *eh_frame, size_t size) { - if (eh_frame == NULL || size < 4) { - return; - } - // Publish the read window so the get*/skip* helpers clamp to it. - _section_start = eh_frame; - _section_end = eh_frame + size; - _ptr = eh_frame; - - while (_ptr + 4 <= _section_end) { - const char *record_start = _ptr; - u32 length = get32(); - if (length == 0) { - break; // terminator - } - if (length == 0xffffffff) { - break; // 64-bit DWARF not supported - } - - if (length > (size_t)(_section_end - record_start) - 4) { - break; - } - const char *record_end = record_start + 4 + length; - - u32 cie_id = get32(); - - if (cie_id == 0) { - // CIE: update code and data alignment factors. - // Layout after cie_id: [1-byte version][augmentation string \0][code_align LEB][data_align SLEB] - // [return_address_register][augmentation data (if 'z')]... - // return_address_register and everything after data_align are not consumed; _ptr = record_end - // at the bottom of the loop skips them. - // - // _has_z_augmentation is overwritten by every CIE encountered. The DWARF spec allows - // multiple CIEs with different augmentation strings in a single .eh_frame section, so - // strictly speaking each FDE should resolve its own CIE via the backward cie_id offset. - // We intentionally skip that: macOS binaries compiled by clang typically emit a single CIE - // per module, and this parser is only called for macOS __eh_frame sections. Multi-CIE - // binaries are not produced by the toolchains we target here. - if (_ptr >= record_end) { - _ptr = record_end; - continue; - } - _ptr++; // skip version - if (_ptr >= record_end) { - _ptr = record_end; - continue; - } - _has_z_augmentation = (*_ptr == 'z'); - while (_ptr < record_end && *_ptr++) { - } // skip null-terminated augmentation string - if (_ptr >= record_end) { - _ptr = record_end; - continue; - } - _code_align = getLeb(record_end); - _data_align = getSLeb(record_end); - } else { - // FDE: parse frame description for the covered PC range. - // After cie_id: [pcrel-range-start 4 bytes][range-len 4 bytes][aug-data-len LEB][aug-data][instructions] - // Assumes DW_EH_PE_pcrel | DW_EH_PE_sdata4 encoding for range-start (clang macOS default). - // The augmentation data length field (and the data itself) is only present when the CIE - // augmentation string starts with 'z'. - if (_ptr + 8 > record_end) { - break; - } - u32 range_start = (u32)(getPtr() - _image_base); - u32 range_len = get32(); - if (_has_z_augmentation) { - _ptr += getLeb(record_end); // getLeb reads the length; advance past the augmentation data bytes - if (_ptr > record_end) { - break; - } - } - parseInstructions(range_start, record_end); - addRecord(range_start + range_len, DW_REG_FP, LINKED_FRAME_CLANG_SIZE, - -LINKED_FRAME_CLANG_SIZE, -LINKED_FRAME_CLANG_SIZE + DW_STACK_SLOT); - } - - _ptr = record_end; - } - - if (_count > 1) { - qsort(_table, _count, sizeof(FrameDesc), FrameDesc::comparator); - } -} - -void DwarfParser::parseCie() { - if (_ptr + 4 > _image_end) return; - u32 cie_len = get32(); - if (cie_len == 0 || cie_len == 0xffffffff) { - return; - } - - const char *cie_start = _ptr; - const char *cie_end = cie_start + cie_len; - if (cie_end > _section_end) return; - - if (!canRead(5)) { _ptr = _section_end; return; } - _ptr += 5; - while (_ptr < cie_end && *_ptr++) { - } - _code_align = getLeb(cie_end); - _data_align = getSLeb(cie_end); - _ptr = cie_end; -} - -void DwarfParser::parseFde() { - if (_ptr + 4 > _image_end) return; - u32 fde_len = get32(); - if (fde_len == 0 || fde_len == 0xffffffff) { - return; - } - - const char *fde_start = _ptr; - const char *fde_end = fde_start + fde_len; - if (fde_end > _image_end) return; - - if (_ptr + 4 > fde_end) return; - u32 cie_offset = get32(); - if (_count == 0) { - if (cie_offset > (size_t)(fde_start - _section_start)) { - return; - } - _ptr = fde_start - cie_offset; - parseCie(); - _ptr = fde_start + 4; - } - - if (_ptr + 8 > fde_end) return; - u32 range_start = getPtr() - _image_base; - u32 range_len = get32(); - _ptr += getLeb(fde_end); - if (_ptr > fde_end) return; - parseInstructions(range_start, fde_end); - addRecord(range_start + range_len, DW_REG_FP, LINKED_FRAME_SIZE, - -LINKED_FRAME_SIZE, -LINKED_FRAME_SIZE + DW_STACK_SLOT); -} - -void DwarfParser::parseInstructions(u32 loc, const char *end) { - // `end` is derived from an untrusted record length; never let it run past - // the section. Reads inside the loop are clamped by the get*/skip* helpers, - // but clamping here keeps the loop bound honest and _ptr in range. - if (end > _section_end) { - end = _section_end; - } - const u32 code_align = _code_align; - const int data_align = _data_align; - - u32 cfa_reg = DW_REG_SP; - int cfa_off = EMPTY_FRAME_SIZE; - int fp_off = DW_SAME_FP; - int pc_off = -EMPTY_FRAME_SIZE; - - u32 rem_cfa_reg = DW_REG_SP; - int rem_cfa_off = EMPTY_FRAME_SIZE; - int rem_fp_off = DW_SAME_FP; - int rem_pc_off = -EMPTY_FRAME_SIZE; - - while (_ptr < end) { - u8 op = get8(); - switch (op >> 6) { - case 0: - switch (op) { - case DW_CFA_nop: - case DW_CFA_set_loc: - _ptr = end; - break; - case DW_CFA_advance_loc1: - addRecord(loc, cfa_reg, cfa_off, fp_off, pc_off); - loc += get8() * code_align; - break; - case DW_CFA_advance_loc2: - addRecord(loc, cfa_reg, cfa_off, fp_off, pc_off); - loc += get16() * code_align; - break; - case DW_CFA_advance_loc4: - addRecord(loc, cfa_reg, cfa_off, fp_off, pc_off); - loc += get32() * code_align; - break; - case DW_CFA_offset_extended: - switch (getLeb()) { - case DW_REG_FP: - fp_off = getLeb() * data_align; - break; - case DW_REG_PC: - pc_off = getLeb() * data_align; - break; - default: - skipLeb(); - } - break; - case DW_CFA_restore_extended: - case DW_CFA_undefined: - case DW_CFA_same_value: - if (getLeb() == DW_REG_FP) { - fp_off = DW_SAME_FP; - } - break; - case DW_CFA_register: - skipLeb(); - skipLeb(); - break; - case DW_CFA_remember_state: - rem_cfa_reg = cfa_reg; - rem_cfa_off = cfa_off; - rem_fp_off = fp_off; - rem_pc_off = pc_off; - break; - case DW_CFA_restore_state: - cfa_reg = rem_cfa_reg; - cfa_off = rem_cfa_off; - fp_off = rem_fp_off; - pc_off = rem_pc_off; - break; - case DW_CFA_def_cfa: - cfa_reg = getLeb(); - cfa_off = getLeb(); - break; - case DW_CFA_def_cfa_register: - cfa_reg = getLeb(); - break; - case DW_CFA_def_cfa_offset: - cfa_off = getLeb(); - break; - case DW_CFA_def_cfa_expression: { - u32 len = getLeb(); - cfa_reg = len == 11 ? DW_REG_PLT : DW_REG_INVALID; - cfa_off = DW_STACK_SLOT; - _ptr += len; - if (_ptr > _section_end) _ptr = _section_end; - break; - } - case DW_CFA_expression: - skipLeb(); - _ptr += getLeb(); - if (_ptr > _section_end) _ptr = _section_end; - break; - case DW_CFA_offset_extended_sf: - switch (getLeb()) { - case DW_REG_FP: - fp_off = getSLeb() * data_align; - break; - case DW_REG_PC: - pc_off = getSLeb() * data_align; - break; - default: - skipLeb(); - } - break; - case DW_CFA_def_cfa_sf: - cfa_reg = getLeb(); - cfa_off = getSLeb() * data_align; - break; - case DW_CFA_def_cfa_offset_sf: - cfa_off = getSLeb() * data_align; - break; - case DW_CFA_val_offset: - case DW_CFA_val_offset_sf: - skipLeb(); - skipLeb(); - break; - case DW_CFA_val_expression: - if (getLeb() == DW_REG_PC) { - int pc_off = parseExpression(); - if (pc_off != 0) { - fp_off = DW_PC_OFFSET | (pc_off << 1); - } - } else { - _ptr += getLeb(); - if (_ptr > _section_end) _ptr = _section_end; - } - break; -#ifdef __aarch64__ - case DW_CFA_AARCH64_negate_ra_state: - break; -#endif - case DW_CFA_GNU_args_size: - skipLeb(); - break; - default: - Log::warn("Unknown DWARF instruction 0x%x in %s", op, _name); - return; - } - break; - case DW_CFA_advance_loc: - addRecord(loc, cfa_reg, cfa_off, fp_off, pc_off); - loc += (op & 0x3f) * code_align; - break; - case DW_CFA_offset: - switch (op & 0x3f) { - case DW_REG_FP: - fp_off = getLeb() * data_align; - break; - case DW_REG_PC: - pc_off = getLeb() * data_align; - break; - default: - skipLeb(); - } - break; - case DW_CFA_restore: - if ((op & 0x3f) == DW_REG_FP) { - fp_off = DW_SAME_FP; - } - break; - } - } - - addRecord(loc, cfa_reg, cfa_off, fp_off, pc_off); -} - -// Parse a limited subset of DWARF expressions, which is used in -// DW_CFA_val_expression to point to the previous PC relative to the current PC. -// Returns the offset of the previous PC from the current PC. -int DwarfParser::parseExpression() { - int pc_off = 0; - int tos = 0; - - u32 len = getLeb(); - const char *end = _ptr + len; - if (end > _section_end) { - end = _section_end; - } - - while (_ptr < end) { - u8 op = get8(); - switch (op) { - case DW_OP_breg_pc: - pc_off = getSLeb(); - break; - case DW_OP_const1u: - tos = get8(); - break; - case DW_OP_const1s: - tos = (signed char)get8(); - break; - case DW_OP_const2u: - tos = get16(); - break; - case DW_OP_const2s: - tos = (short)get16(); - break; - case DW_OP_const4u: - case DW_OP_const4s: - tos = get32(); - break; - case DW_OP_constu: - tos = getLeb(); - break; - case DW_OP_consts: - tos = getSLeb(); - break; - case DW_OP_minus: - pc_off -= tos; - break; - case DW_OP_plus: - pc_off += tos; - break; - default: - Log::warn("Unknown DWARF opcode 0x%x in %s", op, _name); - _ptr = end; - return 0; - } - } - - return pc_off; -} - -void DwarfParser::addRecord(u32 loc, u32 cfa_reg, int cfa_off, int fp_off, - int pc_off) { - // cfa_reg and cfa_off are packed into a single u32 (cfa_off << 8 | cfa_reg), - // so cfa_reg must fit in 8 bits (0..255) and cfa_off in a signed 24-bit range - // (-2^23 .. 2^23-1). Well-formed compiler-generated DWARF always satisfies - // this, but a malformed or corrupt .eh_frame from an untrusted ELF may not. - // Rather than pack a truncated (wrong) descriptor, drop the record: the stack - // walker treats a missing entry for a PC the same as any other unwind miss. - if (cfa_reg > 0xFF || cfa_off < -8388608 || cfa_off > 8388607) { - return; - } - - // cfa_reg and cfa_off can be encoded to a single 32 bit value, considering the existing and supported systems - u32 cfa = static_cast(cfa_off) << 8 | static_cast(cfa_reg & 0xff); - - // Detect the linked frame size from the first FP-based entry with a non-zero offset. - // Both GCC and Clang emit DW_REG_FP with cfa_off = LINKED_FRAME_CLANG_SIZE in function - // bodies after the prologue completes. Terminal records use cfa_off = 0 (LINKED_FRAME_SIZE - // on aarch64) and do not influence detection. - if (_linked_frame_size < 0 && cfa_reg == DW_REG_FP && cfa_off > 0) { - _linked_frame_size = cfa_off; - } - - if (_prev == NULL || (_prev->loc == loc && --_count >= 0) || - _prev->cfa != cfa || _prev->fp_off != fp_off || _prev->pc_off != pc_off) { - _prev = addRecordRaw(loc, cfa, fp_off, pc_off); - } -} - -FrameDesc *DwarfParser::addRecordRaw(u32 loc, int cfa, int fp_off, int pc_off) { - if (_count >= _capacity) { - FrameDesc *frameDesc = - (FrameDesc *)realloc(_table, _capacity * 2 * sizeof(FrameDesc)); - if (frameDesc) { - _capacity *= 2; - _table = frameDesc; - } else { - return NULL; - } - } - - FrameDesc *f = &_table[_count++]; - f->loc = loc; - f->cfa = cfa; - f->fp_off = fp_off; - f->pc_off = pc_off; - return f; -} diff --git a/ddprof-lib/src/main/cpp/dwarf.h b/ddprof-lib/src/main/cpp/dwarf.h deleted file mode 100644 index d591383e8..000000000 --- a/ddprof-lib/src/main/cpp/dwarf.h +++ /dev/null @@ -1,283 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _DWARF_H -#define _DWARF_H - -#include -#include -#include "arch.h" - - -const int DW_REG_PLT = 128; // denotes special rule for PLT entries -const int DW_REG_INVALID = 255; // denotes unsupported configuration - -const int DW_PC_OFFSET = 1; -const int DW_SAME_FP = 0x80000000; -const int DW_LINK_REGISTER = 0x80000000; -const int DW_STACK_SLOT = sizeof(void*); - - -#if defined(__x86_64__) - -#define DWARF_SUPPORTED true - -const int DW_REG_FP = 6; -const int DW_REG_SP = 7; -const int DW_REG_PC = 16; -const int EMPTY_FRAME_SIZE = DW_STACK_SLOT; -const int LINKED_FRAME_SIZE = 2 * DW_STACK_SLOT; -const int LINKED_FRAME_CLANG_SIZE = LINKED_FRAME_SIZE; -const int INITIAL_PC_OFFSET = -EMPTY_FRAME_SIZE; - -#elif defined(__i386__) - -#define DWARF_SUPPORTED true - -const int DW_REG_FP = 5; -const int DW_REG_SP = 4; -const int DW_REG_PC = 8; -const int EMPTY_FRAME_SIZE = DW_STACK_SLOT; -const int LINKED_FRAME_SIZE = 2 * DW_STACK_SLOT; -const int LINKED_FRAME_CLANG_SIZE = LINKED_FRAME_SIZE; -const int INITIAL_PC_OFFSET = -EMPTY_FRAME_SIZE; - -#elif defined(__aarch64__) - -#define DWARF_SUPPORTED true - -const int DW_REG_FP = 29; -const int DW_REG_SP = 31; -const int DW_REG_PC = 30; -const int EMPTY_FRAME_SIZE = 0; -const int LINKED_FRAME_SIZE = 0; -const int LINKED_FRAME_CLANG_SIZE = 2 * DW_STACK_SLOT; // clang uses different frame layout than GCC -const int INITIAL_PC_OFFSET = DW_LINK_REGISTER; - -#else - -#define DWARF_SUPPORTED false - -const int DW_REG_FP = 0; -const int DW_REG_SP = 1; -const int DW_REG_PC = 2; -const int EMPTY_FRAME_SIZE = 0; -const int LINKED_FRAME_SIZE = 0; -const int LINKED_FRAME_CLANG_SIZE = LINKED_FRAME_SIZE; -const int INITIAL_PC_OFFSET = DW_LINK_REGISTER; - -#endif - - -struct FrameDesc { - u32 loc; - u32 cfa; - int fp_off; - int pc_off; - - static FrameDesc empty_frame; - static FrameDesc default_frame; - static FrameDesc default_clang_frame; - static FrameDesc no_dwarf_frame; - - // Best-guess fallback frame layout when a PC doesn't map to any known library. - // Per-library detection overrides this: on macOS via __eh_frame section presence, - // on Linux via DwarfParser::detectedDefaultFrame(). - static const FrameDesc& fallback_default_frame() { -#if defined(__APPLE__) && defined(__aarch64__) - return default_clang_frame; -#else - return default_frame; -#endif - } - - static int comparator(const void* p1, const void* p2) { - FrameDesc* fd1 = (FrameDesc*)p1; - FrameDesc* fd2 = (FrameDesc*)p2; - return (int)(fd1->loc - fd2->loc); - } -}; - - -class DwarfParser { - private: - const char* _name; - const char* _image_base; - const char* _image_end; - const char* _ptr; - // Read window [_section_start, _section_end). Both paths set this window: - // - parseEhFrame(): set to the .eh_frame section bounds. - // - parse(): set to the full ELF image bounds [image_base, image_end) so - // that FDE reads into the adjacent .eh_frame are bounded to mapped memory. - const char* _section_start; - const char* _section_end; - - int _capacity; - int _count; - FrameDesc* _table; - FrameDesc* _prev; - - u32 _code_align; - int _data_align; - int _linked_frame_size; // detected from FP-based DWARF entries; -1 = undetected - bool _has_z_augmentation; - - // True if `size` bytes can be read at _ptr without leaving the section. - // Guards against both over-reads (past _section_end) and under-reads - // (_ptr moved before _section_start by an untrusted offset). - bool canRead(size_t size) const { - return _ptr >= _section_start && _ptr <= _section_end && - size <= (size_t)(_section_end - _ptr); - } - - u8 get8() { - if (!canRead(1)) { - _ptr = _section_end; - return 0; - } - return *_ptr++; - } - - u16 get16() { - if (!canRead(2)) { - _ptr = _section_end; - return 0; - } - const char* ptr = _ptr; - _ptr += 2; - u16 result; - memcpy(&result, ptr, sizeof(u16)); - return result; - } - - u32 get32() { - if (!canRead(4)) { - _ptr = _section_end; - return 0; - } - const char* ptr = _ptr; - _ptr += 4; - u32 result; - memcpy(&result, ptr, sizeof(u32)); - return result; - } - - u32 getLeb() { - u32 result = 0; - for (u32 shift = 0; canRead(1) && shift < 32; shift += 7) { - u8 b = *_ptr++; - result |= (u32)(b & 0x7f) << shift; - if ((b & 0x80) == 0) { - break; - } - } - return result; - } - - u32 getLeb(const char* end) { - if (end > _section_end) end = _section_end; - u32 result = 0; - for (u32 shift = 0; _ptr < end && shift < 32; shift += 7) { - u8 b = *_ptr++; - result |= (u32)(b & 0x7f) << shift; - if ((b & 0x80) == 0) { - return result; - } - } - return result; - } - - int getSLeb() { - int result = 0; - for (u32 shift = 0; canRead(1) && shift < 32; shift += 7) { - u8 b = *_ptr++; - // Compute in unsigned to avoid signed left-shift overflow (UB) for - // large shift values; the result is reinterpreted as signed below. - result |= (int)((u32)(b & 0x7f) << shift); - if ((b & 0x80) == 0) { - if ((b & 0x40) != 0 && (shift += 7) < 32) { - result |= (int)(~0U << shift); - } - break; - } - } - return result; - } - - int getSLeb(const char* end) { - if (end > _section_end) end = _section_end; - int result = 0; - for (u32 shift = 0; _ptr < end && shift < 32; shift += 7) { - u8 b = *_ptr++; - // Compute in unsigned to avoid signed left-shift overflow (UB) for - // large shift values; the result is reinterpreted as signed below. - result |= (int)((u32)(b & 0x7f) << shift); - if ((b & 0x80) == 0) { - if ((b & 0x40) != 0 && (shift += 7) < 32) { - result |= (int)(~0U << shift); - } - return result; - } - } - return result; - } - - void skipLeb() { - while (canRead(1) && (*_ptr++ & 0x80)) {} - } - - const char* getPtr() { - if (_ptr + 4 > _image_end) { _ptr = _image_end; return _image_base; } - const char* ptr = _ptr; - if (!canRead(4)) { - _ptr = _section_end; - return ptr; - } - int offset; - memcpy(&offset, _ptr, sizeof(int)); - _ptr += 4; - return ptr + offset; - } - - void init(const char* name, const char* image_base, const char* image_end); - void parse(const char* eh_frame_hdr, size_t size, const char* image_end); - void parseEhFrame(const char* eh_frame, size_t size); - void parseCie(); - void parseFde(); - void parseInstructions(u32 loc, const char* end); - int parseExpression(); - - void addRecord(u32 loc, u32 cfa_reg, int cfa_off, int fp_off, int pc_off); - FrameDesc* addRecordRaw(u32 loc, int cfa, int fp_off, int pc_off); - - public: - // Tag to disambiguate the .eh_frame_hdr (binary-search index) constructor - // from the raw .eh_frame constructor below: with a size added, the two - // would otherwise share a signature. - struct EhFrameHdrTag {}; - DwarfParser(const char* name, const char* image_base, const char* eh_frame_hdr, size_t eh_frame_hdr_size, EhFrameHdrTag, const char* image_end); - DwarfParser(const char* name, const char* image_base, const char* eh_frame, size_t eh_frame_size); - - // Ownership of the returned pointer transfers to the caller. - // The caller is responsible for freeing it with free() (not delete[]). - // DwarfParser has no destructor; _table is left dangling after this call is used. - FrameDesc* table() const { - return _table; - } - - int count() const { - return _count; - } - - const FrameDesc& detectedDefaultFrame() const { - if (_linked_frame_size == LINKED_FRAME_CLANG_SIZE && LINKED_FRAME_CLANG_SIZE != LINKED_FRAME_SIZE) { - return FrameDesc::default_clang_frame; - } - return FrameDesc::default_frame; - } -}; - -#endif // _DWARF_H diff --git a/ddprof-lib/src/main/cpp/engine.cpp b/ddprof-lib/src/main/cpp/engine.cpp deleted file mode 100644 index ca0c9471f..000000000 --- a/ddprof-lib/src/main/cpp/engine.cpp +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2018 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "engine.h" - -Error Engine::check(Arguments &args) { return Error::OK; } - -Error Engine::start(Arguments &args) { return Error::OK; } - -void Engine::stop() {} diff --git a/ddprof-lib/src/main/cpp/engine.h b/ddprof-lib/src/main/cpp/engine.h deleted file mode 100644 index c9e75e298..000000000 --- a/ddprof-lib/src/main/cpp/engine.h +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2017 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _ENGINE_H -#define _ENGINE_H - -#include "arguments.h" - -class Engine { -protected: - static bool updateCounter(volatile unsigned long long &counter, - unsigned long long value, - unsigned long long interval) { - if (interval <= 1) { - return true; - } - - while (true) { - unsigned long long prev = counter; - unsigned long long next = prev + value; - if (next < interval) { - if (__sync_bool_compare_and_swap(&counter, prev, next)) { - return false; - } - } else { - if (__sync_bool_compare_and_swap(&counter, prev, next % interval)) { - return true; - } - } - } - } - -public: - virtual const char *name() { return "None"; } - - virtual Error check(Arguments &args); - virtual Error start(Arguments &args); - virtual void stop(); - virtual long interval() const { return 0L; } - - virtual int registerThread(int tid) { return -1; } - virtual void unregisterThread(int tid) {} - - virtual void enableEvents(bool enabled) { - // do nothing - } -}; - -#endif // _ENGINE_H diff --git a/ddprof-lib/src/main/cpp/event.h b/ddprof-lib/src/main/cpp/event.h deleted file mode 100644 index 0747f2a4f..000000000 --- a/ddprof-lib/src/main/cpp/event.h +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2020 Andrei Pangin - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _EVENT_H -#define _EVENT_H - -#include "context.h" -#include "os.h" -#include "threadState.h" -#include -#include -#include -using namespace std; - -#define MAX_STRING_LEN 8191 - -// The order is important: look for event_type comparison -enum EventType { - PERF_SAMPLE, - EXECUTION_SAMPLE, - WALL_CLOCK_SAMPLE, - MALLOC_SAMPLE, - INSTRUMENTED_METHOD, - METHOD_TRACE, - ALLOC_SAMPLE, - ALLOC_OUTSIDE_TLAB, - LIVE_OBJECT, - LOCK_SAMPLE, - PARK_SAMPLE, - PROFILING_WINDOW, - USER_EVENT, -}; - -class Event { -public: - u32 _id; - - Event() : _id(0) {} -}; - -class ExecutionEvent : public Event { -public: - OSThreadState _thread_state; - ExecutionMode _execution_mode; - u64 _weight; - u32 _call_trace_id; - - ExecutionEvent() - : Event(), _thread_state(OSThreadState::RUNNABLE), _execution_mode(ExecutionMode::UNKNOWN), - _weight(1), _call_trace_id(0) {} -}; - -class AllocEvent : public Event { -public: - u64 _size; - float _weight; - - AllocEvent() : _size(0), _weight(1) {} -}; - -class LockEvent : public Event { -public: - u64 _start_time; - u64 _end_time; - uintptr_t _address; - long long _timeout; -}; - -class ObjectLivenessEvent : public Event { -public: - AllocEvent _alloc; - u64 _skipped; - u64 _start_time; - u64 _age; - Context _ctx; -}; - -class MallocEvent : public Event { -public: - u64 _start_time; - uintptr_t _address; - u64 _size; - float _weight; - - MallocEvent() : Event(), _start_time(0), _address(0), _size(0), _weight(1.0f) {} -}; - -class NativeSocketEvent : public Event { -public: - u64 _start_time; // TSC ticks at call entry - u64 _end_time; // TSC ticks at call return - u8 _operation; // 0 = SEND, 1 = RECV, 2 = WRITE, 3 = READ - char _remote_addr[64]; // "ip:port" null-terminated string - u64 _bytes; // bytes transferred (return value of send/recv/write/read) - float _weight; // inverse-transform sample weight - - NativeSocketEvent() : Event(), _start_time(0), _end_time(0), _operation(0), - _bytes(0), _weight(1.0f) { _remote_addr[0] = '\0'; } -}; - -class WallClockEpochEvent { -public: - bool _dirty; - u64 _start_time; - u64 _duration_millis; - u32 _num_samplable_threads; - u32 _num_successful_samples; - u32 _num_failed_samples; - u32 _num_exited_threads; - u32 _num_permission_denied; - u64 _num_suppressed_sampled_run; - - WallClockEpochEvent(u64 start_time) - : _dirty(false), _start_time(start_time), _duration_millis(0), - _num_samplable_threads(0), _num_successful_samples(0), - _num_failed_samples(0), _num_exited_threads(0), - _num_permission_denied(0), _num_suppressed_sampled_run(0) {} - - bool hasChanged() { return _dirty; } - - void updateNumSamplableThreads(u32 num_samplable_threads) { - if (_num_samplable_threads != num_samplable_threads) { - _dirty = true; - _num_samplable_threads = num_samplable_threads; - } - } - - void updateNumSuccessfulSamples(u32 num_successful_samples) { - if (_num_successful_samples != num_successful_samples) { - _dirty = true; - _num_successful_samples = num_successful_samples; - } - } - - void updateNumFailedSamples(u32 num_failed_samples) { - if (_num_failed_samples != num_failed_samples) { - _dirty = true; - _num_failed_samples = num_failed_samples; - } - } - - void updateNumExitedThreads(u32 num_exited_threads) { - if (_num_exited_threads != num_exited_threads) { - _dirty = true; - _num_exited_threads = num_exited_threads; - } - } - - void updateNumPermissionDenied(u32 num_permission_denied) { - if (_num_permission_denied != num_permission_denied) { - _dirty = true; - _num_permission_denied = num_permission_denied; - } - } - - void addNumSuppressedSampledRun(u64 n) { - if (n > 0) { - _dirty = true; - _num_suppressed_sampled_run += n; - } - } - - void endEpoch(u64 millis) { _duration_millis = millis; } - - void clean() { _dirty = false; } - - void newEpoch(u64 start_time) { - _dirty = false; - _start_time = start_time; - _num_suppressed_sampled_run = 0; - } -}; - -class TraceRootEvent { -public: - u64 _local_root_span_id; - u32 _label; - u32 _operation; - - TraceRootEvent(u64 local_root_span_id, u32 label, u32 operation) - : _local_root_span_id(local_root_span_id), _label(label), - _operation(operation){}; -}; - -typedef struct QueueTimeEvent { - u64 _start; - u64 _end; - u32 _task; - u32 _scheduler; - u32 _origin; - u32 _queueType; - u32 _queueLength; -} QueueTimeEvent; - -#endif // _EVENT_H diff --git a/ddprof-lib/src/main/cpp/fdtransferClient.h b/ddprof-lib/src/main/cpp/fdtransferClient.h deleted file mode 100644 index 2552c3541..000000000 --- a/ddprof-lib/src/main/cpp/fdtransferClient.h +++ /dev/null @@ -1,14 +0,0 @@ -#ifndef _FD_TRANSFER_CLIENT_H -// async-profiler fdtransferClient.h shim - -class FdTransferClient { - public: - static inline bool hasPeer() { - return false; - } - - static inline int requestKallsymsFd() { - return -1; - } -}; -#endif // _FD_TRANSFER_CLIENT_H diff --git a/ddprof-lib/src/main/cpp/findLibraryImpl.h b/ddprof-lib/src/main/cpp/findLibraryImpl.h deleted file mode 100644 index af63385f1..000000000 --- a/ddprof-lib/src/main/cpp/findLibraryImpl.h +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _FINDLIBRARYIMPL_H -#define _FINDLIBRARYIMPL_H - -// Signal-handler-safe last-hit-index cache for findLibraryByAddress. -// -// Templated on CacheArray and CacheEntry so that the exact same algorithm -// can be exercised from benchmarks (using lightweight fake types) without -// pulling in JVM headers, while production uses CodeCacheArray/CodeCache. -// -// Requirements on CacheArray: -// int count() const — number of live entries -// CacheEntry* operator[](int i) const — entry at index i (may be nullptr) -// -// Requirements on CacheEntry: -// bool contains(const void* address) const -// -// Signal-safety: the last-hit index is a plain static volatile int. -// DTLS initialisation for shared libraries calls calloc internally; if a -// profiler signal fires on a thread whose TLS block has not been set up yet -// while that thread is inside malloc, any thread_local access deadlocks on -// the allocator lock. A plain static volatile int avoids TLS entirely. -// A benign race on the index is acceptable — the worst case is a cache miss -// that falls through to the O(n) linear scan. - -template -inline CacheEntry* findLibraryByAddressImpl(const CacheArray& libs, const void* address) { - static volatile int last_hit = 0; - const int count = libs.count(); - int hint = last_hit; - if (hint < count) { - CacheEntry* lib = libs[hint]; - if (lib != nullptr && lib->contains(address)) { - return lib; - } - } - for (int i = 0; i < count; i++) { - CacheEntry* lib = libs[i]; - if (lib != nullptr && lib->contains(address)) { - last_hit = i; - return lib; - } - } - return nullptr; -} - -#endif // _FINDLIBRARYIMPL_H diff --git a/ddprof-lib/src/main/cpp/flightRecorder.cpp b/ddprof-lib/src/main/cpp/flightRecorder.cpp deleted file mode 100644 index 17560d37a..000000000 --- a/ddprof-lib/src/main/cpp/flightRecorder.cpp +++ /dev/null @@ -1,2206 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -#include "buffers.h" -#include "callTraceHashTable.h" -#include "context.h" -#include "context_api.h" -#include "counters.h" -#include "dictionary.h" -#include "flightRecorder.h" -#include "incbin.h" -#include "jfrMetadata.h" -#include "jniHelper.h" -#include "os.h" -#include "profiler.h" -#include "signalSafety.h" -#include "rustDemangler.h" -#include "safeAccess.h" -#include "spinLock.h" -#include "unwindStats.h" -#include "symbols.h" -#include "threadFilter.h" -#include "threadState.h" -#include "tsc.h" -#include "hotspot/vmStructs.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -static const char *const SETTING_RING[] = {NULL, "kernel", "user", "any"}; -static const char *const SETTING_CSTACK[] = {NULL, "no", "fp", "dwarf", "lbr"}; - -// Compute a non-negative event duration from TSC timestamps. Unsigned u64 -// subtraction wraps to a near-2^64 value when end < start, which can happen if -// the thread migrates cores between the two TSC reads and the per-core counters -// are not perfectly synchronised. Clamp such inversions to 0 so the emitted -// duration is never an absurd outlier. -static inline u64 safeDuration(u64 start_time, u64 end_time) { - return end_time >= start_time ? end_time - start_time : 0; -} - -SharedLineNumberTable::~SharedLineNumberTable() { - // _ptr is a malloc'd copy of the JVMTI line number table (see - // Lookup::fillJavaMethodInfo). Freeing here is independent of class - // unload, preventing use-after-free in ~SharedLineNumberTable and getLineNumber. - if (_ptr != nullptr) { - free(_ptr); - Counters::decrement(LINE_NUMBER_TABLES); - } -} - -void Lookup::fillNativeMethodInfo(MethodInfo *mi, const char *name, - const char *lib_name) { - mi->_class = _classes->lookupDuringDump("", 0); - // TODO return the library name once we figured out how to cooperate with the - // backend - // if (lib_name == NULL) { - // mi->_class = _classes->lookup(""); - // } else if (lib_name[0] == '[' && lib_name[1] != 0) { - // mi->_class = _classes->lookup(lib_name + 1, strlen(lib_name) - - // 2); - // } else { - // mi->_class = _classes->lookup(lib_name); - // } - - mi->_modifiers = 0x100; - mi->_line_number_table = nullptr; - - if (name[0] == '_' && name[1] == 'Z') { - int status; - char *demangled = abi::__cxa_demangle(name, NULL, NULL, &status); - if (demangled != NULL) { - cutArguments(demangled); - mi->_sig = _symbols.lookup("()L;"); - mi->_type = FRAME_CPP; - - // Rust legacy demangling - if (RustDemangler::is_probably_rust_legacy(demangled)) { - std::string rust_demangled = RustDemangler::demangle(demangled); - mi->_name = _symbols.lookup(rust_demangled.c_str()); - } else { - mi->_name = _symbols.lookup(demangled); - } - free(demangled); - return; - } - } - - size_t len = strlen(name); - if (len >= 4 && strcmp(name + len - 4, "_[k]") == 0) { - mi->_name = _symbols.lookup(name, len - 4); - mi->_sig = _symbols.lookup("(Lk;)L;"); - mi->_type = FRAME_KERNEL; - } else { - mi->_name = _symbols.lookup(name); - mi->_sig = _symbols.lookup("()L;"); - mi->_type = FRAME_NATIVE; - } -} - -void Lookup::fillRemoteFrameInfo(MethodInfo *mi, const RemoteFrameInfo *rfi) { - // Store build-id in the class name field - mi->_class = _classes->lookupDuringDump(rfi->build_id, strlen(rfi->build_id)); - - // Store PC offset in hex format in the signature field - char offset_hex[32]; - snprintf(offset_hex, sizeof(offset_hex), "0x%" PRIxPTR, rfi->pc_offset); - mi->_sig = _symbols.lookup(offset_hex); - - // Use same modifiers as regular native frames (0x100 = ACC_NATIVE for consistency) - mi->_modifiers = 0x100; - // Use FRAME_NATIVE_REMOTE type to indicate remote symbolication - mi->_type = FRAME_NATIVE_REMOTE; - mi->_line_number_table = nullptr; - - // Method name indicates need for remote symbolication - mi->_name = _symbols.lookup(""); -} - -void Lookup::cutArguments(char *func) { - char *p = strrchr(func, ')'); - if (p == NULL) - return; - - int balance = 1; - while (--p > func) { - if (*p == '(' && --balance == 0) { - *p = 0; - return; - } else if (*p == ')') { - balance++; - } - } -} - -void Lookup::fillJavaMethodInfo(MethodInfo *mi, jmethodID method, - bool first_time) { - JNIEnv *jni = VM::jni(); - if (jni->PushLocalFrame(64) != 0) { - return; - } - jvmtiEnv *jvmti = VM::jvmti(); - - jvmtiPhase phase; - jclass method_class = NULL; - // invariant: these strings must remain null, or be assigned by JVMTI - char *class_name = nullptr; - char *method_name = nullptr; - char *method_sig = nullptr; - u32 class_name_id = 0; - u32 method_name_id = 0; - u32 method_sig_id = 0; - - jint line_number_table_size = 0; - jvmtiLineNumberEntry *line_number_table = NULL; - - jvmti->GetPhase(&phase); - if ((phase & (JVMTI_PHASE_START | JVMTI_PHASE_LIVE)) != 0) { - bool entry = false; - bool readable = false; - const size_t probe_len = 256; - if (VMMethod::check_jmethodID(method) && - jvmti->GetMethodDeclaringClass(method, &method_class) == 0 && - // GetMethodDeclaringClass may return a jclass wrapping a stale/garbage oop when the class was - // unloaded between sample capture and dump (TOCTOU race with class unloading). Guard against - // null handles before calling GetClassSignature. - method_class != NULL && - // On some older versions of J9, the JVMTI call to GetMethodDeclaringClass will return OK = 0, but when a - // classloader is unloaded they free all JNIIDs. This means that anyone holding on to a jmethodID is - // pointing to corrupt data and the behaviour is undefined. - // The behaviour is adjusted so that when asgct() is used or if `-XX:+KeepJNIIDs` is specified, - // when a classloader is unloaded, the jmethodIDs are not freed, but instead marked as -1. - // The check below mitigates these crashes on J9. - (!VM::isOpenJ9() || method_class != reinterpret_cast(-1)) && - jvmti->GetClassSignature(method_class, &class_name, NULL) == 0 && - jvmti->GetMethodName(method, &method_name, &method_sig, NULL) == 0) { - // The JVMTI strings should be non-null and mapped per spec, but crash - // telemetry shows both `strncmp` and `jvmti_Deallocate` faulting on them. - // Probe each pointer over a range covering the longest prefix - // compared below (~50 bytes) plus headroom for strlen, and NULL any that - // fails so the unconditional Deallocate block at end of this function - // skips it (os::free faults on an unmapped pointer just like strncmp). - // Accept a small leak on the corruption path. Probes run independently - // so a single bad pointer does not leak its siblings. Best-effort only: - // a concurrent munmap between probe and use can still fault; the SIGSEGV - // handler is the second line of defence. - auto probe = [&](char*& ptr) -> bool { - if (ptr == nullptr || !SafeAccess::isReadableRange(ptr, probe_len)) { - ptr = nullptr; - return false; - } - return true; - }; - readable = probe(class_name) & probe(method_name) & probe(method_sig); - } - if (readable) { - const size_t class_name_len = strnlen(class_name, 65536); - const char* normalized_class_name = - class_name_len >= 2 ? class_name + 1 : ""; - const size_t normalized_class_name_len = - class_name_len >= 2 ? class_name_len - 2 : 0; - - if (first_time) { - jvmtiError line_table_error = jvmti->GetLineNumberTable(method, &line_number_table_size, - &line_number_table); - // Defensive: if GetLineNumberTable failed, clean up any potentially allocated memory - // Some buggy JVMTI implementations might allocate despite returning an error - if (line_table_error != JVMTI_ERROR_NONE) { - if (line_number_table != nullptr) { - // Try to deallocate to prevent leak from buggy JVM - jvmti->Deallocate((unsigned char *)line_number_table); - } - line_number_table = nullptr; - line_number_table_size = 0; - } - } - - // Check if the frame is Thread.run or inherits from it - if (strncmp(method_name, "run", 4) == 0 && - strncmp(method_sig, "()V", 3) == 0) { - jclass Thread_class = jni->FindClass("java/lang/Thread"); - jclass Class_class = jni->FindClass("java/lang/Class"); - if (Thread_class != nullptr && Class_class != nullptr) { - jmethodID equals = jni->GetMethodID(Class_class, - "equals", "(Ljava/lang/Object;)Z"); - if (equals != nullptr) { - jclass klass = method_class; - do { - entry = jni->CallBooleanMethod(Thread_class, equals, klass); - if (jniExceptionCheck(jni)) { - entry = false; - break; - } - if (entry) { - break; - } - } while ((klass = jni->GetSuperclass(klass)) != NULL); - } - } - // Clear any exceptions from the reflection calls above - jniExceptionCheck(jni); - } else if (strncmp(method_name, "main", 5) == 0 && - strncmp(method_sig, "(Ljava/lang/String;)V", 21)) { - // public static void main(String[] args) - 'public static' translates - // to modifier bits 0 and 3, hence check for '9' - entry = true; - } - - // maybe we should store the lookups below in initialisation-time - // constants... - if (has_prefix(class_name, - "Ljdk/internal/reflect/GeneratedConstructorAccessor")) { - class_name_id = _classes->lookupDuringDump( - "jdk/internal/reflect/GeneratedConstructorAccessor", - strlen("jdk/internal/reflect/GeneratedConstructorAccessor")); - method_name_id = - _symbols.lookup("Object " - "jdk.internal.reflect.GeneratedConstructorAccessor." - "newInstance(Object[])"); - method_sig_id = _symbols.lookup(method_sig); - } else if (has_prefix(class_name, - "Lsun/reflect/GeneratedConstructorAccessor")) { - class_name_id = - _classes->lookupDuringDump("sun/reflect/GeneratedConstructorAccessor", - strlen("sun/reflect/GeneratedConstructorAccessor")); - method_name_id = _symbols.lookup( - "Object " - "sun.reflect.GeneratedConstructorAccessor.newInstance(Object[])"); - method_sig_id = _symbols.lookup(method_sig); - } else if (has_prefix(class_name, - "Ljdk/internal/reflect/GeneratedMethodAccessor")) { - class_name_id = - _classes->lookupDuringDump("jdk/internal/reflect/GeneratedMethodAccessor", - strlen("jdk/internal/reflect/GeneratedMethodAccessor")); - method_name_id = - _symbols.lookup("Object " - "jdk.internal.reflect.GeneratedMethodAccessor." - "invoke(Object, Object[])"); - method_sig_id = _symbols.lookup(method_sig); - } else if (has_prefix(class_name, - "Lsun/reflect/GeneratedMethodAccessor")) { - class_name_id = _classes->lookupDuringDump("sun/reflect/GeneratedMethodAccessor", - strlen("sun/reflect/GeneratedMethodAccessor")); - method_name_id = _symbols.lookup( - "Object sun.reflect.GeneratedMethodAccessor.invoke(Object, " - "Object[])"); - method_sig_id = _symbols.lookup(method_sig); - } else if (has_prefix(class_name, "Ljava/lang/invoke/LambdaForm$")) { - const int lambdaFormPrefixLength = - strlen("Ljava/lang/invoke/LambdaForm$"); - // we want to normalise to java/lang/invoke/LambdaForm$MH, - // java/lang/invoke/LambdaForm$DMH, java/lang/invoke/LambdaForm$BMH, - if (has_prefix(class_name + lambdaFormPrefixLength, "MH")) { - class_name_id = _classes->lookupDuringDump("java/lang/invoke/LambdaForm$MH", - strlen("java/lang/invoke/LambdaForm$MH")); - } else if (has_prefix(class_name + lambdaFormPrefixLength, "BMH")) { - class_name_id = _classes->lookupDuringDump("java/lang/invoke/LambdaForm$BMH", - strlen("java/lang/invoke/LambdaForm$BMH")); - } else if (has_prefix(class_name + lambdaFormPrefixLength, "DMH")) { - class_name_id = _classes->lookupDuringDump("java/lang/invoke/LambdaForm$DMH", - strlen("java/lang/invoke/LambdaForm$DMH")); - } else { - // don't recognise the suffix, so don't normalise - class_name_id = _classes->lookupDuringDump( - normalized_class_name, normalized_class_name_len); - } - method_name_id = _symbols.lookup(method_name); - method_sig_id = _symbols.lookup(method_sig); - } else { - class_name_id = _classes->lookupDuringDump(normalized_class_name, - normalized_class_name_len); - method_name_id = _symbols.lookup(method_name); - method_sig_id = _symbols.lookup(method_sig); - } - } else { - Counters::increment(JMETHODID_SKIPPED); - class_name_id = _classes->lookupDuringDump("", 0); - method_name_id = _symbols.lookup("jvmtiError"); - method_sig_id = _symbols.lookup("()L;"); - } - - mi->_class = class_name_id; - mi->_name = method_name_id; - mi->_sig = method_sig_id; - mi->_type = FRAME_INTERPRETED; - mi->_is_entry = entry; - if (line_number_table != nullptr) { - // Detach from JVMTI lifetime: copy into our own buffer and deallocate - // the JVMTI-allocated memory immediately. This keeps _ptr valid even - // after the underlying class is unloaded. - void *owned_table = nullptr; - if (line_number_table_size > 0) { - size_t bytes = (size_t)line_number_table_size * sizeof(jvmtiLineNumberEntry); - owned_table = malloc(bytes); - if (owned_table != nullptr) { - memcpy(owned_table, line_number_table, bytes); - } else { - TEST_LOG("Failed to allocate %zu bytes for line number table copy", bytes); - } - } - jvmtiError dealloc_err = jvmti->Deallocate((unsigned char *)line_number_table); - if (dealloc_err != JVMTI_ERROR_NONE) { - TEST_LOG("Unexpected error while deallocating linenumber table: %d", dealloc_err); - } - if (owned_table != nullptr) { - mi->_line_number_table = std::make_shared( - line_number_table_size, owned_table); - // Increment counter for tracking live line number tables - Counters::increment(LINE_NUMBER_TABLES); - } - } - - // strings are null or came from JVMTI - if (method_name) { - jvmti->Deallocate((unsigned char *)method_name); - } - if (method_sig) { - jvmti->Deallocate((unsigned char *)method_sig); - } - if (class_name) { - jvmti->Deallocate((unsigned char *)class_name); - } - } - jni->PopLocalFrame(NULL); -} - -bool Lookup::resolveVTableReceiver(VMSymbol *sym, char *buf, size_t bufsize, - u32 *out_class_id) { - if (sym == nullptr || !SafeAccess::isReadable(sym)) { - return false; - } - // Read the 4-byte word containing the u2 length field. In all HotSpot - // versions we support the length is at offset 0 of Symbol; we still go - // through VMStructs in case that ever changes. The low 16 bits hold the - // length on little-endian targets (all supported platforms). - int32_t *len_word_addr = - (int32_t *)((char *)sym + VMSymbol::lengthOffset()); - int32_t w1 = SafeAccess::safeFetch32(len_word_addr, -1); - int32_t w2 = SafeAccess::safeFetch32(len_word_addr, 0); - if (w1 == -1 && w2 == 0) { - return false; - } - unsigned len = (unsigned)(w1 & 0xFFFF); - // Bounds: a usable internal class name needs at least 1 byte (single-char - // descriptors like "B"/"C" for primitives never appear as vtable receivers - // because primitives can't be receivers of virtual or interface dispatch). - // Upper bound is the caller-provided buffer; class names above this length - // are dropped — operators see VTABLE_RECEIVER_RESOLVE_FAILED rise. - if (len == 0 || len > bufsize) { - return false; - } - const void *body = (const char *)sym + VMSymbol::bodyOffset(); - if (!SafeAccess::safeCopy(buf, body, len)) { - return false; - } - // Reject anything that doesn't look like a JVM internal class name. - // Valid bytes for slash-separated internal names: '/', '$', '[', ';', '_', - // alnum. Rejecting reduces — but does not eliminate — the case where the - // Symbol slot was reused for unrelated data that happens to be printable. - for (unsigned i = 0; i < len; i++) { - unsigned char c = (unsigned char)buf[i]; - if (c < 0x20 || c >= 0x7F) { - return false; - } - } - // lookupDuringDump (not lookup) because this runs inside writeCpool, after - // rotate(): standby holds the pre-rotate snapshot that writeClasses() will - // serialize. Plain lookup() would insert into the new active only, leaving - // the stack frame's class_id absent from this chunk's class pool. - // (Plain lookup() remains correct for non-dump callers — e.g. Profiler:: - // lookupClass on JVM threads — where the next rotate() will propagate.) - u32 class_id = _classes->lookupDuringDump(buf, len); - // Apply synthetic-accessor/LambdaForm normalisation so that the many - // distinct names HotSpot generates for these families (..Accessor1234, - // LambdaForm$MH/0x...) collapse to one bucket each in the JFR class pool. - // Folding the normalisation inside resolveVTableReceiver keeps the call - // site in resolveMethod minimal and ensures the cache stores normalised - // class ids (so MethodMap deduplication works for these families too). - if (has_prefix_n(buf, len, - "jdk/internal/reflect/GeneratedConstructorAccessor")) { - static const char kName[] = "jdk/internal/reflect/GeneratedConstructorAccessor"; - class_id = _classes->lookupDuringDump(kName, sizeof(kName) - 1); - } else if (has_prefix_n(buf, len, "sun/reflect/GeneratedConstructorAccessor")) { - static const char kName[] = "sun/reflect/GeneratedConstructorAccessor"; - class_id = _classes->lookupDuringDump(kName, sizeof(kName) - 1); - } else if (has_prefix_n(buf, len, - "jdk/internal/reflect/GeneratedMethodAccessor")) { - static const char kName[] = "jdk/internal/reflect/GeneratedMethodAccessor"; - class_id = _classes->lookupDuringDump(kName, sizeof(kName) - 1); - } else if (has_prefix_n(buf, len, "sun/reflect/GeneratedMethodAccessor")) { - static const char kName[] = "sun/reflect/GeneratedMethodAccessor"; - class_id = _classes->lookupDuringDump(kName, sizeof(kName) - 1); - } else if (has_prefix_n(buf, len, "java/lang/invoke/LambdaForm$")) { - size_t prefix_len = strlen("java/lang/invoke/LambdaForm$"); - const char *suffix = buf + prefix_len; - size_t suffix_len = len - prefix_len; - if (suffix_len >= 2 && suffix[0] == 'M' && suffix[1] == 'H') { - static const char kName[] = "java/lang/invoke/LambdaForm$MH"; - class_id = _classes->lookupDuringDump(kName, sizeof(kName) - 1); - } else if (suffix_len >= 3 && suffix[0] == 'B' && suffix[1] == 'M' && - suffix[2] == 'H') { - static const char kName[] = "java/lang/invoke/LambdaForm$BMH"; - class_id = _classes->lookupDuringDump(kName, sizeof(kName) - 1); - } else if (suffix_len >= 3 && suffix[0] == 'D' && suffix[1] == 'M' && - suffix[2] == 'H') { - static const char kName[] = "java/lang/invoke/LambdaForm$DMH"; - class_id = _classes->lookupDuringDump(kName, sizeof(kName) - 1); - } - } - *out_class_id = class_id; - return true; -} - -u32 Lookup::resolveVTableReceiverCached(void *sym) { - auto cached = _vtable_receiver_cache.find(sym); - if (cached != _vtable_receiver_cache.end()) { - return cached->second; - } - // Stack buffer sized to fit virtually every real class name. HotSpot - // Symbol length is u2 (max 65535); names beyond 4096 bytes are rare - // (deeply nested LambdaForm signatures, large CGLIB proxies) and are - // recorded as resolve failures via the sentinel below. - char buf[4096]; - u32 class_id = 0; - if (!resolveVTableReceiver((VMSymbol *)sym, buf, sizeof(buf), &class_id)) { - Counters::increment(VTABLE_RECEIVER_RESOLVE_FAILED); - // Explicit sentinel so JFR renders an obvious "we couldn't read it" - // marker instead of an empty class name (which is indistinguishable - // from a parser/encoder error downstream). - static const char kName[] = ""; - class_id = _classes->lookupDuringDump(kName, sizeof(kName) - 1); - } - _vtable_receiver_cache[sym] = class_id; - return class_id; -} - -MethodInfo *Lookup::resolveMethod(ASGCT_CallFrame &frame) { - static const char* UNKNOWN = "unknown"; - unsigned long key; - jint bci = frame.bci; - - jmethodID method = frame.method_id; - - // BCI_VTABLE_RECEIVER: method holds a VMSymbol* (see vmEntry.h). Resolve - // to a class_id via the per-dump cache once, then key MethodMap by the - // resolved class_id so two distinct Symbol addresses for the same class - // name (class unload + reload within a chunk) collapse to one MethodInfo - // row. - u32 vtable_class_id = 0; - if (bci == BCI_VTABLE_RECEIVER) { - vtable_class_id = resolveVTableReceiverCached((void *)method); - } - - if (method == nullptr) { - key = MethodMap::makeKey(UNKNOWN); - } else if (bci == BCI_ERROR || bci == BCI_NATIVE_FRAME) { - key = MethodMap::makeKey(frame.native_function_name); - } else if (bci == BCI_NATIVE_FRAME_REMOTE) { - key = MethodMap::makeKey(frame.packed_remote_frame); - } else if (bci == BCI_VTABLE_RECEIVER) { - key = MethodMap::makeVTableReceiverKey(vtable_class_id); - } else { - FrameTypeId frame_type = FrameType::decode(bci); - assert(frame_type == FRAME_INTERPRETED || frame_type == FRAME_JIT_COMPILED || - frame_type == FRAME_INLINED || frame_type == FRAME_C1_COMPILED || - VM::isOpenJ9()); // OpenJ9 may have bugs that produce invalid frame types - key = MethodMap::makeKey(method); - } - - MethodInfo *mi = &(*_method_map)[key]; - - if (!mi->_mark) { - mi->_mark = true; - bool first_time = mi->_key == 0; - if (first_time) { - // Allocate a method-pool id that is unique among live methods. Must not - // be derived from the map size: cleanupUnreferencedMethods() erases - // entries, so size()+1 would reissue an id still owned by a surviving - // method, producing duplicate ids in the chunk's method constant pool - // (PROF-15130). The allocator recycles ids freed on erase instead. - mi->_key = _method_map->allocId(); - } - if (method == nullptr) { - fillNativeMethodInfo(mi, UNKNOWN, nullptr); - } else if (bci == BCI_ERROR) { - fillNativeMethodInfo(mi, (const char *)method, nullptr); - } else if (bci == BCI_NATIVE_FRAME) { - const char *name = (const char *)method; - fillNativeMethodInfo(mi, name, - Profiler::instance()->getLibraryName(name)); - } else if (bci == BCI_NATIVE_FRAME_REMOTE) { - // Unpack remote symbolication data using utility struct - // Layout: pc_offset (44 bits) | mark (3 bits) | lib_index (15 bits) - unsigned long packed_remote_frame = frame.packed_remote_frame; - uintptr_t pc_offset = Profiler::RemoteFramePacker::unpackPcOffset(packed_remote_frame); - [[maybe_unused]] char mark = Profiler::RemoteFramePacker::unpackMark(packed_remote_frame); - uint32_t lib_index = Profiler::RemoteFramePacker::unpackLibIndex(packed_remote_frame); - - TEST_LOG("Unpacking remote frame: packed=0x%zx, pc_offset=0x%lx, mark=%d, lib_index=%u", - packed_remote_frame, pc_offset, (int)mark, lib_index); - - // Lookup library by index to get build_id - // Note: This is called during JFR serialization with lockAll() held (see Profiler::dump), - // so the library array is stable - no concurrent dlopen_hook calls can modify it. - CodeCache* lib = Libraries::instance()->getLibraryByIndex(lib_index); - if (lib != nullptr && lib->hasBuildId() && Profiler::instance()->isRemoteSymbolication()) { - TEST_LOG("Found library: %s, build_id=%s", lib->name(), lib->buildId()); - // Remote symbolication: defer to backend - RemoteFrameInfo rfi(lib->buildId(), pc_offset, lib_index); - fillRemoteFrameInfo(mi, &rfi); - } else if (lib != nullptr) { - // Locally unsymbolized: render as [libname+0xoffset] - char name_buf[256]; - const char* s = lib->name(); - const char* basename = strrchr(s, '/'); - if (basename) basename++; else basename = s; - snprintf(name_buf, sizeof(name_buf), "[%s+0x%" PRIxPTR "]", basename, pc_offset); - fillNativeMethodInfo(mi, name_buf, nullptr); - } else { - TEST_LOG("WARNING: Library lookup failed for index %u", lib_index); - fillNativeMethodInfo(mi, "unknown_library", nullptr); - } - } else if (bci == BCI_VTABLE_RECEIVER) { - // Synthetic vtable-receiver frame: method_id holds a VMSymbol* - // captured in walkVM. The Symbol -> class_id resolution (with - // synthetic-accessor/LambdaForm normalisation) was already done - // above via resolveVTableReceiverCached, which also handles - // resolution failures by mapping them to "" - // and incrementing VTABLE_RECEIVER_RESOLVE_FAILED. - mi->_class = vtable_class_id; - mi->_name = _symbols.lookup(""); - mi->_sig = _symbols.lookup("()V"); - mi->_type = FRAME_NATIVE; - mi->_is_entry = false; - } else { - fillJavaMethodInfo(mi, method, first_time); - } - } - - return mi; -} - -void Lookup::initClassCache() { - // Snapshot _classes into _class_cache for use by resolveMethod(BCI_ALLOC). - // Must be called before writeStackTraces() so the snapshot covers all - // vtable-receiver classes (pre-registered before profiling starts). - // This snapshot is intentionally NOT used by writeClasses(): regular Java - // classes are inserted into _classes by fillJavaMethodInfo() during - // writeStackTraces/writeMethods, so writeClasses() must re-collect after - // those passes to obtain the complete class pool. - // standby() is the post-rotate snapshot of _classes; collect() copies its - // entries with no concurrent writers (rotate drained them). The shared - // classMapSharedGuard is held for any concurrent #527 vtable readers that - // also touch _classes directly via lookup() on active. - auto guard = Profiler::instance()->classMapSharedGuard(); - _classes->standby()->collect(_class_cache); -} - -u32 Lookup::getPackage(const char *class_name) { - const char *package = strrchr(class_name, '/'); - if (package == NULL) { - return 0; - } - if (package[1] >= '0' && package[1] <= '9') { - // Seems like a hidden or anonymous class, e.g. com/example/Foo/0x012345 - do { - if (package == class_name) - return 0; - } while (*--package != '/'); - } - if (class_name[0] == '[') { - class_name = strchr(class_name, 'L') + 1; - } - return _packages.lookup(class_name, package - class_name); -} - -u32 Lookup::getSymbol(const char *name) { return _symbols.lookup(name); } - -char *Recording::_agent_properties = NULL; -char *Recording::_jvm_args = NULL; -char *Recording::_jvm_flags = NULL; -char *Recording::_java_command = NULL; - -Recording::Recording(int fd, Arguments &args) - : _fd(fd), _method_map() { - - args.save(_args); - _chunk_start = lseek(_fd, 0, SEEK_END); - _start_time = OS::micros(); - _start_ticks = TSC::ticks(); - _recording_start_time = _start_time; - _recording_start_ticks = _start_ticks; - _bytes_written = 0; - - _tid = OS::threadId(); - _active_index.store(0, std::memory_order_relaxed); - - VM::jvmti()->GetAvailableProcessors(&_available_processors); - - writeHeader(_buf); - writeMetadata(_buf); - writeSettings(_buf, args); - if (!args.hasOption(NO_SYSTEM_INFO)) { - writeOsCpuInfo(_buf); - writeJvmInfo(_buf); - } - if (!args.hasOption(NO_SYSTEM_PROPS)) { - writeSystemProperties(_buf); - } - if (!args.hasOption(NO_NATIVE_LIBS)) { - _recorded_lib_count = 0; - writeNativeLibraries(_buf); - } else { - _recorded_lib_count = -1; - } - flush(_buf); - - _cpu_monitor_enabled = !args.hasOption(NO_CPU_LOAD); - if (_cpu_monitor_enabled) { - _last_times.proc.real = - OS::getProcessCpuTime(&_last_times.proc.user, &_last_times.proc.system); - _last_times.total.real = - OS::getTotalCpuTime(&_last_times.total.user, &_last_times.total.system); - } -} - -Recording::~Recording() { - finishChunk(true); - close(_fd); -} - -void Recording::copyTo(int target_fd) { - OS::copyFile(_fd, target_fd, 0, finishChunk(true)); -} - -off_t Recording::finishChunk() { return finishChunk(false); } - -off_t Recording::finishChunk(bool end_recording, bool do_cleanup) { - jvmtiEnv *jvmti = VM::jvmti(); - JNIEnv *env = VM::jni(); - - jclass *classes; - jint count = 0; - // Pin all currently-loaded classes for the duration of finishChunk(). - // resolveMethod() calls GetLineNumberTable/GetClassSignature/GetMethodName on - // jmethodIDs of classes that were loaded when the sample was taken but could - // be unloaded concurrently by the GC before we flush. Holding a local JNI - // reference to each class makes it a GC root, closing that race window. - // Note: this only guards against concurrent unloading that starts AFTER this - // call. Classes already unloaded before finishChunk() was entered are not - // present in the list and receive no protection here. - jvmtiError err = jvmti->GetLoadedClasses(&count, &classes); - - flush(&_cpu_monitor_buf); - - writeNativeLibraries(_buf); - - const ObjectSampler *oSampler = ObjectSampler::instance(); - // write the engine dependent setting - if (oSampler->_record_allocations) { - writeIntSetting(_buf, T_ALLOC, "interval", oSampler->_interval); - } - if (oSampler->_record_liveness) { - writeIntSetting(_buf, T_HEAP_LIVE_OBJECT, "interval", oSampler->_interval); - writeIntSetting(_buf, T_HEAP_LIVE_OBJECT, "capacity", - LivenessTracker::instance()->_table_cap); - writeIntSetting(_buf, T_HEAP_LIVE_OBJECT, "maximum capacity", - LivenessTracker::instance()->_table_max_cap); - } - writeDatadogProfilerConfig( - _buf, Profiler::instance()->cpuEngine()->interval() / 1000000, - Profiler::instance()->wallEngine()->interval() / 1000000, - oSampler->_record_allocations ? oSampler->_interval : 0L, - oSampler->_record_liveness ? oSampler->_interval : 0L, - oSampler->_record_liveness ? LivenessTracker::instance()->_table_cap : 0L, - oSampler->_record_liveness ? LivenessTracker::instance()->_subsample_ratio - : 0.0, - oSampler->_gc_generations, Profiler::instance()->eventMask(), - Profiler::instance()->cpuEngine()->name()); - - _stop_time = OS::micros(); - _stop_ticks = TSC::ticks(); - - if (end_recording) { - writeRecordingInfo(_buf); - } - - // this will not report correct counts for any counters updated during writing - // the constant pool because it just isn't worth the complexity and cost of - // being able to account for the resources used in serialization during - // serialization. Some counters we verify to balance (e.g. the anonymous - // dictionaries) will be reported as positive, others (e.g. the classes - // dictionary) will reflect the previous serialization. That is, some level of - // familiarity with the code base will be required to use this diagnostic - // information for now. - writeCounters(_buf); - - // Keep a simple stats for where we failed to unwind - // For the sakes of simplicity we are not keeping the count of failed unwinds which would also be - // just 'eventually consistent' because we do not want to block the unwinding while writing out the stats. - writeUnwindFailures(_buf); - - for (int i = 0; i < CONCURRENCY_LEVEL; i++) { - flush(&_buf[i]); - } - - off_t cpool_offset = lseek(_fd, 0, SEEK_CUR); - int count_offset_in_cpool = 0; - int pool_count = writeCpool(_buf, &count_offset_in_cpool); - flush(_buf); - - off_t cpool_end = lseek(_fd, 0, SEEK_CUR); - - // Patch cpool size field - _buf->putVar32(0, cpool_end - cpool_offset); - ssize_t result = pwrite(_fd, _buf->data(), 5, cpool_offset); - (void)result; - - // Patch the constant pool count placeholder (written as a 1-byte put8 in - // writeCpool). Done flush-safe via pwrite to the FILE offset, mirroring the - // size patch above: _buf has been flushed/reset, so _buf->data() is scratch. - _buf->put8(0, (char)pool_count); - result = pwrite(_fd, _buf->data(), 1, cpool_offset + count_offset_in_cpool); - (void)result; - - off_t chunk_end = lseek(_fd, 0, SEEK_CUR); - - // // Workaround for JDK-8191415: compute actual TSC frequency, in case JFR is - // wrong - u64 tsc_frequency = TSC::frequency(); - // if (tsc_frequency > 1000000000) { - // tsc_frequency = (u64)(double(_stop_ticks - _start_ticks) / - // double(_stop_time - _start_time) * 1000000); - // } - - // Patch chunk header - _buf->put64(chunk_end - _chunk_start); - _buf->put64(cpool_offset - _chunk_start); - _buf->put64(68); - _buf->put64(_start_time * 1000); - _buf->put64((_stop_time - _start_time) * 1000); - _buf->put64(_start_ticks); - _buf->put64(tsc_frequency); - result = pwrite(_fd, _buf->data(), 56, _chunk_start + 8); - (void)result; - - OS::freePageCache(_fd, _chunk_start); - - _buf->reset(); - - // Run method_map cleanup while the class pins from GetLoadedClasses are still - // held. Line number tables are now malloc'd copies (fillJavaMethodInfo copies - // the JVMTI buffer and calls Deallocate() immediately), so ~SharedLineNumberTable() - // calls free() — safe regardless of class-unload state. Cleanup runs before - // DeleteLocalRef to ensure erased jmethodID keys have not yet been recycled by - // a newly-loaded class. - if (do_cleanup) { - cleanupUnreferencedMethods(); - } - - if (!err) { - // delete all local references - for (int i = 0; i < count; i++) { - env->DeleteLocalRef((jobject)classes[i]); - } - // deallocate the class array - jvmti->Deallocate((unsigned char *)classes); - } - return chunk_end; -} - -// Finish the current chunk, move it to the external file `fd` (must be a valid -// open descriptor), then restart the continuous recording file with a fresh -// chunk header. Callers guarantee fd > -1 (see FlightRecorder::dump). -void Recording::switchChunk(int fd) { - _chunk_start = finishChunk(/*end_recording=*/true, /*do_cleanup=*/true); - - TEST_LOG("MethodMap: %zu methods after cleanup", _method_map.size()); - - _start_time = _stop_time; - _start_ticks = _stop_ticks; - _bytes_written = 0; - - // move the chunk to the external file and reset the continuous recording file - OS::copyFile(_fd, fd, 0, _chunk_start); - OS::truncateFile(_fd); - _chunk_start = 0; - - // the recording file is restarted, so write out all the info events again - writeHeader(_buf); - writeMetadata(_buf); - writeSettings(_buf, _args); - if (!_args.hasOption(NO_SYSTEM_INFO)) { - writeOsCpuInfo(_buf); - writeJvmInfo(_buf); - } - if (!_args.hasOption(NO_SYSTEM_PROPS)) { - writeSystemProperties(_buf); - } - if (!_args.hasOption(NO_NATIVE_LIBS)) { - _recorded_lib_count = 0; - writeNativeLibraries(_buf); - } else { - _recorded_lib_count = -1; - } - flush(_buf); -} - -void Recording::cpuMonitorCycle() { - if (!_cpu_monitor_enabled) - return; - - CpuTimes times; - times.proc.real = OS::getProcessCpuTime(×.proc.user, ×.proc.system); - times.total.real = - OS::getTotalCpuTime(×.total.user, ×.total.system); - - float proc_user = 0, proc_system = 0, machine_total = 0; - - if (times.proc.real != (u64)-1 && times.proc.real > _last_times.proc.real) { - float delta = - (times.proc.real - _last_times.proc.real) * _available_processors; - proc_user = ratio((times.proc.user - _last_times.proc.user) / delta); - proc_system = ratio((times.proc.system - _last_times.proc.system) / delta); - } - - if (times.total.real != (u64)-1 && - times.total.real > _last_times.total.real) { - float delta = times.total.real - _last_times.total.real; - machine_total = - ratio(((times.total.user + times.total.system) - - (_last_times.total.user + _last_times.total.system)) / - delta); - if (machine_total < proc_user + proc_system) { - machine_total = ratio(proc_user + proc_system); - } - } - - recordCpuLoad(&_cpu_monitor_buf, proc_user, proc_system, machine_total); - flushIfNeeded(&_cpu_monitor_buf, BUFFER_LIMIT); - - _last_times = times; -} - -void Recording::cleanupUnreferencedMethods() { - if (!_args._enable_method_cleanup) { - return; // Feature disabled - } - - const int AGE_THRESHOLD = 3; // Remove after 3 consecutive chunks without reference - size_t removed_count = 0; - size_t removed_with_line_tables = 0; - [[maybe_unused]] size_t total_before = _method_map.size(); - - for (MethodMap::iterator it = _method_map.begin(); it != _method_map.end(); ) { - MethodInfo& mi = it->second; - - if (!mi._referenced) { - // Method not referenced in this chunk - mi._age++; - - if (mi._age >= AGE_THRESHOLD) { - // Method hasn't been used for N chunks, safe to remove - // SharedLineNumberTable will be automatically deallocated via shared_ptr destructor - bool has_line_table = (mi._line_number_table != nullptr && mi._line_number_table->_ptr != nullptr); - if (has_line_table) { - removed_with_line_tables++; - } - // Recycle the erased method's pool id so a later method can reuse it - // without colliding with any still-live method (PROF-15130). - _method_map.freeId(mi._key); - it = _method_map.erase(it); - removed_count++; - continue; - } - } else { - // Method was referenced, reset age - mi._age = 0; - } - - ++it; - } - - if (removed_count > 0) { - TEST_LOG("Cleaned up %zu unreferenced methods (age >= %d chunks, %zu with line tables, total: %zu -> %zu)", - removed_count, AGE_THRESHOLD, removed_with_line_tables, total_before, _method_map.size()); - - // Log current count of live line number tables - [[maybe_unused]] long long live_tables = Counters::getCounter(LINE_NUMBER_TABLES); - TEST_LOG("Live line number tables after cleanup: %lld", live_tables); - } -} - -void Recording::appendRecording(const char *target_file, size_t size) { - int append_fd = open(target_file, O_WRONLY); - if (append_fd >= 0) { - lseek(append_fd, 0, SEEK_END); - OS::copyFile(_fd, append_fd, 0, size); - close(append_fd); - } else { - Log::warn("Failed to open JFR recording at %s: %s", target_file, - strerror(errno)); - } -} - -RecordingBuffer *Recording::buffer(int lock_index) { return &_buf[lock_index]; } - -bool Recording::parseAgentProperties() { - JNIEnv *env = VM::jni(); - jclass vm_support = env->FindClass("jdk/internal/vm/VMSupport"); - if (vm_support == NULL) { - env->ExceptionClear(); - vm_support = env->FindClass("sun/misc/VMSupport"); - } - if (vm_support != NULL) { - jmethodID get_agent_props = env->GetStaticMethodID( - vm_support, "getAgentProperties", "()Ljava/util/Properties;"); - jmethodID to_string = env->GetMethodID(env->FindClass("java/lang/Object"), - "toString", "()Ljava/lang/String;"); - if (get_agent_props != NULL && to_string != NULL) { - jobject props = env->CallStaticObjectMethod(vm_support, get_agent_props); - jniExceptionCheck(env); - if (props != NULL && !env->ExceptionCheck()) { - jstring str = (jstring)env->CallObjectMethod(props, to_string); - jniExceptionCheck(env); - if (str != NULL && !env->ExceptionCheck()) { - _agent_properties = (char *)env->GetStringUTFChars(str, NULL); - } - } - } - } - env->ExceptionClear(); - - if (_agent_properties == NULL) { - return false; - } - - char *p = _agent_properties + 1; - p[strlen(p) - 1] = 0; - - while (*p) { - if (strncmp(p, "sun.jvm.args=", 13) == 0) { - _jvm_args = p + 13; - } else if (strncmp(p, "sun.jvm.flags=", 14) == 0) { - _jvm_flags = p + 14; - } else if (strncmp(p, "sun.java.command=", 17) == 0) { - _java_command = p + 17; - } - - if ((p = strstr(p, ", ")) == NULL) { - break; - } - - *p = 0; - p += 2; - } - - return true; -} - -void Recording::flush(Buffer *buf) { - ssize_t result = write(_fd, buf->data(), buf->offset()); - if (result > 0) { - atomicInc(_bytes_written, result); - } - buf->reset(); -} - -void Recording::flushIfNeeded(Buffer *buf, int limit) { - if (buf->offset() >= limit) { - flush(buf); - } -} - -void Recording::writeMetadata(Buffer *buf) { - int metadata_start = buf->skip(5); // size will be patched later - buf->putVar64(T_METADATA); - buf->putVar64(_start_ticks); - buf->put8(0); - buf->put8(1); - - std::vector &strings = JfrMetadata::strings(); - buf->putVar64(strings.size()); - for (size_t i = 0; i < strings.size(); i++) { - const char *string = strings[i].c_str(); - int length = strlen(string); - flushIfNeeded(buf, RECORDING_BUFFER_LIMIT - length); - buf->putUtf8(string, length); - } - - writeElement(buf, JfrMetadata::root()); - - buf->putVar32(metadata_start, buf->offset() - metadata_start); - flushIfNeeded(buf); -} - -void Recording::writeHeader(Buffer *buf) { - buf->put("FLR\0", 4); // magic - buf->put16(2); // major - buf->put16(0); // minor - buf->put64( - 1024 * 1024 * - 1024); // chunk size (initially large, for JMC to skip incomplete chunk) - buf->put64(0); // cpool offset - buf->put64(0); // meta offset - buf->put64(_start_time * 1000); // start time, ns - buf->put64(0); // duration, ns - buf->put64(_start_ticks); // start ticks - buf->put64(TSC::frequency()); // ticks per sec - buf->put32(1); // features - flushIfNeeded(buf); -} - -void Recording::writeElement(Buffer *buf, const Element *e) { - buf->putVar64(e->_name); - - buf->putVar64(e->_attributes.size()); - for (size_t i = 0; i < e->_attributes.size(); i++) { - flushIfNeeded(buf); - buf->putVar64(e->_attributes[i]._key); - buf->putVar64(e->_attributes[i]._value); - } - - buf->putVar64(e->_children.size()); - for (size_t i = 0; i < e->_children.size(); i++) { - flushIfNeeded(buf); - writeElement(buf, e->_children[i]); - } - flushIfNeeded(buf); -} - -void Recording::writeRecordingInfo(Buffer *buf) { - int start = buf->skip(5); - buf->putVar64(T_ACTIVE_RECORDING); - buf->putVar64(_recording_start_ticks); - buf->putVar64(_stop_ticks - _recording_start_ticks); - buf->putVar64(_tid); - buf->put8(0); - buf->put8(1); - buf->putUtf8("java-profiler " PROFILER_VERSION); - buf->putUtf8("java-profiler.jfr"); - buf->putVar64(MAX_JLONG); - if (VM::hotspot_version() >= 14) { - buf->put8(0); - } - buf->put8(0); - buf->putVar64(_recording_start_time / 1000); - buf->putVar64((_stop_time - _recording_start_time) / 1000); - buf->putVar32(start, buf->offset() - start); - flushIfNeeded(buf); -} - -void Recording::writeSettings(Buffer *buf, Arguments &args) { - writeBoolSetting(buf, T_ACTIVE_RECORDING, "asyncprofiler", true); - writeStringSetting(buf, T_ACTIVE_RECORDING, "version", PROFILER_VERSION); - writeIntSetting(buf, T_ACTIVE_RECORDING, "tscfrequency", TSC::frequency()); - writeStringSetting(buf, T_ACTIVE_RECORDING, "loglevel", - Log::LEVEL_NAME[Log::level()]); - writeBoolSetting(buf, T_ACTIVE_RECORDING, "hotspot", VM::isHotspot()); - writeBoolSetting(buf, T_ACTIVE_RECORDING, "openj9", VM::isOpenJ9()); - for (auto attribute : args._context_attributes) { - writeStringSetting(buf, T_ACTIVE_RECORDING, "contextattribute", - attribute.c_str()); - } - - if (!((args._event != NULL && strcmp(args._event, EVENT_NOOP) != 0) || - args._cpu >= 0)) { - writeBoolSetting(buf, T_EXECUTION_SAMPLE, "enabled", false); - } else { - writeBoolSetting(buf, T_EXECUTION_SAMPLE, "enabled", true); - writeIntSetting(buf, T_EXECUTION_SAMPLE, "interval", - args.cpuSamplerInterval()); - } - writeBoolSetting(buf, T_METHOD_SAMPLE, "enabled", args._wall >= 0); - if (args._wall >= 0) { - writeIntSetting(buf, T_METHOD_SAMPLE, "interval", - args._wall ? args._wall : DEFAULT_WALL_INTERVAL); - } - - writeBoolSetting(buf, T_ALLOC, "enabled", args._record_allocations); - writeBoolSetting(buf, T_HEAP_LIVE_OBJECT, "enabled", args._record_liveness); - writeBoolSetting(buf, T_MALLOC, "enabled", args._nativemem >= 0); - if (args._nativemem >= 0) { - writeIntSetting(buf, T_MALLOC, "nativemem", args._nativemem); - // samplingInterval=-1 signals "record every allocation"; mirrors shouldSample's interval<=1 threshold. - writeIntSetting(buf, T_MALLOC, "samplingInterval", args._nativemem <= 1 ? -1 : args._nativemem); - } - - writeBoolSetting(buf, T_ACTIVE_RECORDING, "debugSymbols", - VMStructs::libjvm()->hasDebugSymbols()); - writeBoolSetting(buf, T_ACTIVE_RECORDING, "kernelSymbols", - Symbols::haveKernelSymbols()); - writeStringSetting(buf, T_ACTIVE_RECORDING, "cpuEngine", - Profiler::instance()->cpuEngine()->name()); - writeStringSetting(buf, T_ACTIVE_RECORDING, "wallEngine", - Profiler::instance()->wallEngine()->name()); - writeStringSetting(buf, T_ACTIVE_RECORDING, "cstack", - Profiler::instance()->cstack()); - flushIfNeeded(buf); -} - -void Recording::writeStringSetting(Buffer *buf, int category, const char *key, - const char *value) { - flushIfNeeded(buf, RECORDING_BUFFER_LIMIT - - (2 * MAX_STRING_LENGTH + MAX_JFR_EVENT_SIZE)); - int start = buf->skip(5); - buf->putVar64(T_ACTIVE_SETTING); - buf->putVar64(_start_ticks); - buf->put8(0); - buf->putVar64(_tid); - buf->put8(0); - buf->putVar64(category); - buf->putUtf8(key); - buf->putUtf8(value); - buf->putVar32(start, buf->offset() - start); - flushIfNeeded(buf); -} - -void Recording::writeBoolSetting(Buffer *buf, int category, const char *key, - bool value) { - writeStringSetting(buf, category, key, value ? "true" : "false"); -} - -void Recording::writeIntSetting(Buffer *buf, int category, const char *key, - long long value) { - char str[32]; - snprintf(str, sizeof(str), "%lld", value); - writeStringSetting(buf, category, key, str); -} - -void Recording::writeListSetting(Buffer *buf, int category, const char *key, - const char *base, int offset) { - while (offset != 0) { - writeStringSetting(buf, category, key, base + offset); - offset = ((int *)(base + offset))[-1]; - } - flushIfNeeded(buf); -} - -void Recording::writeDatadogSetting(Buffer *buf, int length, const char *name, - const char *value, const char *unit) { - flushIfNeeded(buf, RECORDING_BUFFER_LIMIT - length); - int start = buf->skip(MAX_VAR32_LENGTH); - buf->putVar64(T_DATADOG_SETTING); - buf->putVar64(_start_ticks); - buf->put8(0); // no duration, but required for compatibility with equivalent - // Java event - buf->putVar32(_tid); - buf->put8(0); // no stacktrace, but required for compatibility with equivalent - // Java event - buf->putUtf8(name); - buf->putUtf8(value); - buf->putUtf8(unit); - buf->putVar32(start, buf->offset() - start); - flushIfNeeded(buf); -} - -void Recording::writeDatadogProfilerConfig( - Buffer *buf, long cpuInterval, long wallInterval, long allocInterval, - long memleakInterval, long memleakCapacity, double memleakRatio, - bool gcGenerations, int modeMask, const char *cpuEngine) { - flushIfNeeded(buf, RECORDING_BUFFER_LIMIT - - (1 + 6 * MAX_VAR64_LENGTH + MAX_VAR32_LENGTH + - 3 * MAX_STRING_LENGTH)); - int start = buf->skip(1); - buf->putVar64(T_DATADOG_PROFILER_CONFIG); - buf->putVar64(_start_ticks); - buf->put8(0); - buf->putVar64(_tid); - buf->putVar64(cpuInterval); - buf->putVar64(wallInterval); - buf->putVar64(allocInterval); - buf->putVar64(memleakInterval); - buf->putVar64(memleakCapacity); - buf->put8(static_cast(memleakRatio * 100)); - buf->put8(gcGenerations); - buf->putVar32(modeMask); - buf->putUtf8(PROFILER_VERSION); - buf->putUtf8(cpuEngine); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::writeHeapUsage(Buffer *buf, long value, bool live) { - int start = buf->skip(1); - buf->putVar64(T_HEAP_USAGE); - buf->putVar64(TSC::ticks()); - buf->putVar64(value); - buf->put8(live); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::writeOsCpuInfo(Buffer *buf) { - struct utsname u; - if (uname(&u) != 0) { - return; - } - - char str[512]; - snprintf(str, sizeof(str) - 1, "uname: %s %s %s %s", u.sysname, u.release, - u.version, u.machine); - str[sizeof(str) - 1] = 0; - - flushIfNeeded(buf, - RECORDING_BUFFER_LIMIT - (2 * strlen(str) + strlen(u.machine))); - - int start = buf->skip(5); - buf->putVar64(T_OS_INFORMATION); - buf->putVar64(_start_ticks); - buf->putUtf8(str); - buf->putVar32(start, buf->offset() - start); - - start = buf->skip(5); - buf->putVar64(T_CPU_INFORMATION); - buf->putVar64(_start_ticks); - buf->putUtf8(u.machine); - buf->putUtf8(OS::getCpuDescription(str, sizeof(str) - 1) ? str : ""); - buf->put8(1); - buf->putVar64(_available_processors); - buf->putVar64(_available_processors); - buf->putVar32(start, buf->offset() - start); - flushIfNeeded(buf); -} - -void Recording::writeJvmInfo(Buffer *buf) { - if (_agent_properties == NULL && !parseAgentProperties()) { - return; - } - - char *jvm_name = NULL; - char *jvm_version = NULL; - - jvmtiEnv *jvmti = VM::jvmti(); - jvmti->GetSystemProperty("java.vm.name", &jvm_name); - jvmti->GetSystemProperty("java.vm.version", &jvm_version); - - flushIfNeeded(buf, RECORDING_BUFFER_LIMIT - 5 * MAX_STRING_LENGTH); - int start = buf->skip(5); - buf->putVar64(T_JVM_INFORMATION); - buf->putVar64(_start_ticks); - buf->putUtf8(jvm_name); - buf->putUtf8(jvm_version); - buf->putUtf8(_jvm_args != nullptr ? _jvm_args : ""); - buf->putUtf8(_jvm_flags != nullptr ? _jvm_flags : ""); - buf->putUtf8(_java_command != nullptr ? _java_command : ""); - buf->putVar64(OS::processStartTime()); - buf->putVar64(OS::processId()); - buf->putVar32(start, buf->offset() - start); - flushIfNeeded(buf); - - jvmti->Deallocate((unsigned char *)jvm_version); - jvmti->Deallocate((unsigned char *)jvm_name); -} - -void Recording::writeSystemProperties(Buffer *buf) { - jvmtiEnv *jvmti = VM::jvmti(); - jint count; - char **keys; - if (jvmti->GetSystemProperties(&count, &keys) != 0) { - return; - } - - for (int i = 0; i < count; i++) { - char *key = keys[i]; - char *value = NULL; - if (jvmti->GetSystemProperty(key, &value) == 0) { - flushIfNeeded(buf, RECORDING_BUFFER_LIMIT - 2 * MAX_STRING_LENGTH); - int start = buf->skip(5); - buf->putVar64(T_INITIAL_SYSTEM_PROPERTY); - buf->putVar64(_start_ticks); - buf->putUtf8(key); - buf->putUtf8(value); - buf->putVar32(start, buf->offset() - start); - jvmti->Deallocate((unsigned char *)value); - } - jvmti->Deallocate((unsigned char *)key); - } - flushIfNeeded(buf); - - jvmti->Deallocate((unsigned char *)keys); -} - -void Recording::writeNativeLibraries(Buffer *buf) { - if (_recorded_lib_count < 0) - return; - - Libraries *libraries = Libraries::instance(); - const CodeCacheArray &native_libs = libraries->native_libs(); - int native_lib_count = native_libs.count(); - - // Emit jdk.NativeLibrary events for newly loaded libraries. - // CodeCacheArray::add() stores the pointer before advancing count(), - // so all indices < native_lib_count are guaranteed non-NULL. - for (int i = _recorded_lib_count; i < native_lib_count; i++) { - CodeCache* lib = native_libs[i]; - - flushIfNeeded(buf, RECORDING_BUFFER_LIMIT - MAX_STRING_LENGTH); - int start = buf->skip(5); - buf->putVar64(T_NATIVE_LIBRARY); - buf->putVar64(_start_ticks); - buf->putUtf8(lib->name()); - buf->putVar64((uintptr_t)lib->minAddress()); - buf->putVar64((uintptr_t)lib->maxAddress()); - buf->putUtf8(lib->hasBuildId() ? lib->buildId() : ""); - buf->putVar64((uintptr_t)lib->loadBias()); - buf->putVar32(start, buf->offset() - start); - flushIfNeeded(buf); - } - - _recorded_lib_count = native_lib_count; -} - -int Recording::writeCpool(Buffer *buf, int *count_offset_in_cpool) { - // Offset of the cpool start within the buffer. The header below is tiny and - // flush-free, so the placeholder offset captured relative to this start is a - // stable cpool-relative offset usable for a flush-safe back-patch by the - // caller (mirrors the cpool SIZE patch). - int cpool_start = buf->offset(); - buf->skip(5); // size will be patched later - buf->putVar64(T_CPOOL); - buf->putVar64(_start_ticks); - buf->put8(0); - buf->put8(0); - buf->put8(1); - // Constant pool count. We cannot precompute it: the Method/Class/Package/Symbol - // pools are only fully populated as a side effect of writeStackTraces/writeMethods - // (fillJavaMethodInfo), and empty variable pools are skipped entirely. Write a - // 1-byte placeholder here and back-patch it flush-safe in the caller. - *count_offset_in_cpool = buf->offset() - cpool_start; - buf->put8(0); - - // Profiler::rotateDictsAndRun() rotates the three dictionaries before this - // path runs, so classMap()->standby() returns an old-active snapshot stable - // for the lifetime of writeCpool(). - // initClassCache() seeds vtable-receiver class names for resolveMethod(BCI_ALLOC). - // writeClasses() then collects the COMPLETE class set from standby(): regular Java - // classes are inserted into the new-active by fillJavaMethodInfo during - // writeStackTraces/writeMethods, and those would not appear in the snapshot — - // standby() captures the pre-rotation state which writeClasses extends. - Lookup lookup(this, &_method_map, Profiler::instance()->classMap()); - lookup.initClassCache(); - // CONSTANT pools: always non-empty, always emitted -> 5 sections. - // writeThreads always emits: it inserts _tid unconditionally before checking. - writeFrameTypes(buf); - writeThreadStates(buf); - writeExecutionModes(buf); - writeLogLevels(buf); - writeThreads(buf); - int pool_count = 5; - // VARIABLE pools: each returns 1 if emitted, 0 if empty (and thus skipped). - pool_count += writeStackTraces(buf, &lookup); - pool_count += writeMethods(buf, &lookup); - pool_count += writeClasses(buf, &lookup); - pool_count += writePackages(buf, &lookup); - pool_count += writeConstantPoolSection(buf, T_SYMBOL, &lookup._symbols); - pool_count += writeConstantPoolSection( - buf, T_STRING, Profiler::instance()->stringLabelMap()->standby()); - pool_count += writeConstantPoolSection( - buf, T_ATTRIBUTE_VALUE, Profiler::instance()->contextValueMap()->standby()); - flushIfNeeded(buf); - return pool_count; -} - -void Recording::writeFrameTypes(Buffer *buf) { - buf->putVar32(T_FRAME_TYPE); - buf->putVar32(7); - buf->putVar32(FRAME_INTERPRETED); - buf->putUtf8("Interpreted"); - buf->putVar32(FRAME_JIT_COMPILED); - buf->putUtf8("JIT compiled"); - buf->putVar32(FRAME_INLINED); - buf->putUtf8("Inlined"); - buf->putVar32(FRAME_NATIVE); - buf->putUtf8("Native"); - buf->putVar32(FRAME_CPP); - buf->putUtf8("C++"); - buf->putVar32(FRAME_KERNEL); - buf->putUtf8("Kernel"); - buf->putVar32(FRAME_C1_COMPILED); - buf->putUtf8("C1 compiled"); - flushIfNeeded(buf); -} - -void Recording::writeThreadStates(Buffer *buf) { - buf->putVar64(T_THREAD_STATE); - buf->put8(10); - buf->put8(static_cast(OSThreadState::UNKNOWN)); - buf->putUtf8("UNKNOWN"); - buf->put8(static_cast(OSThreadState::NEW)); - buf->putUtf8("NEW"); - buf->put8(static_cast(OSThreadState::RUNNABLE)); - buf->putUtf8("RUNNABLE"); - buf->put8(static_cast(OSThreadState::MONITOR_WAIT)); - buf->putUtf8("CONTENDED"); - buf->put8(static_cast(OSThreadState::CONDVAR_WAIT)); - buf->putUtf8("PARKED"); - buf->put8(static_cast(OSThreadState::OBJECT_WAIT)); - buf->putUtf8("WAITING"); - buf->put8(static_cast(OSThreadState::BREAKPOINTED)); - buf->putUtf8("BREAKPOINT"); - buf->put8(static_cast(OSThreadState::SLEEPING)); - buf->putUtf8("SLEEPING"); - buf->put8(static_cast(OSThreadState::TERMINATED)); - buf->putUtf8("TERMINATED"); - buf->put8(static_cast(OSThreadState::SYSCALL)); - buf->putUtf8("SYSCALL"); - flushIfNeeded(buf); -} - -void Recording::writeExecutionModes(Buffer *buf) { - buf->putVar64(T_EXECUTION_MODE); - buf->put8(6); - buf->put8(static_cast(ExecutionMode::UNKNOWN)); - buf->putUtf8("UNKNOWN"); - buf->put8(static_cast(ExecutionMode::JAVA)); - buf->putUtf8("JAVA"); - buf->put8(static_cast(ExecutionMode::JVM)); - buf->putUtf8("JVM"); - buf->put8(static_cast(ExecutionMode::NATIVE)); - buf->putUtf8("NATIVE"); - buf->put8(static_cast(ExecutionMode::SAFEPOINT)); - buf->putUtf8("SAFEPOINT"); - buf->put8(static_cast(ExecutionMode::SYSCALL)); - buf->putUtf8("SYSCALL"); - flushIfNeeded(buf); -} - -void Recording::writeThreads(Buffer *buf) { - int old_index = _active_index.fetch_xor(1, std::memory_order_acq_rel); - // After flip: new samples go into the new active set - // We flush from old_index (the previous active set) - - std::unordered_set threads; - threads.insert(_tid); // always present: the recording thread itself - - for (int i = 0; i < CONCURRENCY_LEVEL; ++i) { - // Collect thread IDs from the fixed-size table into the main set - _thread_ids[i][old_index].collect(threads); - _thread_ids[i][old_index].clear(); - } - - Profiler *profiler = Profiler::instance(); - ThreadInfo *t_info = &profiler->_thread_info; - - char name_buf[32]; - - buf->putVar64(T_THREAD); - buf->putVar64(threads.size()); - for (auto tid : threads) { - const char *thread_name; - jlong thread_id; - std::pair, u64> info = t_info->get(tid); - if (info.first) { - thread_name = info.first->c_str(); - thread_id = info.second; - } else { - snprintf(name_buf, sizeof(name_buf), "[tid=%d]", tid); - thread_name = name_buf; - thread_id = 0; - } - - int length = strlen(thread_name); - int required = RECORDING_BUFFER_LIMIT - - (thread_id == 0 ? length + 1 : 2 * length) - - 3 * 10; // 3x max varint length - flushIfNeeded(buf, required); - buf->putVar64(tid); - buf->putUtf8(thread_name, length); - buf->putVar64(tid); - if (thread_id == 0) { - buf->put8(0); - } else { - buf->putUtf8(thread_name, length); - } - buf->putVar64(thread_id); - flushIfNeeded(buf); - } -} - -int Recording::writeStackTraces(Buffer *buf, Lookup *lookup) { - // Reset all referenced flags before processing - for (MethodMap::iterator it = _method_map.begin(); it != _method_map.end(); ++it) { - it->second._referenced = false; - } - - // Tracks how many traces were written so the empty pool can be skipped. - // Note: even with zero traces, the methods marking pass below must still run - // via processCallTraces, but no T_STACK_TRACE section is emitted in that case. - int trace_count = 0; - // Use safe trace processing with guaranteed lifetime during callback execution - Profiler::instance()->processCallTraces([this, buf, lookup, &trace_count](const std::unordered_set& traces) { - if (traces.empty()) { - return; - } - trace_count = (int)traces.size(); - buf->putVar64(T_STACK_TRACE); - buf->putVar64(traces.size()); - for (std::unordered_set::const_iterator it = traces.begin(); - it != traces.end(); ++it) { - CallTrace *trace = *it; - buf->putVar64(trace->trace_id); - if (trace->num_frames > 0) { - MethodInfo *mi = - lookup->resolveMethod(trace->frames[trace->num_frames - 1]); - mi->_referenced = true; // Mark method as referenced - if (mi->_type < FRAME_NATIVE) { - buf->put8(mi->_is_entry ? 0 : 1); - } else { - buf->put8(trace->truncated); - } - } - buf->putVar64(trace->num_frames); - for (int i = 0; i < trace->num_frames; i++) { - MethodInfo *mi = lookup->resolveMethod(trace->frames[i]); - mi->_referenced = true; // Mark method as referenced - buf->putVar64(mi->_key); - jint bci = trace->frames[i].bci; - if (mi->_type < FRAME_NATIVE) { - FrameTypeId type = FrameType::decode(bci); - bci = (bci & 0x10000) ? 0 : (bci & 0xffff); - buf->putVar32(mi->getLineNumber(bci)); - buf->putVar32(bci); - buf->put8(type); - } else { - buf->putVar32(0); - buf->putVar32(bci); - buf->put8(mi->_type); - } - flushIfNeeded(buf); - } - flushIfNeeded(buf); - } - }); // End of processCallTraces lambda - return trace_count > 0 ? 1 : 0; -} - -int Recording::writeMethods(Buffer *buf, Lookup *lookup) { - MethodMap *method_map = lookup->_method_map; - - u32 marked_count = 0; - for (MethodMap::const_iterator it = method_map->begin(); - it != method_map->end(); ++it) { - if (it->second._mark) { - marked_count++; - } - } - - if (marked_count == 0) { - return 0; - } - - buf->putVar64(T_METHOD); - buf->putVar64(marked_count); - for (MethodMap::iterator it = method_map->begin(); it != method_map->end(); - ++it) { - MethodInfo &mi = it->second; - if (mi._mark) { - mi._mark = false; - buf->putVar64(mi._key); - buf->putVar64(mi._class); - buf->putVar64(mi._name); - buf->putVar64(mi._sig); - buf->putVar64(mi._modifiers); - buf->putVar64(mi.isHidden()); - flushIfNeeded(buf); - } - } - return 1; -} - -int Recording::writeClasses(Buffer *buf, Lookup *lookup) { - DEBUG_ASSERT_NOT_IN_SIGNAL(); - std::map classes; - // standby() returns the dump buffer — the stable snapshot captured by - // rotate() for this recording cycle. No other thread writes to this - // buffer after rotate() completes: rotate() drained all in-flight - // cross-thread writers via waitForRefCountToClear() before returning. - lookup->_classes->standby()->collect(classes); - - if (classes.empty()) { - return 0; - } - - buf->putVar64(T_CLASS); - buf->putVar64(classes.size()); - for (std::map::const_iterator it = classes.begin(); - it != classes.end(); ++it) { - const char *name = it->second; - buf->putVar64(it->first); - buf->putVar64(0); // classLoader - buf->putVar64(lookup->getSymbol(name)); - buf->putVar64(lookup->getPackage(name)); - buf->putVar64(0); // access flags - flushIfNeeded(buf); - } - return 1; -} - -int Recording::writePackages(Buffer *buf, Lookup *lookup) { - std::map packages; - lookup->_packages.collect(packages); - - if (packages.empty()) { - return 0; - } - - buf->putVar32(T_PACKAGE); - buf->putVar32(packages.size()); - for (std::map::const_iterator it = packages.begin(); - it != packages.end(); ++it) { - buf->putVar64(it->first); - buf->putVar64(lookup->getSymbol(it->second)); - flushIfNeeded(buf); - } - return 1; -} - -int Recording::writeConstantPoolSection( - Buffer *buf, JfrType type, std::map &constants) { - if (constants.empty()) { - return 0; - } - flushIfNeeded(buf); - buf->putVar64(type); - buf->putVar64(constants.size()); - for (std::map::const_iterator it = constants.begin(); - it != constants.end(); ++it) { - int length = strlen(it->second); - // 5 is max varint length - flushIfNeeded(buf, RECORDING_BUFFER_LIMIT - length - 5); - buf->putVar64(it->first); - buf->putUtf8(it->second, length); - } - return 1; -} - -int Recording::writeConstantPoolSection(Buffer *buf, JfrType type, - Dictionary *dictionary) { - std::map constants; - dictionary->collect(constants); - return writeConstantPoolSection(buf, type, constants); -} - -int Recording::writeConstantPoolSection(Buffer *buf, JfrType type, - StringDictionaryBuffer *buffer) { - std::map constants; - buffer->collect(constants); - return writeConstantPoolSection(buf, type, constants); -} - -void Recording::writeLogLevels(Buffer *buf) { - buf->putVar64(T_LOG_LEVEL); - buf->putVar64(LOG_ERROR - LOG_TRACE + 1); - for (int i = LOG_TRACE; i <= LOG_ERROR; i++) { - buf->putVar32(i); - buf->putUtf8(Log::LEVEL_NAME[i]); - flushIfNeeded(buf); - } -} - -void Recording::writeCounters(Buffer *buf) { - long long *counters = Counters::getCounters(); - if (counters) { - std::vector names = Counters::describeCounters(); - for (size_t i = 0; i < names.size(); i++) { - int start = buf->skip(1); - buf->putVar64(T_DATADOG_COUNTER); - buf->putVar64(_start_ticks); - buf->putUtf8(names[i]); - buf->putVar64(counters[Counters::address(i)]); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); - } - } -} - -void Recording::writeUnwindFailures(Buffer *buf) { - static UnwindFailures failures; - UnwindStats::collectAndReset(failures); - - failures.forEach([&](UnwindFailureKind kind, const char *name, u64 count) { - int start = buf->skip(1); - buf->putVar64(T_UNWIND_FAILURE); - buf->putVar64(_start_ticks); - buf->putUtf8((kind & UNWIND_FAILURE_STUB) ? "stub" : "other"); - buf->putUtf8(name); - buf->putVar64(count); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); - }); -} - -void Recording::writeContextSnapshot(Buffer *buf, Context &context) { - buf->putVar64(context.spanId); - buf->putVar64(context.rootSpanId); - - for (size_t i = 0; i < Profiler::instance()->numContextAttributes(); i++) { - buf->putVar32(context.get_tag(i).value); - } -} - -void Recording::writeCurrentContext(Buffer *buf) { - u64 spanId = 0; - u64 rootSpanId = 0; - bool hasContext = ContextApi::get(spanId, rootSpanId); - // spanId/rootSpanId are initialized to 0 above; ContextApi::get() only updates them - // on success, so 0s are always written when there is no valid context. - buf->putVar64(spanId); - buf->putVar64(rootSpanId); - - size_t numAttrs = Profiler::instance()->numContextAttributes(); - ProfiledThread* thrd = hasContext ? ProfiledThread::currentSignalSafe() : nullptr; - for (size_t i = 0; i < numAttrs; i++) { - buf->putVar32(thrd != nullptr ? thrd->getOtelTagEncoding(i) : 0); - } -} - -void Recording::writeEventSizePrefix(Buffer *buf, int start) { - int size = buf->offset() - start; - assert(size < MAX_JFR_EVENT_SIZE); - buf->put8(start, size); -} - -void Recording::recordExecutionSample(Buffer *buf, int tid, u64 call_trace_id, - u64 correlation_id, - ExecutionEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_EXECUTION_SAMPLE); - buf->putVar64(TSC::ticks()); - buf->putVar64(tid); - buf->putVar64(call_trace_id); - buf->put8(static_cast(event->_thread_state)); - buf->put8(static_cast(event->_execution_mode)); - buf->putVar64(event->_weight); - buf->putVar64(correlation_id); - writeCurrentContext(buf); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordMethodSample(Buffer *buf, int tid, u64 call_trace_id, - u64 correlation_id, - ExecutionEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_METHOD_SAMPLE); - buf->putVar64(TSC::ticks()); - buf->putVar64(tid); - buf->putVar64(call_trace_id); - buf->put8(static_cast(event->_thread_state)); - buf->put8(static_cast(event->_execution_mode)); - buf->putVar64(event->_weight); - buf->putVar64(correlation_id); - writeCurrentContext(buf); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordWallClockEpoch(Buffer *buf, WallClockEpochEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_WALLCLOCK_SAMPLE_EPOCH); - buf->putVar64(event->_start_time); - buf->putVar64(event->_duration_millis); - buf->putVar64(event->_num_samplable_threads); - buf->putVar64(event->_num_successful_samples); - buf->putVar64(event->_num_failed_samples); - buf->putVar64(event->_num_exited_threads); - buf->putVar64(event->_num_permission_denied); - buf->putVar64(event->_num_suppressed_sampled_run); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordTraceRoot(Buffer *buf, int tid, TraceRootEvent *event) { - flushIfNeeded(buf); - int start = buf->skip(1); - buf->putVar64(T_ENDPOINT); - buf->putVar64(TSC::ticks()); - buf->put8(0); - buf->putVar32(tid); - buf->put8(0); - buf->putVar32(event->_label); - buf->putVar32(event->_operation); - buf->putVar64(event->_local_root_span_id); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordQueueTime(Buffer *buf, int tid, QueueTimeEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_QUEUE_TIME); - buf->putVar64(event->_start); - buf->putVar64(event->_end - event->_start); - buf->putVar64(tid); - buf->putVar64(event->_origin); - buf->putVar64(event->_task); - buf->putVar64(event->_scheduler); - buf->putVar64(event->_queueType); - buf->putVar64(event->_queueLength); - writeCurrentContext(buf); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordAllocation(RecordingBuffer *buf, int tid, - u64 call_trace_id, AllocEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_ALLOC); - buf->putVar64(TSC::ticks()); - buf->putVar64(tid); - buf->putVar64(call_trace_id); - buf->putVar64(event->_id); - buf->putVar64(event->_size); - buf->putFloat(event->_weight); - writeCurrentContext(buf); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordMallocSample(Buffer *buf, int tid, u64 call_trace_id, - MallocEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_MALLOC); - buf->putVar64(event->_start_time); - buf->putVar64(tid); - buf->putVar64(call_trace_id); - buf->putVar64(event->_address); - buf->putVar64(event->_size); - buf->putFloat(event->_weight); - writeCurrentContext(buf); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordNativeSocketSample(Buffer *buf, int tid, u64 call_trace_id, - NativeSocketEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_NATIVE_SOCKET); - buf->putVar64(event->_start_time); - buf->putVar64(tid); - buf->putVar64(call_trace_id); - buf->putVar64(safeDuration(event->_start_time, event->_end_time)); - static const char* const kOpNames[] = {"SEND", "RECV", "WRITE", "READ"}; - buf->putUtf8(event->_operation < 4 ? kOpNames[event->_operation] : "UNKNOWN"); - buf->putUtf8(event->_remote_addr); - buf->putVar64(event->_bytes); - buf->putFloat(event->_weight); - writeCurrentContext(buf); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordHeapLiveObject(Buffer *buf, int tid, u64 call_trace_id, - ObjectLivenessEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_HEAP_LIVE_OBJECT); - buf->putVar64(event->_start_time); - buf->putVar32(tid); - buf->putVar64(call_trace_id); - buf->putVar32(event->_id); - buf->putVar64(event->_age); - buf->putVar64(event->_alloc._size); - // the _alloc._size is 0 only when running in the lightweight mode, only - // tracking surviving generations - buf->putFloat( - event->_alloc._size > 0 - ? ((event->_alloc._weight * event->_alloc._size) + event->_skipped) / - event->_alloc._size - : 0); - writeContextSnapshot(buf, event->_ctx); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordMonitorBlocked(Buffer *buf, int tid, u64 call_trace_id, - LockEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_MONITOR_ENTER); - buf->putVar64(event->_start_time); - buf->putVar64(safeDuration(event->_start_time, event->_end_time)); - buf->putVar64(tid); - buf->putVar64(call_trace_id); - buf->putVar64(event->_id); - buf->put8(0); - buf->putVar64(event->_address); - writeCurrentContext(buf); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordThreadPark(Buffer *buf, int tid, u64 call_trace_id, - LockEvent *event) { - int start = buf->skip(1); - buf->putVar64(T_THREAD_PARK); - buf->putVar64(event->_start_time); - buf->putVar64(safeDuration(event->_start_time, event->_end_time)); - buf->putVar64(tid); - buf->putVar64(call_trace_id); - buf->putVar64(event->_id); - buf->putVar64(event->_timeout); - buf->putVar64(MIN_JLONG); - buf->putVar64(event->_address); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -void Recording::recordCpuLoad(Buffer *buf, float proc_user, float proc_system, - float machine_total) { - int start = buf->skip(1); - buf->putVar64(T_CPU_LOAD); - buf->putVar64(TSC::ticks()); - buf->putFloat(proc_user); - buf->putFloat(proc_system); - buf->putFloat(machine_total); - writeEventSizePrefix(buf, start); - flushIfNeeded(buf); -} - -// assumption is that we hold the lock (with lock_index) -void Recording::addThread(int lock_index, int tid) { - int active = _active_index.load(std::memory_order_acquire); - _thread_ids[lock_index][active].insert(tid); // ThreadIdTable::insert is signal-safe (atomics only) -} - -Error FlightRecorder::start(Arguments &args, bool reset) { - DEBUG_ASSERT_NOT_IN_SIGNAL(); - ExclusiveLockGuard locker(&_rec_lock); - const char *file = args.file(); - if (file == NULL || file[0] == 0) { - _filename = ""; - return Error("Flight Recorder output file is not specified"); - } - _filename = file; - _args = args; - - TSC::enable(args._clock); - - Error ret = newRecording(reset); - return ret; -} - -Error FlightRecorder::newRecording(bool reset) { - int fd = - open(_filename.c_str(), O_CREAT | O_RDWR | (reset ? O_TRUNC : 0), 0644); - if (fd == -1) { - return Error("Could not open Flight Recorder output file"); - } - - _rec = new Recording(fd, _args); - return Error::OK; -} - -void FlightRecorder::stop() { - DEBUG_ASSERT_NOT_IN_SIGNAL(); - ExclusiveLockGuard locker(&_rec_lock); - Recording* rec = _rec; - if (rec != nullptr) { - // NULL first, deallocate later - _rec = nullptr; - delete rec; - } -} - -Error FlightRecorder::dump(const char *filename, const int length) { - DEBUG_ASSERT_NOT_IN_SIGNAL(); - assert(length >= 0); - ExclusiveLockGuard locker(&_rec_lock); - Recording* rec = _rec; - if (rec != nullptr) { - if (_filename.length() != static_cast(length) || - strncmp(filename, _filename.c_str(), length) != 0) { - // if the filename to dump the recording to is specified move the current - // working file there - int copy_fd = open(filename, O_CREAT | O_RDWR | O_TRUNC, 0644); - if (copy_fd == -1) { - return Error("Could not open recording file for dump"); - } - rec->switchChunk(copy_fd); - close(copy_fd); - return Error::OK; - } - return Error( - "Can not dump recording to itself. Provide a different file name!"); - } - return Error("No active recording"); -} - -void FlightRecorder::wallClockEpoch(int lock_index, - WallClockEpochEvent *event) { - OptionalSharedLockGuard locker(&_rec_lock); - if (locker.ownsLock()) { - Recording* rec = _rec; - if (rec != nullptr) { - Buffer *buf = rec->buffer(lock_index); - rec->recordWallClockEpoch(buf, event); - } - } -} - -void FlightRecorder::recordTraceRoot(int lock_index, int tid, - TraceRootEvent *event) { - OptionalSharedLockGuard locker(&_rec_lock); - if (locker.ownsLock()) { - Recording* rec = _rec; - if (rec != nullptr) { - Buffer *buf = rec->buffer(lock_index); - rec->recordTraceRoot(buf, tid, event); - } - } -} - -void FlightRecorder::recordQueueTime(int lock_index, int tid, - QueueTimeEvent *event) { - OptionalSharedLockGuard locker(&_rec_lock); - if (locker.ownsLock()) { - Recording* rec = _rec; - if (rec != nullptr) { - Buffer *buf = rec->buffer(lock_index); - rec->recordQueueTime(buf, tid, event); - } - } -} - -void FlightRecorder::recordDatadogSetting(int lock_index, int length, - const char *name, const char *value, - const char *unit) { - DEBUG_ASSERT_NOT_IN_SIGNAL(); - OptionalSharedLockGuard locker(&_rec_lock); - if (locker.ownsLock()) { - Recording* rec = _rec; - if (rec != nullptr) { - Buffer *buf = rec->buffer(lock_index); - rec->writeDatadogSetting(buf, length, name, value, unit); - } - } -} - -void FlightRecorder::recordHeapUsage(int lock_index, long value, bool live) { - DEBUG_ASSERT_NOT_IN_SIGNAL(); - OptionalSharedLockGuard locker(&_rec_lock); - if (locker.ownsLock()) { - Recording* rec = _rec; - if (rec != nullptr) { - Buffer *buf = rec->buffer(lock_index); - rec->writeHeapUsage(buf, value, live); - } - } -} - -bool FlightRecorder::recordEvent(int lock_index, int tid, u64 call_trace_id, - int event_type, Event *event) { - OptionalSharedLockGuard locker(&_rec_lock); - if (locker.ownsLock()) { - Recording* rec = _rec; - if (rec != nullptr) { - RecordingBuffer *buf = rec->buffer(lock_index); - switch (event_type) { - case BCI_CPU: - rec->recordExecutionSample(buf, tid, call_trace_id, 0, - (ExecutionEvent *)event); - break; - case BCI_WALL: - rec->recordMethodSample(buf, tid, call_trace_id, 0, - (ExecutionEvent *)event); - break; - case BCI_ALLOC: - rec->recordAllocation(buf, tid, call_trace_id, (AllocEvent *)event); - break; - case BCI_LIVENESS: - rec->recordHeapLiveObject(buf, tid, call_trace_id, - (ObjectLivenessEvent *)event); - break; - case BCI_LOCK: - rec->recordMonitorBlocked(buf, tid, call_trace_id, (LockEvent *)event); - break; - case BCI_PARK: - rec->recordThreadPark(buf, tid, call_trace_id, (LockEvent *)event); - break; - case BCI_NATIVE_MALLOC: - rec->recordMallocSample(buf, tid, call_trace_id, (MallocEvent *)event); - break; - case BCI_NATIVE_SOCKET: - rec->recordNativeSocketSample(buf, tid, call_trace_id, (NativeSocketEvent *)event); - break; - default: - return false; - } - rec->flushIfNeeded(buf); - rec->addThread(lock_index, tid); - return true; - } - } else { - Counters::increment(SAMPLES_DROPPED_REC_LOCK); - } - return false; -} - -bool FlightRecorder::recordEventDelegated(int lock_index, int tid, - u64 correlation_id, int event_type, - Event *event) { - OptionalSharedLockGuard locker(&_rec_lock); - if (locker.ownsLock()) { - Recording* rec = _rec; - if (rec != nullptr) { - RecordingBuffer *buf = rec->buffer(lock_index); - switch (event_type) { - case BCI_CPU: - rec->recordExecutionSample(buf, tid, 0, correlation_id, - (ExecutionEvent *)event); - break; - case BCI_WALL: - rec->recordMethodSample(buf, tid, 0, correlation_id, - (ExecutionEvent *)event); - break; - default: - // Delegation is only wired for CPU/wall samples in v1. - return false; - } - rec->flushIfNeeded(buf); - rec->addThread(lock_index, tid); - return true; - } - } else { - Counters::increment(SAMPLES_DROPPED_REC_LOCK); - } - return false; -} - -void FlightRecorder::recordLog(LogLevel level, const char *message, - size_t len) { - OptionalSharedLockGuard locker(&_rec_lock); - if (locker.ownsLock()) { - Recording* rec = _rec; - if (rec != nullptr) { - if (len > MAX_STRING_LENGTH) - len = MAX_STRING_LENGTH; - // cppcheck-suppress obsoleteFunctions - Buffer *buf = (Buffer *)alloca(len + 40); - buf->reset(); - - int start = buf->skip(5); - buf->putVar64(T_LOG); - buf->putVar64(TSC::ticks()); - buf->putVar64(level); - buf->putUtf8(message, len); - buf->putVar32(start, buf->offset() - start); - _rec->flush(buf); - } - } -} diff --git a/ddprof-lib/src/main/cpp/flightRecorder.h b/ddprof-lib/src/main/cpp/flightRecorder.h deleted file mode 100644 index 213292e13..000000000 --- a/ddprof-lib/src/main/cpp/flightRecorder.h +++ /dev/null @@ -1,467 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _FLIGHTRECORDER_H -#define _FLIGHTRECORDER_H - -#include -#include -#include -#include - -#include -#include - -#include "arch.h" -#include "arguments.h" -#include "buffers.h" -#include "counters.h" -#include "dictionary.h" -#include "stringDictionary.h" -#include "event.h" -#include "frame.h" -#include "jfrMetadata.h" -#include "log.h" -#include "mutex.h" -#include "objectSampler.h" -#include "threadFilter.h" -#include "threadIdTable.h" -#include "vmEntry.h" - -class VMSymbol; // hotspot/vmStructs.h - -const u64 MAX_JLONG = 0x7fffffffffffffffULL; -const u64 MIN_JLONG = 0x8000000000000000ULL; -const int MAX_JFR_EVENT_SIZE = 256; -const int JFR_EVENT_FLUSH_THRESHOLD = RECORDING_BUFFER_LIMIT; -const int MAX_VAR64_LENGTH = 10; -const int MAX_VAR32_LENGTH = 5; - -#ifndef CONCURRENCY_LEVEL -const int CONCURRENCY_LEVEL = 16; -#endif -const u16 ACC_SYNTHETIC = 0x1000; -const u16 ACC_BRIDGE = 0x0040; -const u16 ACC_HIDDEN = ACC_SYNTHETIC | ACC_BRIDGE; - -class Profiler; -class Lookup; - -struct CpuTime { - u64 real; - u64 user; - u64 system; -}; - -struct CpuTimes { - CpuTime proc; - CpuTime total; -}; - -class SharedLineNumberTable { -public: - int _size; - // Owned malloc'd buffer holding a copy of the JVMTI line number table. - // Owning the memory (instead of holding the JVMTI-allocated pointer - // directly) keeps lifetime independent of class unload. - void *_ptr; - - SharedLineNumberTable(int size, void *ptr) : _size(size), _ptr(ptr) {} - ~SharedLineNumberTable(); -}; - -class MethodInfo { -public: - MethodInfo() - : _mark(false), _is_entry(false), _referenced(false), _age(0), _key(0), _class(0), - _name(0), _sig(0), _modifiers(0), _line_number_table(nullptr), _type() {} - - bool _mark; - bool _is_entry; - bool _referenced; // Tracked during writeStackTraces() for cleanup - int _age; // Consecutive chunks without reference (0 = recently used) - u32 _key; - u32 _class; - u32 _name; - u32 _sig; - jint _modifiers; - std::shared_ptr _line_number_table; - FrameTypeId _type; - - jint getLineNumber(jint bci) { - // if the shared pointer is not pointing to the line number table, consider - // size 0 - if (!_line_number_table || _line_number_table->_size == 0) { - return 0; - } - - int i = 1; - while (i < _line_number_table->_size && - bci >= ((jvmtiLineNumberEntry *)_line_number_table->_ptr)[i] - .start_location) { - i++; - } - return ((jvmtiLineNumberEntry *)_line_number_table->_ptr)[i - 1] - .line_number; - } - - bool isHidden() { - // 0x1400 = ACC_SYNTHETIC(0x1000) | ACC_BRIDGE(0x0040) - return _modifiers == 0 || (_modifiers & 0x1040); - } -}; - -// MethodMap's key can be derived from 3 sources: -// 1) jmethodID for Java methods -// 2) void* address for native method names -// 3) Encoded RemoteFrameInfo -// The values of the keys are potentially overlapping, so we use -// the highest 2 bits to distinguish them. -// Key encoding (top two bits): -// 00 - jmethodID -// 10 - void* address (native frame names) -// 01 - RemoteFrameInfo (packed remote symbolication) -// 11 - vtable_receiver class_id (BCI_VTABLE_RECEIVER frames) -class MethodMap : public std::map { -public: - static constexpr unsigned long ADDRESS_MARK = 0x8000000000000000ULL; - static constexpr unsigned long REMOTE_FRAME_MARK = 0x4000000000000000ULL; - static constexpr unsigned long VTABLE_RECEIVER_MARK = ADDRESS_MARK | REMOTE_FRAME_MARK; - static constexpr unsigned long KEY_TYPE_MASK = ADDRESS_MARK | REMOTE_FRAME_MARK; - - MethodMap() {} - - static unsigned long makeKey(jmethodID method) { - unsigned long key = (unsigned long)method; - assert((key & KEY_TYPE_MASK) == 0); - return key; - } - - static unsigned long makeKey(const char* addr) { - unsigned long key = (unsigned long)addr; - assert((key & KEY_TYPE_MASK) == 0); - return (key | ADDRESS_MARK); - } - - static unsigned long makeKey(unsigned long packed_remote_frame) { - unsigned long key = packed_remote_frame; - assert((key & KEY_TYPE_MASK) == 0); - return (key | REMOTE_FRAME_MARK);} - - // BCI_VTABLE_RECEIVER frames key by the resolved class_id (not by the - // VMSymbol* captured at sample time), so two distinct Symbol addresses - // for the same class name collapse to a single MethodInfo row. - static unsigned long makeVTableReceiverKey(u32 class_id) { - unsigned long key = (unsigned long)class_id; - assert((key & KEY_TYPE_MASK) == 0); - return (key | VTABLE_RECEIVER_MARK); - } - - // JFR method-pool id allocator. Ids must be unique among the methods written - // in a single chunk, but may be recycled once a method is erased by - // cleanupUnreferencedMethods() — an erased method is never written again, so - // its id is free for reuse. Recycling freed ids bounds the id range to the - // peak number of live methods (keeping LEB128 encoding compact) while - // guaranteeing no two live methods ever share an id. Id 0 stays reserved as - // the "no entry" sentinel. Single-threaded: only touched on the dump thread - // (allocId from resolveMethod under lockAll, freeId from - // cleanupUnreferencedMethods under the recording lock). - u32 allocId() { - if (!_free_ids.empty()) { - u32 id = _free_ids.back(); - _free_ids.pop_back(); - return id; - } - return ++_id_high_water; - } - - void freeId(u32 id) { - if (id != 0) { - _free_ids.push_back(id); - } - } - -private: - u32 _id_high_water = 0; - std::vector _free_ids; -}; - -class Recording { - friend ObjectSampler; - friend Profiler; - friend Lookup; - -private: - static char *_agent_properties; - static char *_jvm_args; - static char *_jvm_flags; - static char *_java_command; - - RecordingBuffer _buf[CONCURRENCY_LEVEL]; - // we have several tables to avoid lock contention - // we have a second dimension to allow a switch in the active table - ThreadIdTable _thread_ids[CONCURRENCY_LEVEL][2]; - std::atomic _active_index{0}; // 0 or 1 globally - - int _fd; - off_t _chunk_start; - MethodMap _method_map; - - Arguments _args; - u64 _start_time; - u64 _recording_start_time; - u64 _start_ticks; - u64 _recording_start_ticks; - u64 _stop_time; - u64 _stop_ticks; - - u64 _bytes_written; - - int _tid; - int _available_processors; - int _recorded_lib_count; - - bool _cpu_monitor_enabled; - Buffer _cpu_monitor_buf; - CpuTimes _last_times; - - static float ratio(float value) { - return value < 0 ? 0 : value > 1 ? 1 : value; - } - -public: - Recording(int fd, Arguments &args); - ~Recording(); - - void copyTo(int target_fd); - off_t finishChunk(); - - off_t finishChunk(bool end_recording, bool do_cleanup = false); - void switchChunk(int fd); - - void cpuMonitorCycle(); - void appendRecording(const char *target_file, size_t size); - - RecordingBuffer *buffer(int lock_index); - - bool parseAgentProperties(); - - void flush(Buffer *buf); - void flushIfNeeded(Buffer *buf, int limit = JFR_EVENT_FLUSH_THRESHOLD); - void writeHeader(Buffer *buf); - - void writeMetadata(Buffer *buf); - - void writeElement(Buffer *buf, const Element *e); - - void writeEventSizePrefix(Buffer *buf, int start); - - void writeRecordingInfo(Buffer *buf); - - void writeSettings(Buffer *buf, Arguments &args); - - void writeStringSetting(Buffer *buf, int category, const char *key, - const char *value); - - void writeBoolSetting(Buffer *buf, int category, const char *key, bool value); - - void writeIntSetting(Buffer *buf, int category, const char *key, - long long value); - void writeListSetting(Buffer *buf, int category, const char *key, - const char *base, int offset); - - void writeDatadogSetting(Buffer *buf, int length, const char *name, - const char *value, const char *unit); - - void writeDatadogProfilerConfig(Buffer *buf, long cpuInterval, - long wallInterval, long allocInterval, - long memleakInterval, long memleakCapacity, - double memleakRatio, bool gcGenerations, - int modeMask, const char *cpuEngine); - - void writeHeapUsage(Buffer *buf, long value, bool live); - void writeOsCpuInfo(Buffer *buf); - void writeJvmInfo(Buffer *buf); - void writeSystemProperties(Buffer *buf); - void writeNativeLibraries(Buffer *buf); - // Writes the cpool checkpoint. Returns the number of pool sections actually - // emitted (empty variable pools are skipped) and reports the byte offset of - // the pool-count placeholder within the cpool via *count_offset_in_cpool, so - // the caller can back-patch it flush-safe alongside the cpool size field. - int writeCpool(Buffer *buf, int *count_offset_in_cpool); - - void writeFrameTypes(Buffer *buf); - - void writeThreadStates(Buffer *buf); - - void writeExecutionModes(Buffer *buf); - // writeThreads always emits: _tid is inserted unconditionally so the thread - // pool is never empty. The following variable-pool writers return 1 if a - // section was emitted, 0 if the pool was empty and skipped. - void writeThreads(Buffer *buf); - - int writeStackTraces(Buffer *buf, Lookup *lookup); - - int writeMethods(Buffer *buf, Lookup *lookup); - - int writeClasses(Buffer *buf, Lookup *lookup); - - int writePackages(Buffer *buf, Lookup *lookup); - - int writeConstantPoolSection(Buffer *buf, JfrType type, - std::map &constants); - - int writeConstantPoolSection(Buffer *buf, JfrType type, - Dictionary *dictionary); - int writeConstantPoolSection(Buffer *buf, JfrType type, - StringDictionaryBuffer *buffer); - - void writeLogLevels(Buffer *buf); - - void writeCounters(Buffer *buf); - - void writeUnwindFailures(Buffer *buf); - - void writeContextSnapshot(Buffer *buf, Context &context); - void writeCurrentContext(Buffer *buf); - - void recordExecutionSample(Buffer *buf, int tid, u64 call_trace_id, - u64 correlation_id, ExecutionEvent *event); - void recordMethodSample(Buffer *buf, int tid, u64 call_trace_id, - u64 correlation_id, ExecutionEvent *event); - void recordWallClockEpoch(Buffer *buf, WallClockEpochEvent *event); - void recordTraceRoot(Buffer *buf, int tid, TraceRootEvent *event); - void recordQueueTime(Buffer *buf, int tid, QueueTimeEvent *event); - void recordAllocation(RecordingBuffer *buf, int tid, u64 call_trace_id, - AllocEvent *event); - void recordMallocSample(Buffer *buf, int tid, u64 call_trace_id, - MallocEvent *event); - void recordNativeSocketSample(Buffer *buf, int tid, u64 call_trace_id, - NativeSocketEvent *event); - void recordHeapLiveObject(Buffer *buf, int tid, u64 call_trace_id, - ObjectLivenessEvent *event); - void recordMonitorBlocked(Buffer *buf, int tid, u64 call_trace_id, - LockEvent *event); - void recordThreadPark(Buffer *buf, int tid, u64 call_trace_id, - LockEvent *event); - void recordCpuLoad(Buffer *buf, float proc_user, float proc_system, - float machine_total); - - void addThread(int lock_index, int tid); - -private: - void cleanupUnreferencedMethods(); -}; - -class Lookup { -public: - Recording *_rec; - MethodMap *_method_map; - StringDictionary *_classes; - std::map _class_cache; // snapshot of _classes->standby() at dump time - // Per-dump VMSymbol* -> resolved class_id cache for BCI_VTABLE_RECEIVER - // frames. Two purposes: (1) amortise the SafeAccess work to once per - // distinct Symbol pointer per dump; (2) the resolved class_id is used - // as the MethodMap key, so distinct Symbol* addresses for the same - // class name (class unload/reload mid-chunk) collapse to a single - // MethodInfo row. - std::unordered_map _vtable_receiver_cache; - Dictionary _packages; - Dictionary _symbols; - -private: - void fillNativeMethodInfo(MethodInfo *mi, const char *name, - const char *lib_name); - void fillRemoteFrameInfo(MethodInfo *mi, const RemoteFrameInfo *rfi); - void cutArguments(char *func); - void fillJavaMethodInfo(MethodInfo *mi, jmethodID method, bool first_time); - bool has_prefix(const char *str, const char *prefix) const { - return strncmp(str, prefix, strlen(prefix)) == 0; - } - // Length-bounded variant for buffers that may not be NUL-terminated. - bool has_prefix_n(const char *buf, size_t buf_len, const char *prefix) const { - size_t plen = strlen(prefix); - return buf_len >= plen && strncmp(buf, prefix, plen) == 0; - } - - // Resolves a VMSymbol* captured at sample time (BCI_VTABLE_RECEIVER) into a - // class id in _classes, applying the synthetic-accessor/LambdaForm - // normalisation inline. Crash-safe under concurrent class unloading: all - // reads of the Symbol go through SafeAccess (safefetch + bounded copy), so - // a Symbol freed and its page unmapped between sample and dump cannot - // SIGSEGV the dump thread. On success returns true and fills *out_class_id - // with the normalised class id. `buf` is a working area used internally; - // its contents on return are unspecified. - bool resolveVTableReceiver(VMSymbol *sym, char *buf, size_t bufsize, - u32 *out_class_id); - - // Cache wrapper: look up Symbol* in _vtable_receiver_cache; on miss, - // resolve via resolveVTableReceiver and cache the result. On any - // resolution failure (SafeAccess fault, length out of range, non-printable - // bytes) returns the sentinel "" class_id and - // increments VTABLE_RECEIVER_RESOLVE_FAILED. - u32 resolveVTableReceiverCached(void *sym); - -public: - Lookup(Recording *rec, MethodMap *method_map, StringDictionary *classes) - : _rec(rec), _method_map(method_map), _classes(classes), _packages(), - _symbols() {} - - // Call once before writeStackTraces. Populates _class_cache from - // _classes->standby() under the shared lock. NOTE: _class_cache is - // currently write-only — writeClasses() re-collects from standby() and - // resolveMethod() inserts via lookupDuringDump() rather than reading - // this cache. Kept for compatibility with #527's API and as a hook - // for future readers; safe to remove if no consumer materialises. - void initClassCache(); - - MethodInfo *resolveMethod(ASGCT_CallFrame &frame); - u32 getPackage(const char *class_name); - u32 getSymbol(const char *name); -}; - -class FlightRecorder { - friend Profiler; - -private: - std::string _filename; - Arguments _args; - - SpinLock _rec_lock; - Recording* _rec; - - Error newRecording(bool reset); - -public: - FlightRecorder() : _rec(NULL) {} - Error start(Arguments &args, bool reset); - void stop(); - Error dump(const char *filename, const int length); - void wallClockEpoch(int lock_index, WallClockEpochEvent *event); - void recordTraceRoot(int lock_index, int tid, TraceRootEvent *event); - void recordQueueTime(int lock_index, int tid, QueueTimeEvent *event); - - bool active() const { return _rec != NULL; } - - bool recordEvent(int lock_index, int tid, u64 call_trace_id, int event_type, - Event *event); - - // Emit a BCI_CPU / BCI_WALL sample with no stack-trace attached to our - // recording. `correlation_id` is the same jlong passed to the HotSpot - // RequestStackTrace extension so downstream tooling can join our event with - // the JVM-emitted jdk.StackTraceRequest. - bool recordEventDelegated(int lock_index, int tid, u64 correlation_id, - int event_type, Event *event); - - void recordLog(LogLevel level, const char *message, size_t len); - - void recordDatadogSetting(int lock_index, int length, const char *name, - const char *value, const char *unit); - - void recordHeapUsage(int lock_index, long value, bool live); -}; - -#endif // _FLIGHTRECORDER_H diff --git a/ddprof-lib/src/main/cpp/frame.h b/ddprof-lib/src/main/cpp/frame.h deleted file mode 100644 index dbd27c2e0..000000000 --- a/ddprof-lib/src/main/cpp/frame.h +++ /dev/null @@ -1,33 +0,0 @@ -#ifndef _FRAME_H -#define _FRAME_H - -enum FrameTypeId { - FRAME_INTERPRETED = 0, - FRAME_JIT_COMPILED = 1, - FRAME_INLINED = 2, - FRAME_NATIVE = 3, - FRAME_CPP = 4, - FRAME_KERNEL = 5, - FRAME_C1_COMPILED = 6, - FRAME_NATIVE_REMOTE = 7, // Native frame with remote symbolication (build-id + pc-offset) - FRAME_TYPE_MAX = FRAME_NATIVE_REMOTE // Maximum valid frame type -}; - -class FrameType { -public: - static inline int encode(int type, int bci) { - return (1 << 24) | (type << 25) | (bci & 0xffffff); - } - - static inline FrameTypeId decode(int bci) { - if ((bci >> 24) <= 0) { - // Unencoded BCI (bit 24 not set) or negative special BCI values - return FRAME_JIT_COMPILED; - } - // Clamp to valid FrameTypeId range to defend against corrupted values - int raw_type = bci >> 25; - return (FrameTypeId)(raw_type <= FRAME_TYPE_MAX ? raw_type : FRAME_TYPE_MAX); - } -}; - -#endif // _FRAME_H diff --git a/ddprof-lib/src/main/cpp/frames.h b/ddprof-lib/src/main/cpp/frames.h deleted file mode 100644 index 15549e6e8..000000000 --- a/ddprof-lib/src/main/cpp/frames.h +++ /dev/null @@ -1,18 +0,0 @@ -#ifndef _FRAMES_H -#define _FRAMES_H - -#include -#include "vmEntry.h" - -inline int makeFrame(ASGCT_CallFrame *frames, jint type, jmethodID id) { - frames[0].bci = type; - frames[0].method_id = id; - return 1; -} - -inline int makeFrame(ASGCT_CallFrame *frames, jint type, - const char *id) { - return makeFrame(frames, type, (jmethodID)id); -} - -#endif // _FRAMES_H diff --git a/ddprof-lib/src/main/cpp/gtest_crash_handler.h b/ddprof-lib/src/main/cpp/gtest_crash_handler.h deleted file mode 100644 index 7bc15bcb1..000000000 --- a/ddprof-lib/src/main/cpp/gtest_crash_handler.h +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef GTEST_CRASH_HANDLER_H -#define GTEST_CRASH_HANDLER_H - -#include -#include -#include -#include -#include -#include - -#include "common.h" // TSAN_ENABLED (toolchain-agnostic sanitizer detection) - -// Platform detection for execinfo.h availability -#if defined(__GLIBC__) || (defined(__APPLE__) && defined(__MACH__)) || defined(__FreeBSD__) || defined(__NetBSD__) || defined(__OpenBSD__) - #define HAVE_EXECINFO_H 1 - #include -#else - #define HAVE_EXECINFO_H 0 - // Fallback declarations for platforms without execinfo.h -#endif - -/** - * Shared crash handler for all gtest files. - * Provides detailed crash reporting with backtrace and register information. - * Use installGtestCrashHandler() to install and restoreDefaultSignalHandlers() to cleanup. - */ - -// Global crash handler for detailed debugging of segfaults -inline void gtestCrashHandler(int sig, siginfo_t *info, void *context, const char* test_name) { - // Prevent recursive calls - static volatile sig_atomic_t in_crash_handler = 0; - if (in_crash_handler) { - // Already in crash handler - just exit to prevent infinite loop - _exit(128 + sig); - } - in_crash_handler = 1; - - // Use async-signal-safe functions only - const char* signal_names[] = { - "UNKNOWN", "SIGHUP", "SIGINT", "SIGQUIT", "SIGILL", "SIGTRAP", "SIGABRT", "SIGBUS", - "SIGFPE", "SIGKILL", "SIGSEGV", "SIGUSR1", "SIGPIPE", "SIGALRM", "SIGTERM", "SIGUSR2" - }; - - const char* signal_name = (sig >= 1 && sig <= 15) ? signal_names[sig] : "UNKNOWN"; - - // Write crash info to stderr (async-signal-safe) - write(STDERR_FILENO, "\n=== GTEST CRASH: ", 19); - write(STDERR_FILENO, test_name, strlen(test_name)); - write(STDERR_FILENO, " ===\n", 6); - - // Signal type - write(STDERR_FILENO, "Signal: ", 8); - write(STDERR_FILENO, signal_name, strlen(signal_name)); - - // Format signal number - char sig_buf[32]; - snprintf(sig_buf, sizeof(sig_buf), " (%d)\n", sig); - write(STDERR_FILENO, sig_buf, strlen(sig_buf)); - - // Fault address for memory access violations - if (sig == SIGSEGV || sig == SIGBUS) { - write(STDERR_FILENO, "Fault address: 0x", 17); - char addr_buf[32]; - snprintf(addr_buf, sizeof(addr_buf), "%lx\n", (unsigned long)info->si_addr); - write(STDERR_FILENO, addr_buf, strlen(addr_buf)); - } - - // Thread ID - write(STDERR_FILENO, "Thread ID: ", 11); - char tid_buf[32]; - snprintf(tid_buf, sizeof(tid_buf), "%d\n", getpid()); - write(STDERR_FILENO, tid_buf, strlen(tid_buf)); - - // Backtrace (if available) - write(STDERR_FILENO, "\nBacktrace:\n", 12); -#if HAVE_EXECINFO_H - void *buffer[64]; - int nptrs = backtrace(buffer, 64); - - // Use backtrace_symbols_fd which is async-signal-safe - backtrace_symbols_fd(buffer, nptrs, STDERR_FILENO); -#else - write(STDERR_FILENO, " [Backtrace not available on this platform]\n", 45); -#endif - - // Register state (platform specific) -#ifdef __APPLE__ - ucontext_t *uctx = (ucontext_t *)context; - if (uctx && uctx->uc_mcontext) { - write(STDERR_FILENO, "\nRegister state:\n", 17); - char reg_buf[128]; - #ifdef __x86_64__ - snprintf(reg_buf, sizeof(reg_buf), "RIP: 0x%llx, RSP: 0x%llx\n", - uctx->uc_mcontext->__ss.__rip, uctx->uc_mcontext->__ss.__rsp); - #elif defined(__aarch64__) - snprintf(reg_buf, sizeof(reg_buf), "PC: 0x%llx, SP: 0x%llx\n", - uctx->uc_mcontext->__ss.__pc, uctx->uc_mcontext->__ss.__sp); - #endif - write(STDERR_FILENO, reg_buf, strlen(reg_buf)); - } -#endif - - write(STDERR_FILENO, "\n=== END CRASH INFO ===\n", 25); - - // Ensure output is flushed - fsync(STDERR_FILENO); - - // Don't interfere with AddressSanitizer - just exit cleanly - _exit(128 + sig); -} - -// Template wrapper to pass test name to crash handler -template -void specificCrashHandler(int sig, siginfo_t *info, void *context) { - gtestCrashHandler(sig, info, context, TestName); -} - -// Install crash handler for debugging. -// No-op under TSan: TSan installs its own SIGSEGV/SIGBUS/SIGABRT interceptors -// and overriding them causes TSan to crash before it can write its report. -template -void installGtestCrashHandler() { -#if !defined(TSAN_ENABLED) - struct sigaction sa; - sa.sa_flags = SA_SIGINFO; // Get detailed info, keep handler active - sigemptyset(&sa.sa_mask); - sa.sa_sigaction = specificCrashHandler; - - // Install for various crash signals - sigaction(SIGSEGV, &sa, nullptr); - sigaction(SIGBUS, &sa, nullptr); - sigaction(SIGABRT, &sa, nullptr); - sigaction(SIGFPE, &sa, nullptr); - sigaction(SIGILL, &sa, nullptr); -#endif -} - -// Restore default signal handlers. -inline void restoreDefaultSignalHandlers() { -#if !defined(TSAN_ENABLED) - signal(SIGSEGV, SIG_DFL); - signal(SIGBUS, SIG_DFL); - signal(SIGABRT, SIG_DFL); - signal(SIGFPE, SIG_DFL); - signal(SIGILL, SIG_DFL); -#endif -} - -#endif // GTEST_CRASH_HANDLER_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/guards.cpp b/ddprof-lib/src/main/cpp/guards.cpp deleted file mode 100644 index 1bfc0b695..000000000 --- a/ddprof-lib/src/main/cpp/guards.cpp +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "guards.h" -#include "common.h" -#include "os.h" -#include "thread.h" - -// Signal-context tracking — backed by ProfiledThread::_signal_depth; see -// the comment block in guards.h for the rationale (initial-exec TLS was -// rejected because of the static TLS surplus on Graal). - -int getInSignalDepth() { - ProfiledThread *pt = ProfiledThread::currentSignalSafe(); - return pt != nullptr ? static_cast(pt->signalDepth()) : 0; -} - -bool isInTrackedSignalContext() { - ProfiledThread *pt = ProfiledThread::currentSignalSafe(); - // null ProfiledThread = no thread context; the SignalHandlerScope - // never ran, so we have no positive evidence of a signal frame. - // See header comment for the rationale of returning false here. - return pt != nullptr && pt->signalDepth() != 0; -} - -SignalHandlerScope::SignalHandlerScope() : _active(true) { - ProfiledThread *pt = ProfiledThread::currentSignalSafe(); - if (pt != nullptr) { - pt->enterSignalScope(); - } else { - // No thread context: nothing to update; mark inactive so destructor - // and release() are no-ops. - _active = false; - } -} - -SignalHandlerScope::~SignalHandlerScope() { - if (!_active) return; - ProfiledThread *pt = ProfiledThread::currentSignalSafe(); - if (pt != nullptr) { - pt->exitSignalScope(); - } -} - -void SignalHandlerScope::release() { - if (!_active) return; - ProfiledThread *pt = ProfiledThread::currentSignalSafe(); - if (pt != nullptr) { - pt->exitSignalScope(); - } - _active = false; -} - -void signalHandlerUnwindAfterLongjmp() { - ProfiledThread *pt = ProfiledThread::currentSignalSafe(); - if (pt != nullptr) { - pt->exitSignalScope(); - } -} - -// Static bitmap storage for fallback cases -uint64_t CriticalSection::_fallback_bitmap[CriticalSection::FALLBACK_BITMAP_WORDS] = {}; - -CriticalSection::CriticalSection() : _entered(false), _using_fallback(false), _word_index(0), _bit_mask(0), _thread_ptr(nullptr) { - _thread_ptr = ProfiledThread::currentSignalSafe(); - if (_thread_ptr != nullptr) { - // Primary path: Use ProfiledThread storage (fast and memory-efficient) - _entered = _thread_ptr->tryEnterCriticalSection(); - } else { - // Fallback path: Use hash-based bitmap for stress tests and edge cases - _using_fallback = true; - int tid = OS::threadId(); - - // Hash TID to distribute across bitmap words, reducing clustering - // We are OK with false collision for the fallback - it should be used only for testing when we don't have full profiler initialized - _word_index = hash_tid(tid) % FALLBACK_BITMAP_WORDS; - uint32_t bit_index = tid % 64; - _bit_mask = 1ULL << bit_index; - - // Use ACQUIRE ordering to ensure visibility of protected data after acquiring critical section - uint64_t old_word = __atomic_fetch_or(&_fallback_bitmap[_word_index], _bit_mask, __ATOMIC_ACQUIRE); - _entered = !(old_word & _bit_mask); // Success if bit was previously 0 - } -} - -CriticalSection::~CriticalSection() { - if (_entered) { - if (_using_fallback) { - // Clear the bit atomically for fallback bitmap - // Use RELEASE ordering to ensure protected data writes are visible before releasing - __atomic_fetch_and(&_fallback_bitmap[_word_index], ~_bit_mask, __ATOMIC_RELEASE); - } else { - // Release ProfiledThread flag using the pointer captured at construction - if (_thread_ptr != nullptr) { - _thread_ptr->exitCriticalSection(); - } - } - } -} - -uint32_t CriticalSection::hash_tid(int tid) { - return static_cast(tid * KNUTH_MULTIPLICATIVE_CONSTANT); -} diff --git a/ddprof-lib/src/main/cpp/guards.h b/ddprof-lib/src/main/cpp/guards.h deleted file mode 100644 index 4addd8d39..000000000 --- a/ddprof-lib/src/main/cpp/guards.h +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _GUARDS_H -#define _GUARDS_H - -#include -#include -#include -#include - -class ProfiledThread; - -// --------------------------------------------------------------------------- -// Signal-context depth tracking — always on. -// -// Profiler::dlopen_hook is the production caller — it queries -// isInTrackedSignalContext() to decide between the synchronous refresh() -// path and the deferred markDirty() path. The debug-only -// DEBUG_ASSERT_NOT_IN_SIGNAL() macro in signalSafety.h asserts on the -// same counter. -// -// Storage: the depth lives in ProfiledThread::_signal_depth. An earlier -// design used a thread_local int, but on Graal aarch64 the lazy DTV slot -// allocation triggered malloc inside our signal handler and deadlocked -// against the JVMCI compiler holding the heap lock. initial-exec fixed -// the malloc but tripped the static TLS surplus and broke dlopen on -// Graal. ProfiledThread is already AS-safe-accessible via -// pthread_getspecific (POSIX guarantees it does not allocate; returns -// nullptr when unset). -// -// When ProfiledThread is null on a thread we don't yet have a thread -// context — uninstrumented JVM-internal threads (VM Thread, JIT, GC) fall -// into this bucket too, and they can receive signals. The -// SignalHandlerScope guard is a no-op on those threads (nothing to -// update), so isInTrackedSignalContext() returns false: production code -// prefers synchronous refresh() on null-PT threads because (a) those -// threads regularly call dlopen during normal JVM operation, and (b) -// wasmtime's broken sigaction patching depends on switchLibraryTrap -// running work inline. The residual risk — an uninstrumented thread -// calling dlopen from inside a foreign signal handler — is small in -// practice: prewarmUnwinder() closes the known libgcc_s lazy-load case -// and mainstream JVM signal handlers are AS-safe by design. -// -// DEBUG_ASSERT_NOT_IN_SIGNAL likewise skips its check when ProfiledThread -// is null so well-behaved non-signal code on uninstrumented threads -// doesn't trip a false abort. -// --------------------------------------------------------------------------- - -// Returns the signal-handler depth for the calling thread, or 0 if the -// thread has no ProfiledThread yet. Intended for tests and diagnostic -// code; production callers should use isInTrackedSignalContext(). -int getInSignalDepth(); - -// Returns true only when we have positively tracked entering one of our -// installed signal handlers on this thread (depth > 0 on a non-null -// ProfiledThread). null ProfiledThread → false, matching the -// SignalHandlerScope semantics (the guard is a no-op there). -// Used by Profiler::dlopen_hook to gate the deferred-refresh branch. -bool isInTrackedSignalContext(); - -// Internal RAII type — do not instantiate directly; use the macros below. -class SignalHandlerScope { -public: - SignalHandlerScope(); - ~SignalHandlerScope(); - void release(); - SignalHandlerScope(const SignalHandlerScope&) = delete; - SignalHandlerScope& operator=(const SignalHandlerScope&) = delete; -private: - bool _active; -}; - -// Declare a scope guard local that increments the depth on entry and -// decrements on scope exit. Use as the very first statement in every -// installed signal handler. -#define SIGNAL_HANDLER_GUARD() SignalHandlerScope _signal_handler_scope - -// Manually release the most recent SIGNAL_HANDLER_GUARD() before chaining to -// another handler that may longjmp through us (e.g. J9's SIGSEGV null-pointer -// check handler). After release(), depth has already been decremented; the -// destructor becomes a no-op. -#define SIGNAL_HANDLER_GUARD_RELEASE() _signal_handler_scope.release() - -// Compensate for a longjmp that bypassed a SignalHandlerScope's destructor. -// Call at the setjmp landing point AFTER a known longjmp originated from -// within a signal handler frame (e.g. HotSpot's checkFault → longjmp recovery -// in walkVM). -void signalHandlerUnwindAfterLongjmp(); -#define SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP() signalHandlerUnwindAfterLongjmp() - -/** - * Race-free critical section using atomic compare-and-swap. - * - * Hybrid implementation: - * - Primary: Uses ProfiledThread storage when available (zero memory overhead) - * - Fallback: Hash-based bitmap for stress tests and cases without ProfiledThread - * - * This approach is async-signal-safe and avoids TLS allocation issues. - * - * Usage: - * { - * CriticalSection cs; // Atomically claim critical section - * if (!cs.entered()) return; // Another thread/signal handler is active - * // Complex data structure operations - * // Signal handlers will be blocked from entering - * } // Critical section automatically released - * - * This eliminates race conditions between signal handlers and normal code - * by ensuring only one can hold the critical section at a time per thread. - * - * !Warning! This is not a generic critical section implementation. - * It relies on the fact that 'put' operations can not be preempted by the 'processing' operation. - * That means that each 'put' operation will fully complete before 'processing' proceeds. - * - * The only preemption sequence is like this: - * - processing enter - * - processing acquire critical section - * - signal interrupts processing; results in calling put - * - put tries to acquire the critical section and fails - * - put bails out - * - processing proceeds and eventually releases the critical section - */ -class CriticalSection { -private: - static constexpr size_t FALLBACK_BITMAP_WORDS = 1024; // 8KB for 64K bits - // Atomic bitmap for thread-safe critical section tracking without TLS - // Must be atomic because multiple signal handlers can run concurrently across - // different threads and attempt to set/clear bits simultaneously. Compare-and-swap - // operations ensure race-free bit manipulation even during signal interruption. - static uint64_t _fallback_bitmap[FALLBACK_BITMAP_WORDS]; - - bool _entered; // Track if this instance successfully entered - bool _using_fallback; // Track which storage mechanism we're using - uint32_t _word_index; // For fallback bitmap cleanup - uint64_t _bit_mask; // For fallback bitmap cleanup - ProfiledThread* _thread_ptr; // ProfiledThread captured at construction - -public: - CriticalSection(); - ~CriticalSection(); - - // Non-copyable, non-movable - CriticalSection(const CriticalSection&) = delete; - CriticalSection& operator=(const CriticalSection&) = delete; - CriticalSection(CriticalSection&&) = delete; - CriticalSection& operator=(CriticalSection&&) = delete; - - // Check if this instance successfully entered the critical section - bool entered() const { return _entered; } - -private: - // Hash function to distribute thread IDs across bitmap words - static uint32_t hash_tid(int tid); -}; - -/** - * RAII guard to block profiling signals during critical operations. - * - * Blocks SIGPROF and SIGVTALRM signals on construction and automatically - * restores the original signal mask on destruction. This prevents signal - * handlers from interrupting operations that are not async-signal-safe, - * such as musl libc's TLS initialization. - * - * !WARNING! - * For guarding access to code running as a signal handler use CriticalSection - * !WARNING! - * - * Usage: - * { - * SignalBlocker blocker; // Blocks profiling signals - * // Perform operations that must not be interrupted by signals - * // (e.g., TLS initialization, malloc, etc.) - * } // Signal mask automatically restored - * - * The blocker is exception-safe: the signal mask will be restored even - * if an exception is thrown within the protected scope. - * - * Note: This only blocks signals for the current thread. Other threads - * continue to receive profiling signals normally. - */ -class SignalBlocker { -private: - sigset_t _old_mask; - bool _active; - -public: - SignalBlocker() : _active(false) { - sigset_t prof_signals; - sigemptyset(&prof_signals); - - // Block only the profiling signals that the profiler actually registers. - // No profiler engine uses RT signals, so blocking them is unnecessary - // and risks interfering with glibc NPTL internals (SIGRTMIN, SIGRTMIN+1) - // or other JVM-internal signal usage. - sigaddset(&prof_signals, SIGPROF); // Used by ITimer and CTimer - sigaddset(&prof_signals, SIGVTALRM); // Used by WallClock - - if (pthread_sigmask(SIG_BLOCK, &prof_signals, &_old_mask) == 0) { - _active = true; - } - } - - ~SignalBlocker() { - if (_active) { - pthread_sigmask(SIG_SETMASK, &_old_mask, nullptr); - } - } - - // Non-copyable - SignalBlocker(const SignalBlocker&) = delete; - SignalBlocker& operator=(const SignalBlocker&) = delete; -}; - -#endif // _GUARDS_H diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotStackFrame.h b/ddprof-lib/src/main/cpp/hotspot/hotspotStackFrame.h deleted file mode 100644 index b631e9077..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/hotspotStackFrame.h +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _HOTSPOT_HOTSPOTSTACKFRAME_H -#define _HOTSPOT_HOTSPOTSTACKFRAME_H - -#include "stackFrame.h" -#include "hotspot/vmStructs.h" - -class HotspotStackFrame : public StackFrame { -public: - explicit HotspotStackFrame(void* ucontext): StackFrame(ucontext) { - } - - bool unwindCompiled(VMNMethod* nm) { - return unwindCompiled(nm, pc(), sp(), fp()); - } - - bool unwindStub(instruction_t* entry, const char* name) { - return unwindStub(entry, name, pc(), sp(), fp()); - } - - bool unwindStub(instruction_t* entry, const char* name, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp); - - // TODO: this function will be removed once `vm` becomes the default stack walking mode - bool unwindCompiled(VMNMethod* nm, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp); - - bool unwindPrologue(VMNMethod* nm, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp); - bool unwindEpilogue(VMNMethod* nm, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp); - - static bool unwindAtomicStub(const StackFrame& frame, const void*& pc); -}; - -#endif // _HOTSPOT_HOTSPOTSTACKFRAME_H - diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotStackFrame_aarch64.cpp b/ddprof-lib/src/main/cpp/hotspot/hotspotStackFrame_aarch64.cpp deleted file mode 100644 index d848cc655..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/hotspotStackFrame_aarch64.cpp +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifdef __aarch64__ - -#include -#include "hotspot/hotspotStackFrame.h" - -static inline bool isSTP(instruction_t insn) { - // stp xn, xm, [sp, #-imm]! - // stp dn, dm, [sp, #-imm]! - return (insn & 0xffe003e0) == 0xa9a003e0 || (insn & 0xffe003e0) == 0x6da003e0; -} - -// Check if this is a well-known leaf stub with a constant size frame -static inline bool isFixedSizeFrame(const char* name) { - // Dispatch by the first character to optimize lookup - switch (name[0]) { - case 'i': - return strncmp(name, "indexof_linear_", 15) == 0; - case 'm': - return strncmp(name, "md5_implCompress", 16) == 0; - case 's': - return strncmp(name, "sha256_implCompress", 19) == 0 - || strncmp(name, "string_indexof_linear_", 22) == 0 - || strncmp(name, "slow_subtype_check", 18) == 0; - default: - return false; - } -} - -// Check if this is a well-known leaf stub that does not change stack pointer -static inline bool isZeroSizeFrame(const char* name) { - // Dispatch by the first character to optimize lookup - switch (name[0]) { - case 'I': - return strcmp(name, "InlineCacheBuffer") == 0; - case 'S': - return strncmp(name, "SafeFetch", 9) == 0; - case 'a': - return strncmp(name, "atomic", 6) == 0; - case 'b': - return strncmp(name, "bigInteger", 10) == 0 - || strcmp(name, "base64_encodeBlock") == 0; - case 'c': - return strncmp(name, "copy_", 5) == 0 - || strncmp(name, "compare_long_string_", 20) == 0; - case 'e': - return strcmp(name, "encodeBlock") == 0; - case 'f': - return strcmp(name, "f2hf") == 0; - case 'g': - return strcmp(name, "ghash_processBlocks") == 0; - case 'h': - return strcmp(name, "hf2f") == 0; - case 'i': - return strncmp(name, "itable", 6) == 0; - case 'l': - return strcmp(name, "large_byte_array_inflate") == 0 - || strncmp(name, "lookup_secondary_supers_", 24) == 0; - case 'm': - return strncmp(name, "md5_implCompress", 16) == 0; - case 's': - return strncmp(name, "sha1_implCompress", 17) == 0 - || strncmp(name, "compare_long_string_same_encoding", 33) == 0 - || strcmp(name, "compare_long_string_LL") == 0 - || strcmp(name, "compare_long_string_UU") == 0; - case 'u': - return strcmp(name, "updateBytesAdler32") == 0; - case 'v': - return strncmp(name, "vtable", 6) == 0; - case 'z': - return strncmp(name, "zero_", 5) == 0; - default: - return false; - } -} - -static inline bool isEntryBarrier(instruction_t* ip) { - // ldr w9, [x28, #32] - // cmp x8, x9 - return ip[0] == 0xb9402389 && ip[1] == 0xeb09011f; -} - -static inline bool isFrameComplete(instruction_t* entry, instruction_t* ip) { - // Frame is fully constructed after sp is decremented by the frame size. - // Check if there is such an instruction anywhere between - // the method entry and the current instruction pointer. - while (--ip >= entry) { - if ((*ip & 0xff8003ff) == 0xd10003ff) { // sub sp, sp, #frame_size - return true; - } - } - return false; -} - - -static inline bool isPollReturn(instruction_t* ip) { - // JDK 17+ - // add sp, sp, #0x30 - // ldr x8, [x28, #832] - // cmp sp, x8 - // b.hi offset - // ret - // - // JDK 11 - // add sp, sp, #0x30 - // ldr x8, [x28, #264] - // ldr wzr, [x8] - // ret - // - // JDK 8 - // add sp, sp, #0x30 - // adrp x8, polling_page - // ldr wzr, [x8] - // ret - // - if ((ip[0] & 0xffc003ff) == 0xf9400388 && (ip[-1] & 0xff8003ff) == 0x910003ff) { - // ldr x8, preceded by add sp - return true; - } else if ((ip[0] & 0x9f00001f) == 0x90000008 && (ip[-1] & 0xff8003ff) == 0x910003ff) { - // adrp x8, preceded by add sp - return true; - } else if (ip[0] == 0xeb2863ff && ip[2] == 0xd65f03c0) { - // cmp sp, x8, followed by ret - return true; - } else if ((ip[0] & 0xff000010) == 0x54000000 && ip[1] == 0xd65f03c0) { - // b.cond, followed by ret - return true; - } else if (ip[0] == 0xb940011f && ip[1] == 0xd65f03c0) { - // ldr wzr, followed by ret - return true; - } - return false; -} - - NOSANALIGSANITIZE bool HotspotStackFrame::unwindCompiled(VMNMethod* nm, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp) { - instruction_t* ip = (instruction_t*)pc; - instruction_t* entry = (instruction_t*)nm->entry(); - if ((*ip & 0xffe07fff) == 0xa9007bfd) { - // stp x29, x30, [sp, #offset] - // SP has been adjusted, but FP not yet stored in a new frame - unsigned int offset = (*ip >> 12) & 0x1f8; - sp += offset + 16; - pc = link(); - } else if (ip > entry && ip[0] == 0x910003fd && ip[-1] == 0xa9bf7bfd) { - // stp x29, x30, [sp, #-16]! - // mov x29, sp - sp += 16; - pc = ((uintptr_t*)sp)[-1]; - } else if (ip > entry + 3 && !nm->isFrameCompleteAt(ip) && - (isEntryBarrier(ip) || isEntryBarrier(ip + 1))) { - // Frame should be complete at this point - sp += nm->frameSize() * sizeof(void*); - fp = ((uintptr_t*)sp)[-2]; - pc = ((uintptr_t*)sp)[-1]; - } else { - // Just try - pc = link(); - } - return true; -} - -NOSANALIGSANITIZE bool HotspotStackFrame::unwindPrologue(VMNMethod* nm, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp) { - // C1/C2 methods: - // {stack_bang} - // sub sp, sp, #0x40 - // stp x29, x30, [sp, #48] - // - // Native wrappers: - // {stack_bang} - // stp x29, x30, [sp, #-16]! - // mov x29, sp - // sub sp, sp, #0x50 - // - instruction_t* ip = (instruction_t*)pc; - instruction_t* entry = (instruction_t*)nm->entry(); - if (ip <= entry) { - pc = link(); - } else if ((*ip & 0xffe07fff) == 0xa9007bfd) { - // stp x29, x30, [sp, #offset] - // SP has been adjusted, but FP not yet stored in a new frame - unsigned int offset = (*ip >> 12) & 0x1f8; - sp += offset + 16; - pc = link(); - } else if (ip[0] == 0x910003fd && ip[-1] == 0xa9bf7bfd) { - // stp x29, x30, [sp, #-16]! - // mov x29, sp - sp += 16; - pc = ((uintptr_t*)sp)[-1]; - } else if (ip <= entry + 16 && isFrameComplete(entry, ip)) { - sp += nm->frameSize() * sizeof(void*); - fp = ((uintptr_t*)sp)[-2]; - pc = ((uintptr_t*)sp)[-1]; - } else { - pc = link(); - } - return true; -} - -NOSANALIGSANITIZE bool HotspotStackFrame::unwindEpilogue(VMNMethod* nm, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp) { - // ldp x29, x30, [sp, #32] - // add sp, sp, #0x30 - // {poll_return} - // ret - instruction_t* ip = (instruction_t*)pc; - if (*ip == 0xd65f03c0 || isPollReturn(ip)) { // ret - pc = link(); - return true; - } - return false; -} -NOSANALIGSANITIZE bool HotspotStackFrame::unwindStub(instruction_t* entry, const char* name, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp) { - instruction_t* ip = (instruction_t*)pc; - if (ip == entry || *ip == 0xd65f03c0) { - pc = link(); - return true; - } else if (entry != NULL && entry[0] == 0xa9bf7bfd) { - // The stub begins with - // stp x29, x30, [sp, #-16]! - // mov x29, sp - if (ip == entry + 1) { - sp += 16; - pc = ((uintptr_t*)sp)[-1]; - return true; - } else if (entry[1] == 0x910003fd && withinCurrentStack(fp)) { - sp = fp + 16; - fp = ((uintptr_t*)sp)[-2]; - pc = ((uintptr_t*)sp)[-1]; - return true; - } - } else if (entry != NULL && isSTP(entry[0]) && isFixedSizeFrame(name)) { - // The stub begins with - // stp xn, xm, [sp, #-imm]! - int offset = int(entry[0] << 10) >> 25; - sp = (intptr_t)sp - offset * 8; - pc = link(); - return true; - } else if (isZeroSizeFrame(name)) { - // Should be done after isSTP check, since frame size may vary between JVM versions - pc = link(); - return true; - } else if (strcmp(name, "forward_copy_longs") == 0 - || strcmp(name, "backward_copy_longs") == 0 - // There is a typo in JDK 8 - || strcmp(name, "foward_copy_longs") == 0) { - // These are called from arraycopy stub that maintains the regular frame link - if (&pc == &this->pc() && withinCurrentStack(fp)) { - // Unwind both stub frames for AsyncGetCallTrace - sp = fp + 16; - fp = ((uintptr_t*)sp)[-2]; - pc = ((uintptr_t*)sp)[-1] - sizeof(instruction_t); - } else { - // When cstack=vm, unwind stub frames one by one - pc = link(); - } - return true; - } - - // Generic fallback: detect standard aarch64 frame prologue - // stp x29, x30, [sp, #-16]! (0xa9bf7bfd) - // mov x29, sp (0x910003fd) - // This catches JVM stubs not in the hardcoded whitelist. - if (entry != NULL && withinCurrentStack(fp)) { - for (int i = 0; i < 8 && entry + i < ip; i++) { - if (entry[i] == 0xa9bf7bfd && i + 1 < 8 && entry + i + 1 < ip && entry[i + 1] == 0x910003fd) { - sp = fp + 16; - fp = ((uintptr_t*)sp)[-2]; - pc = ((uintptr_t*)sp)[-1]; - Counters::increment(WALKVM_STUB_GENERIC_UNWIND); - return true; - } - } - } - - return false; -} - -bool HotspotStackFrame::unwindAtomicStub(const StackFrame& frame, const void*& pc) { - // VM threads may call generated atomic stubs, which are not normally walkable - const void* lr = (const void*)frame.link(); - if (VMStructs::libjvm()->contains(lr)) { - VMNMethod* nm = CodeHeap::findNMethod(pc); - if (nm != NULL && strncmp(nm->name(), "Stub", 4) == 0) { - pc = lr; - return true; - } - } - return false; -} - -#endif // __aarch64__ diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotStackFrame_x64.cpp b/ddprof-lib/src/main/cpp/hotspot/hotspotStackFrame_x64.cpp deleted file mode 100644 index 24ab30b11..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/hotspotStackFrame_x64.cpp +++ /dev/null @@ -1,181 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifdef __x86_64__ - -#include -#include "hotspot/hotspotStackFrame.h" - -__attribute__((no_sanitize("address"))) bool HotspotStackFrame::unwindStub(instruction_t* entry, const char* name, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp) { - instruction_t* ip = (instruction_t*)pc; - if (ip == entry || *ip == 0xc3 - || strncmp(name, "itable", 6) == 0 - || strncmp(name, "vtable", 6) == 0 - || strcmp(name, "InlineCacheBuffer") == 0) - { - pc = ((uintptr_t*)sp)[0] - 1; - sp += 8; - return true; - } else if (entry != NULL && ([&] { unsigned int val; memcpy(&val, entry, sizeof(val)); return val; }()) == 0xec8b4855) { - // The stub begins with - // push rbp - // mov rbp, rsp - if (ip == entry + 1) { - pc = ((uintptr_t*)sp)[1] - 1; - sp += 16; - return true; - } else if (withinCurrentStack(fp)) { - sp = fp + 16; - fp = ((uintptr_t*)sp)[-2]; - pc = ((uintptr_t*)sp)[-1] - 1; - return true; - } - } - return false; -} - - -static inline bool isFrameComplete(instruction_t* entry, instruction_t* ip) { - // Frame is fully constructed after rsp is decremented by the frame size. - // Check if there is such an instruction anywhere between - // the method entry and the current instruction pointer. - for (ip -= 4; ip >= entry; ip--) { - if (ip[0] == 0x48 && ip[2] == 0xec && (ip[1] & 0xfd) == 0x81) { // sub rsp, frame_size - return true; - } - } - return false; -} - -static inline bool isPollReturn(instruction_t* ip) { - // JDK 17+ - // pop %rbp - // cmp 0x348(%r15),%rsp - // ja offset_32 - // ret - if (ip[0] == 0x49 && ip[1] == 0x3b && (ip[2] == 0x67 || ip[2] == 0xa7) && ip[-1] == 0x5d) { - // cmp, preceded by pop rbp - return true; - } else if (ip[0] == 0x0f && ip[1] == 0x87 && ip[6] == 0xc3) { - // ja, followed by ret - return true; - } - - // JDK 11 - // pop %rbp - // mov 0x108(%r15),%r10 - // test %eax,(%r10) - // ret - if (ip[0] == 0x4d && ip[1] == 0x8b && ip[2] == 0x97 && ip[-1] == 0x5d) { - // mov, preceded by pop rbp - return true; - } else if (ip[0] == 0x41 && ip[1] == 0x85 && ip[2] == 0x02 && ip[3] == 0xc3) { - // test, followed by ret - return true; - } - - // JDK 8 - // pop %rbp - // test %eax,offset(%rip) - // ret - if (ip[0] == 0x85 && ip[1] == 0x05 && ip[6] == 0xc3) { - // test, followed by ret - return true; - } - - return false; -} - -bool HotspotStackFrame::unwindCompiled(VMNMethod* nm, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp) { - instruction_t* ip = (instruction_t*)pc; - instruction_t* entry = (instruction_t*)nm->entry(); - if (ip <= entry - || *ip == 0xc3 // ret - || *ip == 0x55 // push rbp - || ip[-1] == 0x5d // after pop rbp - || (ip[0] == 0x41 && ip[1] == 0x85 && ip[2] == 0x02 && ip[3] == 0xc3)) // poll return - { - // Subtract 1 for PC to point to the call instruction, - // otherwise it may be attributed to a wrong bytecode - pc = ((uintptr_t*)sp)[0] - 1; - sp += 8; - return true; - } else if (*ip == 0x5d) { - // pop rbp - fp = ((uintptr_t*)sp)[0]; - pc = ((uintptr_t*)sp)[1] - 1; - sp += 16; - return true; - } else if (ip <= entry + 15 && ((uintptr_t)ip & 0xfff) && ip[-1] == 0x55) { - // push rbp - pc = ((uintptr_t*)sp)[1] - 1; - sp += 16; - return true; - } else if (ip <= entry + 7 && ip[0] == 0x48 && ip[1] == 0x89 && ip[2] == 0x6c && ip[3] == 0x24) { - // mov [rsp + #off], rbp - sp += ip[4] + 16; - pc = ((uintptr_t*)sp)[-1] - 1; - return true; - } else if ((ip[0] == 0x41 && ip[1] == 0x81 && ip[2] == 0x7f && *(u32*)(ip + 4) == 1) || - (ip >= entry + 8 && ip[-8] == 0x41 && ip[-7] == 0x81 && ip[-6] == 0x7f && *(u32*)(ip - 4) == 1)) { - // cmp [r15 + #off], 1 - // nmethod_entry_barrier: frame is fully constructed here - sp += nm->frameSize() * sizeof(void*); - fp = ((uintptr_t*)sp)[-2]; - pc = ((uintptr_t*)sp)[-1]; - return true; - } - return false; -} - -bool HotspotStackFrame::unwindPrologue(VMNMethod* nm, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp) { - // 0: mov %eax,-0x14000(%rsp) - // 7: push %rbp - // 8: mov %rsp,%rbp ; for native methods only - // 11: sub $0x50,%rsp - instruction_t* ip = (instruction_t*)pc; - instruction_t* entry = (instruction_t*)nm->entry(); - if (ip <= entry || *ip == 0x55 || nm->frameSize() == 0) { // push rbp - pc = ((uintptr_t*)sp)[0] - 1; - sp += 8; - return true; - } else if (ip <= entry + 15 && ip[-1] == 0x55) { // right after push rbp - pc = ((uintptr_t*)sp)[1] - 1; - sp += 16; - return true; - } else if (ip <= entry + 31 && isFrameComplete(entry, ip)) { - sp += nm->frameSize() * sizeof(void*); - fp = ((uintptr_t*)sp)[-2]; - pc = ((uintptr_t*)sp)[-1]; - return true; - } - return false; -} - -bool HotspotStackFrame::unwindEpilogue(VMNMethod* nm, uintptr_t& pc, uintptr_t& sp, uintptr_t& fp) { - // add $0x40,%rsp - // pop %rbp - // {poll_return} - // ret - instruction_t* ip = (instruction_t*)pc; - if (*ip == 0xc3 || isPollReturn(ip)) { // ret - pc = ((uintptr_t*)sp)[0] - 1; - sp += 8; - return true; - } else if (*ip == 0x5d) { // pop rbp - fp = ((uintptr_t*)sp)[0]; - pc = ((uintptr_t*)sp)[1] - 1; - sp += 16; - return true; - } - return false; -} - -bool HotspotStackFrame::unwindAtomicStub(const StackFrame& frame, const void*& pc) { - return false; -} - -#endif // __x86_64__ diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp deleted file mode 100644 index b0c034233..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.cpp +++ /dev/null @@ -1,1176 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include "asyncSampleMutex.h" -#include "hotspot/hotspotSupport.h" -#include "hotspot/jitCodeCache.h" -#include "hotspot/vmStructs.inline.h" -#include "jvmSupport.h" -#include "profiler.h" -#include "guards.h" -#include "stackWalker.inline.h" -#include "frames.h" - -using StackWalkValidation::inDeadZone; -using StackWalkValidation::aligned; -using StackWalkValidation::MAX_FRAME_SIZE; -using StackWalkValidation::sameStack; - -static bool isAddressInCode(const void *pc, bool include_stubs = true) { - if (CodeHeap::contains(pc)) { - return CodeHeap::findNMethod(pc) != NULL && - (include_stubs || !JitCodeCache::isCallStub(pc)); - } else { - return Profiler::instance()->libraries()->findLibraryByAddress(pc) != NULL; - } -} - -static jmethodID getMethodId(VMMethod* method) { - if (!inDeadZone(method) && aligned((uintptr_t)method) - && SafeAccess::isReadableRange(method, VMMethod::type_size())) { - return method->validatedId(); - } - return NULL; -} - -/** - * Converts a BCI_* frame type value to the corresponding EventType enum value. - * - * This conversion is necessary because Datadog's implementation uses BCI_* values - * (from ASGCT_CallFrameType) directly as event type identifiers, while upstream - * HotspotSupport::walkVM() expects EventType enum values for its logic. - * - * BCI_* values are special frame types with negative values (except BCI_CPU=0) - * that indicate non-standard frame information in call traces. EventType values - * are positive enum indices used for event categorization in the upstream code. - * - * @param bci_type A BCI_* value (e.g., BCI_CPU, BCI_WALL, BCI_ALLOC) - * @return The corresponding EventType enum value - */ -inline EventType eventTypeFromBCI(jint bci_type) { - switch (bci_type) { - case BCI_CPU: - return EXECUTION_SAMPLE; // CPU samples map to execution samples - case BCI_WALL: - return WALL_CLOCK_SAMPLE; - case BCI_ALLOC: - return ALLOC_SAMPLE; - case BCI_ALLOC_OUTSIDE_TLAB: - return ALLOC_OUTSIDE_TLAB; - case BCI_LIVENESS: - return LIVE_OBJECT; - case BCI_LOCK: - return LOCK_SAMPLE; - case BCI_PARK: - return PARK_SAMPLE; - case BCI_NATIVE_MALLOC: - return MALLOC_SAMPLE; - default: - // For unknown or invalid BCI types, default to EXECUTION_SAMPLE - // This maintains backward compatibility and prevents undefined behavior - return EXECUTION_SAMPLE; - } -} - -static void fillFrameTypes(ASGCT_CallFrame *frames, int num_frames, VMNMethod *nmethod) { - if (nmethod->isNMethod() && nmethod->isAlive()) { - VMMethod *method = nmethod->method(); - if (method == NULL) { - return; - } - - jmethodID current_method_id = method->id(); - if (current_method_id == NULL) { - return; - } - - // Mark current_method as COMPILED and frames above current_method as - // INLINED - for (int i = 0; i < num_frames; i++) { - if (frames[i].method_id == NULL || frames[i].bci <= BCI_NATIVE_FRAME) { - break; - } - if (frames[i].method_id == current_method_id) { - int level = nmethod->level(); - frames[i].bci = FrameType::encode( - level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED, - frames[i].bci); - for (int j = 0; j < i; j++) { - frames[j].bci = FrameType::encode(FRAME_INLINED, frames[j].bci); - } - break; - } - } - } else if (nmethod->isInterpreter()) { - // Mark the first Java frame as INTERPRETED - for (int i = 0; i < num_frames; i++) { - if (frames[i].bci > BCI_NATIVE_FRAME) { - frames[i].bci = FrameType::encode(FRAME_INTERPRETED, frames[i].bci); - break; - } - } - } -} - -static ucontext_t empty_ucontext{}; - -#ifdef NDEBUG -static const bool CONT_UNWIND_DISABLED = false; -#else -// DEBUG-only: when set, both continuation-unwind detection branches -// (cont_entry_return_pc for fully-thawed VTs, cont_returnBarrier for VTs -// with frozen frames) are skipped, reproducing pre-fix behaviour. -// Used by negative integration tests to verify that carrier frames are not -// visible and walk-error sentinels do appear without the fix. -// NOTE: the env var is evaluated once at library load time; it must be set -// in the environment before the profiler agent is attached. -static const bool CONT_UNWIND_DISABLED = (std::getenv("DDPROF_DISABLE_CONT_UNWIND") != nullptr); -#endif - -__attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, - StackWalkFeatures features, EventType event_type, int lock_index, bool* truncated) { - if (ucontext == NULL) { - return walkVM(&empty_ucontext, frames, max_depth, features, event_type, - callerPC(), (uintptr_t)callerSP(), (uintptr_t)callerFP(), lock_index, truncated); - } else { - HotspotStackFrame frame(ucontext); - return walkVM(ucontext, frames, max_depth, features, event_type, - (const void*)frame.pc(), frame.sp(), frame.fp(), lock_index, truncated); - } -} - -__attribute__((no_sanitize("address"))) int HotspotSupport::walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, - StackWalkFeatures features, EventType event_type, - const void* pc, uintptr_t sp, uintptr_t fp, int lock_index, bool* truncated) { - // VMStructs is only available for hotspot JVM - assert(VM::isHotspot()); - HotspotStackFrame frame(ucontext); - uintptr_t bottom = (uintptr_t)&frame + MAX_WALK_SIZE; - - Profiler* profiler = Profiler::instance(); - int bcp_offset = InterpreterFrame::bcp_offset(); - - jmp_buf crash_protection_ctx; - VMThread* vm_thread = VMThread::current(); - if (vm_thread != NULL && !vm_thread->isThreadAccessible()) { - Counters::increment(WALKVM_THREAD_INACCESSIBLE); - vm_thread = NULL; - } - if (vm_thread == NULL) { - Counters::increment(WALKVM_NO_VMTHREAD); - } else { - Counters::increment(WALKVM_VMTHREAD_OK); - } - void* saved_exception = vm_thread != NULL ? vm_thread->exception() : NULL; - - // Should be preserved across setjmp/longjmp - volatile int depth = 0; - int actual_max_depth = truncated ? max_depth + 1 : max_depth; - bool fp_chain_fallback = false; - int fp_chain_depth = 0; - - ProfiledThread* profiled_thread = ProfiledThread::currentSignalSafe(); - - VMJavaFrameAnchor* anchor = NULL; - if (vm_thread != NULL) { - anchor = vm_thread->anchor(); - if (anchor == NULL) { - Counters::increment(WALKVM_ANCHOR_NULL); - } - vm_thread->exception() = &crash_protection_ctx; - if (profiled_thread != nullptr) { - profiled_thread->setCrashProtectionActive(true); - } - if (setjmp(crash_protection_ctx) != 0) { - // checkFault() does a longjmp from inside segvHandler, bypassing - // segvHandler's SignalHandlerScope destructor. Compensate. - SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP(); - if (profiled_thread != nullptr) { - profiled_thread->setCrashProtectionActive(false); - } - vm_thread->exception() = saved_exception; - if (depth < max_depth) { - fillFrame(frames[depth++], BCI_ERROR, "break_not_walkable"); - } - return depth; - } - } - - const void* prev_native_pc = NULL; - - // Last ContinuationEntry crossed; advanced via parent() for nested continuations. - VMContinuationEntry* cont_entry = nullptr; - - // Saved anchor data — preserved across anchor consumption so inline - // recovery can redirect even after the anchor pointer has been set to NULL. - // Recovery is one-shot: once attempted, we do not retry to avoid - // ping-ponging between CodeHeap and unmapped native regions. - const void* saved_anchor_pc = NULL; - uintptr_t saved_anchor_sp = 0; - uintptr_t saved_anchor_fp = 0; - bool anchor_recovery_used = false; - - // Show extended frame types and stub frames for execution-type events - bool details = event_type <= MALLOC_SAMPLE || features.mixed; - - if (details && vm_thread != NULL && VMThread::isJavaThread(vm_thread)) { - anchor = vm_thread->anchor(); - } - - static const char* CONT_ROOT_FRAME = "JVM Continuation"; - - // Advances through a continuation boundary to the carrier frame. - // Without carrier_frames (default, cstack=vm): always stops with a "JVM Continuation" - // synthetic root frame — VT frames are complete, carrier internals are noise. - // With carrier_frames (cstack=vmx): attempts to walk through; failures emit BCI_ERROR - // so the sample is truthfully marked truncated. - // Walks cont_entry->parent() on repeated calls to handle nested continuations - // (_parent not triggered by standard single-level VTs today, but required - // once any runtime layers continuations on top of VTs). - // - // all_frames_thawed: true when the bottom VT frame's return PC is - // cont_entry_return_pc (all VT frames are thawed — CPU-bound VT), - // false when it is cont_returnBarrier (frozen frames remain in the - // StackChunk — VT parked and just remounted). - // Needed to derive entry_fp on JDK 21-26 where ContinuationEntry - // type size is absent from vmStructs and contEntry() returns nullptr. - // - // Returns true to continue the walk, false to break. - auto walkThroughContinuation = [&](bool all_frames_thawed) -> bool { - if (depth >= actual_max_depth) return false; - if (!features.carrier_frames) { - fillFrame(frames[depth++], BCI_NATIVE_FRAME, CONT_ROOT_FRAME); - return false; - } - - uintptr_t entry_fp; - - if (VMContinuationEntry::type_size() > 0) { - // ContinuationEntry is known via vmStructs (JDK 27+, added by - // JDK-8378985). Walk the linked list of entries for nested- - // continuation support and derive the enterSpecial frame FP from - // the struct layout (entry + type_size). - cont_entry = (cont_entry != nullptr) ? cont_entry->parent() : vm_thread->contEntry(); - if (cont_entry == nullptr) { - Counters::increment(WALKVM_CONT_ENTRY_NULL); - fillFrame(frames[depth++], BCI_ERROR, "break_cont_entry_null"); - return false; - } - entry_fp = cont_entry->entryFP(); - } else { - // ContinuationEntry absent from vmStructs (JDK 21-26). - // Derive the enterSpecial frame FP from the current fp: - // all frames thawed (pc == cont_entry_return_pc): fp IS the - // enterSpecial frame FP. - // frozen frames remain (pc == cont_returnBarrier): the saved - // caller FP at *fp leads to the enterSpecial frame on the - // carrier stack. - // Nested continuation tracking is unavailable without type_size(). - entry_fp = all_frames_thawed ? fp : (uintptr_t)SafeAccess::load((void**)fp); - } - - if (!StackWalkValidation::isValidFP(entry_fp)) { - fillFrame(frames[depth++], BCI_ERROR, "break_cont_entry_fp"); - return false; - } - // entry_fp has been range-checked by isValidFP above; any remaining - // SIGSEGV from a stale/concurrently-freed pointer is caught by the - // setjmp crash protection in walkVM (checkFault -> longjmp). - uintptr_t carrier_fp = *(uintptr_t*)entry_fp; - const void* carrier_pc = ((const void**)entry_fp)[FRAME_PC_SLOT]; - uintptr_t carrier_sp = entry_fp + (FRAME_PC_SLOT + 1) * sizeof(void*); - if (!StackWalkValidation::isValidFP(carrier_fp) || - StackWalkValidation::inDeadZone(carrier_pc) || - !StackWalkValidation::isValidSP(carrier_sp, sp, bottom)) { - fillFrame(frames[depth++], BCI_ERROR, "break_cont_carrier_sp"); - return false; - } - sp = carrier_sp; - fp = carrier_fp; - pc = carrier_pc; - return true; - }; - - unwind_loop: - - // Walk until the bottom of the stack or until the first Java frame - while (depth < actual_max_depth) { - if (CodeHeap::contains(pc)) { - Counters::increment(WALKVM_HIT_CODEHEAP); - if (fp_chain_fallback) { - Counters::increment(WALKVM_FP_CHAIN_REACHED_CODEHEAP); - fp_chain_fallback = false; - fp_chain_depth = 0; - } - // If we're in JVM-generated code but don't have a VMThread, we cannot safely - // walk the Java stack because crash protection is not set up. - // - // This can occur during JNI attach/detach transitions: when a thread detaches, - // pthread_setspecific() clears the VMThread TLS, but if a profiling signal arrives - // while PC is still in JVM stubs (JavaCalls, method entry/exit), we see CodeHeap - // code without VMThread context. - // - // Without vm_thread, crash protection via setjmp/longjmp cannot work - // (checkFault() needs vm_thread->exception() to longjmp). Any memory dereference in interpreter - // frame handling or NMethod validation would crash the process with unrecoverable SEGV. - // - // The missing VMThread is a timing issue during thread lifecycle. - if (vm_thread == NULL) { - Counters::increment(WALKVM_CODEH_NO_VM); - fillFrame(frames[depth++], BCI_ERROR, "break_no_vmthread"); - break; - } - prev_native_pc = NULL; // we are in JVM code, no previous 'native' PC - // Both continuation boundary PCs are JVM stubs whose findNMethod() - // returns NULL; detect them by exact-PC match before the nmethod - // dispatch below. - // cont_returnBarrier: bottom thawed frame returns here when frozen - // frames remain in the StackChunk (blocking/remounted VT). - // cont_entry_return_pc: bottom thawed frame returns here when the - // continuation is fully thawed (CPU-bound VT, never yielded). - if (!CONT_UNWIND_DISABLED && VMStructs::isContReturnBarrier(pc)) { - Counters::increment(WALKVM_CONT_BARRIER_HIT); - if (walkThroughContinuation(false)) continue; - break; - } - if (!CONT_UNWIND_DISABLED && VMStructs::isContEntryReturnPc(pc)) { - Counters::increment(WALKVM_ENTER_SPECIAL_HIT); - if (walkThroughContinuation(true)) continue; - break; - } - VMNMethod* nm = CodeHeap::findNMethod(pc); - if (nm == NULL) { - // On JDK 21+ builds, the continuation entry PC may be absent - // from vmStructs OR resolved but pointing to the wrong address - // (some distributions expose the symbol at the wrong address, so - // the exact-PC check above never fires). Attempt a fully-thawed - // continuation walk whenever we see an unknown nmethod after - // collecting Java frames. walkThroughContinuation validates the - // fp chain and emits BCI_ERROR cleanly on mismatch, so false - // positives are safe. - if (!CONT_UNWIND_DISABLED - && features.carrier_frames - && VM::hotspot_version() >= 21 - && depth > 0 - && vm_thread != NULL && vm_thread->isCarryingVirtualThread()) { - Counters::increment(WALKVM_CONT_SPECULATIVE_HIT); - if (walkThroughContinuation(true)) continue; - break; - } - if (anchor == NULL) { - // Add an error frame only if we cannot recover - fillFrame(frames[depth++], BCI_ERROR, "unknown_nmethod"); - } - break; - } - - // Always prefer JavaFrameAnchor when it is available, - // since it provides reliable SP and FP. - // Do not treat the topmost stub as Java frame. - // Exception: when VT carrier-frame unwinding is active, skip the anchor - // redirect — it can bypass the continuation boundary by jumping directly - // into carrier frames, causing walkThroughContinuation to never fire. - // The continuation mechanism finds carrier frames on its own. - bool anchor_eligible = anchor != NULL && (depth > 0 || !nm->isStub()); - bool cont_unwind_active = features.carrier_frames && !CONT_UNWIND_DISABLED - && vm_thread != NULL && vm_thread->isCarryingVirtualThread(); - if (anchor_eligible && !cont_unwind_active) { - Counters::increment(WALKVM_ANCHOR_CONSUMED); - // Preserve anchor data before consumption — getFrame() is read-only - // but we set anchor=NULL below, losing the pointer for later recovery. - if (saved_anchor_sp == 0) { - saved_anchor_pc = anchor->lastJavaPC(); - saved_anchor_sp = anchor->lastJavaSP(); - saved_anchor_fp = anchor->lastJavaFP(); - } - if (anchor->getFrame(pc, sp, fp) && !nm->contains(pc)) { - anchor = NULL; - continue; // NMethod has changed as a result of correction - } - anchor = NULL; - } else if (anchor_eligible && cont_unwind_active) { - // Clear the anchor without redirecting so it doesn't corrupt fp - // for the continuation boundary walk. - anchor = NULL; - } - - if (nm->isInterpreter()) { - if (vm_thread != NULL && vm_thread->inDeopt()) { - fillFrame(frames[depth++], BCI_ERROR, "break_deopt"); - break; - } - - bool is_plausible_interpreter_frame = StackWalkValidation::isPlausibleInterpreterFrame(fp, sp, bcp_offset); - if (is_plausible_interpreter_frame) { - VMMethod* method = ((VMMethod**)fp)[InterpreterFrame::method_offset]; - jmethodID method_id = getMethodId(method); - if (method_id != NULL) { - Counters::increment(WALKVM_JAVA_FRAME_OK); - const char* bytecode_start = method->bytecode(); - const char* bcp = ((const char**)fp)[bcp_offset]; - int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start; - fillFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id); - - sp = ((uintptr_t*)fp)[InterpreterFrame::sender_sp_offset]; - pc = stripPointer(((void**)fp)[FRAME_PC_SLOT]); - fp = *(uintptr_t*)fp; - continue; - } - } - - if (depth == 0) { - VMMethod* method = (VMMethod*)frame.method(); - jmethodID method_id = getMethodId(method); - if (method_id != NULL) { - Counters::increment(WALKVM_JAVA_FRAME_OK); - fillFrame(frames[depth++], FRAME_INTERPRETED, 0, method_id); - - if (is_plausible_interpreter_frame) { - pc = stripPointer(((void**)fp)[FRAME_PC_SLOT]); - sp = frame.senderSP(); - fp = *(uintptr_t*)fp; - } else { - pc = stripPointer(SafeAccess::load((void**)sp)); - sp = frame.senderSP(); - } - continue; - } - } - - Counters::increment(WALKVM_BREAK_INTERPRETED); - fillFrame(frames[depth++], BCI_ERROR, "break_interpreted"); - break; - } else if (nm->isNMethod()) { - // enterSpecial is a generated native nmethod that acts as the - // continuation entry stub on JDK 27+. It has no JavaCallWrapper, so - // isEntryFrame() will not fire for it. Detect it by identity - // and navigate to the carrier thread via ContinuationEntry. - if (!CONT_UNWIND_DISABLED && nm == VMStructs::enterSpecialNMethod()) { - Counters::increment(WALKVM_ENTER_SPECIAL_HIT); - if (walkThroughContinuation(true)) continue; - break; - } - // Check if deoptimization is in progress before walking compiled frames - if (vm_thread != NULL && vm_thread->inDeopt()) { - fillFrame(frames[depth++], BCI_ERROR, "break_deopt_compiled"); - break; - } - - Counters::increment(WALKVM_JAVA_FRAME_OK); - int level = nm->level(); - FrameTypeId type = details && level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED; - fillFrame(frames[depth++], type, 0, nm->method()->id()); - - if (nm->isFrameCompleteAt(pc)) { - if (depth == 1 && frame.unwindEpilogue(nm, (uintptr_t&)pc, sp, fp)) { - continue; - } - - int scope_offset = nm->findScopeOffset(pc); - if (scope_offset > 0) { - depth--; - ScopeDesc scope(nm); - do { - scope_offset = scope.decode(scope_offset); - if (details) { - type = scope_offset > 0 ? FRAME_INLINED : - level >= 1 && level <= 3 ? FRAME_C1_COMPILED : FRAME_JIT_COMPILED; - } - fillFrame(frames[depth++], type, scope.bci(), scope.method()->id()); - } while (scope_offset > 0 && depth < max_depth); - } - - // Handle situations when sp is temporarily changed in the compiled code - frame.adjustSP(nm->entry(), pc, sp); - - // Validate NMethod metadata before using frameSize() - int frame_size = nm->frameSize(); - if (frame_size <= 0 || frame_size > MAX_FRAME_SIZE_WORDS) { - fillFrame(frames[depth++], BCI_ERROR, "break_invalid_framesize"); - break; - } - - sp += frame_size * sizeof(void*); - - // Verify alignment before dereferencing sp as pointer (secondary defense) - if (!aligned(sp)) { - fillFrame(frames[depth++], BCI_ERROR, "break_misaligned_sp"); - break; - } - - fp = ((uintptr_t*)sp)[-FRAME_PC_SLOT - 1]; - pc = ((const void**)sp)[-FRAME_PC_SLOT]; - continue; - } else if (frame.unwindPrologue(nm, (uintptr_t&)pc, sp, fp)) { - continue; - } - - Counters::increment(WALKVM_BREAK_COMPILED); - fillFrame(frames[depth++], BCI_ERROR, "break_compiled"); - break; - } else if (nm->isEntryFrame(pc) && !features.mixed) { - VMJavaFrameAnchor* next_anchor = VMJavaFrameAnchor::fromEntryFrame(fp); - if (next_anchor == NULL) { - fillFrame(frames[depth++], BCI_ERROR, "break_entry_frame"); - break; - } - uintptr_t prev_sp = sp; - if (!next_anchor->getFrame(pc, sp, fp)) { - // End of Java stack - break; - } - if (sp < prev_sp || sp >= bottom || !aligned(sp)) { - fillFrame(frames[depth++], BCI_ERROR, "break_entry_frame"); - break; - } - continue; - } else { - if (features.vtable_target && nm->isVTableStub() && depth == 0) { - uintptr_t receiver = frame.jarg0(); - if (receiver != 0) { - VMSymbol* symbol = VMKlass::fromOop(receiver)->name(); - // Store the raw VMSymbol* in the frame's method_id - // slot. BCI_VTABLE_RECEIVER (vmEntry.h) repurposes - // method_id for this pointer — same precedent as - // BCI_NATIVE_FRAME storing const char* and - // BCI_NATIVE_FRAME_REMOTE storing a packed blob. - // Resolution happens at dump time via SafeAccess so - // a concurrent class-unload + Symbol free cannot - // crash the dump thread (see Lookup::resolveVTableReceiver). - if (symbol != nullptr) { - fillFrame(frames[depth++], BCI_VTABLE_RECEIVER, (void*)symbol); - } - } - } - - CodeBlob* stub = JitCodeCache::findRuntimeStub(pc); - const void* start = stub != NULL ? stub->_start : nm->code(); - const char* name = stub != NULL ? stub->_name : nm->name(); - - if (details) { - fillFrame(frames[depth++], BCI_NATIVE_FRAME, name); - } - - if (frame.unwindStub((instruction_t*)start, name, (uintptr_t&)pc, sp, fp)) { - continue; - } - - if (depth > 0 && nm->frameSize() > 0) { - Counters::increment(WALKVM_STUB_FRAMESIZE_FALLBACK); - // Validate NMethod metadata before using frameSize() - int frame_size = nm->frameSize(); - if (frame_size <= 0 || frame_size > MAX_FRAME_SIZE_WORDS) { - fillFrame(frames[depth++], BCI_ERROR, "break_invalid_framesize"); - break; - } - - sp += frame_size * sizeof(void*); - - // Verify alignment before dereferencing sp as pointer (secondary defense) - if (!aligned(sp)) { - fillFrame(frames[depth++], BCI_ERROR, "break_misaligned_sp"); - break; - } - - fp = ((uintptr_t*)sp)[-FRAME_PC_SLOT - 1]; - pc = ((const void**)sp)[-FRAME_PC_SLOT]; - continue; - } - } - } else { - // Resolve native frame (may use remote symbolication if enabled) - Profiler::NativeFrameResolution resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)pc, lock_index); - if (resolution.is_marked) { - // This is a marked C++ interpreter frame, terminate scan - break; - } - const char* method_name = resolution.method_name; - int frame_bci = resolution.bci; - char mark; - if (frame_bci != BCI_NATIVE_FRAME_REMOTE && method_name != NULL && (mark = NativeFunc::read_mark(method_name)) != 0) { - if (mark == MARK_ASYNC_PROFILER && event_type == MALLOC_SAMPLE) { - // Skip all internal frames above malloc_hook functions, leave the hook itself - depth = 0; - } else if (mark == MARK_COMPILER_ENTRY && features.comp_task && vm_thread != NULL) { - // Insert current compile task as a pseudo Java frame - VMMethod* method = vm_thread->compiledMethod(); - jmethodID method_id = method != NULL ? method->id() : NULL; - if (method_id != NULL) { - fillFrame(frames[depth++], FRAME_JIT_COMPILED, 0, method_id); - } - } else if (mark == MARK_THREAD_ENTRY) { - // Thread entry point detected via pre-computed mark - this is the root frame - // No need for expensive symbol resolution, just stop unwinding - Counters::increment(THREAD_ENTRY_MARK_DETECTIONS); - break; - } - } else if (method_name == NULL && details && !anchor_recovery_used - && profiler->findLibraryByAddress(pc) == NULL) { - // Try anchor recovery — prefer live anchor, fall back to saved data - anchor_recovery_used = true; - const void* recovery_pc = NULL; - uintptr_t recovery_sp = 0; - uintptr_t recovery_fp = 0; - bool have_anchor_data = false; - - if (anchor) { - Counters::increment(WALKVM_ANCHOR_USED_INLINE); - recovery_fp = anchor->lastJavaFP(); - recovery_sp = anchor->lastJavaSP(); - recovery_pc = anchor->lastJavaPC(); - have_anchor_data = true; - } else if (saved_anchor_sp != 0) { - Counters::increment(WALKVM_SAVED_ANCHOR_USED); - recovery_fp = saved_anchor_fp; - recovery_sp = saved_anchor_sp; - recovery_pc = saved_anchor_pc; - have_anchor_data = true; - // Clear saved data after use — one-shot recovery - saved_anchor_sp = 0; - } else { - Counters::increment(WALKVM_ANCHOR_INLINE_NO_ANCHOR); - } - - if (have_anchor_data) { - // Try to read the Java method directly from the anchor's FP, - // treating it as an interpreter frame. - // In HotSpot, lastJavaFP is non-zero only for interpreter frames; - // compiled frames record FP=0 in the anchor. - if (StackWalkValidation::isPlausibleInterpreterFrame(recovery_fp, recovery_sp, bcp_offset)) { - VMMethod* method = ((VMMethod**)recovery_fp)[InterpreterFrame::method_offset]; - jmethodID method_id = getMethodId(method); - if (method_id != NULL) { - anchor = NULL; - prev_native_pc = NULL; - if (depth > 0 && depth + 1 < actual_max_depth) { - fillFrame(frames[depth++], BCI_ERROR, "[skipped frames]"); - } - Counters::increment(WALKVM_JAVA_FRAME_OK); - const char* bytecode_start = method->bytecode(); - const char* bcp = ((const char**)recovery_fp)[bcp_offset]; - int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start; - fillFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id); - - sp = ((uintptr_t*)recovery_fp)[InterpreterFrame::sender_sp_offset]; - pc = stripPointer(((void**)recovery_fp)[FRAME_PC_SLOT]); - fp = *(uintptr_t*)recovery_fp; - continue; - } - } - - // Fallback: redirect via recovery SP/FP/PC - sp = recovery_sp; - fp = recovery_fp; - pc = recovery_pc; - if (pc != NULL && !CodeHeap::contains(pc) && sp != 0 && aligned(sp) && sp < bottom) { - pc = ((const void**)sp)[-1]; - } - if (sp != 0 && pc != NULL) { - anchor = NULL; - if (sp >= bottom || !aligned(sp)) { - Counters::increment(WALKVM_ANCHOR_INLINE_BAD_SP); - fillFrame(frames[depth++], BCI_ERROR, "break_no_anchor"); - break; - } - prev_native_pc = NULL; - if (depth > 0) { - fillFrame(frames[depth++], BCI_ERROR, "[skipped frames]"); - } - continue; - } - Counters::increment(WALKVM_ANCHOR_INLINE_NO_SP); - } - // Check previous frame for thread entry points (Rust, libc/pthread) - // Only check marks for traditionally-resolved frames; packed remote - // frames store an integer in the method_name union, not a valid pointer. - if (prev_native_pc != NULL) { - Profiler::NativeFrameResolution prev_resolution = profiler->resolveNativeFrameForWalkVM((uintptr_t)prev_native_pc, lock_index); - if (prev_resolution.bci != BCI_NATIVE_FRAME_REMOTE) { - const char* prev_method_name = prev_resolution.method_name; - if (prev_method_name != NULL) { - char prev_mark = NativeFunc::read_mark(prev_method_name); - if (prev_mark == MARK_THREAD_ENTRY) { - Counters::increment(THREAD_ENTRY_MARK_DETECTIONS); - break; - } - } - } - } - // Fall through to DWARF section — when findLibraryByAddress(pc) - // returns NULL, default_frame uses FP-chain walking (DW_REG_FP) - // which can bridge symbol-less gaps in libjvm.so. - Counters::increment(WALKVM_FP_CHAIN_ATTEMPT); - fp_chain_fallback = true; - if (++fp_chain_depth > actual_max_depth) { - break; - } - goto dwarf_unwind; - } - fillFrame(frames[depth++], frame_bci, (void*)method_name); - } - - dwarf_unwind: - uintptr_t prev_sp = sp; - CodeCache* cc = profiler->findLibraryByAddress(pc); - FrameDesc f = cc != NULL ? cc->findFrameDesc(pc) : FrameDesc::fallback_default_frame(); - - u8 cfa_reg = (u8)f.cfa; - int cfa_off = f.cfa >> 8; - - // If DWARF is invalid, we cannot continue unwinding reliably - // Thread entry points are detected earlier via MARK_THREAD_ENTRY - if (cfa_reg == DW_REG_INVALID || cfa_reg > DW_REG_PLT) { - break; - } - - if (cfa_reg == DW_REG_SP) { - sp = sp + cfa_off; - } else if (cfa_reg == DW_REG_FP) { - // Sanity-check FP before deriving CFA from it. A corrupted FP can produce a - // phantom CFA and cause the walk to record spurious frames before breaking. - // We cannot check fp < sp here because on aarch64 the frame pointer is set - // to SP at function entry, which is typically less than the previous CFA. - if (fp >= bottom || !aligned(fp)) { - break; - } - sp = fp + cfa_off; - } else if (cfa_reg == DW_REG_PLT) { - sp += ((uintptr_t)pc & 15) >= 11 ? cfa_off * 2 : cfa_off; - } - - // Check if the next frame is below on the current stack - if (sp < prev_sp || sp >= prev_sp + MAX_FRAME_SIZE || sp >= bottom) { - break; - } - - // Stack pointer must be word aligned - if (!aligned(sp)) { - break; - } - - // store the previous pc before unwinding - prev_native_pc = pc; - if (f.fp_off & DW_PC_OFFSET) { - pc = (const char*)pc + (f.fp_off >> 1); - } else { - if (f.fp_off != DW_SAME_FP && f.fp_off < MAX_FRAME_SIZE && f.fp_off > -MAX_FRAME_SIZE) { - fp = (uintptr_t)SafeAccess::load((void**)(sp + f.fp_off)); - } - - if (EMPTY_FRAME_SIZE > 0 || f.pc_off != DW_LINK_REGISTER) { - // Verify alignment before dereferencing sp + offset - uintptr_t pc_addr = sp + f.pc_off; - if (!aligned(pc_addr)) { - break; - } - pc = stripPointer(SafeAccess::load((void**)pc_addr)); - } else if (depth == 1) { - pc = (const void*)frame.link(); - } else { - break; - } - - if (EMPTY_FRAME_SIZE == 0 && cfa_off == 0 && f.fp_off != DW_SAME_FP) { - // AArch64 default_frame - sp = defaultSenderSP(sp, fp); - if (sp < prev_sp || sp >= bottom || !aligned(sp)) { - break; - } - } - } - - if (inDeadZone(pc) || (pc == prev_native_pc && sp == prev_sp)) { - break; - } - } - - // If we did not meet Java frame but current thread has JavaFrameAnchor set, - // try to read the interpreter frame directly from the anchor's FP. - // In HotSpot, lastJavaFP != 0 reliably indicates an interpreter frame. - if (anchor != NULL) { - uintptr_t anchor_fp = anchor->lastJavaFP(); - uintptr_t anchor_sp = anchor->lastJavaSP(); - if (anchor_sp == 0) { - Counters::increment(WALKVM_ANCHOR_NOT_IN_JAVA); - goto done; - } - if (StackWalkValidation::isPlausibleInterpreterFrame(anchor_fp, anchor_sp, bcp_offset)) { - VMMethod* method = ((VMMethod**)anchor_fp)[InterpreterFrame::method_offset]; - jmethodID method_id = getMethodId(method); - if (method_id != NULL) { - Counters::increment(WALKVM_ANCHOR_FALLBACK); - Counters::increment(WALKVM_JAVA_FRAME_OK); - anchor = NULL; - while (depth > 0 && frames[depth - 1].method_id == NULL) depth--; - if (depth < actual_max_depth) { - const char* bytecode_start = method->bytecode(); - const char* bcp = ((const char**)anchor_fp)[bcp_offset]; - int bci = bytecode_start == NULL || bcp < bytecode_start ? 0 : bcp - bytecode_start; - fillFrame(frames[depth++], FRAME_INTERPRETED, bci, method_id); - sp = ((uintptr_t*)anchor_fp)[InterpreterFrame::sender_sp_offset]; - pc = stripPointer(((void**)anchor_fp)[FRAME_PC_SLOT]); - fp = *(uintptr_t*)anchor_fp; - if (sp != 0 && sp < bottom && aligned(sp)) { - goto unwind_loop; - } - } - } - } - // Fallback: redirect via anchor frame and sp[-1] - if (anchor != NULL && anchor->getFrame(pc, sp, fp)) { - if (!CodeHeap::contains(pc) && sp != 0 && aligned(sp) && sp < bottom) { - pc = ((const void**)sp)[-1]; - } - Counters::increment(WALKVM_ANCHOR_FALLBACK); - anchor = NULL; - while (depth > 0 && frames[depth - 1].method_id == NULL) depth--; - if (sp != 0 && sp < bottom && aligned(sp)) { - goto unwind_loop; - } - } else if (anchor != NULL) { - Counters::increment(WALKVM_ANCHOR_FALLBACK_FAIL); - } - } - - done: - if (profiled_thread != nullptr) { - profiled_thread->setCrashProtectionActive(false); - } - if (vm_thread != NULL) { - vm_thread->exception() = saved_exception; - } - - // Drop unknown leaf frame - it provides no useful information and breaks - // aggregation by lumping unrelated samples under a single "unknown" entry - depth = StackWalkValidation::dropUnknownLeaf(frames, depth); - - if (depth == 0) { - Counters::increment(WALKVM_DEPTH_ZERO); - } - - if (truncated) { - if (depth > max_depth) { - *truncated = true; - depth = max_depth; - } else if (depth > 0) { - if (frames[depth - 1].bci == BCI_ERROR) { - // root frame is error; best guess is that the trace is truncated - *truncated = true; - } - } - } - - return depth; -} - -void HotspotSupport::checkFault(ProfiledThread* thrd) { - if (!JVMThread::isInitialized()) { - // JVM has not been loaded or has not been initialized yet - return; - } - - VMThread* vm_thread = VMThread::current(); - if (vm_thread == NULL || !vm_thread->isThreadAccessible()) { - return; - } - - // Prefer the semantic crash protection flag (reliable regardless of stack frame sizes). - // Fall back to sameStack heuristic when ProfiledThread TLS is unavailable (e.g. during - // early init or in crash recovery tests). sameStack uses a fixed 8KB threshold which - // can fail with ASAN-inflated frames, but the crashProtectionActive path handles that. - bool protected_walk = (thrd != nullptr && thrd->isCrashProtectionActive()) - || sameStack(vm_thread->exception(), &vm_thread); - if (!protected_walk) { - return; - } - - if (thrd != nullptr) { - thrd->resetCrashHandler(); - } - longjmp(*(jmp_buf*)vm_thread->exception(), 1); -} - - -int HotspotSupport::getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames, - int max_depth, StackContext *java_ctx, - bool *truncated) { - // Workaround for JDK-8132510: it's not safe to call GetEnv() inside a signal - // handler since JDK 9, so we do it only for threads already registered in - // ThreadLocalStorage - VMThread *vm_thread = VMThread::current(); - if (vm_thread == NULL || !vm_thread->isThreadAccessible()) { - Counters::increment(AGCT_NOT_REGISTERED_IN_TLS); - return 0; - } - - JNIEnv *jni = VM::jni(); - if (jni == NULL) { - // Not a Java thread - Counters::increment(AGCT_NOT_JAVA); - return 0; - } - - HotspotStackFrame frame(ucontext); - uintptr_t saved_pc = 0, saved_sp = 0, saved_fp = 0; - if (ucontext != NULL) { - saved_pc = frame.pc(); - saved_sp = frame.sp(); - saved_fp = frame.fp(); - - if (JitCodeCache::isCallStub((const void *)saved_pc)) { - // call_stub is unsafe to walk - frames->bci = BCI_ERROR; - frames->method_id = (jmethodID) "call_stub"; - return 1; - } - - if (!VMStructs::isSafeToWalk(saved_pc)) { - frames->bci = BCI_NATIVE_FRAME; - CodeBlob *codeBlob = - VMStructs::libjvm()->findBlobByAddress((const void *)saved_pc); - if (codeBlob) { - frames->method_id = (jmethodID)codeBlob->_name; - } else { - frames->method_id = (jmethodID) "unknown_unwalkable"; - } - return 1; - } - } - // Ported from upstream async-profiler (Profiler::getJavaTraceAsync in - // src/profiler.cpp): when ucontext is NULL — as it is for malloc hooks, - // which run outside any signal context — skip the PC-dependent pre-checks - // and fall through to ASGCT. ASGCT then resolves the top Java frame from - // JavaThread::last_Java_sp / last_Java_pc, which the JVM populates on every - // Java → native transition. - - JVMJavaThreadState state = vm_thread->state(); - bool in_java = (state == _thread_in_Java || state == _thread_in_Java_trans); - if (in_java && java_ctx->sp != 0) { - // skip ahead to the Java frames before calling AGCT - frame.restore((uintptr_t)java_ctx->pc, java_ctx->sp, java_ctx->fp); - } else if (state != _thread_uninitialized) { - VMJavaFrameAnchor* a = vm_thread->anchor(); - if (a == nullptr || a->lastJavaSP() == 0) { - // we haven't found the top Java frame ourselves, and the lastJavaSP wasn't - // recorded either when not in the Java state, lastJava ucontext will be - // used by AGCT - Counters::increment(AGCT_NATIVE_NO_JAVA_CONTEXT); - return 0; - } - } - bool blocked_in_vm = (state == _thread_blocked || state == _thread_blocked_trans); - // avoid unwinding during deoptimization - if (blocked_in_vm && vm_thread->osThreadState() == OSThreadState::RUNNABLE) { - Counters::increment(AGCT_BLOCKED_IN_VM); - return 0; - } - - JitWriteProtection jit(false); - // AsyncGetCallTrace writes to ASGCT_CallFrame array - ASGCT_CallTrace trace = {jni, 0, frames}; - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - - if (trace.num_frames > 0) { - frame.restore(saved_pc, saved_sp, saved_fp); - return trace.num_frames; - } - - int safe_mode = Profiler::instance()->safe_mode(); - CStack cstack = Profiler::instance()->cstackMode(); - - if ((trace.num_frames == ticks_unknown_Java || - trace.num_frames == ticks_not_walkable_Java) && - !(safe_mode & UNKNOWN_JAVA) && ucontext != NULL) { - CodeBlob *stub = JitCodeCache::findRuntimeStub((const void *)frame.pc()); - if (stub != NULL) { - if (cstack != CSTACK_NO) { - max_depth -= makeFrame(trace.frames++, BCI_NATIVE_FRAME, stub->_name); - } - if (!(safe_mode & POP_STUB) && - frame.unwindStub((instruction_t *)stub->_start, stub->_name) && - isAddressInCode((const void *)frame.pc())) { - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - } else if (VMStructs::hasMethodStructs()) { - VMNMethod *nmethod = CodeHeap::findNMethod((const void *)frame.pc()); - if (nmethod != NULL && nmethod->isNMethod() && nmethod->isAlive()) { - VMMethod *method = nmethod->method(); - if (method != NULL) { - jmethodID method_id = method->id(); - if (method_id != NULL) { - max_depth -= makeFrame(trace.frames++, 0, method_id); - } - if (!(safe_mode & POP_METHOD) && frame.unwindCompiled(nmethod) && - isAddressInCode((const void *)frame.pc())) { - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - if ((safe_mode & PROBE_SP) && trace.num_frames < 0) { - if (method_id != NULL) { - trace.frames--; - } - for (int i = 0; trace.num_frames < 0 && i < PROBE_SP_LIMIT; i++) { - frame.sp() += sizeof(void*); - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - } - } - } else if (nmethod != NULL) { - if (cstack != CSTACK_NO) { - max_depth -= - makeFrame(trace.frames++, BCI_NATIVE_FRAME, nmethod->name()); - } - if (!(safe_mode & POP_STUB) && - frame.unwindStub(NULL, nmethod->name()) && - isAddressInCode((const void *)frame.pc())) { - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - } - } - } else if (trace.num_frames == ticks_unknown_not_Java && - !(safe_mode & LAST_JAVA_PC)) { - VMJavaFrameAnchor* anchor = vm_thread->anchor(); - if (anchor == NULL) return 0; - uintptr_t sp = anchor->lastJavaSP(); - const void* pc = anchor->lastJavaPC(); - if (sp != 0 && pc == NULL) { - // We have the last Java frame anchor, but it is not marked as walkable. - // Make it walkable here - pc = ((const void**)sp)[-1]; - anchor->setLastJavaPC(pc); - - VMNMethod *m = CodeHeap::findNMethod(pc); - const Libraries* libs = Profiler::instance()->libraries(); - - if (m != NULL) { - // AGCT fails if the last Java frame is a Runtime Stub with an invalid - // _frame_complete_offset. In this case we patch _frame_complete_offset - // manually - if (!m->isNMethod() && m->frameSize() > 0 && - m->frameCompleteOffset() == -1) { - m->setFrameCompleteOffset(0); - } - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } else if (libs->findLibraryByAddress(pc) != NULL) { - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - - anchor->setLastJavaPC(nullptr); - } - } else if (trace.num_frames == ticks_not_walkable_not_Java && - !(safe_mode & LAST_JAVA_PC)) { - VMJavaFrameAnchor* anchor = vm_thread->anchor(); - if (anchor == NULL) return 0; - uintptr_t sp = anchor->lastJavaSP(); - const void* pc = anchor->lastJavaPC(); - if (sp != 0 && pc != NULL) { - // Similar to the above: last Java frame is set, - // but points to a Runtime Stub with an invalid _frame_complete_offset - VMNMethod *m = CodeHeap::findNMethod(pc); - if (m != NULL && !m->isNMethod() && m->frameSize() > 0 && - m->frameCompleteOffset() == -1) { - m->setFrameCompleteOffset(0); - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - } - } - } else if (trace.num_frames == ticks_GC_active && !(safe_mode & GC_TRACES)) { - VMJavaFrameAnchor* anchor = vm_thread->anchor(); - if (anchor == NULL || anchor->lastJavaSP() == 0) { - // Do not add 'GC_active' for threads with no Java frames, e.g. Compiler - // threads - frame.restore(saved_pc, saved_sp, saved_fp); - return 0; - } - } - - frame.restore(saved_pc, saved_sp, saved_fp); - - if (trace.num_frames > 0) { - return trace.num_frames + (trace.frames - frames); - } - - const char *err_string = Profiler::asgctError(trace.num_frames); - if (err_string == NULL) { - // No Java stack, because thread is not in Java context - return 0; - } - - Profiler::instance()->incFailure(-trace.num_frames); - trace.frames->bci = BCI_ERROR; - trace.frames->method_id = (jmethodID)err_string; - return trace.frames - frames + 1; -} - - -int HotspotSupport::walkJavaStack(StackWalkRequest& request) { - CStack cstack = Profiler::instance()->cstackMode(); - StackWalkFeatures features = Profiler::instance()->stackWalkFeatures(); - void* ucontext = request.ucontext; - ASGCT_CallFrame* frames = request.frames; - int max_depth = request.max_depth; - StackContext* java_ctx = request.java_ctx; - bool* truncated = request.truncated; - u32 lock_index = request.lock_index; - - int java_frames = 0; - if (features.mixed) { - java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated); - } else if (request.event_type == BCI_NATIVE_MALLOC || request.event_type == BCI_NATIVE_SOCKET) { - if (cstack >= CSTACK_VM) { - java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated); - } else { - AsyncSampleMutex mutex(ProfiledThread::currentSignalSafe()); - if (mutex.acquired()) { - java_frames = getJavaTraceAsync(ucontext, frames, max_depth, java_ctx, truncated); - if (java_frames > 0 && java_ctx->pc != NULL && VMStructs::hasMethodStructs()) { - VMNMethod* nmethod = CodeHeap::findNMethod(java_ctx->pc); - if (nmethod != NULL) { - fillFrameTypes(frames, java_frames, nmethod); - } - } - } - if (java_frames > 0 && VM::hotspot_version() >= 21 && java_frames < max_depth) { - VMThread* carrier = VMThread::current(); - if (carrier != nullptr && carrier->isCarryingVirtualThread()) { - frames[java_frames].bci = BCI_NATIVE_FRAME; - frames[java_frames].method_id = (jmethodID) "JVM Continuation"; - LP64_ONLY(frames[java_frames].padding = 0;) - java_frames++; - } - } - } - } else if (request.event_type == BCI_CPU || request.event_type == BCI_WALL) { - if (cstack >= CSTACK_VM) { - java_frames = walkVM(ucontext, frames, max_depth, features, eventTypeFromBCI(request.event_type), lock_index, truncated); - } else { - // Async events - AsyncSampleMutex mutex(ProfiledThread::currentSignalSafe()); - if (mutex.acquired()) { - java_frames = getJavaTraceAsync(ucontext, frames, max_depth, java_ctx, truncated); - if (java_frames > 0 && java_ctx->pc != NULL && VMStructs::hasMethodStructs()) { - VMNMethod* nmethod = CodeHeap::findNMethod(java_ctx->pc); - if (nmethod != NULL) { - fillFrameTypes(frames, java_frames, nmethod); - } - } - } - // ASGCT stops at the continuation boundary for virtual threads (JDK 21+). - // Append a synthetic root frame so the UI does not show "Missing Frames". - if (java_frames > 0 && VM::hotspot_version() >= 21 && java_frames < max_depth) { - VMThread* carrier = VMThread::current(); - if (carrier != nullptr && carrier->isCarryingVirtualThread()) { - frames[java_frames].bci = BCI_NATIVE_FRAME; - frames[java_frames].method_id = (jmethodID) "JVM Continuation"; - LP64_ONLY(frames[java_frames].padding = 0;) - java_frames++; - } - } - } - } - return java_frames; -} diff --git a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.h b/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.h deleted file mode 100644 index 47245d30a..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/hotspotSupport.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _HOTSPOT_HOTSPOTSUPPORT_H -#define _HOTSPOT_HOTSPOTSUPPORT_H - -#include "hotspot/hotspotStackFrame.h" -#include "hotspot/jitCodeCache.h" -#include "stackFrame.h" -#include "stackWalker.h" - -class ProfiledThread; - -class HotspotSupport { -private: - static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, - StackWalkFeatures features, EventType event_type, - const void* pc, uintptr_t sp, uintptr_t fp, int lock_index, bool* truncated); - static int walkVM(void* ucontext, ASGCT_CallFrame* frames, int max_depth, - StackWalkFeatures features, EventType event_type, - int lock_index, bool* truncated = nullptr); - - static int getJavaTraceAsync(void *ucontext, ASGCT_CallFrame *frames, - int max_depth, StackContext *java_ctx, - bool *truncated); - -public: - static void checkFault(ProfiledThread* thrd = nullptr); - static int walkJavaStack(StackWalkRequest& request); - static inline bool canUnwind(const StackFrame& frame, const void*& pc) { - return HotspotStackFrame::unwindAtomicStub(frame, pc); - } - - static inline bool isJitCode(const void* p) { - return JitCodeCache::isJitCode(p); - } -}; - -#endif // _HOTSPOT_HOTSPOTSUPPORT_H diff --git a/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.cpp b/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.cpp deleted file mode 100644 index 85af3d19f..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.cpp +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "jitCodeCache.h" - -#include "hotspot/vmStructs.h" - -SpinLock JitCodeCache::_stubs_lock; -CodeCache JitCodeCache::_runtime_stubs("[stubs]"); -std::atomic JitCodeCache::_call_stub_begin = { nullptr }; -std::atomic JitCodeCache::_call_stub_end = { nullptr }; - -// CompiledMethodLoad is also needed to enable DebugNonSafepoints info by -// default -void JNICALL JitCodeCache::CompiledMethodLoad(jvmtiEnv *jvmti, jmethodID method, - jint code_size, const void *code_addr, - jint map_length, - const jvmtiAddrLocationMap *map, - const void *compile_info) { - CodeHeap::updateBounds(code_addr, (const char *)code_addr + code_size); -} - -void JNICALL JitCodeCache::DynamicCodeGenerated(jvmtiEnv *jvmti, const char *name, - const void *address, jint length) { - _stubs_lock.lock(); - _runtime_stubs.add(address, length, name, true); - _stubs_lock.unlock(); - - if (name[0] == 'I' && strcmp(name, "Interpreter") == 0) { - CodeHeap::setInterpreterStart(address); - } else if (strcmp(name, "call_stub") == 0) { - _call_stub_begin.store(address, std::memory_order_relaxed); - // This fence ensures that _call_stub_begin is visible before _call_stub_end, so that isCallStub() works correctly - std::atomic_thread_fence(std::memory_order_release); - _call_stub_end.store((const char *)address + length, std::memory_order_relaxed); - } - - CodeHeap::updateBounds(address, (const char *)address + length); -} - -CodeBlob* JitCodeCache::findRuntimeStub(const void *address) { - CodeBlob *stub = nullptr; - _stubs_lock.lockShared(); - if (_runtime_stubs.contains(address)) { - stub = _runtime_stubs.findBlobByAddress(address); - } - _stubs_lock.unlockShared(); - return stub; -} diff --git a/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.h b/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.h deleted file mode 100644 index 005223384..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/jitCodeCache.h +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _HOTSPOT_JITCODECACHE_H -#define _HOTSPOT_JITCODECACHE_H - -#include - -#include "codeCache.h" -#include "spinLock.h" - -#include "hotspot/vmStructs.h" - -// The class tracks JIT-compiled code and ranges -class JitCodeCache { -private: - static SpinLock _stubs_lock; - static CodeCache _runtime_stubs; - static std::atomic _call_stub_begin; - static std::atomic _call_stub_end; - -public: - static void JNICALL CompiledMethodLoad(jvmtiEnv *jvmti, jmethodID method, - jint code_size, const void *code_addr, - jint map_length, - const jvmtiAddrLocationMap *map, - const void *compile_info); - static void JNICALL DynamicCodeGenerated(jvmtiEnv *jvmti, const char *name, - const void *address, jint length); - - static inline bool isCallStub(const void *address) { - const void* stub_end = _call_stub_end.load(std::memory_order_acquire); - return stub_end != nullptr && - address >= _call_stub_begin.load(std::memory_order_relaxed) && - address < stub_end; - } - - static CodeBlob* findRuntimeStub(const void *address); - static bool isJitCode(const void* pc) { - return CodeHeap::contains(pc); - } -}; - -#endif // _HOTSPOT_JITCODECACHE_H diff --git a/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp b/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp deleted file mode 100644 index 41b1540b8..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/vmStructs.cpp +++ /dev/null @@ -1,1197 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include "hotspot/vmStructs.inline.h" -#include "vmEntry.h" -#include "jniHelper.h" -#include "jvmHeap.h" -#include "jvmThread.h" -#include "safeAccess.h" -#include "spinLock.h" -#include "threadState.h" - -CodeCache* VMStructs::_libjvm = nullptr; -bool VMStructs::_has_class_names = false; -bool VMStructs::_has_method_structs = false; -bool VMStructs::_has_compiler_structs = false; -bool VMStructs::_has_stack_structs = false; -bool VMStructs::_has_class_loader_data = false; -bool VMStructs::_has_native_thread_id = false; -bool VMStructs::_can_dereference_jmethod_id = false; -bool VMStructs::_compact_object_headers = false; - -char* VMStructs::_code_heap[3] = {}; -const void* VMStructs::_code_heap_low = NO_MIN_ADDRESS; -const void* VMStructs::_code_heap_high = NO_MAX_ADDRESS; -char* VMStructs::_narrow_klass_base = nullptr; -int VMStructs::_narrow_klass_shift = -1; -int VMStructs::_interpreter_frame_bcp_offset = 0; -unsigned char VMStructs::_unsigned5_base = 0; -const void* VMStructs::_call_stub_return = nullptr; -const void* VMStructs::_cont_return_barrier = nullptr; -const void* VMStructs::_cont_entry_return_pc = nullptr; -VMNMethod* VMStructs::_enter_special_nm = nullptr; -const void* VMStructs::_interpreter_start = nullptr; -VMNMethod* VMStructs::_interpreter_nm = nullptr; -const void* VMStructs::_interpreted_frame_valid_start = nullptr; -const void* VMStructs::_interpreted_frame_valid_end = nullptr; - - -// Initialize type size to 0 -#define INIT_TYPE_SIZE(name, names) uint64_t VMStructs::TYPE_SIZE_NAME(name) = 0; -DECLARE_TYPES_DO(INIT_TYPE_SIZE) -DECLARE_V27_TYPES_DO(INIT_TYPE_SIZE) -#undef INIT_TYPE_SIZE - -#define offset_value -1 -#define address_value nullptr - -// Initialize field variables -// offset = -1 -// address = nullptr - -// Do nothing macro -#define DO_NOTHING(...) -#define INIT_FIELD(var, field_type, names) \ - field_type VMStructs::var = field_type##_value; - -#define INIT_FIELD_WITH_VERSION(var, field_type, min_version, max_version, names) \ - field_type VMStructs::var = field_type##_value; - -DECLARE_TYPE_FIELD_DO(DO_NOTHING, INIT_FIELD, INIT_FIELD_WITH_VERSION, DO_NOTHING) -DECLARE_V27_TYPE_FIELD_DO(DO_NOTHING, INIT_FIELD, INIT_FIELD_WITH_VERSION, DO_NOTHING) - -#undef INIT_FIELD -#undef INIT_FIELD_WITH_VERSION -#undef DO_NOTHING -#undef offset_value -#undef address_value - -// Initialize constant variables to -1 -#define INIT_INT_CONSTANT(type, field, ...) \ - int VMStructs::_##type##_##field = -1; -#define INIT_LONG_CONSTANT(type, field, ...) \ - long VMStructs::_##type##_##field = -1; - -DECLARE_INT_CONSTANTS_DO(INIT_INT_CONSTANT, INIT_INT_CONSTANT) -DECLARE_LONG_CONSTANTS_DO(INIT_LONG_CONSTANT, INIT_LONG_CONSTANT) -#undef INIT_INT_CONSTANT -#undef INIT_LONG_CONSTANT - -jfieldID VMStructs::_eetop = NULL; -jfieldID VMStructs::_klass = NULL; -intptr_t VMStructs::_env_offset = -1; -void* VMStructs::_java_thread_vtbl[6]; - -VMStructs::LockFunc VMStructs::_lock_func; -VMStructs::LockFunc VMStructs::_unlock_func; - -// Datadog-specific static variables -CodeCache VMStructs::_unsafe_to_walk("unwalkable code"); -VMStructs::HeapUsageFunc VMStructs::_heap_usage_func = NULL; -VMStructs::MemoryUsageFunc VMStructs::_memory_usage_func = NULL; -VMStructs::GCHeapSummaryFunc VMStructs::_gc_heap_summary_func = NULL; -VMStructs::IsValidMethodFunc VMStructs::_is_valid_method_func = NULL; - - -uintptr_t VMStructs::readSymbol(const char* symbol_name) { - const void* symbol = _libjvm->findSymbol(symbol_name); - if (symbol == NULL) { - // Avoid JVM crash in case of missing symbols - return 0; - } - return *(uintptr_t*)symbol; -} - -// Run at agent load time -void VMStructs::init(CodeCache* libjvm) { - if (libjvm != NULL) { - _libjvm = libjvm; - initOffsets(); - initJvmFunctions(); - initUnsafeFunctions(); - initCriticalJNINatives(); - } -} - -// Run when VM is initialized and JNI is available -void VMStructs::ready() { - resolveOffsets(); - patchSafeFetch(); -} - -bool matchAny(const char* target_name, std::initializer_list names) { - for (const char* name : names) { - if (strcmp(target_name, name) == 0) { - return true; - } - } - return false; -} - -void VMStructs::init_offsets_and_addresses() { - uintptr_t entry = readSymbol("gHotSpotVMStructs"); - uintptr_t stride = readSymbol("gHotSpotVMStructEntryArrayStride"); - uintptr_t type_offset = readSymbol("gHotSpotVMStructEntryTypeNameOffset"); - uintptr_t field_offset = readSymbol("gHotSpotVMStructEntryFieldNameOffset"); - uintptr_t offset_offset = readSymbol("gHotSpotVMStructEntryOffsetOffset"); - uintptr_t isStatic_offset = readSymbol("gHotSpotVMStructEntryIsStaticOffset"); - uintptr_t address_offset = readSymbol("gHotSpotVMStructEntryAddressOffset"); - bool isStatic; - - auto read_offset = [&]() -> int { - assert(!isStatic); - return *(int*)(entry + offset_offset); - }; - - auto read_address = [&]() -> address { - assert(isStatic); - return *(address*)(entry + address_offset); - }; - - if (entry != 0 && stride != 0) { - for (;; entry += stride) { - const char* type_name = *(const char**)(entry + type_offset); - const char* field_name = *(const char**)(entry + field_offset); - isStatic = *(int32_t*)(entry + isStatic_offset) != 0; - - if (type_name == nullptr || field_name == nullptr) { - break; - } -#define MATCH_TYPE_NAMES(type, type_names) \ - if (matchAny(type_name, type_names)) { -#define READ_FIELD_VALUE(var, field_type, field_names) \ - if (matchAny(field_name, field_names)) { \ - var = read_##field_type(); \ - continue; \ - } -#define READ_FIELD_VALUE_WITH_VERSION(var, field_type, min_version, max_version, field_names) \ - if (matchAny(field_name, field_names)) { \ - var = read_##field_type(); \ - continue; \ - } - -#define END_TYPE() continue; } - DECLARE_TYPE_FIELD_DO(MATCH_TYPE_NAMES, READ_FIELD_VALUE, READ_FIELD_VALUE_WITH_VERSION, END_TYPE) - DECLARE_V27_TYPE_FIELD_DO(MATCH_TYPE_NAMES, READ_FIELD_VALUE, READ_FIELD_VALUE_WITH_VERSION, END_TYPE) -#undef MATCH_TYPE_NAMES -#undef READ_FIELD_VALUE -#undef READ_FIELD_VALUE_WITH_VERSION -#undef END_TYPE - } - } -} - -void VMStructs::init_type_sizes() { - uintptr_t entry = readSymbol("gHotSpotVMTypes"); - uintptr_t stride = readSymbol("gHotSpotVMTypeEntryArrayStride"); - uintptr_t type_offset = readSymbol("gHotSpotVMTypeEntryTypeNameOffset"); - uintptr_t size_offset = readSymbol("gHotSpotVMTypeEntrySizeOffset"); - - if (entry != 0 && stride != 0) { - for (;; entry += stride) { - const char* type = *(const char**)(entry + type_offset); - if (type == NULL) { - break; - } - - uint64_t size = *(uint64_t*)(entry + size_offset); - - #define READ_TYPE_SIZE(name, names) \ - if (matchAny(type, names)) { \ - TYPE_SIZE_NAME(name) = size; \ - continue; \ - } - - DECLARE_TYPES_DO(READ_TYPE_SIZE) - DECLARE_V27_TYPES_DO(READ_TYPE_SIZE) - -#undef READ_TYPE_SIZE - - } - } -} - -#define READ_CONSTANT(type, field, ...) \ - if (strcmp(type_name, #type "::" #field) == 0) { \ - _##type##_##field = value; \ - continue; \ - } - - -void VMStructs::init_constants() { - // Int constants - uintptr_t entry = readSymbol("gHotSpotVMIntConstants"); - uintptr_t stride = readSymbol("gHotSpotVMIntConstantEntryArrayStride"); - uintptr_t name_offset = readSymbol("gHotSpotVMIntConstantEntryNameOffset"); - uintptr_t value_offset = readSymbol("gHotSpotVMIntConstantEntryValueOffset"); - if (entry != 0 && stride != 0) { - for (;; entry += stride) { - const char* type_name = *(const char**)(entry + name_offset); - if (type_name == nullptr) { - break; - } - int value = *(int*)(entry + value_offset); - DECLARE_INT_CONSTANTS_DO(READ_CONSTANT, READ_CONSTANT) - } - } - // Special case - _frame_entry_frame_call_wrapper_offset *= sizeof(uintptr_t); - - - // Long constants - entry = readSymbol("gHotSpotVMLongConstants"); - stride = readSymbol("gHotSpotVMLongConstantEntryArrayStride"); - name_offset = readSymbol("gHotSpotVMLongConstantEntryNameOffset"); - value_offset = readSymbol("gHotSpotVMLongConstantEntryValueOffset"); - - if (entry != 0 && stride != 0) { - for (;; entry += stride) { - const char* type_name = *(const char**)(entry + name_offset); - if (type_name == nullptr) { - break; - } - - long value = *(long*)(entry + value_offset); - DECLARE_LONG_CONSTANTS_DO(READ_CONSTANT, READ_CONSTANT) - } - } -} - -#undef READ_CONSTANT - - -#ifdef DEBUG -void VMStructs::verify_offsets() { - int hotspot_version = VM::hotspot_version(); - // Hotspot only - // NOTE: disabled for JDK25, due to a weird failure in CI - if (!VM::isHotspot() || hotspot_version == 25) { - return; - } - -// Verify type sizes -// Note: DECLARE_V27_TYPES_DO (VMContinuationEntry) is intentionally excluded here. -// ContinuationEntry is not exported in gHotSpotVMTypes before JDK 27 (added via JDK-8378985); -// asserting type_size() > 0 would SIGABRT on any JDK 21-26 build. -#define VERIFY_TYPE_SIZE(name, names) assert(TYPE_SIZE_NAME(name) > 0); - DECLARE_TYPES_DO(VERIFY_TYPE_SIZE); -#undef VERIFY_TYPE_SIZE - - -// Verify offsets and addresses -// Note: DECLARE_V27_TYPE_FIELD_DO is intentionally excluded here. -// Continuation-related fields (_cont_entry_offset, _cont_return_barrier_addr, -// _cont_entry_return_pc_addr, _cont_entry_parent_offset) are absent from -// gHotSpotVMStructs in all JDK 21-26 builds: ContinuationEntry was not -// exported in the vmStructs table until JDK 27 (JDK-8378985). walkVM degrades -// gracefully when they are missing. -#define offset_value -1 -#define address_value nullptr - -// Do nothing macro -#define DO_NOTHING(...) -#define VERIFY_FIELD(var, field_type, names) \ - assert(var != field_type##_value); -#define VERSION_FIELD_WITH_VERSION(var, field_type, min_ver, max_ver, names) \ - assert(hotspot_version < min_ver || hotspot_version > max_ver || var != field_type##_value); - DECLARE_TYPE_FIELD_DO(DO_NOTHING, VERIFY_FIELD, VERSION_FIELD_WITH_VERSION, DO_NOTHING) -#undef VERSION_FIELD_WITH_VERSION -#undef VERIFY_FIELD -#undef DO_NOTHING -#undef offset_value -#undef address_value - -// Verify constants -// Initialize constant variables to -1 -#define VERIFY_CONSTANT(type, field) \ - assert(_##type##_##field != -1); -#define VERIFY_CONSTANT_WITH_VERSION(type, field, min_ver, max_ver) \ - assert(hotspot_version < min_ver || hotspot_version > max_ver || _##type##_##field != -1); - - DECLARE_INT_CONSTANTS_DO(VERIFY_CONSTANT, VERIFY_CONSTANT_WITH_VERSION) - DECLARE_LONG_CONSTANTS_DO(VERIFY_CONSTANT, VERIFY_CONSTANT_WITH_VERSION) -#undef VERIFY_CONSTANT -} - -#endif // DEBUG - -void VMStructs::initOffsets() { - // J9 does not support vmStructs - if (VM::isOpenJ9() || VM::isZing()) { - return; - } - - init_type_sizes(); - init_offsets_and_addresses(); - init_constants(); - - -#ifdef DEBUG - // Disable verifier for now - verify_offsets(); -#endif -} - -void VMStructs::resolveOffsets() { - if (VM::isOpenJ9() || VM::isZing()) { - return; - } - - if (_klass_offset_addr != NULL) { - _klass = (jfieldID)(uintptr_t)(*(int*)_klass_offset_addr << 2 | 2); - } - - VMFlag* ccp = VMFlag::find("UseCompressedClassPointers"); - if (ccp != NULL && ccp->get() && _narrow_klass_base_addr != NULL && _narrow_klass_shift_addr != nullptr) { - _narrow_klass_base = *(char**)_narrow_klass_base_addr; - _narrow_klass_shift = *(int*)_narrow_klass_shift_addr; - } - - VMFlag* coh = VMFlag::find("UseCompactObjectHeaders"); - if (coh != NULL && coh->get()) { - _compact_object_headers = true; - } - - _has_class_names = _klass_name_offset >= 0 - && (_compact_object_headers ? (_markWord_klass_shift >= 0 && _markWord_monitor_value == MONITOR_BIT) - : _oop_klass_offset >= 0) - && _symbol_length_offset >= 0 - && _symbol_body_offset >= 0 - && _klass != NULL; - - _has_method_structs = _jmethod_ids_offset >= 0 - && _nmethod_method_offset >= 0 - && (_nmethod_entry_offset != -1 || _nmethod_entry_address != -1) - && _nmethod_state_offset >= 0 - && _method_constmethod_offset >= 0 - && _method_code_offset >= 0 - && _constmethod_constants_offset >= 0 - && _constmethod_idnum_offset >= 0 - && VMConstMethod::type_size() > 0 - && _pool_holder_offset >= 0; - - _has_compiler_structs = _comp_env_offset >= 0 - && _comp_task_offset >= 0 - && _comp_method_offset >= 0; - - _has_class_loader_data = _class_loader_data_offset >= 0 - && _class_loader_data_next_offset == sizeof(uintptr_t) * 8 + 8 - && _methods_offset >= 0 - && _klass != NULL - && _lock_func != NULL && _unlock_func != NULL; - -#if defined(__x86_64__) || defined(__i386__) - _interpreter_frame_bcp_offset = VM::hotspot_version() >= 11 ? -8 : VM::hotspot_version() == 8 ? -7 : 0; -#elif defined(__aarch64__) - _interpreter_frame_bcp_offset = VM::hotspot_version() >= 11 ? -9 : VM::hotspot_version() == 8 ? -7 : 0; - // The constant is missing on ARM, but fortunately, it has been stable for years across all JDK versions - _frame_entry_frame_call_wrapper_offset = -64; -#elif defined(__arm__) || defined(__thumb__) - _interpreter_frame_bcp_offset = VM::hotspot_version() >= 11 ? -8 : 0; - _frame_entry_frame_call_wrapper_offset = 0; -#endif - - // JDK-8292758 has slightly changed ScopeDesc encoding - if (VM::hotspot_version() >= 20) { - _unsigned5_base = 1; - } - - if (_call_stub_return_addr != NULL) { - _call_stub_return = *(const void**)_call_stub_return_addr; - } - if (_cont_return_barrier_addr != NULL) { - _cont_return_barrier = *(const void**)_cont_return_barrier_addr; - } - if (_cont_entry_return_pc_addr != NULL) { - _cont_entry_return_pc = *(const void**)_cont_entry_return_pc_addr; - } - // Fallback for JDK 21-26: StubRoutines::_cont_returnBarrier and - // ContinuationEntry::_return_pc are absent from gHotSpotVMStructs before - // JDK 27 (added via JDK-8378985). Resolve them via C++ symbol lookup. - // Symbol names use Itanium C++ ABI mangling (GCC/Clang), which matches - // the HotSpot build toolchain on all supported platforms. - if (_cont_return_barrier == nullptr && VM::hotspot_version() >= 21) { - const void** sym = (const void**)_libjvm->findSymbol("_ZN12StubRoutines19_cont_returnBarrierE"); - if (sym != nullptr) _cont_return_barrier = *sym; - } - if (_cont_entry_return_pc == nullptr && VM::hotspot_version() >= 21) { - const void** sym = (const void**)_libjvm->findSymbol("_ZN17ContinuationEntry10_return_pcE"); - if (sym != nullptr) _cont_entry_return_pc = *sym; - } - - // Since JDK 23, _metadata_offset is relative to _data_offset. See metadata() - if (_nmethod_immutable_offset < 0) { - _data_offset = 0; - } - - _has_stack_structs = _has_method_structs - && _call_wrapper_anchor_offset >= 0 - && _frame_entry_frame_call_wrapper_offset != -1 - && _interpreter_frame_bcp_offset != 0 - && (_code_offset != -1 || _code_address != -1) - && _data_offset >= 0 - && (_scopes_data_offset != -1 || _scopes_data_address != -1) - && _scopes_pcs_offset >= 0 - && ((_mutable_data_offset >= 0 && _relocation_size_offset >= 0) || _nmethod_metadata_offset >= 0) - && _thread_vframe_offset >= 0 - && _thread_exception_offset >= 0 - && VMThread::type_size() > 0; - - // Since JDK-8268406, it is no longer possible to get VMMethod* by dereferencing jmethodID - _can_dereference_jmethod_id = _has_method_structs && VM::hotspot_version() <= 25; - - if (_code_heap_addr != NULL && _code_heap_low_addr != NULL && _code_heap_high_addr != NULL) { - char* code_heaps = *(char**)_code_heap_addr; - unsigned int code_heap_count = *(unsigned int*)(code_heaps + _array_len_offset); - if (code_heap_count <= 3 && _array_data_offset >= 0) { - char* code_heap_array = *(char**)(code_heaps + _array_data_offset); - memcpy(_code_heap, code_heap_array, code_heap_count * sizeof(_code_heap[0])); - } - _code_heap_low = *(const void**)_code_heap_low_addr; - _code_heap_high = *(const void**)_code_heap_high_addr; - } else if (_code_heap_addr != NULL && _code_heap_memory_offset >= 0) { - _code_heap[0] = *(char**)_code_heap_addr; - _code_heap_low = *(const void**)(_code_heap[0] + _code_heap_memory_offset + _vs_low_bound_offset); - _code_heap_high = *(const void**)(_code_heap[0] + _code_heap_memory_offset + _vs_high_bound_offset); - } - - // Invariant: _code_heap[i] != NULL iff all CodeHeap structures are available - if (_code_heap[0] != NULL && _code_heap_segment_shift >= 0) { - _code_heap_segment_shift = *(int*)(_code_heap[0] + _code_heap_segment_shift); - } - if (_code_heap_memory_offset < 0 || _code_heap_segmap_offset < 0 || - _code_heap_segment_shift < 0 || _code_heap_segment_shift > 16 || - _heap_block_used_offset < 0) { - memset(_code_heap, 0, sizeof(_code_heap)); - } - if (_interpreter_nm == NULL && _interpreter_start != NULL) { - _interpreter_nm = CodeHeap::findNMethod(_interpreter_start); - } - if (_enter_special_nm == NULL && _cont_entry_return_pc != NULL) { - // On JDK 27+, enterSpecial is a proper nmethod; findNMethod succeeds. - // On JDK 21-26, it is a RuntimeBlob; findNMethod returns NULL and - // _enter_special_nm stays NULL. The cont_entry_return_pc boundary is - // then detected via isContEntryReturnPc() in the walk loop rather than - // nmethod identity. - _enter_special_nm = CodeHeap::findNMethod(_cont_entry_return_pc); - } -} - -void VMStructs::initJvmFunctions() { - if (VM::hotspot_version() == 8) { - _lock_func = (LockFunc)_libjvm->findSymbol("_ZN7Monitor28lock_without_safepoint_checkEv"); - _unlock_func = (LockFunc)_libjvm->findSymbol("_ZN7Monitor6unlockEv"); - } - - if (VM::hotspot_version() > 0) { - CodeBlob* blob = _libjvm->findBlob("_ZNK5frame26is_interpreted_frame_validEP10JavaThread"); - if (blob != NULL) { - _interpreted_frame_valid_start = blob->_start; - _interpreted_frame_valid_end = blob->_end; - } - } - - // Datadog-specific function pointer resolution - _heap_usage_func = (HeapUsageFunc)findHeapUsageFunc(); - _gc_heap_summary_func = (GCHeapSummaryFunc)_libjvm->findSymbol( - "_ZN13CollectedHeap19create_heap_summaryEv"); - _is_valid_method_func = (IsValidMethodFunc)_libjvm->findSymbol( - "_ZN6Method15is_valid_methodEPKS_"); -} - -void VMStructs::patchSafeFetch() { - // Workarounds for JDK-8307549 and JDK-8321116 - if (WX_MEMORY && VM::hotspot_version() == 17) { - void** entry = (void**)_libjvm->findSymbol("_ZN12StubRoutines18_safefetch32_entryE"); - if (entry != NULL) { - *entry = (void*)SafeAccess::load32; - } - } else if (WX_MEMORY && VM::hotspot_version() == 11) { - void** entry = (void**)_libjvm->findSymbol("_ZN12StubRoutines17_safefetchN_entryE"); - if (entry != NULL) { - *entry = (void*)SafeAccess::load; - } - } -} - -// ===== Datadog-specific VMStructs extensions ===== - -void VMStructs::initUnsafeFunctions() { - // see - // https://github.com/openjdk/jdk/blob/master/src/hotspot/share/gc/z/zBarrierSetRuntime.hpp#L33 - // https://bugs.openjdk.org/browse/JDK-8302317 - std::vector unsafeMangledPrefixes{"_ZN18ZBarrierSetRuntime", - "_ZN9JavaCalls11call_helper", - "_ZN14MM_RootScanner"}; - - std::vector symbols; - _libjvm->findSymbolsByPrefix(unsafeMangledPrefixes, symbols); - for (const void *symbol : symbols) { - CodeBlob *blob = _libjvm->findBlobByAddress(symbol); - if (blob) { - _unsafe_to_walk.add(blob->_start, - ((uintptr_t)blob->_end - (uintptr_t)blob->_start), - blob->_name, true); - } - } -} - -void VMStructs::initCriticalJNINatives() { -#ifdef __aarch64__ - // aarch64 does not support CriticalJNINatives - VMFlag* flag = VMFlag::find("CriticalJNINatives", {VMFlag::Type::Bool}); - if (flag != nullptr && flag->get()) { - flag->set(0); - } -#endif // __aarch64__ -} - -const void *VMStructs::findHeapUsageFunc() { - if (VM::hotspot_version() < 17) { - // For JDK 11 it is really unreliable to find the memory_usage function - - // just disable it - return nullptr; - } else { - VMFlag* flag = VMFlag::find("UseG1GC", {VMFlag::Type::Bool}); - if (flag != NULL && flag->get()) { - // The CollectedHeap::memory_usage function is a virtual one - - // G1, Shenandoah and ZGC are overriding it and calling the base class - // method results in asserts triggering. Therefore, we try to locate the - // concrete overridden method form. - return _libjvm->findSymbol("_ZN15G1CollectedHeap12memory_usageEv"); - } - flag = VMFlag::find("UseShenandoahGC", {VMFlag::Type::Bool}); - if (flag != NULL && flag->get()) { - return _libjvm->findSymbol("_ZN14ShenandoahHeap12memory_usageEv"); - } - flag = VMFlag::find("UseZGC", {VMFlag::Type::Bool}); - if (flag != NULL && flag->get() && VM::hotspot_version() < 21) { - // accessing this method in JDK 21 (generational ZGC) will cause SIGSEGV - return _libjvm->findSymbol("_ZN14ZCollectedHeap12memory_usageEv"); - } - return _libjvm->findSymbol("_ZN13CollectedHeap12memory_usageEv"); - } -} - -bool VMStructs::isSafeToWalk(uintptr_t pc) { - // Check if PC is in the unsafe-to-walk code region - // Note: findFrameDesc now returns by value instead of pointer, but it always returns - // a valid FrameDesc (either from table or default_frame), so the old pointer check - // was always true. The effective logic is simply checking if pc is in _unsafe_to_walk. - return !_unsafe_to_walk.contains((const void *)pc); -} - -void JNICALL VMStructs::NativeMethodBind(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread, - jmethodID method, void *address, - void **new_address_ptr) { - static SpinLock _lock; - static int delayedCounter = 0; - static void **delayed = (void **)malloc(512 * sizeof(void *) * 2); - - if (_memory_usage_func == NULL) { - if (jvmti != NULL && jni != NULL) { - checkNativeBinding(jvmti, jni, method, address); - void **tmpDelayed = NULL; - int tmpCounter = 0; - _lock.lock(); - if (delayed != NULL && delayedCounter > 0) { - // in order to minimize the lock time, we copy the delayed list, free it - // and release the lock - tmpCounter = delayedCounter; - tmpDelayed = (void **)malloc(tmpCounter * sizeof(void *) * 2); - memcpy(tmpDelayed, delayed, tmpCounter * sizeof(void *) * 2); - delayedCounter = 0; - free(delayed); - delayed = NULL; - } - _lock.unlock(); - // if there was a delayed list, we check it now, not blocking on the lock - if (tmpDelayed != NULL) { - for (int i = 0; i < tmpCounter; i += 2) { - checkNativeBinding(jvmti, jni, (jmethodID)tmpDelayed[i], - tmpDelayed[i + 1]); - } - // don't forget to free the tmp list - free(tmpDelayed); - } - } else { - _lock.lock(); - if (delayed != NULL) { - delayed[delayedCounter] = method; - delayed[delayedCounter + 1] = address; - delayedCounter += 2; - } - _lock.unlock(); - } - } -} - -void VMStructs::checkNativeBinding(jvmtiEnv *jvmti, JNIEnv *jni, - jmethodID method, void *address) { - char *method_name; - char *method_sig; - int error = jvmti->GetMethodName(method, &method_name, &method_sig, NULL); - if (error == 0) { - if (strcmp(method_name, "getMemoryUsage0") == 0 && - strcmp(method_sig, "(Z)Ljava/lang/management/MemoryUsage;") == 0) { - _memory_usage_func = (MemoryUsageFunc)address; - } - } - jvmti->Deallocate((unsigned char *)method_sig); - jvmti->Deallocate((unsigned char *)method_name); -} - -void* VMThread::initialize(jthread thread) { - JNIEnv* env = VM::jni(); - jclass thread_class = env->FindClass("java/lang/Thread"); - if (thread_class == NULL) { - env->ExceptionClear(); - return nullptr; - } - - // Get eetop field - a bridge from Java Thread to VMThread - if ((_eetop = env->GetFieldID(thread_class, "eetop", "J")) == NULL) { - // No such field - probably not a HotSpot JVM - env->ExceptionClear(); - return nullptr; - } - - void* vm_thread = fromJavaThread(env, thread); - assert(vm_thread != nullptr); - _has_native_thread_id = _thread_osthread_offset >= 0 && _osthread_id_offset >= 0; - _env_offset = (intptr_t)env - (intptr_t)vm_thread; - memcpy(_java_thread_vtbl, VMThread::cast(vm_thread)->vtable(), sizeof(_java_thread_vtbl)); - return vm_thread; -} - -bool VMThread::isJavaThread(VMThread* vm_thread) { - // Not a JVM thread - native thread, e.g. thread launched by JNI code - if (vm_thread == nullptr) { - return false; - } - - // Must be called from current thread - assert(vm_thread == VMThread::current()); - - // JVMTI ThreadStart callback may have set the flag, which is reliable. - // Or we may already compute and cache it, so use it instead. - ProfiledThread *prof_thread = ProfiledThread::currentSignalSafe(); - if (prof_thread != nullptr) { - ProfiledThread::ThreadType type = prof_thread->threadType(); - if (type != ProfiledThread::ThreadType::TYPE_UNKNOWN) { - return type == ProfiledThread::ThreadType::TYPE_JAVA_THREAD; - } - } - - // jvmti ThreadStart does not callback to JVM internal threads, e.g. Compiler threads, which are also JavaThreads, - // let's check the vtable pointer to make sure it is a Java thread. - // A Java thread should have the same vtable as the one we got from a known Java thread during initialization - bool is_java_thread = vm_thread->hasJavaThreadVtable(); - // Cache the thread type for future quick check - if (prof_thread != nullptr) { - prof_thread->setJavaThread(is_java_thread); - } - if (!is_java_thread) { - Counters::increment(WALKVM_CACHED_NOT_JAVA); - } - return is_java_thread; -} - -static ExecutionMode convertJvmExecutionState(JVMJavaThreadState state) { - switch (state) { - case _thread_in_native: - case _thread_in_native_trans: - return ExecutionMode::NATIVE; - case _thread_uninitialized: - case _thread_new: - case _thread_new_trans: - case _thread_in_vm: - case _thread_in_vm_trans: - return ExecutionMode::JVM; - case _thread_in_Java: - case _thread_in_Java_trans: - return ExecutionMode::JAVA; - case _thread_blocked: - case _thread_blocked_trans: - return ExecutionMode::SAFEPOINT; - default: - return ExecutionMode::UNKNOWN; - } -} - -ExecutionMode VMThread::getExecutionMode() { - assert(VM::isHotspot()); - VMThread* vm_thread = VMThread::current(); - if (vm_thread == nullptr) { - return ExecutionMode::NATIVE; - } - - if (isJavaThread(vm_thread)) { - JVMJavaThreadState thread_state = vm_thread->state(); - return convertJvmExecutionState(thread_state); - } else { - return ExecutionMode::JVM; - } -} - -OSThreadState VMThread::getOSThreadState() { - VMThread* vm_thread = VMThread::current(); - if (vm_thread == nullptr) { - return OSThreadState::UNKNOWN; - } - // All hotspot JVM threads have osThread fields - return vm_thread->osThreadState(); -} - -int VMThread::osThreadId() { - const char* osthread = (const char*) SafeAccess::load((void**) at(_thread_osthread_offset)); - if (osthread != NULL) { - // Java thread may be in the middle of termination, and its osthread structure just released - return SafeAccess::load32((int32_t*)(osthread + _osthread_id_offset), -1); - } - return -1; -} - -JNIEnv* VMThread::jni() { - if (_env_offset < 0) { - return VM::jni(); // fallback for non-HotSpot JVM - } - return isJavaThread(this) ? (JNIEnv*) at(_env_offset) : NULL; -} - -jmethodID VMMethod::id() { - // We may find a bogus NMethod during stack walking, it does not always point to a valid VMMethod - const char* const_method = (const char*) SafeAccess::load((void**) at(_method_constmethod_offset)); - if (!goodPtr(const_method)) { - return NULL; - } - - const char* cpool = (const char*) SafeAccess::load((void**)(const_method + _constmethod_constants_offset)); - unsigned short num = (unsigned short) SafeAccess::load32((int32_t*)(const_method + _constmethod_idnum_offset), 0); - if (goodPtr(cpool)) { - VMKlass* holder = (VMKlass*) SafeAccess::loadPtr((void**)(cpool + _pool_holder_offset), nullptr); - if (goodPtr(holder)) { - jmethodID* ids = (jmethodID*) SafeAccess::loadPtr((void**)((char*)holder + _jmethod_ids_offset), nullptr); - if (ids != NULL) { - size_t len = (size_t) SafeAccess::load32((int32_t*)ids, 0); - if (num < len) { - return (jmethodID) SafeAccess::loadPtr((void**)(ids + num + 1), nullptr); - } - } - } - } - return NULL; -} - -jmethodID VMMethod::validatedId() { - jmethodID method_id = id(); - if (!_can_dereference_jmethod_id || - ((goodPtr(method_id) && SafeAccess::loadPtr((void**)method_id, nullptr) == this))) { - return method_id; - } - return NULL; -} - -VMNMethod* CodeHeap::findNMethod(char* heap, const void* pc) { - unsigned char* heap_start = *(unsigned char**)(heap + _code_heap_memory_offset + _vs_low_offset); - unsigned char* segmap = *(unsigned char**)(heap + _code_heap_segmap_offset + _vs_low_offset); - size_t idx = ((unsigned char*)pc - heap_start) >> _code_heap_segment_shift; - - if (segmap[idx] == 0xff) { - return NULL; - } - while (segmap[idx] > 0) { - idx -= segmap[idx]; - } - - unsigned char* block = heap_start + (idx << _code_heap_segment_shift) + _heap_block_used_offset; - if (!*block) { - return NULL; - } - VMNMethod* nm = align(block + sizeof(uintptr_t)); - // Validate the nmethod memory is still readable. findNMethod is called from - // signal-handler context (walkVM) where nmethods can be freed concurrently - // during class unloading. Unlike other VMStructs casts that go through - // cast_to with readability validation, align<> provides none. - if (!SafeAccess::isReadableRange(nm, VMNMethod::type_size())) { - return NULL; - } - return nm; -} - -int VMNMethod::findScopeOffset(const void* pc) { - intptr_t pc_offset = (const char*)pc - code(); - if (pc_offset < 0 || pc_offset > 0x7fffffff) { - return -1; - } - - const int* scopes_pcs = (const int*) at(_scopes_pcs_offset); - PcDesc* pcd = (PcDesc*) immutableDataAt(scopes_pcs[0]); - PcDesc* pcd_end = (PcDesc*) immutableDataAt(scopes_pcs[1]); - int low = 0; - int high = (pcd_end - pcd) - 1; - - while (low <= high) { - int mid = (unsigned int)(low + high) >> 1; - if (pcd[mid]._pc < pc_offset) { - low = mid + 1; - } else if (pcd[mid]._pc > pc_offset) { - high = mid - 1; - } else { - return pcd[mid]._scope_offset; - } - } - - return pcd + low < pcd_end ? pcd[low]._scope_offset : -1; -} - -int ScopeDesc::readInt() { - unsigned char c = *_stream++; - unsigned int n = c - _unsigned5_base; - if (c >= 192) { - for (int shift = 6; ; shift += 6) { - c = *_stream++; - n += (c - _unsigned5_base) << shift; - if (c < 192 || shift >= 24) break; - } - } - return n; -} - -VMFlag* VMFlag::find(const char* name) { - if (_flags_addr != NULL && VMFlag::type_size() > 0) { - size_t count = *(size_t*)_flag_count; - - for (size_t i = 0; i < count; i++) { - VMFlag* f = VMFlag::cast(*(const char**)_flags_addr + i * VMFlag::type_size()); - if (f->name() != NULL && strcmp(f->name(), name) == 0 && f->addr() != NULL) { - return f; - } - } - } - return NULL; -} - -VMFlag *VMFlag::find(const char *name, std::initializer_list types) { - int mask = 0; - for (int type : types) { - mask |= 0x1 << type; - } - return find(name, mask); -} - -VMFlag *VMFlag::find(const char *name, int type_mask) { - if (_flags_addr != NULL && VMFlag::type_size() > 0) { - size_t count = *(size_t*)_flag_count; - for (size_t i = 0; i < count; i++) { - VMFlag* f = VMFlag::cast(*(const char**)_flags_addr + i * VMFlag::type_size()); - if (f->name() != NULL && strcmp(f->name(), name) == 0) { - int masked = 0x1 << f->type(); - if (masked & type_mask) { - return (VMFlag*)f; - } - } - } - } - return NULL; -} - -int VMFlag::type() { - if (VM::hotspot_version() < 16) { // in JDK 16 the JVM flag implementation has changed - char* type_name = *(char **)at(_flag_type_offset); - if (type_name == NULL) { - return Unknown; - } - if (strcmp(type_name, "bool") == 0) { - return Bool; - } - if (strcmp(type_name, "int") == 0) { - return Int; - } - if (strcmp(type_name, "uint") == 0) { - return Uint; - } - if (strcmp(type_name, "intx") == 0) { - return Intx; - } - if (strcmp(type_name, "uintx") == 0) { - return Uintx; - } - if (strcmp(type_name, "uint64_t") == 0) { - return Uint64_t; - } - if (strcmp(type_name, "size_t") == 0) { - return Size_t; - } - if (strcmp(type_name, "double") == 0) { - return Double; - } - if (strcmp(type_name, "ccstr") == 0) { - return String; - } - if (strcmp(type_name, "ccstrlist") == 0) { - return Stringlist; - } - return Unknown; - } - return *(int *)at(_flag_type_offset); -} - -/** - * jmethodIDs are unreliable, even if the profiler has created strong global JNI - * references to the classes containing methods with those jmethodIDs. This is - * affecting particularly hard the 'record-on-shutdown' feature when the VM - * class structures seem to be aggressively cleaned-up despite JNI global - * references pointing to them are still there. This check is attempting to - * validate that all data structures required to reconstruct the method metadata - * from a jmethodID are still readable at this point. - */ -bool VMMethod::check_jmethodID(jmethodID id) { - if (id == NULL) { - return false; - } - if (VM::isOpenJ9()) { - return check_jmethodID_J9(id); - } - return check_jmethodID_hotspot(id); -} - -bool VMMethod::check_jmethodID_hotspot(jmethodID id) { - if (VM::hotspot_version() > 25) { - // In https://bugs.openjdk.org/browse/JDK-8268406 the jmethodids are completely reworked - // The assumption that jmethodid can be resolved to Method* is false and we really can - // not do any extra checks here - return true; - } - const char *method_ptr = (const char *)SafeAccess::load((void **)id); - // check for NULL ptr and 'empty' ptr (JNIMethodBlock::_free_method) - if (method_ptr == NULL || (size_t)method_ptr == 55) { - return false; - } - VMStructs::IsValidMethodFunc func = VMStructs::is_valid_method_func(); - if (func != NULL) { - if (!func((void *)method_ptr)) { - return false; - } - } - // we can only perform the extended validity check if there are expected - // vmStructs in place - - const char *const_method = NULL; - const char *cpool = NULL; - const char *cpool_holder = NULL; - const char *cl_data = NULL; - - if (VMStructs::method_constmethod_offset() >= 0) { - const_method = (const char *)SafeAccess::load( - (void **)(method_ptr + VMStructs::method_constmethod_offset())); - if (const_method == NULL) { - return false; - } - } - if (VMStructs::constmethod_constants_offset() >= 0) { - cpool = (const char *)SafeAccess::load( - (void **)(const_method + VMStructs::constmethod_constants_offset())); - if (cpool == NULL) { - return false; - } - } - if (VMStructs::pool_holder_offset() >= 0) { - cpool_holder = - (const char *)SafeAccess::load((void **)(cpool + VMStructs::pool_holder_offset())); - if (cpool_holder == NULL) { - return false; - } - } - if (VMStructs::class_loader_data_offset() >= 0) { - // Verify the Klass at cpool_holder is readable over the full range we're about to access, - // catching freed/reclaimed Klass memory between check_jmethodID and the JVMTI calls (PROF-14003). - if (!SafeAccess::isReadableRange(cpool_holder, - (size_t)VMStructs::class_loader_data_offset() + sizeof(void*))) { - return false; - } - cl_data = (const char *)SafeAccess::load( - (void **)(cpool_holder + VMStructs::class_loader_data_offset())); - if (cl_data == NULL) { - return false; - } - } - return true; -} - -bool VMMethod::check_jmethodID_J9(jmethodID id) { - // the J9 jmethodid check is not working properly, so we just check for NULL - return id != NULL && *((void **)id) != NULL; -} - -OSThreadState VMThread::osThreadState() { - if (VMStructs::thread_osthread_offset() >= 0 && VMStructs::osthread_state_offset() >= 0) { - const char *osthread = (const char*) SafeAccess::load((void**) at(VMStructs::thread_osthread_offset())); - if (osthread != nullptr) { - // If the location is not accessible, the thread must have been terminated - int value = SafeAccess::safeFetch32((int*)(osthread + VMStructs::osthread_state_offset()), - static_cast(OSThreadState::TERMINATED)); - // Checking for bad data - if (value > static_cast(OSThreadState::SYSCALL)) { - return OSThreadState::TERMINATED; - } - return static_cast(value); - } - } - return OSThreadState::UNKNOWN; -} - -JVMJavaThreadState VMThread::state() { - assert(isJavaThread(this) && "Must be a Java thread"); - int state = 0; - int offset = VMStructs::thread_state_offset(); - if (offset >= 0) { - int* state_addr = (int*)at(offset); - if (state_addr != nullptr) { - state = SafeAccess::safeFetch32(state_addr, 0); - // Checking for bad data - if (state >= _thread_max_state || state < 0) { - state = 0; - } - } - } - return static_cast(state); -} - -bool HeapUsage::is_jmx_attempted = false; -bool HeapUsage::is_jmx_supported = false; // default to not-supported - -void HeapUsage::initJMXUsage(JNIEnv *env) { - if (is_jmx_attempted) { - // do not re-run the initialization - return; - } - is_jmx_attempted = true; - jclass factory = env->FindClass("java/lang/management/ManagementFactory"); - if (!jniExceptionCheck(env) || factory == nullptr) { - return; - } - jclass memoryBeanClass = env->FindClass("java/lang/management/MemoryMXBean"); - if (!jniExceptionCheck(env) || memoryBeanClass == nullptr) { - return; - } - jmethodID get_memory = env->GetStaticMethodID( - factory, "getMemoryMXBean", "()Ljava/lang/management/MemoryMXBean;"); - if (!jniExceptionCheck(env) || get_memory == nullptr) { - return; - } - jobject memoryBean = env->CallStaticObjectMethod(factory, get_memory); - if (!jniExceptionCheck(env) || memoryBean == nullptr) { - return; - } - jmethodID get_heap = env->GetMethodID(memoryBeanClass, "getHeapMemoryUsage", - "()Ljava/lang/management/MemoryUsage;"); - if (!jniExceptionCheck(env) || get_heap == nullptr) { - return; - } - env->CallObjectMethod(memoryBean, get_heap); - if (!jniExceptionCheck(env)) { - return; - } - // mark JMX as supported only after we were able to retrieve the memory usage - is_jmx_supported = true; -} - -bool HeapUsage::isLastGCUsageSupported() { - // only supported for JDK 17+ - // the CollectedHeap structure is vastly different in JDK 11 and earlier so - // we can't support it - return _collected_heap_addr != NULL && _heap_usage_func != NULL; -} - -bool HeapUsage::needsNativeBindingInterception() { - return _collected_heap_addr == NULL || - (_heap_usage_func == NULL && _gc_heap_summary_func == NULL); -} - -jlong HeapUsage::getMaxHeap(JNIEnv *env) { - static jclass _rt; - static jmethodID _get_rt; - static jmethodID _max_memory; - - if (!(_rt = env->FindClass("java/lang/Runtime"))) { - jniExceptionCheck(env); - return -1; - } - - if (!(_get_rt = env->GetStaticMethodID(_rt, "getRuntime", - "()Ljava/lang/Runtime;"))) { - jniExceptionCheck(env); - return -1; - } - - if (!(_max_memory = env->GetMethodID(_rt, "maxMemory", "()J"))) { - jniExceptionCheck(env); - return -1; - } - - jobject rt = (jobject)env->CallStaticObjectMethod(_rt, _get_rt); - jlong ret = (jlong)env->CallLongMethod(rt, _max_memory); - if (jniExceptionCheck(env)) { - return -1; - } - return ret; -} - -HeapUsage HeapUsage::get() { - return get(true); -} - -HeapUsage HeapUsage::get(bool allow_jmx) { - HeapUsage usage; - if (_collected_heap_addr != NULL) { - if (_heap_usage_func != NULL) { - // this is the JDK 17+ path - usage = _heap_usage_func(*(char**)_collected_heap_addr); - usage._used_at_last_gc = - ((CollectedHeapWrapper *)*(char**)_collected_heap_addr)->_used_at_last_gc; - } else if (_gc_heap_summary_func != NULL) { - // this is the JDK 11 path - // we need to collect GCHeapSummary information first - GCHeapSummary summary = _gc_heap_summary_func(*(char**)_collected_heap_addr); - usage._initSize = -1; - usage._used = summary.used(); - usage._committed = -1; - usage._maxSize = summary.maxSize(); - } - } - if (usage._maxSize == size_t(-1) && _memory_usage_func != NULL && allow_jmx && isJMXSupported()) { - // this path is for non-hotspot JVMs - // we need to patch the native method binding for JMX GetMemoryUsage to - // capture the native method pointer first also, it requires JMX and - // allocating new objects so it really should not be used in a GC callback - JNIEnv *env = VM::jni(); - if (env == NULL) { - return usage; - } - jobject m_usage = - (jobject)_memory_usage_func(env, (jobject)NULL, (jboolean) true); - jclass cls = env->GetObjectClass(m_usage); - jfieldID init_fid = env->GetFieldID(cls, "init", "J"); - jfieldID max_fid = env->GetFieldID(cls, "max", "J"); - jfieldID used_fid = env->GetFieldID(cls, "used", "J"); - jfieldID committed_fid = env->GetFieldID(cls, "committed", "J"); - if (init_fid == NULL || max_fid == NULL || used_fid == NULL || - committed_fid == NULL) { - return usage; - } - usage._initSize = env->GetLongField(m_usage, init_fid); - usage._maxSize = env->GetLongField(m_usage, max_fid); - usage._used = env->GetLongField(m_usage, used_fid); - usage._committed = env->GetLongField(m_usage, committed_fid); - } - return usage; -} diff --git a/ddprof-lib/src/main/cpp/hotspot/vmStructs.h b/ddprof-lib/src/main/cpp/hotspot/vmStructs.h deleted file mode 100644 index 7459aef40..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/vmStructs.h +++ /dev/null @@ -1,1212 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _HOTSPOT_VMSTRUCTS_H -#define _HOTSPOT_VMSTRUCTS_H - -#include -#include -#include -#include -#include -#include "codeCache.h" -#include "counters.h" -#include "jvmThread.h" -#include "safeAccess.h" -#include "thread.h" -#include "threadState.h" -#include "vmEntry.h" - -class GCHeapSummary; -class HeapUsage; -class VMNMethod; - - -// During stack walking in the profiler's signal handler, GC or class unloading -// on another thread can free VMNMethod/VMMethod memory concurrently, making -// pointers stale between the readability check and the actual dereference. -// In release builds the setjmp/longjmp crash protection in walkVM catches the -// resulting SIGSEGV. In debug builds the assert(isReadable) fires first, -// sending SIGABRT which is uncatchable by crash protection. -// When crash protection is active the assert is redundant — any bad read will -// be caught by the SIGSEGV handler and recovered via longjmp — so we skip it. -// -// Defined at the bottom of this file after VMThread is declared so that the -// VMThread fallback path (isExceptionActive) is accessible without forward- -// declaring the full class. -inline bool crashProtectionActive(); - -template -inline T* cast_to(const void* ptr) { - assert(VM::isHotspot()); // This should only be used in HotSpot-specific code - assert(T::type_size() > 0); // Ensure type size has been initialized - assert(crashProtectionActive() || ptr == nullptr || SafeAccess::isReadableRange(ptr, T::type_size())); - return reinterpret_cast(const_cast(ptr)); -} - -#define TYPE_SIZE_NAME(name) _##name##_size - -// MATCH_SYMBOLS macro expands into a string list, that is consumed by matchAny() method -#define MATCH_SYMBOLS(...) std::initializer_list{ __VA_ARGS__ } - -/** - * This macro defines a counterpart of a JVM class, e.g. VMKlass -> Klass. - * By the convention, we prefix the class name with 'VM' to avoid namespace collision - * with JVM inside a debug session. E.g. - * gdb > p this - * gdb > (VMKlass*)0x123456 - * gdb > p (Klass*)this - * .... - * - * The macro implicitly defines three static functions: - * - type_size() Return class size defined in JVM. - * - cast() It performs memory readability check before casts a void* pointer to this type. - * It ensures the memory range [ptr, ptr + type_size()) is readable. - * - load_then_cast() It loads a pointer from specified location, then does above cast. - */ -#define DECLARE(name) \ - class name : VMStructs { \ - public: \ - static uint64_t type_size() { return TYPE_SIZE_NAME(name); } \ - static name * cast(const void* ptr) { return cast_to(ptr); } \ - static name * cast_raw(const void* ptr) { return (name *)ptr; } \ - static name * load_then_cast(const void* ptr) { \ - assert(ptr != nullptr); \ - return cast(*(const void**)ptr); } - -#define DECLARE_END }; - -/** - * Defines a type and its matching symbols in vmStructs. - * A type may match multiple names in different JVM versions. - * Macro expansion: - * - Declaration phase - * static uint64_t _TYPE_size; - * - * For example: - * f(VMClassLoaderData) -> static uint64_t _VMClassLoaderData_size; - * - * - Initialization phase - * uint64_t VMStructs::_TYPE_size = 0; - * - * For example: - * f(VMClassLoaderData) -> uint64_t VMStructs::_VMClassLoaderData_size = 0; - * - * - Value population phase - * if (matchAny((char*)[]) { typeName, nullptr}) { - * _TYPE_size = size; - * continue; - * } - * - * For example: - * f(VMClassLoaderData, MATCH_SYMBOLS("ClassLoaderData")) -> - * if (matchAny((char*)[] {"ClassLoaderData", nullptr})) { - * _ClassLoaderData_size = size; - * continue; - * } - * - * - Value verification phase - * assert(_TYPE_size > 0); - * - * For example: - * f(VMClassLoaderData, MATCH_SYMBOLS("ClassLoaderData")) -> - * assert(_VMClassLoaderData_size > 0); - */ - -#define DECLARE_TYPES_DO(f) \ - f(VMClassLoaderData, MATCH_SYMBOLS("ClassLoaderData")) \ - f(VMConstantPool, MATCH_SYMBOLS("ConstantPool")) \ - f(VMConstMethod, MATCH_SYMBOLS("ConstMethod")) \ - f(VMFlag, MATCH_SYMBOLS("JVMFlag", "Flag")) \ - f(VMJavaFrameAnchor, MATCH_SYMBOLS("JavaFrameAnchor")) \ - f(VMKlass, MATCH_SYMBOLS("Klass")) \ - f(VMMethod, MATCH_SYMBOLS("Method")) \ - f(VMNMethod, MATCH_SYMBOLS("nmethod")) \ - f(VMSymbol, MATCH_SYMBOLS("Symbol")) \ - f(VMThread, MATCH_SYMBOLS("Thread")) - -// ContinuationEntry type. Only exported via gHotSpotVMTypes starting in -// JDK 27 (JDK-8378985); there is no mangled-symbol fallback for its size. -// On JDK <27 type_size() stays 0 and any code that needs the layout -// (contEntry(), parent()) bails out. -#define DECLARE_V27_TYPES_DO(f) \ - f(VMContinuationEntry, MATCH_SYMBOLS("ContinuationEntry")) - -// Fields for virtual-thread / continuation support, all added to -// gHotSpotVMStructs / gHotSpotVMTypes in JDK 27 (JDK-8378985): -// - JavaThread::_cont_entry -> _cont_entry_offset -// - StubRoutines::_cont_returnBarrier -> _cont_return_barrier_addr -// - ContinuationEntry::_return_pc -> _cont_entry_return_pc_addr -// - ContinuationEntry::_parent -> _cont_entry_parent_offset -// On JDK <27 these stay at their default (-1 / nullptr). The two stub -// *addresses* (_cont_return_barrier, _cont_entry_return_pc) are separately -// resolved via Itanium-mangled symbol lookup in resolveOffsets() so the -// cont_returnBarrier / cont_entry_return_pc frame-boundary checks still work -// on JDK 21-26; the _offset fields have no such fallback. All entries are -// intentionally excluded from verify_offsets() so a missing entry causes -// graceful degradation rather than SIGABRT. -#define DECLARE_V27_TYPE_FIELD_DO(type_begin, field, field_with_version, type_end) \ - type_begin(VMJavaThread, MATCH_SYMBOLS("JavaThread", "Thread")) \ - field_with_version(_cont_entry_offset, offset, 27, MAX_VERSION, MATCH_SYMBOLS("_cont_entry")) \ - type_end() \ - type_begin(VMStubRoutine, MATCH_SYMBOLS("StubRoutines")) \ - field_with_version(_cont_return_barrier_addr, address, 27, MAX_VERSION, MATCH_SYMBOLS("_cont_returnBarrier")) \ - type_end() \ - type_begin(VMContinuationEntry, MATCH_SYMBOLS("ContinuationEntry")) \ - field_with_version(_cont_entry_return_pc_addr, address, 27, MAX_VERSION, MATCH_SYMBOLS("_return_pc")) \ - field_with_version(_cont_entry_parent_offset, offset, 27, MAX_VERSION, MATCH_SYMBOLS("_parent")) \ - type_end() - -/** - * Following macros define field offsets, addresses or values of JVM classes that are exported by - * vmStructs. - * - type_begin() Start a definition of a type. The type name is not used at this moment, but - * improves readability. - * - field() Define a field of a class, can be either an offset, an address or a value - * - field_with_version A field that only exits in the specified JVM version range - * - type_end() End of a type definition -*/ - -typedef int offset; -typedef void* address; - -#define MIN_VERSION 8 - -// JDK 128 :-) -#define MAX_VERSION 128 - -#define DECLARE_TYPE_FIELD_DO(type_begin, field, field_with_version, type_end) \ - type_begin(VMMemRegion, MATCH_SYMBOLS("MemRegion")) \ - field(_region_start_offset, offset, MATCH_SYMBOLS("_start")) \ - field(_region_size_offset, offset, MATCH_SYMBOLS("_word_size")) \ - type_end() \ - type_begin(VMNMethod, MATCH_SYMBOLS("CompiledMethod", "nmethod")) \ - field(_nmethod_method_offset, offset,MATCH_SYMBOLS("_method")) \ - field_with_version(_nmethod_entry_offset, offset, 23, MAX_VERSION, MATCH_SYMBOLS("_verified_entry_offset")) \ - field_with_version(_nmethod_entry_address, offset, 8, 22, MATCH_SYMBOLS("_verified_entry_point")) \ - field(_nmethod_state_offset, offset, MATCH_SYMBOLS("_state")) \ - field(_nmethod_level_offset, offset, MATCH_SYMBOLS("_comp_level")) \ - field_with_version(_nmethod_metadata_offset, offset, MIN_VERSION, 24, MATCH_SYMBOLS("_metadata_offset")) \ - field_with_version(_nmethod_immutable_offset, offset, 23, MAX_VERSION, MATCH_SYMBOLS("_immutable_data")) \ - field(_scopes_pcs_offset, offset, MATCH_SYMBOLS("_scopes_pcs_offset")) \ - field_with_version(_scopes_data_offset, offset, 23, MAX_VERSION, MATCH_SYMBOLS("_scopes_data_offset")) \ - field_with_version(_scopes_data_address, offset, 9, 22, MATCH_SYMBOLS("_scopes_data_begin")) \ - type_end() \ - type_begin(VMMethod, MATCH_SYMBOLS("Method")) \ - field(_method_constmethod_offset, offset, MATCH_SYMBOLS("_constMethod")) \ - field(_method_code_offset, offset, MATCH_SYMBOLS("_code")) \ - type_end() \ - type_begin(VMConstMethod, MATCH_SYMBOLS("ConstMethod")) \ - field(_constmethod_constants_offset, offset, MATCH_SYMBOLS("_constants")) \ - field(_constmethod_idnum_offset, offset, MATCH_SYMBOLS("_method_idnum")) \ - type_end() \ - type_begin(VMConstantPool, MATCH_SYMBOLS("ConstantPool")) \ - field(_pool_holder_offset, offset, MATCH_SYMBOLS("_pool_holder")) \ - type_end() \ - type_begin(VMKlass, MATCH_SYMBOLS("Klass", "InstanceKlass")) \ - field(_klass_name_offset, offset, MATCH_SYMBOLS("_name")) \ - field(_class_loader_data_offset, offset, MATCH_SYMBOLS("_class_loader_data")) \ - field(_methods_offset, offset, MATCH_SYMBOLS("_methods")) \ - field(_jmethod_ids_offset, offset, MATCH_SYMBOLS("_methods_jmethod_ids")) \ - type_end() \ - type_begin(VMClassLoaderData, MATCH_SYMBOLS("ClassLoaderData")) \ - field(_class_loader_data_next_offset, offset, MATCH_SYMBOLS("_next")) \ - type_end() \ - type_begin(VMJavaClass, MATCH_SYMBOLS("java_lang_Class")) \ - field(_klass_offset_addr, address, MATCH_SYMBOLS("_klass_offset")) \ - type_end() \ - type_begin(VMSymbol, MATCH_SYMBOLS("Symbol")) \ - field(_symbol_length_offset, offset, MATCH_SYMBOLS("_length")) \ - field(_symbol_body_offset, offset, MATCH_SYMBOLS("_body")) \ - type_end() \ - type_begin(VMJavaThread, MATCH_SYMBOLS("JavaThread", "Thread")) \ - field(_thread_osthread_offset, offset, MATCH_SYMBOLS("_osthread")) \ - field(_thread_anchor_offset, offset, MATCH_SYMBOLS("_anchor")) \ - field(_thread_state_offset, offset, MATCH_SYMBOLS("_thread_state")) \ - field(_thread_vframe_offset, offset, MATCH_SYMBOLS("_vframe_array_head")) \ - type_end() \ - type_begin(VMOSThread, MATCH_SYMBOLS("OSThread")) \ - field(_osthread_id_offset, offset, MATCH_SYMBOLS("_thread_id")) \ - field_with_version(_osthread_state_offset, offset, 10, MAX_VERSION, MATCH_SYMBOLS("_state")) \ - type_end() \ - type_begin(VMThreadShadow, MATCH_SYMBOLS("ThreadShadow")) \ - field(_thread_exception_offset, offset, MATCH_SYMBOLS("_exception_file")) \ - type_end() \ - type_begin(VMCompilerThread, MATCH_SYMBOLS("CompilerThread")) \ - field(_comp_env_offset, offset, MATCH_SYMBOLS("_env")) \ - type_end() \ - type_begin(VMciEnv, MATCH_SYMBOLS("ciEnv")) \ - field(_comp_task_offset, offset, MATCH_SYMBOLS("_task")) \ - type_end() \ - type_begin(VMCompileTask, MATCH_SYMBOLS("CompileTask")) \ - field(_comp_method_offset, offset, MATCH_SYMBOLS("_method")) \ - type_end() \ - type_begin(VMJavaCallWrapper, MATCH_SYMBOLS("JavaCallWrapper")) \ - field(_call_wrapper_anchor_offset, offset, MATCH_SYMBOLS("_anchor")) \ - type_end() \ - type_begin(VMJavaFrameAnchor, MATCH_SYMBOLS("JavaFrameAnchor")) \ - field(_anchor_sp_offset, offset, MATCH_SYMBOLS("_last_Java_sp")) \ - field(_anchor_pc_offset, offset, MATCH_SYMBOLS("_last_Java_pc")) \ - field(_anchor_fp_offset, offset, MATCH_SYMBOLS("_last_Java_fp")) \ - type_end() \ - type_begin(VMCodeBlob, MATCH_SYMBOLS("CodeBlob")) \ - field(_blob_size_offset, offset, MATCH_SYMBOLS("_size")) \ - field(_frame_size_offset, offset, MATCH_SYMBOLS("_frame_size")) \ - field(_frame_complete_offset, offset, MATCH_SYMBOLS("_frame_complete_offset")) \ - field_with_version(_code_offset, offset, 23, MAX_VERSION, MATCH_SYMBOLS("_code_offset")) \ - field_with_version(_code_address, offset, 9, 22, MATCH_SYMBOLS("_code_begin")) \ - field(_data_offset, offset, MATCH_SYMBOLS("_data_offset")) \ - field_with_version(_mutable_data_offset, offset, 25, MAX_VERSION, MATCH_SYMBOLS("_mutable_data")) \ - field_with_version(_relocation_size_offset, offset, 23, MAX_VERSION, MATCH_SYMBOLS("_relocation_size")) \ - field(_nmethod_name_offset, offset, MATCH_SYMBOLS("_name")) \ - type_end() \ - type_begin(VMCodeCache, MATCH_SYMBOLS("CodeCache")) \ - field(_code_heap_addr, address, MATCH_SYMBOLS("_heap", "_heaps")) \ - field_with_version(_code_heap_low_addr, address, 9, MAX_VERSION, MATCH_SYMBOLS("_low_bound")) \ - field_with_version(_code_heap_high_addr, address, 9, MAX_VERSION, MATCH_SYMBOLS("_high_bound")) \ - type_end() \ - type_begin(VMCodeHeap, MATCH_SYMBOLS("CodeHeap")) \ - field(_code_heap_memory_offset, offset, MATCH_SYMBOLS("_memory")) \ - field(_code_heap_segmap_offset, offset, MATCH_SYMBOLS("_segmap")) \ - field(_code_heap_segment_shift, offset, MATCH_SYMBOLS("_log2_segment_size")) \ - type_end() \ - type_begin(VMHeapBlock, MATCH_SYMBOLS("HeapBlock::Header")) \ - field(_heap_block_used_offset, offset, MATCH_SYMBOLS("_used")) \ - type_end() \ - type_begin(VMVirtualSpace, MATCH_SYMBOLS("VirtualSpace")) \ - field(_vs_low_bound_offset, offset, MATCH_SYMBOLS("_low_boundary")) \ - field(_vs_high_bound_offset, offset, MATCH_SYMBOLS("_high_boundary")) \ - field(_vs_low_offset, offset, MATCH_SYMBOLS("_low")) \ - field(_vs_high_offset, offset, MATCH_SYMBOLS("_high")) \ - type_end() \ - type_begin(VMStubRoutine, MATCH_SYMBOLS("StubRoutines")) \ - field(_call_stub_return_addr, address, MATCH_SYMBOLS("_call_stub_return_address")) \ - type_end() \ - type_begin(VMGrowableArray, MATCH_SYMBOLS("GrowableArrayBase", "GenericGrowableArray")) \ - field(_array_len_offset, offset, MATCH_SYMBOLS("_len")) \ - type_end() \ - type_begin(VMGrowableArrayInt, MATCH_SYMBOLS("GrowableArray")) \ - field(_array_data_offset, offset, MATCH_SYMBOLS("_data")) \ - type_end() \ - type_begin(VMFlag, MATCH_SYMBOLS("JVMFlag", "Flag")) \ - field(_flag_name_offset, offset, MATCH_SYMBOLS("_name", "name")) \ - field(_flag_addr_offset, offset, MATCH_SYMBOLS("_addr", "addr")) \ - field(_flag_origin_offset, offset, MATCH_SYMBOLS("_flags", "origin")) \ - field(_flags_addr, address, MATCH_SYMBOLS("flags")) \ - field(_flag_count, address, MATCH_SYMBOLS("numFlags")) \ - field(_flag_type_offset, offset, MATCH_SYMBOLS("_type", "type")) \ - type_end() \ - type_begin(VMOop, MATCH_SYMBOLS("oopDesc")) \ - field(_oop_klass_offset, offset, MATCH_SYMBOLS("_metadata._klass", "_compressed_klass")) \ - type_end() \ - type_begin(VMUniverse, MATCH_SYMBOLS("Universe", "CompressedKlassPointers")) \ - field(_narrow_klass_base_addr, address, MATCH_SYMBOLS("_narrow_klass._base", "_base")) \ - field(_narrow_klass_shift_addr, address, MATCH_SYMBOLS("_narrow_klass._shift", "_shift")) \ - field(_collected_heap_addr, address, MATCH_SYMBOLS("_collectedHeap")) \ - type_end() - -/** - * The following macros declare JVM constants that are exported by vmStructs - * - constant defines a constant of a class - */ - -#define DECLARE_INT_CONSTANTS_DO(constant, constant_with_version) \ - constant(frame, entry_frame_call_wrapper_offset) - -#define DECLARE_LONG_CONSTANTS_DO(constant, constant_with_version) \ - constant_with_version(markWord, klass_shift, 24, MAX_VERSION) \ - constant_with_version(markWord, monitor_value, 24, MAX_VERSION) - -class VMStructs { - public: - typedef bool (*IsValidMethodFunc)(void *); - - protected: - enum { MONITOR_BIT = 2 }; - - static CodeCache* _libjvm; - - static bool _has_class_names; - static bool _has_method_structs; - static bool _has_compiler_structs; - static bool _has_stack_structs; - static bool _has_class_loader_data; - static bool _has_native_thread_id; - static bool _can_dereference_jmethod_id; - static bool _compact_object_headers; - - static int _narrow_klass_shift; - static char* _code_heap[3]; - static const void* _code_heap_low; - static const void* _code_heap_high; - static char* _narrow_klass_base; - static int _interpreter_frame_bcp_offset; - static unsigned char _unsigned5_base; - static const void* _call_stub_return; - static const void* _cont_return_barrier; - static const void* _cont_entry_return_pc; - static VMNMethod* _enter_special_nm; - static const void* _interpreter_start; - static VMNMethod* _interpreter_nm; - static const void* _interpreted_frame_valid_start; - static const void* _interpreted_frame_valid_end; - - -// Declare type size variables - #define DECLARE_TYPE_SIZE_VAR(name, names) \ - static uint64_t TYPE_SIZE_NAME(name); - - DECLARE_TYPES_DO(DECLARE_TYPE_SIZE_VAR) - DECLARE_V27_TYPES_DO(DECLARE_TYPE_SIZE_VAR) -#undef DECLARE_TYPE_SIZE_VAR - -// Declare vmStructs' field offsets and addresses - -// Do nothing macro -#define DO_NOTHING(...) -#define DECLARE_TYPE_FIELD(var, field_type, names) \ - static field_type var; -#define DECLARE_TYPE_FIELD_WITH_VERSION(var, field_type, min_version, max_version, names) \ - static field_type var; - - DECLARE_TYPE_FIELD_DO(DO_NOTHING, DECLARE_TYPE_FIELD, DECLARE_TYPE_FIELD_WITH_VERSION, DO_NOTHING) - DECLARE_V27_TYPE_FIELD_DO(DO_NOTHING, DECLARE_TYPE_FIELD, DECLARE_TYPE_FIELD_WITH_VERSION, DO_NOTHING) -#undef DECLARE_TYPE_FIELD -#undef DECLARE_TYPE_FIELD_WITH_VERSION -#undef DO_NOTHING - -// Declare int constant variables -#define DECLARE_INT_CONSTANT_VAR(type, field, ...) \ - static int _##type##_##field; - DECLARE_INT_CONSTANTS_DO(DECLARE_INT_CONSTANT_VAR, DECLARE_INT_CONSTANT_VAR) -#undef DECLARE_INT_CONSTANT_VAR - -// Declare long constant variables -#define DECLARE_LONG_CONSTANT_VAR(type, field, ...) \ - static long _##type##_##field; - DECLARE_LONG_CONSTANTS_DO(DECLARE_LONG_CONSTANT_VAR, DECLARE_LONG_CONSTANT_VAR) -#undef DECLARE_LONG_CONSTANT_VAR - - - static jfieldID _eetop; - static jfieldID _klass; - static intptr_t _env_offset; - static void* _java_thread_vtbl[6]; - - typedef void (*LockFunc)(void*); - static LockFunc _lock_func; - static LockFunc _unlock_func; - - // Datadog-specific extensions - static CodeCache _unsafe_to_walk; - typedef HeapUsage (*HeapUsageFunc)(const void *); - static HeapUsageFunc _heap_usage_func; - typedef void *(*MemoryUsageFunc)(void *, void *, bool); - static MemoryUsageFunc _memory_usage_func; - typedef GCHeapSummary (*GCHeapSummaryFunc)(void *); - static GCHeapSummaryFunc _gc_heap_summary_func; - static IsValidMethodFunc _is_valid_method_func; - - static uintptr_t readSymbol(const char* symbol_name); - - // Read VM information from vmStructs - static void init_type_sizes(); - static void init_offsets_and_addresses(); - static void init_constants(); - static void initOffsets(); - -#ifdef DEBUG - static void verify_offsets(); -#endif - - static void resolveOffsets(); - static void patchSafeFetch(); - static void initJvmFunctions(); - static void initTLS(void* vm_thread); - static void initThreadBridge(); - - // Datadog-specific private methods - static void initUnsafeFunctions(); - static void initCriticalJNINatives(); - static void checkNativeBinding(jvmtiEnv *jvmti, JNIEnv *jni, jmethodID method, void *address); - static const void *findHeapUsageFunc(); - - const char* at(int offset) { - const char* ptr = (const char*)this + offset; - assert(crashProtectionActive() || SafeAccess::isReadable(ptr)); - return ptr; - } - - static bool goodPtr(const void* ptr) { - return (uintptr_t)ptr >= 0x1000 && ((uintptr_t)ptr & (sizeof(uintptr_t) - 1)) == 0; - } - - template - static T align(const void* ptr) { - static_assert(std::is_pointer::value, "T must be a pointer type"); - return (T)((uintptr_t)ptr & ~(sizeof(T) - 1)); - } - - public: - static void init(CodeCache* libjvm); - static void ready(); - - static CodeCache* libjvm() { - return _libjvm; - } - - static bool hasClassNames() { - return _has_class_names; - } - - static bool hasMethodStructs() { - return _has_method_structs; - } - - static bool hasCompilerStructs() { - return _has_compiler_structs; - } - - static bool hasStackStructs() { - return _has_stack_structs; - } - - static bool hasClassLoaderData() { - return _has_class_loader_data; - } - - static bool hasNativeThreadId() { - return _has_native_thread_id; - } - - static bool isInterpretedFrameValidFunc(const void* pc) { - return pc >= _interpreted_frame_valid_start && pc < _interpreted_frame_valid_end; - } - - static bool isContReturnBarrier(const void* pc) { - return _cont_return_barrier != nullptr && pc == _cont_return_barrier; - } - - // True when the bottom VT frame's return PC is cont_entry_return_pc, meaning all - // VT frames are thawed (CPU-bound VT that never yielded). - // Available on JDK 21+ via vmStructs or symbol fallback. - static bool isContEntryReturnPc(const void* pc) { - return _cont_entry_return_pc != nullptr && pc == _cont_entry_return_pc; - } - - static VMNMethod* enterSpecialNMethod() { - return _enter_special_nm; - } - - // Datadog-specific extensions - static bool isSafeToWalk(uintptr_t pc); - static void JNICALL NativeMethodBind(jvmtiEnv *jvmti, JNIEnv *jni, - jthread thread, jmethodID method, - void *address, void **new_address_ptr); - - static int thread_osthread_offset() { - return _thread_osthread_offset; - } - - static int osthread_state_offset() { - return _osthread_state_offset; - } - - static int osthread_id_offset() { - return _osthread_id_offset; - } - - static int thread_state_offset() { - return _thread_state_offset; - } - - static int flag_type_offset() { - return _flag_type_offset; - } - - static int method_constmethod_offset() { - return _method_constmethod_offset; - } - - static int constmethod_constants_offset() { - return _constmethod_constants_offset; - } - - static int pool_holder_offset() { - return _pool_holder_offset; - } - - static int class_loader_data_offset() { - return _class_loader_data_offset; - } - - static IsValidMethodFunc is_valid_method_func() { - return _is_valid_method_func; - } -}; - -class HeapUsage : VMStructs { -private: - static bool is_jmx_attempted; - static bool is_jmx_supported; // default to not-supported -public: - size_t _initSize = -1; - size_t _used = -1; - size_t _committed = -1; - size_t _maxSize = -1; - size_t _used_at_last_gc = -1; - - static void initJMXUsage(JNIEnv* env); - - static bool isJMXSupported() { - initJMXUsage(VM::jni()); - return is_jmx_supported; - } - - static bool isLastGCUsageSupported(); - static bool needsNativeBindingInterception(); - static jlong getMaxHeap(JNIEnv *env); - static HeapUsage get(); - static HeapUsage get(bool allow_jmx); -}; - -class MethodList { - public: - enum { SIZE = 8 }; - - private: - intptr_t _method[SIZE]; - int _ptr; - MethodList* _next; - int _padding; - - public: - MethodList(MethodList* next) : _ptr(0), _next(next), _padding(0) { - for (int i = 0; i < SIZE; i++) { - _method[i] = 0x37; - } - } -}; - - -class VMNMethod; -class VMMethod; - -DECLARE(VMSymbol) - public: - unsigned short length() { - assert(_symbol_length_offset >= 0); - return *(unsigned short*) at(_symbol_length_offset); - } - - const char* body() { - assert(_symbol_body_offset >= 0); - return at(_symbol_body_offset); - } - - // Public accessors for safefetch-based dump-time resolution (no `this` - // deref): used to compute the address of the length/body fields without - // touching the Symbol's memory, so callers can probe with SafeAccess. - static int lengthOffset() { return _symbol_length_offset; } - static int bodyOffset() { return _symbol_body_offset; } -DECLARE_END - -DECLARE(VMClassLoaderData) - private: - void* mutex() { - return *(void**) at(sizeof(uintptr_t) * 3); - } - - public: - void lock() { - _lock_func(mutex()); - } - - void unlock() { - _unlock_func(mutex()); - } - - MethodList** methodList() { - return (MethodList**) at(sizeof(uintptr_t) * 6 + 8); - } -DECLARE_END - -DECLARE(VMKlass) - public: - static VMKlass* fromJavaClass(JNIEnv* env, jclass cls) { - if (sizeof(VMKlass*) == 8) { - return VMKlass::cast((const void*)(intptr_t)env->GetLongField(cls, _klass)); - } else { - return VMKlass::cast((const void*)(intptr_t)env->GetIntField(cls, _klass)); - } - } - - static VMKlass* fromHandle(uintptr_t handle) { - return VMKlass::cast((const void*)handle); - } - - static VMKlass* fromOop(uintptr_t oop) { - if (_narrow_klass_shift >= 0) { - uintptr_t narrow_klass; - if (_compact_object_headers) { - uintptr_t mark = *(uintptr_t*)oop; - if (mark & MONITOR_BIT) { - mark = *(uintptr_t*)(mark ^ MONITOR_BIT); - } - narrow_klass = mark >> _markWord_klass_shift; - } else { - narrow_klass = *(unsigned int*)(oop + _oop_klass_offset); - } - return VMKlass::cast((const void*)(_narrow_klass_base + (narrow_klass << _narrow_klass_shift))); - } else { - return VMKlass::load_then_cast((const void*)(oop + _oop_klass_offset)); - } - } - - VMSymbol* name() { - assert(_klass_name_offset >= 0); - return VMSymbol::load_then_cast(at(_klass_name_offset)); - } - - VMClassLoaderData* classLoaderData() { - assert(_class_loader_data_offset >= 0); - return VMClassLoaderData::load_then_cast(at(_class_loader_data_offset)); - } - - int methodCount() { - assert(_methods_offset >= 0); - int* methods = *(int**) at(_methods_offset); - return methods == NULL ? 0 : *methods & 0xffff; - } - - jmethodID* jmethodIDs() { - assert(_jmethod_ids_offset >= 0); - return __atomic_load_n((jmethodID**) at(_jmethod_ids_offset), __ATOMIC_ACQUIRE); - } -DECLARE_END - -DECLARE(VMJavaFrameAnchor) - private: - enum { MAX_CALL_WRAPPER_DISTANCE = 512 }; - - public: - NOADDRSANITIZE static VMJavaFrameAnchor* fromEntryFrame(uintptr_t fp) { - assert(_frame_entry_frame_call_wrapper_offset != -1); - assert(_call_wrapper_anchor_offset >= 0); - const char* call_wrapper = (const char*) SafeAccess::loadPtr((void**)(fp + _frame_entry_frame_call_wrapper_offset), nullptr); - if (!goodPtr(call_wrapper) || (uintptr_t)call_wrapper - fp > MAX_CALL_WRAPPER_DISTANCE) { - return NULL; - } - return VMJavaFrameAnchor::cast((const void*)(call_wrapper + _call_wrapper_anchor_offset)); - } - - NOADDRSANITIZE uintptr_t lastJavaSP() { - assert(_anchor_sp_offset >= 0); - return (uintptr_t) SafeAccess::loadPtr((void**) at(_anchor_sp_offset), nullptr); - } - - NOADDRSANITIZE uintptr_t lastJavaFP() { - assert(_anchor_fp_offset >= 0); - return (uintptr_t) SafeAccess::loadPtr((void**) at(_anchor_fp_offset), nullptr); - } - - NOADDRSANITIZE const void* lastJavaPC() { - assert(_anchor_pc_offset >= 0); - return SafeAccess::loadPtr((void**) at(_anchor_pc_offset), nullptr); - } - - void setLastJavaPC(const void* pc) { - assert(_anchor_pc_offset >= 0); - *(const void**) at(_anchor_pc_offset) = pc; - } - - NOADDRSANITIZE bool getFrame(const void*& pc, uintptr_t& sp, uintptr_t& fp) { - if (lastJavaPC() != NULL && lastJavaSP() != 0) { - pc = lastJavaPC(); - sp = lastJavaSP(); - fp = lastJavaFP(); - return true; - } - return false; - } -DECLARE_END - -DECLARE(VMContinuationEntry) - public: - // Address of the enterSpecial frame's {saved_fp, return_addr} pair. - // Layout above this address: [saved_fp][return_addr_to_carrier][carrier_sp...] - // The ContinuationEntry struct is embedded on the carrier stack immediately - // below enterSpecial's saved-fp slot; its size() equals the JVM's - // ContinuationEntry::size() static method, confirmed at: - // https://github.com/openjdk/jdk/blob/master/src/hotspot/share/runtime/continuationEntry.hpp - // https://github.com/openjdk/jdk/blob/master/src/hotspot/share/runtime/continuationEntry.cpp - uintptr_t entryFP() const { - assert(type_size() > 0); // must not be called before ContinuationEntry is resolved - return (uintptr_t)this + type_size(); - } - - // Returns the enclosing ContinuationEntry when continuations are nested - // (e.g. a Continuation.run() call inside a virtual thread). Returns - // nullptr when there is no enclosing entry or the field is unavailable. - VMContinuationEntry* parent() const { - if (_cont_entry_parent_offset < 0) return nullptr; - void* ptr = SafeAccess::loadPtr((void**) const_cast(this)->at(_cont_entry_parent_offset), nullptr); - return ptr != nullptr ? VMContinuationEntry::cast(ptr) : nullptr; - } -DECLARE_END - -// Copied from JDK's globalDefinitions.hpp 'JavaThreadState' enum -enum JVMJavaThreadState { - _thread_uninitialized = 0, // should never happen (missing initialization) - _thread_new = 2, // just starting up, i.e., in process of being initialized - _thread_new_trans = 3, // corresponding transition state (not used, included for completeness) - _thread_in_native = 4, // running in native code - _thread_in_native_trans = 5, // corresponding transition state - _thread_in_vm = 6, // running in VM - _thread_in_vm_trans = 7, // corresponding transition state - _thread_in_Java = 8, // running in Java or in stub code - _thread_in_Java_trans = 9, // corresponding transition state (not used, included for completeness) - _thread_blocked = 10, // blocked in vm - _thread_blocked_trans = 11, // corresponding transition state - _thread_max_state = 12 // maximum thread state+1 - used for statistics allocation -}; - -DECLARE(VMThread) - friend class JVMThread; - public: - static void* initialize(jthread thread); - - static inline VMThread* current(); - static inline VMThread* fromJavaThread(JNIEnv* env, jthread thread); - static bool isJavaThread(VMThread* thread); - static ExecutionMode getExecutionMode(); - static OSThreadState getOSThreadState(); - - int osThreadId(); - JNIEnv* jni(); - - OSThreadState osThreadState(); - - JVMJavaThreadState state(); - - bool inDeopt() { - if (!isJavaThread(this)) return false; - assert(_thread_vframe_offset >= 0); - return SafeAccess::loadPtr((void**) at(_thread_vframe_offset), nullptr) != NULL; - } - - // Check if the thread object memory is readable up to the largest used - // offset. On some JVMs (e.g. GraalVM 25 aarch64), a wall-clock signal - // can hit a thread whose memory is only partially mapped — the vtable - // at offset 0 may be readable while fields deeper in the object are not. - // On non-HotSpot JVMs (J9, Zing) offsets stay at -1; skip the check. - bool isThreadAccessible() { - int max_offset = -1; - if (_thread_exception_offset > max_offset) max_offset = _thread_exception_offset; - if (_thread_state_offset > max_offset) max_offset = _thread_state_offset; - if (_thread_osthread_offset > max_offset) max_offset = _thread_osthread_offset; - if (_thread_anchor_offset > max_offset) max_offset = _thread_anchor_offset; - if (_thread_vframe_offset > max_offset) max_offset = _thread_vframe_offset; - if (max_offset < 0) return true; - return SafeAccess::isReadableRange(this, max_offset + sizeof(void*)); - } - - void*& exception() { - if (_thread_exception_offset < 0) { - static void* _null_exception = nullptr; - return _null_exception; - } - return *(void**) at(_thread_exception_offset); - } - - // Returns true if setjmp crash protection is currently active for this thread. - // Reads the exception field via direct pointer arithmetic, deliberately bypassing - // at() and its crashProtectionActive() assertion to avoid infinite recursion. - // Safe because 'this' is the current live thread (we are in its signal handler). - static bool isExceptionActive() { - if (_thread_exception_offset < 0) return false; - void* vt = JVMThread::current(); - if (vt == nullptr) return false; - return *(const void* const*)((const char*)vt + _thread_exception_offset) != nullptr; - } - - NOADDRSANITIZE VMJavaFrameAnchor* anchor() { - if (!isJavaThread(this)) return NULL; - assert(_thread_anchor_offset >= 0); - return VMJavaFrameAnchor::cast(at(_thread_anchor_offset)); - } - - // Returns true when this thread is currently executing a virtual thread - // (i.e. JavaThread::_cont_entry is non-null). _cont_entry_offset is - // only populated on JDK 27+ (from gHotSpotVMStructs, JDK-8378985); there - // is no symbol fallback, so this returns false on JDK <27. - // Does NOT require ContinuationEntry type_size(). - bool isCarryingVirtualThread() const { - if (_cont_entry_offset < 0) return false; - return SafeAccess::loadPtr((void**) const_cast(this)->at(_cont_entry_offset), nullptr) != nullptr; - } - - // Returns the innermost active ContinuationEntry for this thread, or nullptr - // if none exists or ContinuationEntry layout is unavailable (JDK <27, where - // neither _cont_entry_offset nor ContinuationEntry are in gHotSpotVMStructs/ - // gHotSpotVMTypes so type_size() == 0). - // Used by stackWalker to locate the enterSpecial frame when crossing the - // virtual-thread continuation boundary. - VMContinuationEntry* contEntry() { - if (_cont_entry_offset < 0 || VMContinuationEntry::type_size() == 0) return nullptr; - void* ptr = SafeAccess::loadPtr((void**) at(_cont_entry_offset), nullptr); - return ptr != nullptr ? VMContinuationEntry::cast(ptr) : nullptr; - } - - inline VMMethod* compiledMethod(); -private: - static inline int nativeThreadId(JNIEnv* jni, jthread thread); - inline void** vtable(); - inline bool hasJavaThreadVtable(); - -DECLARE_END - -DECLARE(VMConstMethod) -DECLARE_END - - -DECLARE(VMMethod) - private: - static bool check_jmethodID_J9(jmethodID id); - static bool check_jmethodID_hotspot(jmethodID id); - - public: - jmethodID id(); - - // Performs extra validation when VMMethod comes from incomplete frame - jmethodID validatedId(); - - // Workaround for JDK-8313816 - static bool isStaleMethodId(jmethodID id) { - if (!_can_dereference_jmethod_id) return false; - - VMMethod* vm_method = VMMethod::load_then_cast((const void*)id); - return vm_method == NULL || vm_method->id() == NULL; - } - - const char* bytecode() { - assert(_method_constmethod_offset >= 0); - return *(const char**) at(_method_constmethod_offset) + VMConstMethod::type_size(); - } - - inline VMNMethod* code(); - - static bool check_jmethodID(jmethodID id); -DECLARE_END - -// Inline string comparison to avoid indirect call to strncmp -template -static inline bool startsWith(const char* s, const char (&pattern)[N]) { - for (size_t i = 0; i < N - 1; i++) { - if (s[i] != pattern[i]) return false; - } - return true; -} - -DECLARE(VMNMethod) - public: - int size() { - assert(_blob_size_offset >= 0); - return *(int*) at(_blob_size_offset); - } - - int frameSize() { - assert(_frame_size_offset >= 0); - return *(int*) at(_frame_size_offset); - } - - short frameCompleteOffset() { - assert(_frame_complete_offset >= 0); - return *(short*) at(_frame_complete_offset); - } - - void setFrameCompleteOffset(int offset) { - if (_nmethod_immutable_offset > 0) { - // _frame_complete_offset is short on JDK 23+ - *(short*) at(_frame_complete_offset) = offset; - } else { - *(int*) at(_frame_complete_offset) = offset; - } - } - - const char* immutableDataAt(int offset) { - if (_nmethod_immutable_offset > 0) { - return *(const char**) at(_nmethod_immutable_offset) + offset; - } - return at(offset); - } - - const char* code() { - if (_code_offset != -1) { // JDK23+ - return at(*(int*) at(_code_offset)); - } else { - return *(const char**) at(_code_address); - } - } - - const char* scopes() { - if (_scopes_data_offset != -1) { // JDK23+ - return immutableDataAt(*(int*) at(_scopes_data_offset)); - } else { - return *(const char**) at(_scopes_data_address); - } - } - - const void* entry() { - if (_nmethod_entry_offset != -1) { // JDK23+ - return at(*(int*) at(_code_offset) + *(unsigned short*) at(_nmethod_entry_offset)); - } else { - return *(void**) at(_nmethod_entry_address); - } - } - - bool contains(const void* pc) { - return pc >= this && pc < at(size()); - } - - bool isFrameCompleteAt(const void* pc) { - return pc >= code() + frameCompleteOffset(); - } - - bool isEntryFrame(const void* pc) { - return pc == _call_stub_return; - } - - const char* name() { - assert(_nmethod_name_offset >= 0); - return *(const char**) at(_nmethod_name_offset); - } - - bool isInterpreter() { - return this == _interpreter_nm; - } - - bool isNMethod() { - const char* n = name(); - return n != NULL && (startsWith(n, "nmethod\0") || startsWith(n, "native nmethod\0")); - } - - bool isStub() { - const char* n = name(); - return n != NULL && startsWith(n, "StubRoutines"); - } - - bool isVTableStub() { - const char* n = name(); - return n != NULL && startsWith(n, "vtable chunks"); - } - - VMMethod* method() { - assert(_nmethod_method_offset >= 0); - return VMMethod::load_then_cast((const void*)at(_nmethod_method_offset)); - } - - char state() { - return *at(_nmethod_state_offset); - } - - bool isAlive() { - return state() >= 0 && state() <= 1; - } - - int level() { - return _nmethod_level_offset >= 0 ? *(signed char*) at(_nmethod_level_offset) : 0; - } - - VMMethod** metadata() { - if (_mutable_data_offset >= 0) { - // Since JDK 25 - assert(_relocation_size_offset >= 0); - return (VMMethod**) (*(char**) at(_mutable_data_offset) + *(int*) at(_relocation_size_offset)); - } else if (_data_offset > 0) { - // since JDK 23 - assert(_nmethod_metadata_offset >= 0); - assert(_data_offset >= 0); - return (VMMethod**) at(*(int*) at(_data_offset) + *(unsigned short*) at(_nmethod_metadata_offset)); - } - assert(_nmethod_metadata_offset >= 0); - return (VMMethod**) at(*(int*) at(_nmethod_metadata_offset)); - } - - int findScopeOffset(const void* pc); -DECLARE_END - -class CodeHeap : VMStructs { - private: - static bool contains(char* heap, const void* pc) { - return heap != NULL && - pc >= *(const void**)(heap + _code_heap_memory_offset + _vs_low_offset) && - pc < *(const void**)(heap + _code_heap_memory_offset + _vs_high_offset); - } - - static VMNMethod* findNMethod(char* heap, const void* pc); - - public: - static bool available() { - return _code_heap_addr != NULL; - } - - static bool contains(const void* pc) { - return _code_heap_low <= pc && pc < _code_heap_high; - } - - static void updateBounds(const void* start, const void* end) { - for (const void* low = _code_heap_low; - start < low && !__sync_bool_compare_and_swap(&_code_heap_low, low, start); - low = _code_heap_low); - for (const void* high = _code_heap_high; - end > high && !__sync_bool_compare_and_swap(&_code_heap_high, high, end); - high = _code_heap_high); - } - - static void setInterpreterStart(const void* start) { - _interpreter_start = start; - _interpreter_nm = findNMethod(start); - } - - static VMNMethod* findNMethod(const void* pc) { - if (contains(_code_heap[0], pc)) return findNMethod(_code_heap[0], pc); - if (contains(_code_heap[1], pc)) return findNMethod(_code_heap[1], pc); - if (contains(_code_heap[2], pc)) return findNMethod(_code_heap[2], pc); - return NULL; - } -}; - -DECLARE(VMFlag) - private: - enum { - ORIGIN_DEFAULT = 0, - ORIGIN_MASK = 15, - SET_ON_CMDLINE = 1 << 17 - }; - static VMFlag* find(const char *name, int type_mask); - - public: - enum Type { - Bool = 0, - Int = 1, - Uint = 2, - Intx = 3, - Uintx = 4, - Uint64_t = 5, - Size_t = 6, - Double = 7, - String = 8, - Stringlist = 9, - Unknown = -1 - }; - - static VMFlag* find(const char* name); - static VMFlag *find(const char* name, std::initializer_list types); - - const char* name() { - assert(_flag_name_offset >= 0); - return *(const char**) at(_flag_name_offset); - } - - int type(); - - void* addr() { - assert(_flag_addr_offset >= 0); - return *(void**) at(_flag_addr_offset); - } - - char origin() { - return _flag_origin_offset >= 0 ? (*(char*) at(_flag_origin_offset)) & 15 : 0; - } - - bool isDefault() { - return _flag_origin_offset < 0 || (*(int*) at(_flag_origin_offset) & ORIGIN_MASK) == ORIGIN_DEFAULT; - } - - void setCmdline() { - if (_flag_origin_offset >= 0) { - *(int*) at(_flag_origin_offset) |= SET_ON_CMDLINE; - } - } - - char get() { - return *((char*)addr()); - } - - void set(char value) { - *((char*)addr()) = value; - } -DECLARE_END - -class PcDesc { - public: - int _pc; - int _scope_offset; - int _obj_offset; - int _flags; -}; - -class ScopeDesc : VMStructs { - private: - const unsigned char* _scopes; - VMMethod** _metadata; - const unsigned char* _stream; - int _method_offset; - int _bci; - - int readInt(); - - public: - ScopeDesc(VMNMethod* nm) { - _scopes = (const unsigned char*)nm->scopes(); - _metadata = nm->metadata(); - } - - int decode(int offset) { - _stream = _scopes + offset; - int sender_offset = readInt(); - _method_offset = readInt(); - _bci = readInt() - 1; - return sender_offset; - } - - VMMethod* method() { - return _method_offset > 0 ? _metadata[_method_offset - 1] : NULL; - } - - int bci() { - return _bci; - } -}; - -class InterpreterFrame : VMStructs { - public: - enum { - sender_sp_offset = -1, - method_offset = -3 - }; - - static int bcp_offset() { - return _interpreter_frame_bcp_offset; - } -}; - -// Defined here (after VMThread) so the VMThread::isExceptionActive() fallback -// is accessible. The forward declaration at the top of this file allows cast_to() -// to reference it before VMThread is declared. -inline bool crashProtectionActive() { - ProfiledThread* pt = ProfiledThread::currentSignalSafe(); - if (pt != nullptr && pt->isCrashProtectionActive()) return true; - // Fallback for threads without ProfiledThread TLS (e.g. JVM internal threads): - // if walkVM has set up setjmp protection via vm_thread->exception(), the assert - // is equally redundant — any bad read will be caught by the SIGSEGV handler. - // Uses VMThread::isExceptionActive() which reads the field directly without - // going through at() to avoid recursive assertion. - return JVMThread::key() != pthread_key_t(-1) && VMThread::isExceptionActive(); -} - -#endif // _HOTSPOT_VMSTRUCTS_H diff --git a/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h b/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h deleted file mode 100644 index bb84fa257..000000000 --- a/ddprof-lib/src/main/cpp/hotspot/vmStructs.inline.h +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _HOTSPOT_VMSTRUCTS_INLINE_H -#define _HOTSPOT_VMSTRUCTS_INLINE_H - -#include "hotspot/vmStructs.h" -#include "jvmThread.h" - -VMThread* VMThread::current() { - assert(VM::isHotspot()); - return VMThread::cast(JVMThread::current()); -} - -VMThread* VMThread::fromJavaThread(JNIEnv* env, jthread thread) { - assert(VM::isHotspot()); - assert(_eetop != nullptr); - if (_eetop != nullptr) { - jlong eetop = env->GetLongField(thread, _eetop); - if (env->ExceptionCheck()) { - env->ExceptionClear(); - return nullptr; - } - return eetop != 0 ? VMThread::cast((void*)eetop) : nullptr; - } else { - return nullptr; - } -} - -int VMThread::nativeThreadId(JNIEnv* jni, jthread thread) { - if (_has_native_thread_id) { - VMThread* vm_thread = fromJavaThread(jni, thread); - return vm_thread != NULL ? vm_thread->osThreadId() : -1; - } - return -1; -} - -void** VMThread::vtable() { - assert(SafeAccess::isReadable(this)); - return *(void***)this; -} - -// This thread is considered a JavaThread if at least 2 of the selected 3 vtable entries -// match those of a known JavaThread (which is either application thread or AttachListener). -// Indexes were carefully chosen to work on OpenJDK 8 to 25, both product an debug builds. -bool VMThread::hasJavaThreadVtable() { - void** vtbl = vtable(); - return (SafeAccess::load(&vtbl[1]) == _java_thread_vtbl[1]) + - (SafeAccess::load(&vtbl[3]) == _java_thread_vtbl[3]) + - (SafeAccess::load(&vtbl[5]) == _java_thread_vtbl[5]) >= 2; -} - -VMNMethod* VMMethod::code() { - assert(_method_code_offset >= 0); - const void* code_ptr = *(const void**) at(_method_code_offset); - return VMNMethod::cast(code_ptr); -} - -VMMethod* VMThread::compiledMethod() { - if (!isJavaThread(this)) return NULL; - assert(_comp_method_offset >= 0); - assert(_comp_env_offset >= 0); - assert(_comp_task_offset >= 0); - const char* env = *(const char**) at(_comp_env_offset); - if (env != NULL) { - const char* task = *(const char**) (env + _comp_task_offset); - if (task != NULL) { - return VMMethod::load_then_cast((const void*)(task + _comp_method_offset)); - } - } - return NULL; -} - -#endif // _HOTSPOT_VMSTRUCTS_INLINE_H diff --git a/ddprof-lib/src/main/cpp/incbin.h b/ddprof-lib/src/main/cpp/incbin.h deleted file mode 100644 index afbc7629e..000000000 --- a/ddprof-lib/src/main/cpp/incbin.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _INCBIN_H -#define _INCBIN_H - -#ifdef __APPLE__ -# define INCBIN_SECTION ".const_data" -# define INCBIN_SYMBOL "_" -#else -# define INCBIN_SECTION ".section \".rodata\", \"a\"" -# define INCBIN_SYMBOL -#endif - -#define INCBIN(NAME, FILE) \ - extern "C" const char NAME[];\ - extern "C" const char NAME##_END[];\ - asm(INCBIN_SECTION "\n"\ - ".globl " INCBIN_SYMBOL #NAME "\n"\ - INCBIN_SYMBOL #NAME ":\n"\ - ".incbin \"" FILE "\"\n"\ - ".globl " INCBIN_SYMBOL #NAME "_END\n"\ - INCBIN_SYMBOL #NAME "_END:\n"\ - ".byte 0x00\n"\ - ".previous\n"\ - ); - -#define INCBIN_SIZEOF(NAME) (NAME##_END - NAME) - -#define INCLUDE_HELPER_CLASS(NAME_VAR, DATA_VAR, NAME) \ - static const char* const NAME_VAR = NAME;\ - INCBIN(DATA_VAR, "src/helper/" NAME ".class") - -#endif // _INCBIN_H diff --git a/ddprof-lib/src/main/cpp/itimer.cpp b/ddprof-lib/src/main/cpp/itimer.cpp deleted file mode 100644 index f041e1885..000000000 --- a/ddprof-lib/src/main/cpp/itimer.cpp +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Copyright 2018 Andrei Pangin - * Copyright 2025, 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "itimer.h" -#include "debugSupport.h" -#include "jvmThread.h" -#include "os.h" -#include "profiler.h" -#include "stackWalker.h" -#include "thread.h" -#include "threadState.inline.h" -#include "guards.h" -#include - -volatile bool ITimer::_enabled = false; -long ITimer::_interval; -CStack ITimer::_cstack; - -void ITimer::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { - SIGNAL_HANDLER_GUARD(); - // NOTE: ITimer uses setitimer(ITIMER_PROF) which delivers signals with - // si_code==SI_KERNEL — no sival payload is available. The signal-origin - // check implemented in CTimer/WallClock cannot be applied here. ITimer - // is therefore vulnerable to the foreign-SIGPROF deadlock scenario this - // feature addresses. Use CTimer (the default) when signal-origin - // validation is required. - if (!_enabled) - return; - - // Atomically try to enter critical section - prevents all reentrancy races - CriticalSection cs; - if (!cs.entered()) { - return; // Another critical section is active, defer profiling - } - int tid = 0; - ProfiledThread *current = ProfiledThread::currentSignalSafe(); - if (current != NULL) { - current->noteCPUSample(Profiler::instance()->recordingEpoch()); - tid = current->tid(); - } else { - tid = OS::threadId(); - } - Shims::instance().setSighandlerTid(tid); - - ExecutionEvent event; - event._execution_mode = getThreadExecutionMode(); - Profiler::instance()->recordSample(ucontext, _interval, tid, BCI_CPU, 0, - &event); - Shims::instance().setSighandlerTid(-1); -} - -Error ITimer::check(Arguments &args) { - OS::installSignalHandler(SIGPROF, NULL, SIG_IGN); - - struct itimerval tv_on = {{1, 0}, {1, 0}}; - if (setitimer(ITIMER_PROF, &tv_on, NULL) != 0) { - return Error("ITIMER_PROF is not supported on this system"); - } - - struct itimerval tv_off = {{0, 0}, {0, 0}}; - setitimer(ITIMER_PROF, &tv_off, NULL); - - return Error::OK; -} - -Error ITimer::start(Arguments &args) { - _interval = args.cpuSamplerInterval(); - _cstack = args._cstack; - - OS::installSignalHandler(SIGPROF, signalHandler); - - time_t sec = _interval / 1000000000; - suseconds_t usec = (_interval % 1000000000) / 1000; - struct itimerval tv = {{sec, usec}, {sec, usec}}; - - if (setitimer(ITIMER_PROF, &tv, NULL) != 0) { - return Error("ITIMER_PROF is not supported on this system"); - } - - return Error::OK; -} - -void ITimer::stop() { - struct itimerval tv = {{0, 0}, {0, 0}}; - setitimer(ITIMER_PROF, &tv, NULL); -} - -volatile bool ITimerJvmti::_enabled = false; -long ITimerJvmti::_interval = 0; - -void ITimerJvmti::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { - SIGNAL_HANDLER_GUARD(); - CriticalSection cs; - if (!cs.entered()) { - return; - } - int saved_errno = errno; - if (!__atomic_load_n(&_enabled, __ATOMIC_ACQUIRE)) { - errno = saved_errno; - return; - } - ProfiledThread *current = ProfiledThread::currentSignalSafe(); - if (current != nullptr && JVMThread::isInitialized() && JVMThread::current() == nullptr - && current->inInitWindow()) { - current->tickInitWindow(); - errno = saved_errno; - return; - } - int tid = current ? current->tid() : OS::threadId(); - if (current) { - current->noteCPUSample(Profiler::instance()->recordingEpoch()); - } - Shims::instance().setSighandlerTid(tid); - - ExecutionEvent event; - event._execution_mode = getThreadExecutionMode(); - // setitimer(ITIMER_PROF) delivers SIGPROF to an arbitrary thread chosen by - // the OS, so ucontext may be from a JVM-internal thread. Pass nullptr to - // force the JVM into safepoint-based stack walking instead. - Profiler::instance()->recordSampleDelegated(nullptr, _interval, tid, - BCI_CPU, &event); - Shims::instance().setSighandlerTid(-1); - errno = saved_errno; -} - -Error ITimerJvmti::check(Arguments &args) { - if (!VM::canRequestStackTrace()) { - return Error("HotSpot RequestStackTrace JVMTI extension not available"); - } - - struct sigaction oldsa; - struct sigaction ign = {}; - sigemptyset(&ign.sa_mask); - ign.sa_handler = SIG_IGN; - sigaction(SIGPROF, &ign, &oldsa); - - struct itimerval tv_on = {{1, 0}, {1, 0}}; - Error err = setitimer(ITIMER_PROF, &tv_on, nullptr) != 0 - ? Error("ITIMER_PROF is not supported on this system") - : Error::OK; - struct itimerval tv_off = {{0, 0}, {0, 0}}; - setitimer(ITIMER_PROF, &tv_off, nullptr); - - sigaction(SIGPROF, &oldsa, nullptr); - return err; -} - -Error ITimerJvmti::start(Arguments &args) { - if (args._interval < 0) { - return Error("interval must be positive"); - } - _interval = args.cpuSamplerInterval(); - - OS::installSignalHandler(SIGPROF, signalHandler); - - time_t sec = _interval / 1000000000; - suseconds_t usec = (_interval % 1000000000) / 1000; - struct itimerval tv = {{sec, usec}, {sec, usec}}; - if (setitimer(ITIMER_PROF, &tv, nullptr) != 0) { - return Error("ITIMER_PROF is not supported on this system"); - } - return Error::OK; -} - -void ITimerJvmti::stop() { - struct itimerval tv = {}; - setitimer(ITIMER_PROF, &tv, nullptr); - OS::installSignalHandler(SIGPROF, nullptr, SIG_IGN); -} diff --git a/ddprof-lib/src/main/cpp/itimer.h b/ddprof-lib/src/main/cpp/itimer.h deleted file mode 100644 index 2a216e28f..000000000 --- a/ddprof-lib/src/main/cpp/itimer.h +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2018 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _ITIMER_H -#define _ITIMER_H - -#include "engine.h" -#include - -class ITimer : public Engine { -private: - static volatile bool _enabled; - static long _interval; - static CStack _cstack; - - static void signalHandler(int signo, siginfo_t *siginfo, void *ucontext); - -public: - const char *units() { return "ns"; } - - const char *name() { return "ITimer"; } - - long interval() const { return _interval; } - - Error check(Arguments &args); - Error start(Arguments &args); - void stop(); - - inline void enableEvents(bool enabled) { _enabled = enabled; } -}; - -// CPU-time engine identical to ITimer in its timer mechanism (process-wide -// setitimer(ITIMER_PROF) / SIGPROF) but delegates stack collection to the -// HotSpot JFR RequestStackTrace JVMTI extension instead of ASGCT. Used on -// platforms where per-thread CPU timers are unavailable (e.g. macOS), as a -// macOS-compatible alternative to CTimerJvmti. Because SIGPROF may land on -// any thread, nullptr is passed as ucontext so the JVM uses safepoint-based -// stack walking rather than relying on the signal-frame PC. -class ITimerJvmti : public Engine { -private: - static volatile bool _enabled; - static long _interval; - - static void signalHandler(int signo, siginfo_t *siginfo, void *ucontext); - -public: - const char *units() { return "ns"; } - const char *name() { return "ITimerJvmti"; } - long interval() const { return _interval; } - - Error check(Arguments &args); - Error start(Arguments &args); - void stop(); - - inline void enableEvents(bool enabled) { - __atomic_store_n(&_enabled, enabled, __ATOMIC_RELEASE); - } -}; - -#endif // _ITIMER_H diff --git a/ddprof-lib/src/main/cpp/j9/j9Support.cpp b/ddprof-lib/src/main/cpp/j9/j9Support.cpp deleted file mode 100644 index 87afb3ce2..000000000 --- a/ddprof-lib/src/main/cpp/j9/j9Support.cpp +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2022 Andrei Pangin - * Copyright 2024, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "j9/j9Support.h" -#include "os.h" -#include - -jvmtiEnv *J9Support::_jvmti; - -void *(*J9Support::_j9thread_self)() = NULL; - -jvmtiExtensionFunction J9Support::_GetOSThreadID = NULL; -jvmtiExtensionFunction J9Support::_GetJ9vmThread = NULL; -jvmtiExtensionFunction J9Support::_GetStackTraceExtended = NULL; -jvmtiExtensionFunction J9Support::_GetAllStackTracesExtended = NULL; - -int J9Support::InstrumentableObjectAlloc_id = -1; - -// Look for OpenJ9-specific JVM TI extension -bool J9Support::initialize(jvmtiEnv *jvmti, const void *j9thread_self) { - _jvmti = jvmti; - _j9thread_self = (void *(*)())j9thread_self; - - jint ext_count; - jvmtiExtensionFunctionInfo *ext_functions; - if (jvmti->GetExtensionFunctions(&ext_count, &ext_functions) == 0) { - for (int i = 0; i < ext_count; i++) { - if (strcmp(ext_functions[i].id, "com.ibm.GetOSThreadID") == 0) { - _GetOSThreadID = ext_functions[i].func; - } else if (strcmp(ext_functions[i].id, "com.ibm.GetJ9vmThread") == 0) { - _GetJ9vmThread = ext_functions[i].func; - } else if (strcmp(ext_functions[i].id, "com.ibm.GetStackTraceExtended") == - 0) { - _GetStackTraceExtended = ext_functions[i].func; - } else if (strcmp(ext_functions[i].id, - "com.ibm.GetAllStackTracesExtended") == 0) { - _GetAllStackTracesExtended = ext_functions[i].func; - } - } - jvmti->Deallocate((unsigned char *)ext_functions); - } - - return _GetOSThreadID != NULL && _GetStackTraceExtended != NULL && - _GetAllStackTracesExtended != NULL; -} diff --git a/ddprof-lib/src/main/cpp/j9/j9Support.h b/ddprof-lib/src/main/cpp/j9/j9Support.h deleted file mode 100644 index 7031fb8f5..000000000 --- a/ddprof-lib/src/main/cpp/j9/j9Support.h +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright 2022 Andrei Pangin - * Copyright 2024, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _J9_J9SUPPORT_H -#define _J9_J9SUPPORT_H - -#include - -#include "log.h" -#include "vmEntry.h" - -#define JVMTI_EXT(f, ...) ((jvmtiError(*)(jvmtiEnv *, __VA_ARGS__))f) - -struct jvmtiFrameInfoExtended { - jmethodID method; - jlocation location; - jlocation machinepc; - jint type; - void *native_frame_address; -}; - -struct jvmtiStackInfoExtended { - jthread thread; - jint state; - jvmtiFrameInfoExtended *frame_buffer; - jint frame_count; -}; - -enum { SHOW_COMPILED_FRAMES = 4, SHOW_INLINED_FRAMES = 8 }; - -/** - * J9 frame type constants from ibmjvmti.h. - * These are the expected values returned in jvmtiFrameInfoExtended.type. - */ -enum J9FrameType { - J9_FRAME_NOT_JITTED = 0, // COM_IBM_STACK_FRAME_EXTENDED_NOT_JITTED - J9_FRAME_JITTED = 1, // COM_IBM_STACK_FRAME_EXTENDED_JITTED - J9_FRAME_INLINED = 2 // COM_IBM_STACK_FRAME_EXTENDED_INLINED -}; - -/** - * Validates and maps J9 frame type to FrameTypeId. - * J9's JVMTI extension may return unexpected values in the type field. - * This function ensures we only pass valid values to FrameType::encode(). - * - * @param j9_type The frame type value from jvmtiFrameInfoExtended.type - * @return A valid FrameTypeId (FRAME_INTERPRETED, FRAME_JIT_COMPILED, or FRAME_INLINED) - */ -static inline int sanitizeJ9FrameType(jint j9_type) { - // J9 should only return 0, 1, or 2 for the frame type. - // Any other value is unexpected and we default to JIT compiled. - if (j9_type >= J9_FRAME_NOT_JITTED && j9_type <= J9_FRAME_INLINED) { - return j9_type; // Direct mapping: J9 values match our FrameTypeId values - } - // Unexpected value - default to JIT compiled as the safest assumption - return FRAME_JIT_COMPILED; -} - -class J9Support { - friend class JVMThread; - friend class J9WallClock; -private: - static jvmtiEnv *_jvmti; - - static void *(*_j9thread_self)(); - - static jvmtiExtensionFunction _GetOSThreadID; - static jvmtiExtensionFunction _GetJ9vmThread; - static jvmtiExtensionFunction _GetStackTraceExtended; - static jvmtiExtensionFunction _GetAllStackTracesExtended; - -public: - static bool can_use_ASGCT() { - // J9's ASGCT is not async-signal-safe prior to OpenJ9 0.51. Calling ASGCT - // from a signal handler that interrupted a thread inside jitMethodMonitorEntry - // causes a livelock: jitWalkStackFrames calls getSendSlotsFromSignature which - // acquires a non-reentrant JIT lock already held by the interrupted thread. - // Fixed upstream in eclipse-openj9/openj9#20577 (0.51, April 2025 refresh): - // JDK 8u451 (IBM SDK SR8 FP45) / 8u452 (IBM Semeru), JDK 11.0.27, - // JDK 17.0.15, JDK 21.0.7 - return (VM::java_version() == 8 && VM::java_update_version() >= 451) || - (VM::java_version() == 11 && VM::java_update_version() >= 27) || - (VM::java_version() == 17 && VM::java_update_version() >= 15) || - (VM::java_version() == 21 && VM::java_update_version() >= 7); - } - - static bool is_jvmti_jmethodid_safe() { - // only JDK 8 is safe to use jmethodID in JVMTI for deferred resolution - // unless -XX:+KeepJNIIDs=true is provided - return VM::java_version() == 8; - } - - static bool shouldUseAsgct() { - char *sampler = NULL; - - jvmtiEnv *jvmti = VM::jvmti(); - jvmti->GetSystemProperty("dd.profiling.ddprof.j9.sampler", &sampler); - - bool asgct = true; - if (sampler != nullptr) { - if (!strncmp("asgct", sampler, 5)) { - asgct = true; - } else if (!strncmp("jvmti", sampler, 5)) { - asgct = false; - } else { - fprintf(stdout, "[ddprof] [WARN] Invalid J9 sampler: %s, supported values are [jvmti, asgct]", sampler); - } - } - jvmti->Deallocate((unsigned char *)sampler); - return asgct; - } - - static bool initialize(jvmtiEnv *jvmti, const void *j9thread_self); - - static JNIEnv *GetJ9vmThread(jthread thread) { - JNIEnv *result; - return JVMTI_EXT(_GetJ9vmThread, jthread, JNIEnv **)(_jvmti, thread, - &result) == 0 - ? result - : NULL; - } - - static jvmtiError GetStackTraceExtended(jthread thread, jint start_depth, - jint max_frame_count, - void *frame_buffer, jint *count_ptr) { - return JVMTI_EXT(_GetStackTraceExtended, jint, jthread, jint, jint, void *, - jint *)(_jvmti, SHOW_COMPILED_FRAMES | SHOW_INLINED_FRAMES, - thread, start_depth, max_frame_count, frame_buffer, - count_ptr); - } - - static jvmtiError GetAllStackTracesExtended(jint max_frame_count, - void **stack_info_ptr, - jint *thread_count_ptr) { - return JVMTI_EXT(_GetAllStackTracesExtended, jint, jint, void **, - jint *)(_jvmti, SHOW_COMPILED_FRAMES | SHOW_INLINED_FRAMES, - max_frame_count, stack_info_ptr, thread_count_ptr); - } - - static jvmtiError GetStackTrace(jthread thread, jint start_depth, - jint max_frame_count, - ASGCT_CallFrame *frame_buffer, - jint *count_ptr) { - jvmtiFrameInfoExtended buffer[max_frame_count]; - - jvmtiError err = GetStackTraceExtended(thread, start_depth, max_frame_count, - buffer, count_ptr); - if (err) { - return err; - } - for (int j = 0; j < *count_ptr; j++) { - jvmtiFrameInfoExtended *fi = &buffer[j]; - frame_buffer[j].method_id = fi->method; - frame_buffer[j].bci = FrameType::encode(sanitizeJ9FrameType(fi->type), fi->location); - } - return JVMTI_ERROR_NONE; - } - - static void *j9thread_self() { - return _j9thread_self != NULL ? _j9thread_self() : NULL; - } - - static int InstrumentableObjectAlloc_id; - -private: - static int GetOSThreadID(jthread thread) { - jlong thread_id; - return JVMTI_EXT(_GetOSThreadID, jthread, jlong *)(_jvmti, thread, - &thread_id) == 0 - ? (int)thread_id - : -1; - } -}; - -#endif // _J9_J9SUPPORT_H diff --git a/ddprof-lib/src/main/cpp/j9/j9WallClock.cpp b/ddprof-lib/src/main/cpp/j9/j9WallClock.cpp deleted file mode 100644 index 9928ee838..000000000 --- a/ddprof-lib/src/main/cpp/j9/j9WallClock.cpp +++ /dev/null @@ -1,147 +0,0 @@ -/* - * Copyright 2021 Andrei Pangin - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "j9WallClock.h" -#include "j9/j9Support.h" -#include "profiler.h" -#include "threadState.h" -#include - -volatile bool J9WallClock::_enabled = false; - -long J9WallClock::_interval; - -Error J9WallClock::start(Arguments &args) { - if (_running) { - // only one instance should be running - return Error::OK; - } - - if (args._wall >= 0) { - _sample_idle_threads = true; - } - - _interval = args._wall > 0 ? args._wall : DEFAULT_WALL_INTERVAL; - if (_interval < 0) { - return Error("interval must be non-negative"); - } - _max_stack_depth = args._jstackdepth; - - _running = true; - - if (pthread_create(&_thread, NULL, threadEntry, this) != 0) { - return Error("Unable to create timer thread"); - } - - return Error::OK; -} - -void J9WallClock::stop() { - _running = false; - pthread_kill(_thread, WAKEUP_SIGNAL); - pthread_join(_thread, NULL); -} - -void J9WallClock::timerLoop() { - // IBM J9 may cancel this thread via forced unwinding during JVM shutdown. - // glibc raises abi::__forced_unwind for pthread_cancel; libc++ does not declare - // that type, so we cannot name it in a catch. An RAII cleanup struct works - // under any C++ stdlib because destructors run during forced unwind regardless - // of whether the unwind exception type is C++-named. This also avoids the - // libc++ build break on macOS where abi::__forced_unwind is not available. - // - // PushLocalFrame / PopLocalFrame balance: if the forced unwind fires between - // PushLocalFrame and PopLocalFrame, the outstanding JNI local frame is released - // by DetachCurrentThread (called via VM::detachThread()), which is specified to - // destroy the current stack frame's local references. The common cancellation - // point is OS::sleep() which runs after PopLocalFrame, so no imbalance occurs - // in the typical case. - struct Cleanup { - ASGCT_CallFrame *frames = nullptr; - ~Cleanup() { - free(frames); // free(nullptr) is a no-op - VM::detachThread(); // DetachCurrentThread releases any outstanding JNI local frames - } - } cleanup; - - JNIEnv *jni = VM::attachThread("java-profiler Sampler"); - jvmtiEnv *jvmti = VM::jvmti(); - - int max_frames = _max_stack_depth + MAX_NATIVE_FRAMES + RESERVED_FRAMES; - cleanup.frames = (ASGCT_CallFrame *)malloc(max_frames * sizeof(ASGCT_CallFrame)); - ASGCT_CallFrame *frames = cleanup.frames; - - while (_running) { - if (!_enabled) { - OS::sleep(_interval); - continue; - } - - jni->PushLocalFrame(64); - - jvmtiStackInfoExtended *stack_infos; - jint thread_count; - if (J9Support::GetAllStackTracesExtended( - _max_stack_depth, (void **)&stack_infos, &thread_count) == 0) { - for (int i = 0; i < thread_count; i++) { - jvmtiStackInfoExtended *si = &stack_infos[i]; - if (si->frame_count <= 0) { - // no frames recorded - continue; - } - OSThreadState ts = (si->state & JVMTI_THREAD_STATE_RUNNABLE) - ? OSThreadState::RUNNABLE - : OSThreadState::SLEEPING; - if (!_sample_idle_threads && ts != OSThreadState::RUNNABLE) { - // in execution profiler mode the non-running threads are skipped - continue; - } - for (int j = 0; j < si->frame_count; j++) { - jvmtiFrameInfoExtended *fi = &si->frame_buffer[j]; - frames[j].method_id = fi->method; - frames[j].bci = FrameType::encode(sanitizeJ9FrameType(fi->type), fi->location); - } - - int tid = J9Support::GetOSThreadID(si->thread); - if (tid == -1) { - // clearly an invalid TID; skip the thread - continue; - } - ExecutionEvent event; - event._thread_state = ts; - if (ts == OSThreadState::RUNNABLE) { - Profiler::instance()->recordExternalSample( - _interval, tid, si->frame_count, frames, /*truncated=*/false, - BCI_CPU, &event); - } - if (_sample_idle_threads) { - Profiler::instance()->recordExternalSample( - _interval, tid, si->frame_count, frames, /*truncated=*/false, - BCI_WALL, &event); - } - } - jvmti->Deallocate((unsigned char *)stack_infos); - } - - jni->PopLocalFrame(NULL); - - OS::sleep(_interval); - } - // Cleanup destructor runs here on normal exit; on a forced unwind it runs - // automatically as the stack unwinds out of timerLoop, regardless of the - // active C++ stdlib. -} diff --git a/ddprof-lib/src/main/cpp/j9/j9WallClock.h b/ddprof-lib/src/main/cpp/j9/j9WallClock.h deleted file mode 100644 index f0815a420..000000000 --- a/ddprof-lib/src/main/cpp/j9/j9WallClock.h +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2021 Andrei Pangin - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _J9_J9WALLCLOCK_H -#define _J9_J9WALLCLOCK_H - -#include "engine.h" -#include - -class J9WallClock : public Engine { -private: - static volatile bool _enabled; - static long _interval; - - bool _sample_idle_threads; - int _max_stack_depth; - volatile bool _running; - pthread_t _thread; - - static void *threadEntry(void *wall_clock) { - ((J9WallClock *)wall_clock)->timerLoop(); - return NULL; - } - - void timerLoop(); - -public: - const char *units() { return "ns"; } - - const char *name() { - return _sample_idle_threads ? "J9WallClock" : "J9Execution"; - } - - virtual long interval() const { return _interval; } - - inline void sampleIdleThreads() { _sample_idle_threads = true; } - - Error start(Arguments &args); - void stop(); - - inline void enableEvents(bool enabled) { _enabled = enabled; } -}; - -#endif // _J9_J9WALLCLOCK_H diff --git a/ddprof-lib/src/main/cpp/javaApi.cpp b/ddprof-lib/src/main/cpp/javaApi.cpp deleted file mode 100644 index dcb44c535..000000000 --- a/ddprof-lib/src/main/cpp/javaApi.cpp +++ /dev/null @@ -1,751 +0,0 @@ -/* - * Copyright 2016 Andrei Pangin - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#include "arch.h" -#include "context.h" -#include "context_api.h" -#include "guards.h" -#include "counters.h" -#include "common.h" -#include "engine.h" -#include "hotspot/vmStructs.inline.h" -#include "incbin.h" -#include "jvmThread.h" -#include "os.h" -#include "otel_process_ctx.h" -#include "profiler.h" -#include "thread.h" -#include "tsc.h" -#include "vmEntry.h" -#include -#include -#include -#include - -static void throwNew(JNIEnv *env, const char *exception_class, - const char *message) { - jclass cls = env->FindClass(exception_class); - if (cls != NULL) { - env->ThrowNew(cls, message); - } -} - -class JniString { -private: - JNIEnv *_env; - const char *_c_string; - jstring _java_string; - int _length; - -public: - JniString(JNIEnv *env, jstring java_string) { - _env = env; - _c_string = _env->GetStringUTFChars(java_string, NULL); - _length = _env->GetStringUTFLength(java_string); - _java_string = java_string; - } - JniString(JniString &jniString) = delete; - ~JniString() { _env->ReleaseStringUTFChars(_java_string, _c_string); } - const char *c_str() const { return _c_string; } - int length() const { return _length; } -}; - -extern "C" DLLEXPORT jboolean JNICALL -Java_com_datadoghq_profiler_JavaProfiler_init0(JNIEnv *env, jclass unused) { - // JavaVM* has already been stored when the native library was loaded so we can pass nullptr here - return VM::initProfilerBridge(nullptr, true); -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_stop0(JNIEnv *env, jobject unused) { - Error error = Profiler::instance()->stop(); - - if (error) { - throwNew(env, "java/lang/IllegalStateException", error.message()); - } -} - -extern "C" DLLEXPORT jint JNICALL -Java_com_datadoghq_profiler_JavaProfiler_getTid0(JNIEnv *env, jclass unused) { - return OS::threadId(); -} - -extern "C" DLLEXPORT jstring JNICALL -Java_com_datadoghq_profiler_JavaProfiler_execute0(JNIEnv *env, jobject unused, - jstring command) { - Arguments args; - JniString command_str(env, command); - Error error = args.parse(command_str.c_str()); - - if (error) { - throwNew(env, "java/lang/IllegalArgumentException", error.message()); - return NULL; - } - - Log::open(args); - - std::ostringstream out; - error = Profiler::instance()->runInternal(args, out); - if (!error) { - if (out.tellp() >= 0x3fffffff) { - throwNew(env, "java/lang/IllegalStateException", - "Output exceeds string size limit"); - return NULL; - } - return env->NewStringUTF(out.str().c_str()); - } - - throwNew(env, "java/lang/IllegalStateException", error.message()); - return NULL; -} - -extern "C" DLLEXPORT jstring JNICALL -Java_com_datadoghq_profiler_JavaProfiler_getStatus0(JNIEnv* env, - jclass unused) { - char msg[2048]; - Profiler::instance()->status((char*)msg, sizeof(msg) - 1); - return env->NewStringUTF(msg); -} - -extern "C" DLLEXPORT jlong JNICALL -Java_com_datadoghq_profiler_JavaProfiler_getSamples(JNIEnv *env, - jclass unused) { - return (jlong)Profiler::instance()->total_samples(); -} - -// some duplication between add and remove, though we want to avoid having an extra branch in the hot path - -// JavaCritical is faster JNI, but more restrictive - parameters and return value have to be -// primitives or arrays of primitive types. -// We direct corresponding JNI calls to JavaCritical to make sure the parameters/return value -// still compatible in the event of signature changes in the future. -extern "C" DLLEXPORT void JNICALL -JavaCritical_com_datadoghq_profiler_JavaProfiler_filterThreadAdd0() { - ProfiledThread *current = ProfiledThread::current(); - assert(current != nullptr); - int tid = current->tid(); - if (unlikely(tid < 0)) { - return; - } - ThreadFilter *thread_filter = Profiler::instance()->threadFilter(); - if (unlikely(!thread_filter->enabled())) { - return; - } - - int slot_id = current->filterSlotId(); - if (unlikely(slot_id == -1)) { - // Thread doesn't have a slot ID yet (e.g., main thread), so register it - // Happens when we are not enabled before thread start - slot_id = thread_filter->registerThread(); - current->setFilterSlotId(slot_id); - } - - if (unlikely(slot_id == -1)) { - return; // Failed to register thread - } - // Reset suppression state so a new thread occupying this slot does not inherit - // stale state from its predecessor. Must happen before add(). - thread_filter->resetSlotRunState(slot_id); - thread_filter->add(tid, slot_id); -} - -extern "C" DLLEXPORT void JNICALL -JavaCritical_com_datadoghq_profiler_JavaProfiler_filterThreadRemove0() { - ProfiledThread *current = ProfiledThread::current(); - assert(current != nullptr); - int tid = current->tid(); - if (unlikely(tid < 0)) { - return; - } - ThreadFilter *thread_filter = Profiler::instance()->threadFilter(); - if (unlikely(!thread_filter->enabled())) { - return; - } - - int slot_id = current->filterSlotId(); - if (unlikely(slot_id == -1)) { - // Thread doesn't have a slot ID yet - nothing to remove - return; - } - thread_filter->remove(slot_id); -} - - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_filterThreadAdd0(JNIEnv *env, - jclass unused) { - JavaCritical_com_datadoghq_profiler_JavaProfiler_filterThreadAdd0(); -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_filterThreadRemove0(JNIEnv *env, - jclass unused) { - JavaCritical_com_datadoghq_profiler_JavaProfiler_filterThreadRemove0(); -} - -extern "C" DLLEXPORT jboolean JNICALL -Java_com_datadoghq_profiler_JavaProfiler_recordTrace0( - JNIEnv *env, jclass unused, jlong rootSpanId, jstring endpoint, - jstring operation, jint sizeLimit) { - JniString endpoint_str(env, endpoint); - u32 endpointLabel = Profiler::instance()->stringLabelMap()->bounded_lookup( - endpoint_str.c_str(), endpoint_str.length(), sizeLimit); - // StringDictionary reserves 0 as "no entry"; valid IDs start at 1. - bool acceptValue = endpointLabel != 0; - if (acceptValue) { - u32 operationLabel = 0; - if (operation != NULL) { - JniString operation_str(env, operation); - operationLabel = Profiler::instance()->contextValueMap()->bounded_lookup( - operation_str.c_str(), operation_str.length(), 1 << 16); - } - TraceRootEvent event(rootSpanId, endpointLabel, operationLabel); - int tid = ProfiledThread::currentTid(); - Profiler::instance()->recordTraceRoot(tid, &event); - } - return acceptValue; -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_dump0(JNIEnv *env, jclass unused, - jstring path) { - JniString path_str(env, path); - Profiler::instance()->dump(path_str.c_str(), path_str.length()); -} - -extern "C" DLLEXPORT jobject JNICALL -Java_com_datadoghq_profiler_JavaProfiler_getDebugCounters0(JNIEnv *env, - jclass unused) { -#ifdef COUNTERS - return env->NewDirectByteBuffer((void *)Counters::getCounters(), - (jlong)Counters::size()); -#else - return env->NewDirectByteBuffer(nullptr, 0); -#endif // COUNTERS -} - -extern "C" DLLEXPORT jobjectArray JNICALL -Java_com_datadoghq_profiler_JavaProfiler_describeDebugCounters0( - JNIEnv *env, jclass unused) { -#ifdef COUNTERS - std::vector counter_names = Counters::describeCounters(); - jobjectArray array = (jobjectArray)env->NewObjectArray( - counter_names.size(), env->FindClass("java/lang/String"), - env->NewStringUTF("")); - for (int i = 0; i < counter_names.size(); i++) { - env->SetObjectArrayElement(array, i, - env->NewStringUTF(counter_names.at(i))); - } - return array; -#else - return nullptr; -#endif // COUNTERS -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_recordSettingEvent0( - JNIEnv *env, jclass unused, jstring name, jstring value, jstring unit) { - int tid = ProfiledThread::currentTid(); - if (tid < 0) { - return; - } - int length = 0; - JniString name_str(env, name); - length += name_str.length(); - JniString value_str(env, value); - length += value_str.length(); - JniString unit_str(env, unit); - length += unit_str.length(); - Profiler::instance()->writeDatadogProfilerSetting( - tid, length, name_str.c_str(), value_str.c_str(), unit_str.c_str()); -} - -static int dictionarizeClassName(JNIEnv* env, jstring className) { - JniString str(env, className); - return Profiler::instance()->lookupClass(str.c_str(), str.length()); -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_recordQueueEnd0( - JNIEnv *env, jclass unused, jlong startTime, jlong endTime, jstring task, - jstring scheduler, jthread origin, jstring queueType, jint queueLength) { - int tid = ProfiledThread::currentTid(); - if (tid < 0) { - return; - } - int origin_tid = JVMThread::nativeThreadId(env, origin); - if (origin_tid < 0) { - return; - } - JniString queue_type_str(env, queueType); - int task_offset = dictionarizeClassName(env, task); - if (task_offset < 0) { - return; - } - int scheduler_offset = dictionarizeClassName(env, scheduler); - if (scheduler_offset < 0) { - return; - } - int queue_type_offset = dictionarizeClassName(env, queueType); - if (queue_type_offset < 0) { - return; - } - QueueTimeEvent event; - event._start = startTime; - event._end = endTime; - event._task = task_offset; - event._scheduler = scheduler_offset; - event._origin = origin_tid; - event._queueType = queue_type_offset; - event._queueLength = queueLength; - Profiler::instance()->recordQueueTime(tid, &event); -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_parkEnter0(JNIEnv *env, jclass unused) { - ProfiledThread *current = ProfiledThread::current(); - if (current == nullptr) { - return; - } - bool first_park = current->parkEnter(); - ThreadFilter *tf = Profiler::instance()->threadFilter(); - if (first_park && tf->enabled()) { - ThreadFilter::SlotID slot_id = current->filterSlotId(); - if (slot_id >= 0) { - current->setParkBlockToken( - tf->enterBlockedRun(slot_id, OSThreadState::CONDVAR_WAIT)); - } - } -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_parkExit0( - JNIEnv *env, jclass unused, jlong blocker, jlong unblockingSpanId) { - ProfiledThread *current = ProfiledThread::current(); - if (current == nullptr) { - return; - } - u64 park_block_token = 0; - if (!current->parkExit(park_block_token) || park_block_token == 0) { - return; - } - ThreadFilter *tf = Profiler::instance()->threadFilter(); - if (tf->enabled()) { - ThreadFilter::SlotID slot_id = ThreadFilter::tokenSlotId(park_block_token); - if (current->filterSlotId() == slot_id) { - tf->exitBlockedRun(slot_id, ThreadFilter::tokenGeneration(park_block_token)); - } - } -} - -static bool decodeJavaBlockState(jint state, OSThreadState &decoded) { - if (state == static_cast(OSThreadState::SLEEPING)) { - decoded = OSThreadState::SLEEPING; - return true; - } - decoded = OSThreadState::UNKNOWN; - return false; -} - -extern "C" DLLEXPORT jlong JNICALL -Java_com_datadoghq_profiler_JavaProfiler_blockEnter0( - JNIEnv *env, jclass unused, jint state) { - OSThreadState decoded; - if (!decodeJavaBlockState(state, decoded)) { - return 0; - } - ProfiledThread *current = ProfiledThread::current(); - if (current == nullptr) { - return 0; - } - ThreadFilter *tf = Profiler::instance()->threadFilter(); - if (!tf->enabled()) { - return 0; - } - ThreadFilter::SlotID slot_id = current->filterSlotId(); - if (slot_id < 0) { - return 0; - } - return static_cast(tf->enterBlockedRun(slot_id, decoded)); -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_blockExit0( - JNIEnv *env, jclass unused, jlong token) { - u64 block_token = static_cast(token); - if (block_token == 0) { - return; - } - ProfiledThread *current = ProfiledThread::current(); - if (current == nullptr) { - return; - } - ThreadFilter::SlotID slot_id = ThreadFilter::tokenSlotId(block_token); - if (current->filterSlotId() != slot_id) { - return; - } - ThreadFilter *tf = Profiler::instance()->threadFilter(); - if (tf->enabled()) { - tf->exitBlockedRun(slot_id, ThreadFilter::tokenGeneration(block_token)); - } -} - -extern "C" DLLEXPORT jlong JNICALL -Java_com_datadoghq_profiler_JavaProfiler_currentTicks0(JNIEnv *env, - jclass unused) { - return TSC::ticks(); -} - -extern "C" DLLEXPORT jlong JNICALL -Java_com_datadoghq_profiler_JavaProfiler_tscFrequency0(JNIEnv *env, - jclass unused) { - return TSC::frequency(); -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_mallocArenaMax0(JNIEnv *env, - jclass unused, - jint maxArenas) { - OS::mallocArenaMax(maxArenas); -} - -extern "C" DLLEXPORT jstring JNICALL -Java_com_datadoghq_profiler_JVMAccess_findStringJVMFlag0(JNIEnv *env, - jobject unused, - jstring flagName) { - JniString flag_str(env, flagName); - VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::String, VMFlag::Type::Stringlist}); - if (f) { - char** value = static_cast(f->addr()); - if (value != NULL && *value != NULL) { - return env->NewStringUTF(*value); - } - } - return NULL; -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JVMAccess_setStringJVMFlag0(JNIEnv *env, - jobject unused, - jstring flagName, - jstring flagValue) { - JniString flag_str(env, flagName); - JniString value_str(env, flagValue); - VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::String, VMFlag::Type::Stringlist}); - if (f) { - char** value = static_cast(f->addr()); - if (value != NULL) { - *value = strdup(value_str.c_str()); - } - } -} - -extern "C" DLLEXPORT jboolean JNICALL -Java_com_datadoghq_profiler_JVMAccess_findBooleanJVMFlag0(JNIEnv *env, - jobject unused, - jstring flagName) { - JniString flag_str(env, flagName); - VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::Bool}); - if (f) { - char* value = static_cast(f->addr()); - if (value != NULL) { - return ((*value) & 0xff) == 1; - } - } - return false; -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JVMAccess_setBooleanJVMFlag0(JNIEnv *env, - jobject unused, - jstring flagName, - jboolean flagValue) { - JniString flag_str(env, flagName); - VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::Bool}); - if (f) { - char* value = static_cast(f->addr()); - if (value != NULL) { - *value = flagValue ? 1 : 0; - } - } -} - -extern "C" DLLEXPORT jlong JNICALL -Java_com_datadoghq_profiler_JVMAccess_findIntJVMFlag0(JNIEnv *env, - jobject unused, - jstring flagName) { - JniString flag_str(env, flagName); - VMFlag *f = VMFlag::find(flag_str.c_str(), {VMFlag::Type::Int, VMFlag::Type::Uint, VMFlag::Type::Intx, VMFlag::Type::Uintx, VMFlag::Type::Uint64_t, VMFlag::Type::Size_t}); - if (f) { - long* value = static_cast(f->addr()); - if (value != NULL) { - return *value; - } - } - return 0; -} - -extern "C" DLLEXPORT jdouble JNICALL -Java_com_datadoghq_profiler_JVMAccess_findFloatJVMFlag0(JNIEnv *env, - jobject unused, - jstring flagName) { - JniString flag_str(env, flagName); - VMFlag *f = VMFlag::find(flag_str.c_str(),{ VMFlag::Type::Double}); - if (f) { - double* value = static_cast(f->addr()); - if (value != NULL) { - return *value; - } - } - return 0.0; -} - -extern "C" DLLEXPORT jboolean JNICALL -Java_com_datadoghq_profiler_JVMAccess_healthCheck0(JNIEnv *env, - jobject unused) { - return true; -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_OTelContext_setProcessCtx0(JNIEnv *env, - jclass unused, - jstring env_data, - jstring hostname, - jstring runtime_id, - jstring service, - jstring version, - jstring tracer_version, - jobjectArray attribute_keys - ) { - JniString env_str(env, env_data); - JniString hostname_str(env, hostname); - JniString runtime_id_str(env, runtime_id); - JniString service_str(env, service); - JniString version_str(env, version); - JniString tracer_version_str(env, tracer_version); - - const char *host_name_attrs[] = {"host.name", hostname_str.c_str(), NULL}; - - // Build the thread context attribute_key_map published alongside the process - // context: index 0 is the reserved datadog.local_root_span_id slot, followed by - // the caller-provided keys (clipped to DD_TAGS_CAPACITY) - int count = (attribute_keys != nullptr) ? env->GetArrayLength(attribute_keys) : 0; - int n = count < (int)DD_TAGS_CAPACITY ? count : (int)DD_TAGS_CAPACITY; - if (count > n) { - Log::warn("setProcessContext: %d attribute keys requested but capacity is %d; extra keys will be ignored", - count, (int)DD_TAGS_CAPACITY); - } - - const char *key_ptrs[DD_TAGS_CAPACITY + 2]; // +1 reserved slot, +1 NULL terminator - JniString *jni_keys[DD_TAGS_CAPACITY]; - int built = 0; - key_ptrs[0] = "datadog.local_root_span_id"; - for (int i = 0; i < n; i++) { - jstring jstr = (jstring)env->GetObjectArrayElement(attribute_keys, i); - if (jstr == nullptr) { - // A null key would corrupt the index mapping; abort the publish. - for (int j = 0; j < built; j++) delete jni_keys[j]; - Log::warn("setProcessContext: null attribute key at index %d; skipping publish", i); - return; - } - jni_keys[built] = new JniString(env, jstr); - if (jni_keys[built]->c_str() == nullptr) { - // GetStringUTFChars failed (e.g. OOM); a NULL key pointer would truncate - // the published map mid-array, so abort the publish. - delete jni_keys[built]; - for (int j = 0; j < built; j++) delete jni_keys[j]; - Log::warn("setProcessContext: failed to read attribute key at index %d; skipping publish", i); - return; - } - key_ptrs[i + 1] = jni_keys[built]->c_str(); - built++; - } - key_ptrs[n + 1] = nullptr; - - otel_thread_ctx_config_data thread_ctx_config = { - .schema_version = "tlsdesc_v1_dev", - .attribute_key_map = key_ptrs, - }; - - otel_process_ctx_data data = { - .deployment_environment_name = env_str.c_str(), - .service_instance_id = runtime_id_str.c_str(), - .service_name = service_str.c_str(), - .service_version = version_str.c_str(), - .telemetry_sdk_language = "java", - .telemetry_sdk_version = tracer_version_str.c_str(), - .telemetry_sdk_name = "dd-trace-java", - .resource_attributes = host_name_attrs, - .extra_attributes = NULL, - .thread_ctx_config = &thread_ctx_config - }; - - otel_process_ctx_result result = otel_process_ctx_publish(&data); - if (!result.success) { - Log::warn("Failed to publish process context: %s", result.error_message); - } - - for (int i = 0; i < built; i++) delete jni_keys[i]; -} - -extern "C" DLLEXPORT jobject JNICALL -Java_com_datadoghq_profiler_OTelContext_readProcessCtx0(JNIEnv *env, jclass unused) { -#ifndef OTEL_PROCESS_CTX_NO_READ - otel_process_ctx_read_result result = otel_process_ctx_read(); - - if (!result.success) { - // Return null if reading failed - return nullptr; - } - - // Convert C strings to Java strings - jstring jDeploymentEnvironmentName = result.data.deployment_environment_name ? - env->NewStringUTF(result.data.deployment_environment_name) : nullptr; - jstring jServiceInstanceId = result.data.service_instance_id ? - env->NewStringUTF(result.data.service_instance_id) : nullptr; - jstring jServiceName = result.data.service_name ? - env->NewStringUTF(result.data.service_name) : nullptr; - jstring jServiceVersion = result.data.service_version ? - env->NewStringUTF(result.data.service_version) : nullptr; - jstring jTelemetrySdkLanguage = result.data.telemetry_sdk_language ? - env->NewStringUTF(result.data.telemetry_sdk_language) : nullptr; - jstring jTelemetrySdkVersion = result.data.telemetry_sdk_version ? - env->NewStringUTF(result.data.telemetry_sdk_version) : nullptr; - jstring jTelemetrySdkName = result.data.telemetry_sdk_name ? - env->NewStringUTF(result.data.telemetry_sdk_name) : nullptr; - - // Extract host.name from resource_attributes - jstring jHostName = nullptr; - if (result.data.resource_attributes != NULL) { - for (int i = 0; result.data.resource_attributes[i] != NULL; i += 2) { - if (strcmp(result.data.resource_attributes[i], "host.name") == 0 && result.data.resource_attributes[i + 1] != NULL) { - jHostName = env->NewStringUTF(result.data.resource_attributes[i + 1]); - break; - } - } - } - - // Extract attribute_key_map from thread_ctx_config (NULL if no config was published) - jobjectArray jAttributeKeyMap = nullptr; - if (result.data.thread_ctx_config != NULL && result.data.thread_ctx_config->attribute_key_map != NULL) { - int n = 0; - while (result.data.thread_ctx_config->attribute_key_map[n] != NULL) n++; - jclass stringClass = env->FindClass("java/lang/String"); - if (stringClass != nullptr) { - jAttributeKeyMap = env->NewObjectArray(n, stringClass, nullptr); - for (int i = 0; i < n; i++) { - jstring jKey = env->NewStringUTF(result.data.thread_ctx_config->attribute_key_map[i]); - env->SetObjectArrayElement(jAttributeKeyMap, i, jKey); - env->DeleteLocalRef(jKey); - } - } - } - - otel_process_ctx_read_drop(&result); - - // Find the ProcessContext class - jclass processContextClass = env->FindClass("com/datadoghq/profiler/OTelContext$ProcessContext"); - if (!processContextClass) { - return nullptr; - } - - // Find the constructor - jmethodID constructor = env->GetMethodID(processContextClass, "", - "(Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String;)V"); - if (!constructor) { - return nullptr; - } - - // Create the ProcessContext object - jobject processContext = env->NewObject(processContextClass, constructor, - jDeploymentEnvironmentName, jHostName, jServiceInstanceId, jServiceName, jServiceVersion, jTelemetrySdkLanguage, jTelemetrySdkVersion, jTelemetrySdkName, jAttributeKeyMap); - - return processContext; -#else - // If OTEL_PROCESS_CTX_NO_READ is defined, return null - return nullptr; -#endif -} - -extern "C" DLLEXPORT jobject JNICALL -Java_com_datadoghq_profiler_JavaProfiler_initializeContextTLS0(JNIEnv* env, jclass unused, jlongArray metadata) { - ProfiledThread* thrd = ProfiledThread::current(); - assert(thrd != nullptr); - - if (!thrd->isContextInitialized()) { - ContextApi::initializeContextTLS(thrd); - } - - OtelThreadContextRecord* record = thrd->getOtelContextRecord(); - - // Contiguity of record + tag_encodings + LRS is enforced by alignas(8) on _otel_ctx_record - // plus sizeof(OtelThreadContextRecord) being a multiple of 8 (see thread.h). - // Compile-time alignment check always runs; runtime pointer-layout check is debug-only. - static_assert(DD_TAGS_CAPACITY * sizeof(u32) % alignof(u64) == 0, - "tag encodings array size must be aligned to u64 for contiguous sidecar layout"); -#ifdef DEBUG - uint8_t* record_start = reinterpret_cast(record); - uint8_t* sidecar_start = reinterpret_cast(thrd->getOtelTagEncodingsPtr()); - assert(sidecar_start == record_start + OTEL_MAX_RECORD_SIZE - && "_otel_ctx_record and _otel_tag_encodings must be contiguous"); -#endif - - // Fill metadata[6]: [VALID_OFFSET, TRACE_ID_OFFSET, SPAN_ID_OFFSET, - // ATTRS_DATA_SIZE_OFFSET, ATTRS_DATA_OFFSET, LRS_OFFSET]. - // All offsets are absolute within the unified buffer returned below. - if (metadata != nullptr && env->GetArrayLength(metadata) >= 6) { - jlong meta[6]; - meta[0] = (jlong)offsetof(OtelThreadContextRecord, valid); - meta[1] = (jlong)offsetof(OtelThreadContextRecord, trace_id); - meta[2] = (jlong)offsetof(OtelThreadContextRecord, span_id); - meta[3] = (jlong)offsetof(OtelThreadContextRecord, attrs_data_size); - meta[4] = (jlong)offsetof(OtelThreadContextRecord, attrs_data); - meta[5] = (jlong)(OTEL_MAX_RECORD_SIZE + DD_TAGS_CAPACITY * sizeof(u32)); - env->SetLongArrayRegion(metadata, 0, 6, meta); - } - - // Single contiguous view over [record | tag_encodings | LRS] — used for per-field - // access and for bulk snapshot/restore. All three regions are in one ProfiledThread - // memory block. - size_t totalSize = OTEL_MAX_RECORD_SIZE + DD_TAGS_CAPACITY * sizeof(u32) + sizeof(u64); - return env->NewDirectByteBuffer((void*)record, (jlong)totalSize); -} - -extern "C" DLLEXPORT jint JNICALL -Java_com_datadoghq_profiler_ThreadContext_registerConstant0(JNIEnv* env, jclass unused, jstring value) { - JniString value_str(env, value); - u32 encoding = Profiler::instance()->contextValueMap()->bounded_lookup( - value_str.c_str(), value_str.length(), 1 << 16); - return encoding == 0 ? -1 : (jint)encoding; -} - -// ---- test and debug utilities -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_testlog(JNIEnv* env, jclass unused, jstring msg) { - JniString msg_str(env, msg); - - TEST_LOG("%s", msg_str.c_str()); -} - -extern "C" DLLEXPORT void JNICALL -Java_com_datadoghq_profiler_JavaProfiler_dumpContext(JNIEnv* env, jclass unused) { - u64 spanId = 0, rootSpanId = 0; - ContextApi::get(spanId, rootSpanId); - TEST_LOG("===> Context: tid:%lu, spanId=%lu, rootSpanId=%lu", OS::threadId(), spanId, rootSpanId); -} diff --git a/ddprof-lib/src/main/cpp/jfrMetadata.cpp b/ddprof-lib/src/main/cpp/jfrMetadata.cpp deleted file mode 100644 index f0f425ef7..000000000 --- a/ddprof-lib/src/main/cpp/jfrMetadata.cpp +++ /dev/null @@ -1,407 +0,0 @@ -/* - * Copyright 2020 Andrei Pangin - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "jfrMetadata.h" -#include "counters.h" -#include "vmEntry.h" - -std::map Element::_string_map; -std::vector Element::_strings; - -JfrMetadata JfrMetadata::_root; -bool JfrMetadata::_initialized = false; - -JfrMetadata::JfrMetadata() : Element("root") {} - -// Must only be called after all profiler engines are stopped and no signal -// handlers can fire. std::vector/std::map are not async-signal-safe. -void JfrMetadata::reset() { - _root._children.clear(); - _root._attributes.clear(); - _strings.clear(); - _string_map.clear(); - // Re-register "root" at ID 0 so _root._name (const 0) stays valid - getId("root"); - _initialized = false; -} - -void JfrMetadata::initialize( - const std::vector &contextAttributes) { - if (_initialized) { - return; - } - - _root - << (element("metadata") - - << type("boolean", T_BOOLEAN) << type("char", T_CHAR) - << type("float", T_FLOAT) << type("double", T_DOUBLE) - << type("byte", T_BYTE) << type("short", T_SHORT) - << type("int", T_INT) << type("long", T_LONG) - - << type("java.lang.String", T_STRING) - - << (type("java.lang.Class", T_CLASS, "Java Class") - << field("classLoader", T_CLASS_LOADER, "Class Loader", F_CPOOL) - << field("name", T_SYMBOL, "Name", F_CPOOL) - << field("package", T_PACKAGE, "Package", F_CPOOL) - << field("modifiers", T_INT, "Access Modifiers")) - - << (type("java.lang.Thread", T_THREAD, "Thread") - << field("osName", T_STRING, "OS Thread Name") - << field("osThreadId", T_LONG, "OS Thread Id") - << field("javaName", T_STRING, "Java Thread Name") - << field("javaThreadId", T_LONG, "Java Thread Id")) - - << (type("jdk.types.ClassLoader", T_CLASS_LOADER, "Java Class Loader") - << field("type", T_CLASS, "Type", F_CPOOL) - << field("name", T_SYMBOL, "Name", F_CPOOL)) - - << (type("jdk.types.FrameType", T_FRAME_TYPE, "Frame type", true) - << field("description", T_STRING, "Description")) - - << (type("jdk.types.ThreadState", T_THREAD_STATE, "Java Thread State", - true) - << field("name", T_STRING, "Name")) - - << (type("datadog.types.ExecutionMode", T_EXECUTION_MODE, - "Execution Mode", true) - << field("name", T_STRING, "Name")) - - << (type("jdk.types.StackTrace", T_STACK_TRACE, "Stacktrace") - << field("truncated", T_BOOLEAN, "Truncated") - << field("frames", T_STACK_FRAME, "Stack Frames", F_ARRAY)) - - << (type("jdk.types.StackFrame", T_STACK_FRAME) - << field("method", T_METHOD, "Java Method", F_CPOOL) - << field("lineNumber", T_INT, "Line Number") - << field("bytecodeIndex", T_INT, "Bytecode Index") - << field("type", T_FRAME_TYPE, "Frame Type", F_CPOOL)) - - << (type("jdk.types.Method", T_METHOD, "Java Method") - << field("type", T_CLASS, "Type", F_CPOOL) - << field("name", T_SYMBOL, "Name", F_CPOOL) - << field("descriptor", T_SYMBOL, "Descriptor", F_CPOOL) - << field("modifiers", T_INT, "Access Modifiers") - << field("hidden", T_BOOLEAN, "Hidden")) - - << (type("jdk.types.Package", T_PACKAGE, "Package") - << field("name", T_SYMBOL, "Name", F_CPOOL)) - - << (type("jdk.types.Symbol", T_SYMBOL, "Symbol", true) - << field("string", T_STRING, "String")) - - << (type("profiler.types.LogLevel", T_LOG_LEVEL, "Log Level", true) - << field("name", T_STRING, "Name")) - - << (type("profiler.types.AttributeValue", T_ATTRIBUTE_VALUE, "Value", - true) - << field("value", T_STRING, "Value")) - - << (type("profiler.types.CounterName", T_COUNTER_NAME, "Value", true) - << field("value", T_STRING, "Value")) - - << (type("datadog.ExecutionSample", T_EXECUTION_SAMPLE, - "Method CPU Profiling Sample") - << category("Datadog", "Profiling") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("eventThread", T_THREAD, "Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("state", T_THREAD_STATE, "Thread State", F_CPOOL) - << field("mode", T_EXECUTION_MODE, "Execution Mode", F_CPOOL) - << field("weight", T_LONG, "Sample weight") - << field("correlationId", T_LONG, "Async Stack Trace Correlation ID") - << field("spanId", T_LONG, "Span ID") - << field("localRootSpanId", T_LONG, "Local Root Span ID") || - contextAttributes) - - << (type("datadog.MethodSample", T_METHOD_SAMPLE, - "Method Wall Profiling Sample") - << category("Datadog", "Profiling") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("eventThread", T_THREAD, "Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("state", T_THREAD_STATE, "Thread State", F_CPOOL) - << field("mode", T_EXECUTION_MODE, "Execution Mode", F_CPOOL) - << field("weight", T_LONG, "Sample weight") - << field("correlationId", T_LONG, "Async Stack Trace Correlation ID") - << field("spanId", T_LONG, "Span ID") - << field("localRootSpanId", T_LONG, "Local Root Span ID") || - contextAttributes) - - << (type("datadog.WallClockSamplingEpoch", T_WALLCLOCK_SAMPLE_EPOCH, - "WallClock Sampling Epoch") - << category("Datadog", "Profiling") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("duration", T_LONG, "Duration", F_DURATION_MILLIS) - << field("samplePoolSize", T_INT, "Sample Pool Size") - << field("numSuccessfulSamples", T_INT, - "Number of Successful Samples") - << field("numFailedSamples", T_INT, "Number of Failed Samples") - << field("numExitedThreads", T_INT, - "Number of Exited Threads Before Handling Signal") - << field("numPermissionDenied", T_INT, - "Number of Permission Denied Errors") - << field("numSuppressedSampledRun", T_LONG, - "Signals suppressed by the wall-clock once-per-run filter")) - - << (type("datadog.ObjectSample", T_ALLOC, "Allocation sample") - << category("Datadog", "Profiling") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("objectClass", T_CLASS, "Object Class", F_CPOOL) - << field("size", T_LONG, "Original Size", F_BYTES) - << field("weight", T_FLOAT, "Sample weight") - << field("spanId", T_LONG, "Span ID") - << field("localRootSpanId", T_LONG, "Local Root Span ID") || - contextAttributes) - - << (type("datadog.HeapLiveObject", T_HEAP_LIVE_OBJECT, - "Heap Live Object") - << category("Datadog", "Profiling") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("objectClass", T_CLASS, "Object Class", F_CPOOL) - << field("age", T_LONG, "Age", F_UNSIGNED) - << field("size", T_LONG, "Original Size", F_BYTES) - << field("weight", T_FLOAT, "Sample weight") - << field("spanId", T_LONG, "Span ID") - << field("localRootSpanId", T_LONG, "Local Root Span ID") || - contextAttributes) - - << (type("datadog.Endpoint", T_ENDPOINT, "Endpoint") - << category("Datadog") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("duration", T_LONG, "Duration", F_DURATION_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("endpoint", T_STRING, "Endpoint", F_CPOOL) - << field("operation", T_ATTRIBUTE_VALUE, "Operation", F_CPOOL) - << field("localRootSpanId", T_LONG, "Local Root Span ID")) - - << (type("datadog.QueueTime", T_QUEUE_TIME, "Queue Time") - << category("Datadog") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("duration", T_LONG, "Duration", F_DURATION_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("origin", T_THREAD, "Origin Thread", F_CPOOL) - << field("task", T_CLASS, "Task", F_CPOOL) - << field("scheduler", T_CLASS, "Scheduler", F_CPOOL) - << field("queueType", T_CLASS, "Queue Type", F_CPOOL) - << field("queueLength", T_INT, "Queue Length on Entry") - << field("spanId", T_LONG, "Span ID") - << field("localRootSpanId", T_LONG, "Local Root Span ID") || - contextAttributes) - - << (type("datadog.HeapUsage", T_HEAP_USAGE, "JVM Heap Usage") - << category("Datadog") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("size", T_LONG, "Current Heap Usage", F_BYTES) - << field("isLive", T_BOOLEAN, "After GC")) - - << (type("jdk.CPULoad", T_CPU_LOAD, "CPU Load") - << category("Operating System", "Processor") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("jvmUser", T_FLOAT, "JVM User", F_PERCENTAGE) - << field("jvmSystem", T_FLOAT, "JVM System", F_PERCENTAGE) - << field("machineTotal", T_FLOAT, "Machine Total", F_PERCENTAGE)) - - << (type("jdk.ActiveRecording", T_ACTIVE_RECORDING, - "java-profiler Recording") - << category("Flight Recorder") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("duration", T_LONG, "Duration", F_DURATION_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("id", T_LONG, "Id") << field("name", T_STRING, "Name") - << field("destination", T_STRING, "Destination") - << field("maxAge", T_LONG, "Max Age", F_DURATION_MILLIS) - << field("flushInterval", T_LONG, "Flush Interval", - F_DURATION_MILLIS, VM::hotspot_version() >= 14) - << field("maxSize", T_LONG, "Max Size", F_BYTES) - << field("recordingStart", T_LONG, "Start Time", F_TIME_MILLIS) - << field("recordingDuration", T_LONG, "Recording Duration", - F_DURATION_MILLIS)) - - << (type("jdk.ActiveSetting", T_ACTIVE_SETTING, - "java-profiler Setting") - << category("Flight Recorder") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("duration", T_LONG, "Duration", F_DURATION_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("id", T_LONG, "Event Id") - << field("name", T_STRING, "Setting Name") - << field("value", T_STRING, "Setting Value")) - - << (type("datadog.ProfilerSetting", T_DATADOG_SETTING, - "Profiler Configuration Setting") - << category("Datadog") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("duration", T_LONG, "Duration", F_DURATION_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("name", T_STRING, "Setting Name") - << field("value", T_STRING, "Setting Value") - << field("unit", T_STRING, "Setting Unit")) - - << (type("datadog.DatadogProfilerConfig", T_DATADOG_PROFILER_CONFIG, - "Datadog Profiler Configuration") - << category("Datadog") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("duration", T_LONG, "Duration", F_DURATION_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("cpuInterval", T_LONG, "CPU Sampling Interval", - F_DURATION_MILLIS) - << field("wallInterval", T_LONG, "Wall Sampling Interval", - F_DURATION_MILLIS) - << field("allocInterval", T_LONG, "Allocation Sampling Interval", - F_BYTES) - << field("memleakInterval", T_LONG, "MemLeak Sampling Interval", - F_BYTES) - << field("memleakCapacity", T_LONG, "MemLeak Sampling Capacity") - << field("memleakTrackPercent", T_LONG, - "MemLeak Subsampling Percentage") - << field("gcGenerations", T_BOOLEAN, "GC Generations Tracking") - << field("modeMask", T_INT, "Profiling mode bitmask") - << field("version", T_STRING, "Version") - << field("cpuEngine", T_STRING, "CPU Engine")) - - << (type("datadog.DatadogProfilerClassRefCache", - T_DATADOG_CLASSREF_CACHE, "Datadog ClassRef Cache Stats") - << category("Datadog") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("size", T_LONG, "Cache Size", F_BYTES)) - - << (type("datadog.ProfilerCounter", T_DATADOG_COUNTER, - "Datadog Profiler Internal Counter") - << category("Datadog") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("name", T_COUNTER_NAME, "Name") - << field("count", T_LONG, "Count")) - - << (type("datadog.UnwindFailure", T_UNWIND_FAILURE, "Unwind Failure") - << category("Datadog") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("kind", T_STRING, "Kind") - << field("name", T_STRING, "Name") - << field("count", T_LONG, "Count")) - - << (type("profiler.Malloc", T_MALLOC, "malloc") - << category("Java Virtual Machine", "Native Memory") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("address", T_LONG, "Address", F_ADDRESS) - << field("size", T_LONG, "Size", F_BYTES) - << field("weight", T_FLOAT, "Sample weight") - << field("spanId", T_LONG, "Span ID") - << field("localRootSpanId", T_LONG, "Local Root Span ID") || - contextAttributes) - - << (type("datadog.NativeSocketEvent", T_NATIVE_SOCKET, "Native Socket I/O") - << category("Datadog", "Profiling") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("eventThread", T_THREAD, "Event Thread", F_CPOOL) - << field("stackTrace", T_STACK_TRACE, "Stack Trace", F_CPOOL) - << field("duration", T_LONG, "Duration", F_DURATION_TICKS) - << field("operation", T_STRING, "Operation") - << field("remoteAddress", T_STRING, "Remote Address") - << field("bytesTransferred", T_LONG, "Bytes Transferred", F_BYTES) - << field("weight", T_FLOAT, "Sample weight") - << field("spanId", T_LONG, "Span ID") - << field("localRootSpanId", T_LONG, "Local Root Span ID") || - contextAttributes) - - << (type("jdk.OSInformation", T_OS_INFORMATION, "OS Information") - << category("Operating System") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("osVersion", T_STRING, "OS Version")) - - << (type("jdk.CPUInformation", T_CPU_INFORMATION, "CPU Information") - << category("Operating System", "Processor") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("cpu", T_STRING, "Type") - << field("description", T_STRING, "Description") - << field("sockets", T_INT, "Sockets", F_UNSIGNED) - << field("cores", T_INT, "Cores", F_UNSIGNED) - << field("hwThreads", T_INT, "Hardware Threads", F_UNSIGNED)) - - << (type("jdk.JVMInformation", T_JVM_INFORMATION, "JVM Information") - << category("Java Virtual Machine") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("jvmName", T_STRING, "JVM Name") - << field("jvmVersion", T_STRING, "JVM Version") - << field("jvmArguments", T_STRING, "JVM Command Line Arguments") - << field("jvmFlags", T_STRING, "JVM Settings File Arguments") - << field("javaArguments", T_STRING, "Java Application Arguments") - << field("jvmStartTime", T_LONG, "JVM Start Time", F_TIME_MILLIS) - << field("pid", T_LONG, "Process Identifier")) - - << (type("jdk.InitialSystemProperty", T_INITIAL_SYSTEM_PROPERTY, - "Initial System Property") - << category("Java Virtual Machine") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("key", T_STRING, "Key") - << field("value", T_STRING, "Value")) - - << (type("jdk.NativeLibrary", T_NATIVE_LIBRARY, "Native Library") - << category("Java Virtual Machine", "Runtime") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("name", T_STRING, "Name") - << field("baseAddress", T_LONG, "Base Address", F_ADDRESS) - << field("topAddress", T_LONG, "Top Address", F_ADDRESS) - << field("buildId", T_STRING, "GNU Build ID") - << field("loadBias", T_LONG, "Load Bias", F_ADDRESS)) - - << (type("profiler.Log", T_LOG, "Log Message") - << category("Profiler") - << field("startTime", T_LONG, "Start Time", F_TIME_TICKS) - << field("level", T_LOG_LEVEL, "Level", F_CPOOL) - << field("message", T_STRING, "Message")) - - << (type("jdk.jfr.Label", T_LABEL, NULL) << field("value", T_STRING)) - - << (type("jdk.jfr.Category", T_CATEGORY, NULL) - << field("value", T_STRING, NULL, F_ARRAY)) - - << (type("jdk.jfr.Timestamp", T_TIMESTAMP, "Timestamp") - << field("value", T_STRING)) - - << (type("jdk.jfr.Timespan", T_TIMESPAN, "Timespan") - << field("value", T_STRING)) - - << (type("jdk.jfr.DataAmount", T_DATA_AMOUNT, "Data Amount") - << field("value", T_STRING)) - - << type("jdk.jfr.MemoryAddress", T_MEMORY_ADDRESS, "Memory Address") - - << type("jdk.jfr.Unsigned", T_UNSIGNED, "Unsigned Value") - - << type("jdk.jfr.Percentage", T_PERCENTAGE, "Percentage")) - - << element("region") - .attribute("locale", "en_US") - .attribute("gmtOffset", "0"); - - // The map is used only during construction - _string_map.clear(); - - _initialized = true; -} diff --git a/ddprof-lib/src/main/cpp/jfrMetadata.h b/ddprof-lib/src/main/cpp/jfrMetadata.h deleted file mode 100644 index ac241a7a8..000000000 --- a/ddprof-lib/src/main/cpp/jfrMetadata.h +++ /dev/null @@ -1,272 +0,0 @@ -/* - * Copyright 2020 Andrei Pangin - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _JFRMETADATA_H -#define _JFRMETADATA_H - -#include -#include -#include -#include - -enum JfrType { - T_METADATA = 0, - T_CPOOL = 1, - - T_BOOLEAN = 4, - T_CHAR = 5, - T_FLOAT = 6, - T_DOUBLE = 7, - T_BYTE = 8, - T_SHORT = 9, - T_INT = 10, - T_LONG = 11, - - T_STRING = 20, - T_CLASS = 21, - T_THREAD = 22, - T_CLASS_LOADER = 23, - T_FRAME_TYPE = 24, - T_THREAD_STATE = 25, - T_STACK_TRACE = 26, - T_STACK_FRAME = 27, - T_METHOD = 28, - T_PACKAGE = 29, - T_SYMBOL = 30, - T_LOG_LEVEL = 31, - T_ATTRIBUTE_VALUE = 32, - T_EXECUTION_MODE = 33, - T_COUNTER_NAME = 34, - - T_EVENT = 100, - T_EXECUTION_SAMPLE = 101, - T_METHOD_SAMPLE = 102, - T_ALLOC = 103, - // = 104, - T_HEAP_LIVE_OBJECT = 105, - T_MONITOR_ENTER = 106, - T_THREAD_PARK = 107, - T_CPU_LOAD = 108, - T_ACTIVE_RECORDING = 109, - T_ACTIVE_SETTING = 110, - T_OS_INFORMATION = 111, - T_CPU_INFORMATION = 112, - T_CPU_TSC = 113, - T_JVM_INFORMATION = 114, - T_INITIAL_SYSTEM_PROPERTY = 115, - T_NATIVE_LIBRARY = 116, - T_LOG = 117, - T_WALLCLOCK_SAMPLE_EPOCH = 118, - T_ENDPOINT = 119, - T_DATADOG_SETTING = 120, - T_HEAP_USAGE = 121, - T_DATADOG_PROFILER_CONFIG = 122, - T_QUEUE_TIME = 123, - T_DATADOG_CLASSREF_CACHE = 124, - T_DATADOG_COUNTER = 125, - T_UNWIND_FAILURE = 126, - T_MALLOC = 127, - T_NATIVE_SOCKET = 128, - T_ANNOTATION = 200, - T_LABEL = 201, - T_CATEGORY = 202, - T_TIMESTAMP = 203, - T_TIMESPAN = 204, - T_DATA_AMOUNT = 205, - T_MEMORY_ADDRESS = 206, - T_UNSIGNED = 207, - T_PERCENTAGE = 208 -}; - -class Attribute { -public: - int _key; - int _value; - - Attribute(int key, int value) : _key(key), _value(value) {} -}; - -class Element { -protected: - static std::map _string_map; - static std::vector _strings; - - static int getId(const char *s) { - std::string str(s); - int id = _string_map[str]; - if (id == 0) { - id = _string_map[str] = _string_map.size(); - _strings.push_back(str); - } - return id - 1; - } - -public: - const int _name; - std::vector _attributes; - std::vector _children; - - Element(const char *name) : _name(getId(name)), _attributes(), _children() {} - - Element &attribute(const char *key, const char *value) { - _attributes.push_back(Attribute(getId(key), getId(value))); - return *this; - } - - Element &attribute(const char *key, JfrType value) { - char value_str[16]; - snprintf(value_str, sizeof(value_str), "%i", value); - return attribute(key, value_str); - } - - Element &operator<<(const Element &child) { - if (!const_cast(child).is_nofield()) { - _children.push_back(&child); - } - return *this; - } - - Element &operator||(const std::vector &customAttributes) { - for (auto name : customAttributes) { - Element *child = new Element("field"); - child->attribute("name", name.c_str()); - child->attribute("class", T_ATTRIBUTE_VALUE); - child->attribute("constantPool", "true"); - _children.push_back(child); - } - return *this; - } - - virtual const bool is_nofield() { return false; } -}; - -class NoField : public Element { -public: - NoField(const char *name) : Element(name) {} - - virtual const bool is_nofield() { return true; } -}; - -class JfrMetadata : Element { -private: - static JfrMetadata _root; - static bool _initialized; - - enum FieldFlags { - F_CPOOL = 0x1, - F_ARRAY = 0x2, - F_UNSIGNED = 0x4, - F_BYTES = 0x8, - F_TIME_TICKS = 0x10, - F_TIME_MILLIS = 0x20, - F_DURATION_TICKS = 0x40, - F_DURATION_NANOS = 0x80, - F_DURATION_MILLIS = 0x100, - F_ADDRESS = 0x200, - F_PERCENTAGE = 0x400, - }; - - static Element &element(const char *name) { return *new Element(name); } - - static Element &type(const char *name, JfrType id, const char *label = NULL, - bool simple = false) { - Element &e = element("class"); - e.attribute("name", name); - e.attribute("id", id); - if (simple) { - e.attribute("simpleType", "true"); - } else if (id > T_ANNOTATION) { - e.attribute("superType", "java.lang.annotation.Annotation"); - } else if (id > T_EVENT) { - e.attribute("superType", "jdk.jfr.Event"); - } - if (label != NULL) { - e << annotation(T_LABEL, label); - } - return e; - } - - static Element &field(const char *name, JfrType type, - const char *label = NULL, int flags = 0, - bool condition = true) { - if (!condition) { - return *new NoField(name); - } - Element &e = element("field"); - e.attribute("name", name); - e.attribute("class", type); - if (flags & F_CPOOL) { - e.attribute("constantPool", "true"); - } - if (flags & F_ARRAY) { - e.attribute("dimension", "1"); - } - if (label != NULL) { - e << annotation(T_LABEL, label); - } - if (flags & F_UNSIGNED) { - e << annotation(T_UNSIGNED); - } else if (flags & F_BYTES) { - e << annotation(T_UNSIGNED) << annotation(T_DATA_AMOUNT, "BYTES"); - } else if (flags & F_TIME_TICKS) { - e << annotation(T_TIMESTAMP, "TICKS"); - } else if (flags & F_TIME_MILLIS) { - e << annotation(T_TIMESTAMP, "MILLISECONDS_SINCE_EPOCH"); - } else if (flags & F_DURATION_TICKS) { - e << annotation(T_TIMESPAN, "TICKS"); - } else if (flags & F_DURATION_NANOS) { - e << annotation(T_TIMESPAN, "NANOSECONDS"); - } else if (flags & F_DURATION_MILLIS) { - e << annotation(T_TIMESPAN, "MILLISECONDS"); - } else if (flags & F_ADDRESS) { - e << annotation(T_UNSIGNED) << annotation(T_MEMORY_ADDRESS); - } else if (flags & F_PERCENTAGE) { - e << annotation(T_PERCENTAGE); - } - return e; - } - - static Element &annotation(JfrType type, const char *value = NULL) { - Element &e = element("annotation"); - e.attribute("class", type); - if (value != NULL) { - e.attribute("value", value); - } - return e; - } - - static Element &category(const char *value0, const char *value1 = NULL) { - Element &e = annotation(T_CATEGORY); - e.attribute("value-0", value0); - if (value1 != NULL) { - e.attribute("value-1", value1); - } - return e; - } - -public: - JfrMetadata(); - - static void initialize(const std::vector &contextAttributes); - static void reset(); - - static Element *root() { return &_root; } - - static std::vector &strings() { return _strings; } -}; - -#endif // _JFRMETADATA_H diff --git a/ddprof-lib/src/main/cpp/jniHelper.h b/ddprof-lib/src/main/cpp/jniHelper.h deleted file mode 100644 index 2ccda72dd..000000000 --- a/ddprof-lib/src/main/cpp/jniHelper.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef JAVA_PROFILER_JNIHELPER_H -#define JAVA_PROFILER_JNIHELPER_H - -#include - -inline bool jniExceptionCheck(JNIEnv *jni, bool describe = false) { - if (jni->ExceptionCheck()) { - if (describe) { - jni->ExceptionDescribe(); - } - jni->ExceptionClear(); - return true; - } - return false; -} - -#endif diff --git a/ddprof-lib/src/main/cpp/jvmHeap.h b/ddprof-lib/src/main/cpp/jvmHeap.h deleted file mode 100644 index 8d56ae72f..000000000 --- a/ddprof-lib/src/main/cpp/jvmHeap.h +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2023 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _JVMHEAP_H -#define _JVMHEAP_H - -#include - -/** - * This class only defines a layout compatible with the JDKs VirtualSpaceSummary - * class and particularly its subclasses - */ -class VirtualSpaceSummary { -private: - void *_start; - void *_committed_end; - void *_reserved_end; - -public: - long maxSize() { return (long)_reserved_end - (long)_start; } -}; - -/** - * This class only defines a layout compatible with the JDKs GCHeapSummary class - * and particularly its subclasses - */ -class GCHeapSummary { -private: - void *vptr; // only 1-st level subclasses are used so we need to define the - // 'synthetic' vptr field here - VirtualSpaceSummary _heap; - size_t _used; - -public: - long used() { return (long)_used; } - - long maxSize() { return _heap.maxSize(); } -}; - -/** - * This class only defines a layout compatible with the JDKs CollectedHeap - * class and particularly its subclasses - */ -class CollectedHeapWrapper { -private: - void *vptr; // only 1-st level subclasses are used so we need to define the - // 'synthetic' vptr field here - void *_gc_heap_log; // ignored -public: - // Historic gc information - size_t _capacity_at_last_gc; - size_t _used_at_last_gc; -}; - -#endif // _JVMHEAP_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/jvmSupport.cpp b/ddprof-lib/src/main/cpp/jvmSupport.cpp deleted file mode 100644 index 93bbe00a0..000000000 --- a/ddprof-lib/src/main/cpp/jvmSupport.cpp +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "jvmSupport.h" - -#include "asyncSampleMutex.h" -#include "frames.h" -#include "os.h" -#include "profiler.h" -#include "thread.h" -#include "vmEntry.h" - -#include "hotspot/hotspotSupport.h" - -#include - -int JVMSupport::walkJavaStack(StackWalkRequest& request) { - if (VM::isHotspot()) { - return HotspotSupport::walkJavaStack(request); - } else if (VM::isOpenJ9() || VM::isZing()) { - assert(request.event_type == BCI_CPU || - request.event_type == BCI_WALL || - request.event_type == BCI_NATIVE_MALLOC || - request.event_type == BCI_NATIVE_SOCKET); - return asyncGetCallTrace(request.frames, request.max_depth, request.ucontext); - } - assert(false && "Unsupported JVM"); - return 0; -} - -int JVMSupport::asyncGetCallTrace(ASGCT_CallFrame *frames, int max_depth, void* ucontext) { - JNIEnv *jni = VM::jni(); - if (jni == nullptr) { - return 0; - } - - AsyncSampleMutex mutex(ProfiledThread::currentSignalSafe()); - if (!mutex.acquired()) { - return 0; - } - - JitWriteProtection jit(false); - // AsyncGetCallTrace writes to ASGCT_CallFrame array - ASGCT_CallTrace trace = {jni, 0, frames}; - VM::_asyncGetCallTrace(&trace, max_depth, ucontext); - if (trace.num_frames > 0) { - return trace.num_frames; - } - - const char* err_string = Profiler::asgctError(trace.num_frames); - if (err_string == NULL) { - // No Java stack, because thread is not in Java context - return 0; - } - - Profiler::instance()->incFailure(-trace.num_frames); - return makeFrame(frames, BCI_ERROR, err_string); -} diff --git a/ddprof-lib/src/main/cpp/jvmSupport.h b/ddprof-lib/src/main/cpp/jvmSupport.h deleted file mode 100644 index 1cd387511..000000000 --- a/ddprof-lib/src/main/cpp/jvmSupport.h +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _JVMSUPPORT_H -#define _JVMSUPPORT_H - -#include "stackFrame.h" -#include "stackWalker.h" - -// Stack recovery techniques used to workaround AsyncGetCallTrace flaws. -// Can be disabled with 'safemode' option. -enum StackRecovery { - UNKNOWN_JAVA = (1 << 0), - POP_STUB = (1 << 1), - POP_METHOD = (1 << 2), - LAST_JAVA_PC = (1 << 4), - GC_TRACES = (1 << 5), - PROBE_SP = 0x100, -}; - - -class JVMSupport { - static int asyncGetCallTrace(ASGCT_CallFrame *frames, int max_depth, void* ucontext); -public: - static int walkJavaStack(StackWalkRequest& request); - static inline bool canUnwind(const StackFrame& frame, const void*& pc); - static inline bool isJitCode(const void* pc); -}; - -#endif // _JVMSUPPORT_H diff --git a/ddprof-lib/src/main/cpp/jvmSupport.inline.h b/ddprof-lib/src/main/cpp/jvmSupport.inline.h deleted file mode 100644 index 643847e42..000000000 --- a/ddprof-lib/src/main/cpp/jvmSupport.inline.h +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _JVMSUPPORT_INLINE_H -#define _JVMSUPPORT_INLINE_H - -#include "hotspot/hotspotSupport.h" -#include "jvmSupport.h" -#include "vmEntry.h" - -bool JVMSupport::canUnwind(const StackFrame& frame, const void*& pc) { - if (VM::isHotspot()) { - return HotspotSupport::canUnwind(frame, pc); - } else { - return false; - } -} - -bool JVMSupport::isJitCode(const void* pc) { - if (VM::isHotspot()) { - return HotspotSupport::isJitCode(pc); - } else { - return false; - } -} - -#endif // _JVMSUPPORT_INLINE_H diff --git a/ddprof-lib/src/main/cpp/jvmThread.cpp b/ddprof-lib/src/main/cpp/jvmThread.cpp deleted file mode 100644 index 782ed79e6..000000000 --- a/ddprof-lib/src/main/cpp/jvmThread.cpp +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "jvmThread.h" -#include "hotspot/vmStructs.inline.h" -#include "j9/j9Support.h" -#include "zing/zingSupport.h" -#include "vmEntry.h" - -pthread_key_t JVMThread::_thread_key = pthread_key_t(-1); -jfieldID JVMThread::_tid = nullptr; - -bool JVMThread::initialize() { - void* current_thread = currentThreadSlow(); - if (current_thread == nullptr) { - return false; - } - - for (int i = 0; i < 1024; i++) { - if (pthread_getspecific((pthread_key_t)i) == current_thread) { - _thread_key = pthread_key_t(i); - break; - } - } - // _tid is initialized in currentThreadSlow() - assert(_tid != nullptr); - return _thread_key != pthread_key_t(-1); -} - -int JVMThread::nativeThreadId(JNIEnv* jni, jthread thread) { - return VM::isOpenJ9() ? J9Support::GetOSThreadID(thread) : VMThread::nativeThreadId(jni, thread); -} - -void* JVMThread::currentThreadSlow() { - jthread thread; - if (VM::jvmti()->GetCurrentThread(&thread) != JVMTI_ERROR_NONE) { - return nullptr; - } - - JNIEnv* env = VM::jni(); - jclass thread_class = env->FindClass("java/lang/Thread"); - if (thread_class == NULL || (_tid = env->GetFieldID(thread_class, "tid", "J")) == NULL) { - env->ExceptionClear(); - return nullptr; - } - - if (VM::isOpenJ9()) { - return J9Support::j9thread_self(); - } else if (VM::isZing()) { - return ZingSupport::initialize(thread); - } else { - assert(VM::isHotspot()); - return VMThread::initialize(thread); - } -} diff --git a/ddprof-lib/src/main/cpp/jvmThread.h b/ddprof-lib/src/main/cpp/jvmThread.h deleted file mode 100644 index abc156a08..000000000 --- a/ddprof-lib/src/main/cpp/jvmThread.h +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _JVMTHREAD_H -#define _JVMTHREAD_H - -#include -#include -#include -#include - -/** - * JVMThread represents a native JVM thread that is JVM implementation agnostic - */ -class JVMThread { -private: - static pthread_key_t _thread_key; - static jfieldID _tid; - -public: - static bool isInitialized() { - return _thread_key != pthread_key_t(-1); - } - - /* - * The initialization happens in early startup, in single-threaded mode, - * no synchronization is needed - */ - static bool initialize(); - static inline void* current() { - assert(isInitialized()); - return pthread_getspecific(_thread_key); - } - - static inline pthread_key_t key() { - return _thread_key; - } - - static int nativeThreadId(JNIEnv* jni, jthread thread); - - static inline jlong javaThreadId(JNIEnv* env, jthread thread) { - return env->GetLongField(thread, _tid); - } - - static inline bool hasJavaThreadId() { - return _tid != nullptr; - } - -private: - static void* currentThreadSlow(); -}; - -#endif // _JVMTHREAD_H diff --git a/ddprof-lib/src/main/cpp/libraries.cpp b/ddprof-lib/src/main/cpp/libraries.cpp deleted file mode 100644 index a0c8e83ca..000000000 --- a/ddprof-lib/src/main/cpp/libraries.cpp +++ /dev/null @@ -1,217 +0,0 @@ -#include "codeCache.h" -#include "common.h" -#include "findLibraryImpl.h" -#include "hotspot/vmStructs.h" -#include "libraries.h" -#include "libraryPatcher.h" -#include "log.h" -#include "mallocTracer.h" -#include "os.h" -#include "profiler.h" -#include "symbols.h" -#include "symbols_linux.h" -#include "vmEntry.h" - -// Cadence for the background refresher thread. Bounds the window during -// which a library lazily loaded from signal context (and therefore unable -// to call refresh() synchronously) is unresolvable by the stack walker. -// 500 ms is short enough that frame resolution gaps are barely observable -// in typical sampling, and the refresher only wakes once per tick (cheap). -static constexpr u64 REFRESH_INTERVAL_NS = 500ULL * 1'000'000ULL; - -// Cadence for native (non-Java) thread-name resolution piggy-backed on the -// refresher thread (PROF-15139). Each pass enumerates /proc/self/task and -// reads comm for unknown tids, so it is decimated relative to the 500 ms -// library-refresh tick to bound that cost on high-thread-count processes. -// 2 s is well under the lifetime of long-lived JIT/GC threads we want to name. -static constexpr u64 NATIVE_THREAD_NAME_INTERVAL_NS = 2ULL * 1000ULL * 1'000'000ULL; - -void Libraries::mangle(const char *name, char *buf, size_t size) { - char *buf_end = buf + size; - strcpy(buf, "_ZN"); - buf += 3; - - const char *c; - while ((c = strstr(name, "::")) != NULL && buf + (c - name) + 4 < buf_end) { - int n = snprintf(buf, buf_end - buf, "%d", (int)(c - name)); - if (n < 0 || n >= buf_end - buf) { - if (n < 0) { - Log::debug("Error in snprintf."); - } - goto end; - } - buf += n; - memcpy(buf, name, c - name); - buf += c - name; - name = c + 2; - } - if (buf < buf_end) { - snprintf(buf, buf_end - buf, "%d%sE*", (int)strlen(name), name); - } - -end: - buf_end[-1] = '\0'; -} - -void Libraries::updateSymbols(bool kernel_symbols) { - Symbols::parseLibraries(&_native_libs, kernel_symbols); - LibraryPatcher::patch_libraries(); -} - -void Libraries::refresh() { - // Clear the flag before scanning so any concurrent markDirty() between - // now and the end of this call re-arms us for the next tick. All - // downstream operations are idempotent (parseLibraries tracks - // _parsed_inodes, patch_sigaction checks _sigaction_entries, - // installHooks uses monotonic _patched_libs, updateBuildIds tracks - // _build_id_processed), so redundant invocations are cheap. - _dirty.store(false, std::memory_order_release); - updateSymbols(false); - LibraryPatcher::patch_sigaction(); - LibraryPatcher::install_socket_hooks(); - if (MallocTracer::running()) { - MallocTracer::installHooks(); - } - if (_remote_symbolication) { - updateBuildIds(); - } -} - -void *Libraries::refresherLoop(void *arg) { - Libraries *self = static_cast(arg); - - // Block profiling signals BEFORE publishing our TID. Otherwise a - // SIGPROF / SIGVTALRM armed for this thread (e.g. a stale per-thread - // timer from a previous lifecycle, or an in-flight signal queued during - // pthread_create) could fire on us between TID-publish and sigmask, and - // run the full stack-walk path here — pure overhead, and in debug - // builds with DDPROF_FORCE_STACKWALK_CRASH it inflates the SIGSEGV - // recovery count enough to starve the test work thread and break - // vmStackwalkerCrashRecoveryTest. SIGIO (WAKEUP_SIGNAL) is left - // unmasked because stopRefresher() relies on it to interrupt sleep. - sigset_t mask; - sigemptyset(&mask); - sigaddset(&mask, SIGPROF); - sigaddset(&mask, SIGVTALRM); - pthread_sigmask(SIG_BLOCK, &mask, nullptr); - - // Publish our TID so sampler thread-list enumerations can skip us. - self->_refresher_tid.store(OS::threadId(), std::memory_order_release); - - // Timestamp of the last native-thread-name pass; 0 makes the first eligible - // tick run it. Tracked with a monotonic clock so it is robust to early - // wakeups from stopRefresher()'s SIGIO. - u64 last_native_name_ns = 0; - - while (self->_refresher_running.load(std::memory_order_acquire)) { - // Absolute-deadline sleep that resumes across EINTR (SIGCHLD, debugger - // SIGSTOP/SIGCONT, etc.) and wakes early when stopRefresher() flips - // _refresher_running false and sends SIGIO. See OS::sleepWhile. - OS::sleepWhile(REFRESH_INTERVAL_NS, self->_refresher_running); - if (!self->_refresher_running.load(std::memory_order_acquire)) { - break; - } - if (self->_dirty.load(std::memory_order_acquire)) { - self->refresh(); - } - // Name native (non-Java) threads while they are still alive. JIT/GC and - // other non-Java threads get no JVMTI ThreadStart, so they are otherwise - // named only at dump time; transient ones that exit before the dump fall - // back to "[tid=N]" (PROF-15139). Gated on isRunning() so we do no work - // before the profiler reaches RUNNING (startRefresher precedes that), and - // decimated to NATIVE_THREAD_NAME_INTERVAL_NS to bound the /proc scan cost. - u64 now = OS::nanotime(); - if (Profiler::instance()->isRunning() && - now - last_native_name_ns >= NATIVE_THREAD_NAME_INTERVAL_NS) { - last_native_name_ns = now; - // Defer threads still showing the inherited process name; the dump-time - // pass (which does not defer) records any that never set a real name. - Profiler::instance()->updateNativeThreadNames(true); - } - } - return nullptr; -} - -void Libraries::startRefresher() { - if (_refresher_running.exchange(true, std::memory_order_acq_rel)) { - return; // already running - } - if (pthread_create(&_refresher_thread, nullptr, refresherLoop, this) != 0) { - _refresher_running.store(false, std::memory_order_release); - Log::warn("Unable to start Libraries refresher thread"); - } -} - -void Libraries::stopRefresher() { - if (!_refresher_running.exchange(false, std::memory_order_acq_rel)) { - return; // not running - } - pthread_kill(_refresher_thread, WAKEUP_SIGNAL); - pthread_join(_refresher_thread, nullptr); - // Clear the published TID so a later sampler doesn't skip an unrelated - // thread that may have inherited the same TID after we exited. - _refresher_tid.store(-1, std::memory_order_release); -} - -// Platform-specific implementation of updateBuildIds() is in libraries_linux.cpp (Linux) -// or stub implementation for other platforms - -const void *Libraries::resolveSymbol(const char *name) { - char mangled_name[256]; - if (strstr(name, "::") != NULL) { - mangle(name, mangled_name, sizeof(mangled_name)); - name = mangled_name; - } - - size_t len = strlen(name); - int native_lib_count = _native_libs.count(); - if (len > 0 && name[len - 1] == '*') { - for (int i = 0; i < native_lib_count; i++) { - CodeCache *lib = _native_libs[i]; - if (lib != NULL) { - const void *address = lib->findSymbolByPrefix(name, len - 1); - if (address != NULL) { - return address; - } - } - } - } else { - for (int i = 0; i < native_lib_count; i++) { - CodeCache *lib = _native_libs[i]; - if (lib != NULL) { - const void *address = lib->findSymbol(name); - if (address != NULL) { - return address; - } - } - } - } - - return NULL; -} - -CodeCache *Libraries::findJvmLibrary(const char *j9_lib_name) { - return VM::isOpenJ9() ? findLibraryByName(j9_lib_name) : VMStructs::libjvm(); -} - -CodeCache *Libraries::findLibraryByName(const char *lib_name) { - const size_t lib_name_len = strlen(lib_name); - const int native_lib_count = _native_libs.count(); - for (int i = 0; i < native_lib_count; i++) { - CodeCache *lib = _native_libs[i]; - if (lib != NULL) { - const char *s = lib->name(); - if (s != NULL) { - const char *p = strrchr(s, '/'); - if (p != NULL && strncmp(p + 1, lib_name, lib_name_len) == 0) { - return lib; - } - } - } - } - return NULL; -} - -CodeCache *Libraries::findLibraryByAddress(const void *address) const { - return findLibraryByAddressImpl(_native_libs, address); -} diff --git a/ddprof-lib/src/main/cpp/libraries.h b/ddprof-lib/src/main/cpp/libraries.h deleted file mode 100644 index 18b59c963..000000000 --- a/ddprof-lib/src/main/cpp/libraries.h +++ /dev/null @@ -1,90 +0,0 @@ -#ifndef _LIBRARIES_H -#define _LIBRARIES_H - -#include -#include - -#include "codeCache.h" - -class Libraries { - private: - CodeCacheArray _native_libs; - CodeCache _runtime_stubs; - bool _remote_symbolication; // set via setRemoteSymbolication() - - // Pending-refresh flag set by dlopen_hook when it cannot call refresh() - // synchronously (signal context). Polled by the refresher thread. - std::atomic _dirty; - - // Background refresher thread: periodically (every REFRESH_INTERVAL_NS) - // checks _dirty and runs refresh() outside signal context, narrowing the - // window during which newly-loaded libraries are unresolvable. - pthread_t _refresher_thread; - std::atomic _refresher_running; - std::atomic _refresher_tid; // captured from OS::threadId() on entry - static void *refresherLoop(void *arg); - - static void mangle(const char *name, char *buf, size_t size); - public: - Libraries() : _native_libs(), _runtime_stubs("runtime stubs"), - _remote_symbolication(false), _dirty(false), - _refresher_thread(), _refresher_running(false), - _refresher_tid(-1) {} - - void setRemoteSymbolication(bool enabled) { _remote_symbolication = enabled; } - - // Refresh symbol tables and reinstall hooks/patches for any libraries - // loaded since the last refresh. Idempotent and cheap when no new - // libraries have been loaded (parseLibraries tracks _parsed_inodes). - // Clears the dirty flag. Must be called from non-signal context: - // updateSymbols acquires a Mutex and reads /proc/self/maps. - void refresh(); - - // Async-signal-safe: just sets a flag. The refresher thread will pick - // up the change on its next tick. - void markDirty() { _dirty.store(true, std::memory_order_release); } - - // Start/stop the background refresher thread. Called from - // Profiler::start/stop. - void startRefresher(); - void stopRefresher(); - - // TID of the refresher thread once it has captured its own ID, or -1 if - // the thread is not currently running. Used by sampler thread-list - // enumeration to skip this profiler-internal thread. - int refresherTid() const { return _refresher_tid.load(std::memory_order_acquire); } - - void updateSymbols(bool kernel_symbols); - void updateBuildIds(); // Extract build-ids for all loaded libraries - const void *resolveSymbol(const char *name); - // In J9 the 'libjvm' functionality is spread across multiple libraries - // This function will return the 'libjvm' on non-J9 VMs and the library with the given name on J9 VMs - CodeCache *findJvmLibrary(const char *j9_lib_name); - CodeCache *findLibraryByName(const char *lib_name); - CodeCache *findLibraryByAddress(const void *address) const; - - // Get library by index (used for remote symbolication unpacking) - // Note: Parameter is uint32_t to match lib_index packing (17 bits = max 131K libraries) - CodeCache *getLibraryByIndex(uint32_t index) const { - assert(_native_libs.count() >= 0); - if (index < static_cast(_native_libs.count())) { - return _native_libs[index]; - } - return nullptr; - } - - static Libraries *instance() { - static Libraries instance; - return &instance; - } - - const CodeCacheArray& native_libs() const { - return _native_libs; - } - - // Delete copy constructor and assignment operator to prevent copies - Libraries(const Libraries&) = delete; - Libraries& operator=(const Libraries&) = delete; -}; - -#endif // _LIBRARIES_H diff --git a/ddprof-lib/src/main/cpp/libraries_linux.cpp b/ddprof-lib/src/main/cpp/libraries_linux.cpp deleted file mode 100644 index 6fb5a664e..000000000 --- a/ddprof-lib/src/main/cpp/libraries_linux.cpp +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ - -#include "common.h" -#include "counters.h" -#include "libraries.h" -#include "symbols_linux.h" - -#include -#include - -// Track which libraries have had build-ID extraction attempted -// Mirrors the _parsed_inodes pattern from symbols_linux.cpp for O(1) skip -static std::unordered_set _build_id_processed; -static std::mutex _build_id_lock; - -void Libraries::updateBuildIds() { - std::lock_guard lock(_build_id_lock); - - int lib_count = _native_libs.count(); - - for (int i = 0; i < lib_count; i++) { - CodeCache* lib = _native_libs.at(i); - if (lib == nullptr) { - continue; - } - - // O(1) check: Skip if already processed - // Mirrors _parsed_inodes pattern from symbols_linux.cpp for optimal performance - if (_build_id_processed.find(lib) != _build_id_processed.end()) { - Counters::increment(REMOTE_SYMBOLICATION_BUILD_ID_CACHE_HITS); - continue; - } - - // Mark as processed immediately to avoid re-processing on errors - _build_id_processed.insert(lib); - - // Skip if already has build-id (e.g., from JFR metadata load) - if (lib->hasBuildId()) { - continue; - } - - const char* lib_name = lib->name(); - if (lib_name == nullptr) { - continue; - } - - // Extract build-id from library file (only happens once per library) - size_t build_id_len; - char* build_id = SymbolsLinux::extractBuildId(lib_name, &build_id_len); - - if (build_id != nullptr) { - // Set build-id and calculate load bias - lib->setBuildId(build_id, build_id_len); - - // Calculate load bias: difference between runtime address and file base - // For now, use image_base as the load bias base - if (lib->imageBase() != nullptr) { - lib->setLoadBias((uintptr_t)lib->imageBase()); - } - - free(build_id); // setBuildId makes its own copy - - // Track libraries with build-IDs - Counters::increment(REMOTE_SYMBOLICATION_LIBS_WITH_BUILD_ID); - } else { - TEST_LOG("updateBuildIds: NO build-id found for %s", lib_name); - } - } -} - -#endif // __linux__ diff --git a/ddprof-lib/src/main/cpp/libraries_macos.cpp b/ddprof-lib/src/main/cpp/libraries_macos.cpp deleted file mode 100644 index b1270b2d0..000000000 --- a/ddprof-lib/src/main/cpp/libraries_macos.cpp +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef __linux__ - -#include "libraries.h" - -// Stub implementation for non-Linux platforms -// Remote symbolication with build-ID extraction is Linux-only -void Libraries::updateBuildIds() { - // No-op on non-Linux platforms -} - -#endif // !__linux__ diff --git a/ddprof-lib/src/main/cpp/libraryPatcher.h b/ddprof-lib/src/main/cpp/libraryPatcher.h deleted file mode 100644 index 70be3659b..000000000 --- a/ddprof-lib/src/main/cpp/libraryPatcher.h +++ /dev/null @@ -1,79 +0,0 @@ -#ifndef _LIBRARYPATCHER_H -#define _LIBRARYPATCHER_H - -#include "codeCache.h" -#include "spinLock.h" -#include - -#ifdef __linux__ - -// Patch libraries' @plt entries -typedef struct _patchEntry { - CodeCache* _lib; - // library's @plt location - void** _location; - // original function - void* _func; -} PatchEntry; - - -class LibraryPatcher { -private: - static SpinLock _lock; - static const char* _profiler_name; - static PatchEntry _patched_entries[MAX_NATIVE_LIBS]; - static int _size; - static bool _patch_pthread_create; - - // Separate tracking for sigaction patches - static PatchEntry _sigaction_entries[MAX_NATIVE_LIBS]; - static int _sigaction_size; - - // Separate tracking for socket (send/recv/write/read) patches. - // Each library can contribute up to 4 GOT slots (send/recv/write/read). - static PatchEntry _socket_entries[4 * MAX_NATIVE_LIBS]; - static int _socket_size; - - static void patch_library_unlocked(CodeCache* lib); - static void patch_pthread_create(); - static void patch_pthread_setspecific(); - static void patch_sigaction_in_library(CodeCache* lib); -public: - // True while socket hooks are installed; read by Profiler::dlopen_hook - // to decide whether to re-patch after a new library is loaded. - // Set to true after the first batch of libraries is patched in patch_socket_functions(). - // Libraries loaded after profiler start are picked up on the next dlopen_hook call, - // which calls install_socket_hooks() to patch them if _socket_active is true. - // Low-probability race: stop() is called only on JVM exit; atomic is zero-cost insurance. - static std::atomic _socket_active; - static void initialize(); - static void patch_libraries(); - static void unpatch_libraries(); - static void patch_sigaction(); - static bool patch_socket_functions(); - static void unpatch_socket_functions(); - // Called from Profiler::dlopen_hook after a new library is loaded. - // No-op when socket hooks are not active. - static inline void install_socket_hooks() { - if (_socket_active.load(std::memory_order_acquire)) { - patch_socket_functions(); - } - } -}; - -#else - -class LibraryPatcher { -public: - static void initialize() { } - static void patch_libraries() { } - static void unpatch_libraries() { } - static void patch_sigaction() { } - static bool patch_socket_functions() { return false; } - static void unpatch_socket_functions() { } - static void install_socket_hooks() { } -}; - -#endif - -#endif // _LIBRARYPATCHER_H diff --git a/ddprof-lib/src/main/cpp/libraryPatcher_linux.cpp b/ddprof-lib/src/main/cpp/libraryPatcher_linux.cpp deleted file mode 100644 index ae1168bfe..000000000 --- a/ddprof-lib/src/main/cpp/libraryPatcher_linux.cpp +++ /dev/null @@ -1,714 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "libraryPatcher.h" - -#ifdef __linux__ -#include "counters.h" -#include "guards.h" -#include "nativeSocketSampler.h" -#include "profiler.h" - -#include -#include -#include -#include -#include -#include -#include - -typedef void* (*func_start_routine)(void*); - -SpinLock LibraryPatcher::_lock; -const char* LibraryPatcher::_profiler_name = nullptr; -PatchEntry LibraryPatcher::_patched_entries[MAX_NATIVE_LIBS]; -int LibraryPatcher::_size = 0; -PatchEntry LibraryPatcher::_sigaction_entries[MAX_NATIVE_LIBS]; -int LibraryPatcher::_sigaction_size = 0; -PatchEntry LibraryPatcher::_socket_entries[4 * MAX_NATIVE_LIBS]; -int LibraryPatcher::_socket_size = 0; -std::atomic LibraryPatcher::_socket_active{false}; - -void LibraryPatcher::initialize() { - if (_profiler_name == nullptr) { - Dl_info info; - void* caller_address = __builtin_return_address(0); // Get return address of caller - bool ret = dladdr(caller_address, &info); - assert(ret); - _profiler_name = realpath(info.dli_fname, nullptr); - _size = 0; - } -} - -class RoutineInfo { -private: - func_start_routine _routine; - void* _args; -public: - RoutineInfo(func_start_routine routine, void* args) : - _routine(routine), _args(args) { - } - - func_start_routine routine() const { - return _routine; - } - - void* args() const { - return _args; - } -}; - -// Unregister the current thread from the profiler and release its TLS under a -// single SignalBlocker to close the race window between unregisterThread() -// returning and release() acquiring its internal guard (PROF-14603). Without -// this, a SIGVTALRM delivered in that window could call currentSignalSafe() -// and dereference a now-freed ProfiledThread. Kept noinline so the -// SignalBlocker's sigset_t does not appear in the caller's stack frame on -// musl/aarch64 where the deopt blob may corrupt the wrapper's stack guard. -__attribute__((noinline)) -static void unregister_and_release(int tid) { - SignalBlocker blocker; - Profiler::unregisterThread(tid); - ProfiledThread::release(); -} - -// pthread_cleanup_push callback for thread wrappers. -// Fires when the wrapped routine calls pthread_exit() or the thread is -// canceled. Kept noinline so its stack frame (which may hold a SignalBlocker -// via unregister_and_release) lives outside the DEOPT-corruption zone of the -// caller on musl/aarch64, and so that the SignalBlocker's sigset_t does not -// appear in the caller's frame on platforms with stack-protector canaries. -__attribute__((noinline)) -static void cleanup_unregister(void*) { - unregister_and_release(ProfiledThread::currentTid()); -} - -// Thread-cleanup wrapper that avoids the static-libgcc / forced-unwind crash. -// -// The crash: on glibc, pthread_cleanup_push in C++ mode expands to -// __pthread_cleanup_class (RAII), which adds a cleanup entry to the LSDA of -// this frame. When libjavaProfiler.so is built with -static-libgcc, the -// embedded __gxx_personality_v0 is called by the dynamic libgcc_s.so.1's -// _Unwind_ForcedUnwind. The two libgcc versions have incompatible -// _Unwind_Context layouts; calling _Unwind_SetGR (which happens when the -// personality finds a cleanup action) with a cross-version context triggers -// the cold/error path, which calls abort(). -// -// The fix: use __pthread_register_cancel / __pthread_unregister_cancel -// directly — the same thing the C macro form of pthread_cleanup_push does. -// This registers cleanup via a setjmp buffer in a runtime linked-list, NOT -// via an LSDA destructor. _Unwind_ForcedUnwind's stop function -// (__pthread_unwind_stop) handles the cleanup without ever calling -// __gxx_personality_v0 for this frame, so _Unwind_SetGR is never called and -// the cross-version incompatibility is never triggered. -// -// On musl: pthread_cleanup_push already uses the C/setjmp form (no RAII), -// and pthread_exit does not use _Unwind_ForcedUnwind, so there is no issue. -// The __GLIBC__ guard keeps the musl path unchanged. -#ifdef __GLIBC__ -// On glibc, declares __pthread_register_cancel etc. only inside -// the C (non-C++) conditional, so they're invisible in C++ code. Redeclare -// them with extern "C" so we can call them directly without the header guard. -extern "C" { - extern void __pthread_register_cancel(__pthread_unwind_buf_t*); - extern void __pthread_unregister_cancel(__pthread_unwind_buf_t*); - [[noreturn]] extern void __pthread_unwind_next(__pthread_unwind_buf_t*); -} -#endif - -__attribute__((visibility("hidden"), noinline, no_stack_protector)) -void run_with_cleanup(func_start_routine routine, void* params, - void (*cleanup_fn)(void*), void* cleanup_arg) { -#ifdef __GLIBC__ - __pthread_unwind_buf_t cancel_buf = {}; - // With savemask=0, __sigsetjmp only writes __jmp_buf + int __mask_was_saved; - // it never touches __saved_mask. The inner struct of __pthread_unwind_buf_t - // must cover exactly that writable prefix of struct __jmp_buf_tag. - static_assert(offsetof(__pthread_unwind_buf_t, __cancel_jmp_buf) == 0 && - sizeof(cancel_buf.__cancel_jmp_buf[0]) == offsetof(struct __jmp_buf_tag, __saved_mask), - "glibc __pthread_unwind_buf_t inner layout incompatible with struct __jmp_buf_tag"); - // __sigsetjmp/longjmp only intercepts _Unwind_ForcedUnwind (pthread_exit / - // cancellation). routine(params) must NOT throw a regular C++ exception - // across this boundary: an escaping exception would skip both - // __pthread_unregister_cancel and cleanup_fn below, leaking the thread - // registration and leaving cancel_buf linked against this (unwound) frame. - // We cannot defend with a try/catch here — a handler frame adds an LSDA - // action, which is exactly what triggers the static-libgcc abort this - // function exists to avoid. Production routines are JVM/native start - // routines that handle their own exceptions and do not throw across here. - if (__builtin_expect( - // set __sigsetjmp's savemask=0 (the second parameter, noting that the signal mask is NOT - // saved/restored, which is correct because the cancel mechanism does not depend on signal mask state. - __sigsetjmp((struct __jmp_buf_tag*)(void*)cancel_buf.__cancel_jmp_buf, 0), 0)) { - // Reached via longjmp from glibc's stop function when pthread_exit - // (or cancellation) fires. Run cleanup and continue unwinding. - cleanup_fn(cleanup_arg); - __pthread_unwind_next(&cancel_buf); - // __pthread_unwind_next is [[noreturn]]; this fails loudly rather than - // falling through into __pthread_register_cancel on a torn-down frame - // should a future/variant glibc ever return from it. - __builtin_unreachable(); - } - // Callers must not have a pending pthread_cancel when they enter - // run_with_cleanup: a cancellation arriving between __sigsetjmp returning - // and __pthread_register_cancel below would unwind this frame before - // cancel_buf is registered, silently skipping cleanup_fn. All current - // callers are JVM/native start routines with no pending cancellation. - __pthread_register_cancel(&cancel_buf); - routine(params); - __pthread_unregister_cancel(&cancel_buf); - cleanup_fn(cleanup_arg); -#else - // musl / non-glibc: pthread_cleanup_push uses the C/setjmp form, no RAII. - pthread_cleanup_push(cleanup_fn, cleanup_arg); - routine(params); - pthread_cleanup_pop(1); -#endif -} - -#ifdef UNIT_TEST -// Integration test entry point: exercises the full start_routine_wrapper → -// run_with_cleanup chain without calling Profiler::registerThread or -// Profiler::unregisterThread, which dereference _cpu_engine/_wall_engine and -// crash when the profiler is not started (as in gtest). -// -// The caller supplies cleanup_fn/cleanup_arg so the test can verify cleanup -// fires and observe ProfiledThread::release() without coupling to Profiler state. -// -// Thread lifecycle: -// pthread_create_wrapped_for_test → start_routine_for_test -// → ProfiledThread::initCurrentThread() -// → run_with_cleanup(routine, params, cleanup_fn, cleanup_arg) -// → pthread_exit(nullptr) -struct WrapperTestCtx { - func_start_routine routine; - void* params; - void (*cleanup_fn)(void*); - void* cleanup_arg; -}; - -__attribute__((visibility("hidden"), noinline, no_stack_protector)) -static void* start_routine_for_test(void* raw) { - auto* ctx = static_cast(raw); - func_start_routine routine = ctx->routine; - void* params = ctx->params; - void (*cleanup_fn)(void*) = ctx->cleanup_fn; - void* cleanup_arg = ctx->cleanup_arg; - { - SignalBlocker blocker; - delete ctx; - ProfiledThread::initCurrentThread(); - } - run_with_cleanup(routine, params, cleanup_fn, cleanup_arg); - pthread_exit(nullptr); - __builtin_unreachable(); -} - -int pthread_create_wrapped_for_test(pthread_t* thread, - func_start_routine routine, void* params, - void (*cleanup_fn)(void*), void* cleanup_arg) { - WrapperTestCtx* ctx; - { - SignalBlocker blocker; - ctx = new WrapperTestCtx{routine, params, cleanup_fn, cleanup_arg}; - } - int ret = pthread_create(thread, nullptr, start_routine_for_test, ctx); - if (ret != 0) { - SignalBlocker blocker; - delete ctx; - } - return ret; -} - -// Variant that passes the production cleanup_unregister as the cleanup function. -// Exercises the full chain: start_routine_for_test → run_with_cleanup → -// cleanup_unregister → Profiler::unregisterThread + ProfiledThread::release. -// Profiler::unregisterThread is null-safe under UNIT_TEST (see profiler.cpp). -int pthread_create_with_cleanup_unregister_for_test(pthread_t* thread, - func_start_routine routine, - void* params) { - return pthread_create_wrapped_for_test(thread, routine, params, - cleanup_unregister, nullptr); -} -#endif // UNIT_TEST - -#ifdef __aarch64__ -// Delete RoutineInfo with profiling signals blocked to prevent ASAN -// allocator lock reentrancy. Kept noinline so SignalBlocker's sigset_t -// does not trigger stack-protector canary in the caller on aarch64. -__attribute__((noinline)) -static void delete_routine_info(RoutineInfo* thr) { - SignalBlocker blocker; - delete thr; -} - -// Initialize the current thread's TLS, open the init window (PROF-13072), and -// register the thread with the profiler — all under a single SignalBlocker so -// profiling signals cannot fire in the gap between initCurrentThread() and -// startInitWindow(). Kept noinline for the same stack-protector reason as -// delete_routine_info: SignalBlocker's sigset_t must not appear in -// start_routine_wrapper_spec's own stack frame on musl/aarch64. -__attribute__((noinline)) -static void init_tls_and_register() { - SignalBlocker blocker; - ProfiledThread::initCurrentThread(); - if (ProfiledThread *pt = ProfiledThread::currentSignalSafe()) { - pt->startInitWindow(); - } - Profiler::registerThread(ProfiledThread::currentTid()); -} - -// Wrapper around the real start routine. -// The wrapper: -// 1. Register the newly created thread to profiler -// 2. Call real start routine -// 3. Unregister the thread from profiler once the routine is completed. -// This version works around stack corruption observed on musl/aarch64/JDK11: -// -// Empirical observation (hs_err analysis): after DEOPT PACKING fires on a -// thread running compiled lambda$measureContention$0 at sp=0x...49d0, this -// wrapper's frame (sp=0x...5020, ~144 bytes below thread stack top) shows a -// corrupted FP (odd address 0x...5001) and a corrupted stack canary. The -// corruption is confined to the top ~224 bytes of the stack (the region between -// DEOPT PACKING sp and the thread stack top). -// -// The source of the corruption is the interpreter-frame rebuild sequence in -// HotSpot's deoptimization blob (generate_deopt_blob in -// sharedRuntime_aarch64.cpp, openjdk/jdk11u). After popping the compiled -// frame the blob executes "sub sp, sp, caller_adjustment" followed by a loop -// of enter() calls (each doing "stp rfp, lr, [sp, #-16]!") to lay down -// replacement interpreter frames. When musl's small thread stack places this -// wrapper immediately above the compiled frame, the enter() writes can reach -// into this wrapper's frame, corrupting the saved FP and stack canary. -// The mechanism is the same "precarious stack guard corruption" the noinline -// helpers above already defend against for SignalBlocker's sigset_t. -// -// Two symptoms arise from this corruption: -// -// (a) Stack-canary crash: -fstack-protector-strong inserts a canary whenever -// the frame has a non-trivially destructed local (e.g. a Cleanup struct). -// That canary lands in the corruption zone; the epilogue fires -// __stack_chk_fail. no_stack_protector removes the canary. -// -// (b) Corrupted-LR crash: even without a canary, `return` loads the saved LR -// from the corrupted frame and jumps to a garbage address. pthread_exit() -// terminates the thread without using LR. HotSpot on musl returns normally -// from java_start (no forced-unwind), so no exception-based cleanup path -// is needed. -// -// Cleanup reads tid from TLS (via ProfiledThread::currentTid()) rather than -// from a stack variable, so it is correct even after the frame is corrupted. -// pthread_cleanup_push/pop ensures unregister_and_release() also runs when the -// wrapped routine calls pthread_exit() or the thread is canceled. -__attribute__((visibility("hidden"), no_stack_protector)) -static void* start_routine_wrapper_spec(void* args) { - RoutineInfo* thr = (RoutineInfo*)args; - func_start_routine routine = thr->routine(); - void* params = thr->args(); - delete_routine_info(thr); - init_tls_and_register(); - // cleanup_unregister fires on pthread_exit() or cancellation from within - // routine(params). The push/pop pair lives inside run_with_cleanup so - // that __pthread_unwind_buf_t (glibc) / struct __ptcb (musl) does not land - // in this frame's DEOPT-corruption zone. - run_with_cleanup(routine, params, cleanup_unregister, nullptr); - // pthread_exit instead of 'return': the saved LR in this frame is corrupted - // by DEOPT PACKING; returning would jump to a garbage address. - // cleanup_unregister has already run via run_with_cleanup's normal return - // path, so there is no registered cancel handler left. The forced unwind - // raised by pthread_exit walks this frame, but it is safe because no - // destructor-bearing local (and hence no LSDA cleanup/handler action) is - // live at this call site: __gxx_personality_v0 returns continue-unwind - // without ever calling _Unwind_SetGR, avoiding the static-libgcc abort. - // WARNING: adding any RAII local with a destructor between run_with_cleanup - // and pthread_exit would reintroduce that crash. - pthread_exit(nullptr); - __builtin_unreachable(); -} - -static int pthread_create_hook_spec(pthread_t* thread, - const pthread_attr_t* attr, - func_start_routine start_routine, - void* args) { - RoutineInfo* thr; - { - SignalBlocker blocker; - thr = new RoutineInfo(start_routine, args); - } - int ret = pthread_create(thread, attr, start_routine_wrapper_spec, (void*)thr); - if (ret != 0) { - SignalBlocker blocker; - delete thr; - } - return ret; -} - -#endif // __aarch64__ - -// Wrapper around the real start routine. -// See comments for start_routine_wrapper_spec() for details -__attribute__((visibility("hidden"), no_stack_protector)) -static void* start_routine_wrapper(void* args) { - RoutineInfo* thr = (RoutineInfo*)args; - func_start_routine routine; - void* params; - { - // Block profiling signals while accessing and freeing RoutineInfo - // and during TLS initialization. Under ASAN, new/delete/ - // pthread_setspecific are intercepted and acquire ASAN's internal - // allocator lock. A profiling signal during any of these calls - // runs ASAN-instrumented code that tries to acquire the same - // lock, causing deadlock. - // registerThread is also kept inside the blocker so that the CPU - // timer is armed while SIGPROF/SIGVTALRM are masked. Any pending - // signal fires only after signals are re-enabled (when the blocker - // scope exits), at which point JVMThread::current() is still null - // and the guard in CTimer::signalHandler discards the sample safely. - SignalBlocker blocker; - routine = thr->routine(); - params = thr->args(); - delete thr; - ProfiledThread::initCurrentThread(); - ProfiledThread::currentSignalSafe()->startInitWindow(); - Profiler::registerThread(ProfiledThread::currentTid()); - } - // Use POSIX cleanup instead of C++ RAII to handle pthread_exit(): see run_with_cleanup. - // cleanup_unregister has already run on run_with_cleanup's normal return path. - // The pthread_exit forced unwind is safe here for the same reason as in - // start_routine_wrapper_spec: no destructor-bearing local is live at this - // call site, so __gxx_personality_v0 never calls _Unwind_SetGR. - run_with_cleanup(routine, params, cleanup_unregister, nullptr); - pthread_exit(nullptr); - __builtin_unreachable(); -} - -static int pthread_create_hook(pthread_t* thread, - const pthread_attr_t* attr, - func_start_routine start_routine, - void* args) { - RoutineInfo* thr; - { - SignalBlocker blocker; - thr = new RoutineInfo(start_routine, args); - } - int ret = pthread_create(thread, attr, start_routine_wrapper, (void*)thr); - if (ret != 0) { - SignalBlocker blocker; - delete thr; - } - return ret; -} - -void LibraryPatcher::patch_libraries() { - // LibraryPatcher has yet initialized, only happens in Gtest - if (_profiler_name == nullptr) { - return; - } - - TEST_LOG("Patching libraries"); - patch_pthread_create(); - TEST_LOG("%d libraries patched", _size); -} - -void LibraryPatcher::patch_library_unlocked(CodeCache* lib) { - if (lib->name() == nullptr) return; - - char path[PATH_MAX]; - char* resolved_path = realpath(lib->name(), path); - if (resolved_path != nullptr && // filter out virtual file, e.g. [vdso], etc. - strcmp(resolved_path, _profiler_name) == 0) { // Don't patch self - return; - } - - // Don't patch sanitizer runtime libraries — intercepting their internal - // pthread_create calls causes reentrancy and heap corruption under ASAN. - const char* base = strrchr(lib->name(), '/'); - base = (base != nullptr) ? base + 1 : lib->name(); - if (strncmp(base, "libasan", 7) == 0 || - strncmp(base, "libtsan", 7) == 0 || - strncmp(base, "libubsan", 8) == 0) { - return; - } - - void** pthread_create_location = (void**)lib->findImport(im_pthread_create); - if (pthread_create_location == nullptr) { - return; - } - - for (int index = 0; index < _size; index++) { - // Already patched - if (_patched_entries[index]._lib == lib) { - return; - } - } - TEST_LOG("Patching: %s", lib->name()); - void* func = (void*)pthread_create_hook; - -#ifdef __aarch64__ - // Workaround stack guard corruption in Linux/aarch64/musl/jdk11 - if (VM::isHotspot() && OS::isMusl() && VM::java_version() == 11) { - func = (void*)pthread_create_hook_spec; - } -#endif - _patched_entries[_size]._lib = lib; - _patched_entries[_size]._location = pthread_create_location; - _patched_entries[_size]._func = (void*)__atomic_load_n(pthread_create_location, __ATOMIC_RELAXED); - __atomic_store_n(pthread_create_location, func, __ATOMIC_RELAXED); - _size++; -} - -void LibraryPatcher::unpatch_libraries() { - TEST_LOG("Restore libraries"); - ExclusiveLockGuard locker(&_lock); - for (int index = 0; index < _size; index++) { - __atomic_store_n(_patched_entries[index]._location, _patched_entries[index]._func, __ATOMIC_RELAXED); - } - _size = 0; -} - -void LibraryPatcher::patch_pthread_create() { - const CodeCacheArray& native_libs = Libraries::instance()->native_libs(); - int num_of_libs = native_libs.count(); - ExclusiveLockGuard locker(&_lock); - for (int index = 0; index < num_of_libs; index++) { - CodeCache* lib = native_libs.at(index); - if (lib != nullptr) { - patch_library_unlocked(lib); - } - } -} - -// Patch sigaction in all libraries to prevent any library from overwriting -// our SIGSEGV/SIGBUS handlers. This protects against misbehaving libraries -// (like wasmtime) that install broken signal handlers calling malloc(). -void LibraryPatcher::patch_sigaction_in_library(CodeCache* lib) { - if (lib->name() == nullptr) return; - if (_profiler_name == nullptr) return; // Not initialized yet - - // Don't patch ourselves - char path[PATH_MAX]; - char* resolved_path = realpath(lib->name(), path); - if (resolved_path != nullptr && strcmp(resolved_path, _profiler_name) == 0) { - return; - } - - // Note: We intentionally patch sanitizer libraries (libasan, libtsan, libubsan) here. - // This keeps our handler on top for recoverable SIGSEGVs (e.g., safefetch) while - // still chaining to the sanitizer's handler for unexpected crashes. - - void** sigaction_location = (void**)lib->findImport(im_sigaction); - if (sigaction_location == nullptr) { - return; - } - - // Check if already patched or array is full - if (_sigaction_size >= MAX_NATIVE_LIBS) { - return; - } - for (int index = 0; index < _sigaction_size; index++) { - if (_sigaction_entries[index]._lib == lib) { - return; - } - } - - void* hook = OS::getSigactionHook(); - _sigaction_entries[_sigaction_size]._lib = lib; - _sigaction_entries[_sigaction_size]._location = sigaction_location; - _sigaction_entries[_sigaction_size]._func = (void*)__atomic_load_n(sigaction_location, __ATOMIC_RELAXED); - __atomic_store_n(sigaction_location, hook, __ATOMIC_RELAXED); - _sigaction_size++; - Counters::increment(SIGACTION_PATCHED_LIBS); -} - -void LibraryPatcher::patch_sigaction() { - const CodeCacheArray& native_libs = Libraries::instance()->native_libs(); - int num_of_libs = native_libs.count(); - ExclusiveLockGuard locker(&_lock); - for (int index = 0; index < num_of_libs; index++) { - CodeCache* lib = native_libs.at(index); - if (lib != nullptr) { - patch_sigaction_in_library(lib); - } - } -} - -bool LibraryPatcher::patch_socket_functions() { - // Resolve the real libc symbols ONCE at first call and cache them. On a - // restart cycle (stop()→start()) we MUST NOT re-resolve via RTLD_NEXT: if - // any GOT slot in another DSO was missed during unpatch (e.g. its CodeCache - // disappeared), dlsym(RTLD_NEXT) could now resolve to the still-installed - // hook in that other DSO's GOT — the assignment to _orig_* would become - // self-referential and the next hook call would infinite-loop. - // - // RTLD_NEXT finds the first definition after this DSO in load order, - // bypassing unresolved lazy-binding stubs that would otherwise trigger - // _dl_runtime_resolve and silently overwrite the hook in the GOT. - // May resolve to an LD_PRELOAD interposer (e.g. libasan) — intentional. - // On musl, RTLD_NEXT returns NULL when libc is loaded before this DSO in the - // link map; fall back to RTLD_DEFAULT which finds symbols globally. - // The four statics and the `cached` flag are written once and then - // read-only. They live outside the ExclusiveLockGuard intentionally (dlsym - // must not be called while holding _lock because dlsym may acquire the - // linker lock, which is also acquired during dlopen — inverting the order - // would deadlock). Guard the one-time init with a dedicated once_flag so - // that concurrent callers serialise on the dlsym block rather than racing - // to write the statics. - static NativeSocketSampler::send_fn cached_send = nullptr; - static NativeSocketSampler::recv_fn cached_recv = nullptr; - static NativeSocketSampler::write_fn cached_write = nullptr; - static NativeSocketSampler::read_fn cached_read = nullptr; - static std::once_flag dlsym_once; - std::call_once(dlsym_once, [&]() { - cached_send = (NativeSocketSampler::send_fn) dlsym(RTLD_NEXT, "send"); - if (!cached_send) cached_send = (NativeSocketSampler::send_fn) dlsym(RTLD_DEFAULT, "send"); - cached_recv = (NativeSocketSampler::recv_fn) dlsym(RTLD_NEXT, "recv"); - if (!cached_recv) cached_recv = (NativeSocketSampler::recv_fn) dlsym(RTLD_DEFAULT, "recv"); - cached_write = (NativeSocketSampler::write_fn) dlsym(RTLD_NEXT, "write"); - if (!cached_write) cached_write = (NativeSocketSampler::write_fn) dlsym(RTLD_DEFAULT, "write"); - cached_read = (NativeSocketSampler::read_fn) dlsym(RTLD_NEXT, "read"); - if (!cached_read) cached_read = (NativeSocketSampler::read_fn) dlsym(RTLD_DEFAULT, "read"); - // If dlsym resolves to one of our own hooks the linker is already serving - // the patched copy. Null the pointers so the early-return below fires. - if (cached_send == &NativeSocketSampler::send_hook || - cached_recv == &NativeSocketSampler::recv_hook || - cached_write == &NativeSocketSampler::write_hook || - cached_read == &NativeSocketSampler::read_hook) { - TEST_LOG("patch_socket_functions dlsym returned hook address; refusing to self-reference"); - cached_send = nullptr; cached_recv = nullptr; - cached_write = nullptr; cached_read = nullptr; - } - }); - auto pre_send = cached_send; - auto pre_recv = cached_recv; - auto pre_write = cached_write; - auto pre_read = cached_read; - TEST_LOG("patch_socket_functions dlsym send=%p recv=%p write=%p read=%p", - (void*)pre_send, (void*)pre_recv, (void*)pre_write, (void*)pre_read); - if (!pre_send || !pre_recv || !pre_write || !pre_read) { - TEST_LOG("patch_socket_functions EARLY RETURN: at least one dlsym returned NULL"); - return false; - } - - const CodeCacheArray& native_libs = Libraries::instance()->native_libs(); - int num_of_libs = native_libs.count(); - - // Pre-resolve all library paths before acquiring the lock: realpath() may - // block on I/O and must not be called while holding _lock. - // We only need the is-self flag per library, so avoid a huge stack allocation. - static_assert(MAX_NATIVE_LIBS > 0, "MAX_NATIVE_LIBS must be positive"); - bool is_self[MAX_NATIVE_LIBS]; - int capped = (num_of_libs <= MAX_NATIVE_LIBS) ? num_of_libs : MAX_NATIVE_LIBS; - for (int index = 0; index < capped; index++) { - CodeCache* lib = native_libs.at(index); - is_self[index] = false; - if (lib == nullptr || lib->name() == nullptr) continue; - char path[PATH_MAX]; - char* rp = realpath(lib->name(), path); - is_self[index] = (rp != nullptr && strcmp(rp, _profiler_name) == 0); - } - - ExclusiveLockGuard locker(&_lock); - // Re-check under the lock only on re-entry (when hooks are already installed): - // a concurrent unpatch_socket_functions() may have cleared _socket_active - // between the acquire-load in install_socket_hooks() and this lock acquisition. - // The initial call from NativeSocketSampler::start() always has _socket_size == 0 - // and must proceed regardless of _socket_active. - if (_socket_size > 0 && !_socket_active.load(std::memory_order_relaxed)) { - return false; - } - // Only assign orig pointers on the first call (no hooks installed yet). - // On re-entry via dlopen, RTLD_NEXT would resolve to the hook itself. - if (_socket_size == 0) { - NativeSocketSampler::setOriginalFunctions(pre_send, pre_recv, pre_write, pre_read); - } - // TODO: hook table (name + hook fn) should be owned by NativeSocketSampler; - // LibraryPatcher should iterate an externally-provided table rather than - // hardcoding the four socket hooks here. - auto try_patch_slot = [&](void** location, void* hook_fn, const char* fn_name, CodeCache* lib) { - if (location == nullptr) return; - for (int i = 0; i < _socket_size; i++) { - if (_socket_entries[i]._location == location) return; - } - if (_socket_size < 4 * MAX_NATIVE_LIBS) { - void* orig = (void*)__atomic_load_n(location, __ATOMIC_ACQUIRE); - _socket_entries[_socket_size]._lib = lib; - _socket_entries[_socket_size]._location = location; - _socket_entries[_socket_size]._func = orig; - __atomic_store_n(location, hook_fn, __ATOMIC_RELEASE); - _socket_size++; - } else { - Log::warn("socket patch table full (%d slots), skipping %s in %s", 4 * MAX_NATIVE_LIBS, fn_name, lib ? lib->name() : "?"); - } - }; - for (int index = 0; index < capped; index++) { - CodeCache* lib = native_libs.at(index); - if (lib == nullptr) continue; - if (lib->name() == nullptr) continue; - - if (is_self[index]) { - continue; - } - - void** send_location = (void**)lib->findImport(im_send); - void** recv_location = (void**)lib->findImport(im_recv); - void** write_location = (void**)lib->findImport(im_write); - void** read_location = (void**)lib->findImport(im_read); - - if (send_location == nullptr && recv_location == nullptr - && write_location == nullptr && read_location == nullptr) continue; - - TEST_LOG("patch_socket_functions PATCH %s send=%p recv=%p write=%p read=%p", - lib->name(), (void*)send_location, (void*)recv_location, - (void*)write_location, (void*)read_location); - - // The _lock is held during patching to protect _socket_entries and _socket_size. - // Concurrent dlopen_hook calls serialize via the same lock in install_socket_hooks(), - // ensuring slot_patched checks and updates are atomic with respect to each other. - try_patch_slot(send_location, (void*)NativeSocketSampler::send_hook, "send", lib); - try_patch_slot(recv_location, (void*)NativeSocketSampler::recv_hook, "recv", lib); - try_patch_slot(write_location, (void*)NativeSocketSampler::write_hook, "write", lib); - try_patch_slot(read_location, (void*)NativeSocketSampler::read_hook, "read", lib); - } - - TEST_LOG("patch_socket_functions DONE total_slots=%d num_libs_scanned=%d", - _socket_size, capped); - _socket_active.store(true, std::memory_order_release); - return true; -} - -void LibraryPatcher::unpatch_socket_functions() { - ExclusiveLockGuard locker(&_lock); - // Clear _socket_active FIRST so that any concurrent install_socket_hooks() - // thread that already passed the acquire-load on _socket_active (before we - // acquired the lock) will see false when it checks again after acquiring the - // lock — preventing it from re-patching slots we are about to restore. - // Hooks that already entered the hook body before this store are benign: they - // hold no lock and will complete normally using the still-valid orig pointers. - // - // ASSUMPTION (dlclose UAF): we write through _socket_entries[i]._location - // without checking that the owning library is still mapped. If a patched - // DSO were actually unmapped between patch and unpatch, this store would - // corrupt freed memory or SEGV. In practice this is benign because (a) the - // host JVM does not dlclose libc-importing DSOs, (b) glibc's dlclose - // refcounts and only unmaps when the final reference is dropped, and - // (c) the same risk is already accepted by unpatch_libraries() and - // unpatch_socket_functions has the same trust model. If a host that - // routinely unmaps libc-importing libraries is ever supported, gate each - // store on a /proc/self/maps lookup or hold a dlopen handle on each lib - // for the patch lifetime. - _socket_active.store(false, std::memory_order_release); - TEST_LOG("unpatch_socket_functions restoring %d slot(s)", _socket_size); - for (int index = 0; index < _socket_size; index++) { - __atomic_store_n(_socket_entries[index]._location, _socket_entries[index]._func, __ATOMIC_RELEASE); - } - _socket_size = 0; - // _orig_send/_orig_recv/_orig_write/_orig_read are intentionally NOT nulled. - // In-flight hook invocations that entered before PLT entries were restored - // above may still be executing and will dereference these pointers. - // They remain valid (pointing to the real libc functions) until the next - // patch_socket_functions() call. -} - -#endif // __linux__ diff --git a/ddprof-lib/src/main/cpp/linearAllocator.cpp b/ddprof-lib/src/main/cpp/linearAllocator.cpp deleted file mode 100644 index 4aea288fe..000000000 --- a/ddprof-lib/src/main/cpp/linearAllocator.cpp +++ /dev/null @@ -1,314 +0,0 @@ -/* - * Copyright 2020 Andrei Pangin - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "linearAllocator.h" -#include "counters.h" -#include "os.h" -#include "common.h" -#include -#ifdef TSAN_ENABLED - #include - #include -#endif - -// ASAN_ENABLED / TSAN_ENABLED are defined in common.h (toolchain-agnostic). -#ifdef ASAN_ENABLED - #include -#endif - -#ifdef TSAN_ENABLED - #include -#endif - -LinearAllocator::LinearAllocator(size_t chunk_size) { - _chunk_size = chunk_size; - _reserve = _tail = allocateChunk(NULL); -} - -LinearAllocator::~LinearAllocator() { - clear(); - freeChunk(_tail); -} - -void LinearAllocator::clear() { - // OS::safeAlloc/safeFree use raw syscalls not intercepted by TSan, so TSan - // never clears shadow memory on munmap. Add explicit acquire/release around - // every plain prev-field read so the happens-before chain from freeChunk's - // __tsan_release reaches any thread that later reuses the same VA. - #ifdef TSAN_ENABLED - __tsan_acquire(_reserve); - #endif - if (_reserve->prev == _tail) { - freeChunk(_reserve); // __tsan_release inside - } - #ifdef TSAN_ENABLED - else { - __tsan_release(_reserve); // not freed here; release for future VA-reuse acquirers - } - #endif - - // ASAN POISONING: Mark all allocated memory as poisoned BEFORE freeing chunks - // This catches use-after-free even when memory isn't munmap'd (kept in _tail) - #ifdef ASAN_ENABLED - int chunk_count = 0; - size_t total_poisoned = 0; - for (Chunk *chunk = _tail; chunk != NULL; chunk = chunk->prev) { - // Poison from the start of usable data to the current offset - size_t used_size = chunk->offs - sizeof(Chunk); - if (used_size > 0) { - void* data_start = (char*)chunk + sizeof(Chunk); - ASAN_POISON_MEMORY_REGION(data_start, used_size); - chunk_count++; - total_poisoned += used_size; - } - } - if (chunk_count > 0) { - TEST_LOG("[LinearAllocator::clear] ASan poisoned %d chunks, %zu bytes total", chunk_count, total_poisoned); - } - #endif - - // Walk the chain freeing all chunks except the last (prev==NULL). - // Acquire each chunk BEFORE reading its prev field so the TSan happens-before - // chain covers the condition check, not just the assignment inside the body. - { - Chunk *current = _tail; - #ifdef TSAN_ENABLED - __tsan_acquire(current); // before the first current->prev read (loop condition) - #endif - while (current->prev != NULL) { - Chunk *next = current->prev; - freeChunk(current); // __tsan_release(current) + safeFree inside - current = next; - #ifdef TSAN_ENABLED - __tsan_acquire(current); // before the next iteration's current->prev read - #endif - } - // current is the last chunk (prev==NULL); keep it as the new allocator base. - _reserve = current; - _tail = current; - } - _tail->offs = sizeof(Chunk); - - // DON'T UNPOISON HERE - let alloc() do it on-demand! - // This ensures ASan can catch use-after-free bugs when code accesses - // memory that was cleared but not yet reallocated. -} - -ChunkList LinearAllocator::detachChunks() { - // Capture current state before detaching - ChunkList result(_tail, _chunk_size); - - // Handle reserve chunk: if it's ahead of tail, it becomes part of detached list. - // Acquire TSan ownership before reading _reserve->prev: the reserve chunk may - // have been allocated by another thread via reserveChunk() → allocateChunk(), - // which released ownership with __tsan_release after writing chunk->prev. - if (_reserve != _tail) { - #ifdef TSAN_ENABLED - __tsan_acquire(_reserve); - #endif - if (_reserve->prev == _tail) { - result.head = _reserve; - } - #ifdef TSAN_ENABLED - __tsan_release(_reserve); - #endif - } - - // Allocate a fresh chunk for new allocations - Chunk* fresh = allocateChunk(NULL); - if (fresh != NULL) { - _tail = fresh; - _reserve = fresh; - } else { - // CRITICAL FIX: Allocation failed, but we MUST still detach to prevent double-free. - // Leave the allocator in an unusable state (nullptr) rather than keeping old chunks - // attached. This is safer than silently returning empty while chunks remain attached. - // The allocator will need fresh allocation before it can be used again. - _tail = nullptr; - _reserve = nullptr; - // Note: We still return the detached chunks in result, which will be freed by caller - } - - return result; -} - -void LinearAllocator::freeChunks(ChunkList& chunks) { - if (chunks.head == nullptr || chunks.chunk_size == 0) { - return; - } - - Chunk* current = chunks.head; - while (current != nullptr) { - // Acquire TSan ownership before reading chunk->prev: pairs with the - // __tsan_release in allocateChunk() that published the initialized chunk. - // Without this, TSan cannot connect the writer's (e.g. reserveChunk thread) - // initialization of chunk->prev to this read, and reports a false data race. - #ifdef TSAN_ENABLED - __tsan_acquire(current); - #endif - Chunk* prev = current->prev; - #ifdef TSAN_ENABLED - __tsan_release(current); - #endif - OS::safeFree(current, chunks.chunk_size); - Counters::decrement(LINEAR_ALLOCATOR_BYTES, chunks.chunk_size); - Counters::decrement(LINEAR_ALLOCATOR_CHUNKS); - current = prev; - } - - chunks.head = nullptr; - chunks.chunk_size = 0; -} - -void *LinearAllocator::alloc(size_t size) { - Chunk *chunk = __atomic_load_n(&_tail, __ATOMIC_ACQUIRE); - - // CRITICAL FIX: After detachChunks() fails, _tail may be nullptr. - // We must handle this gracefully to prevent crash. - if (chunk == nullptr) { - return nullptr; - } - - do { - // Fast path: bump a pointer with CAS - for (size_t offs = __atomic_load_n(&chunk->offs, __ATOMIC_ACQUIRE); - offs + size <= _chunk_size; - offs = __atomic_load_n(&chunk->offs, __ATOMIC_ACQUIRE)) { - if (__sync_bool_compare_and_swap(&chunk->offs, offs, offs + size)) { - void* allocated_ptr = (char *)chunk + offs; - - // ASAN UNPOISONING: Unpoison ONLY the allocated region on-demand - // This allows ASan to detect use-after-free of memory that was cleared - // but not yet reallocated - #ifdef ASAN_ENABLED - ASAN_UNPOISON_MEMORY_REGION(allocated_ptr, size); - #endif - - if (_chunk_size / 2 - offs < size) { - // Stepped over a middle of the chunk - it's time to prepare a new one - reserveChunk(chunk); - } - return allocated_ptr; - } - } - } while ((chunk = getNextChunk(chunk)) != NULL); - - return NULL; -} - -Chunk *LinearAllocator::allocateChunk(Chunk *current) { - Chunk *chunk = (Chunk *)OS::safeAlloc(_chunk_size); - if (chunk != NULL) { - // OS::safeAlloc uses a raw mmap syscall that bypasses ASan and TSan - // interceptors by design (to avoid self-instrumentation in the profiler). - // When the OS reuses a VA from a prior munmap, TSan's shadow memory for - // that VA still holds stale access history from previous chunk users, - // causing false-positive data-race reports on user-data addresses. - // - // Fix: re-map the same VA through the libc mmap() wrapper. TSan intercepts - // mmap() and calls MemoryRangeImitateWrite, which sets all 4 shadow slots - // for the entire chunk range to the current thread's write — completely - // overwriting any stale entries. safeAlloc itself is unchanged. - // - // This MUST stay TSan-build-only: the libc mmap() wrapper and TSan's - // interceptor are not async-signal-safe, and allocateChunk() is reachable - // from a signal handler in the full profiler. TSAN_ENABLED is defined for - // any TSan-instrumented translation unit (it tracks the compiler's - // sanitizer detection in common.h, not the build target), but the build - // only applies -fsanitize=thread to the isolated gtest binaries — never to - // the shared library the JVM loads. Those gtest binaries never drive the - // allocator from a signal handler, so the non-async-signal-safe mmap() call - // here is safe in practice. - #ifdef ASAN_ENABLED - ASAN_UNPOISON_MEMORY_REGION(chunk, _chunk_size); - #endif - #ifdef TSAN_ENABLED - void *remap = mmap(chunk, _chunk_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); - // MAP_FIXED unmaps before it maps, so a failure would leave a hole at - // `chunk` and the writes below would fault. Abort unconditionally rather - // than via assert(), which an NDEBUG build would strip — turning a clean, - // diagnosable failure into a stale-shadow or use-after-unmap crash. - if (remap != chunk) { - perror("TSan shadow re-map (mmap MAP_FIXED) failed"); - abort(); - } - #endif - - chunk->prev = current; - chunk->offs = sizeof(Chunk); - - // Publish the initialized chunk: any thread that later acquires this chunk - // (via __tsan_acquire in freeChunks/detachChunks) will see these writes. - #ifdef TSAN_ENABLED - __tsan_release(chunk); - #endif - - Counters::increment(LINEAR_ALLOCATOR_BYTES, _chunk_size); - Counters::increment(LINEAR_ALLOCATOR_CHUNKS); - } - return chunk; -} - -void LinearAllocator::freeChunk(Chunk *current) { - // Release TSan ownership before munmap so the sanitizer knows this thread is - // done with the memory. The mmap(MAP_FIXED) re-map in allocateChunk() resets - // the shadow for whichever thread later reuses this VA (after OS VA reuse), so - // it starts from a clean baseline rather than seeing stale access history. - #ifdef TSAN_ENABLED - __tsan_release(current); - #endif - OS::safeFree(current, _chunk_size); - Counters::decrement(LINEAR_ALLOCATOR_BYTES, _chunk_size); - Counters::decrement(LINEAR_ALLOCATOR_CHUNKS); -} - -void LinearAllocator::reserveChunk(Chunk *current) { - Chunk *reserve = allocateChunk(current); - if (reserve != NULL && - !__sync_bool_compare_and_swap(&_reserve, current, reserve)) { - // Unlikely case that we are too late - freeChunk(reserve); - } -} - -Chunk *LinearAllocator::getNextChunk(Chunk *current) { - // _reserve is written via CAS in reserveChunk(); load it atomically so TSan - // sees the acquire-release relationship with the CAS store. - Chunk *reserve = __atomic_load_n(&_reserve, __ATOMIC_ACQUIRE); - - if (reserve == current) { - // Unlikely case: no reserve yet. - // It's probably being allocated right now, so let's compete - reserve = allocateChunk(current); - if (reserve == NULL) { - // Not enough memory - return NULL; - } - - Chunk *prev_reserve = - __sync_val_compare_and_swap(&_reserve, current, reserve); - if (prev_reserve != current) { - freeChunk(reserve); - reserve = prev_reserve; - } - } - - // Expected case: a new chunk is already reserved - Chunk *tail = __sync_val_compare_and_swap(&_tail, current, reserve); - return tail == current ? reserve : tail; -} diff --git a/ddprof-lib/src/main/cpp/linearAllocator.h b/ddprof-lib/src/main/cpp/linearAllocator.h deleted file mode 100644 index 8d201dd17..000000000 --- a/ddprof-lib/src/main/cpp/linearAllocator.h +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Copyright 2020 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _LINEARALLOCATOR_H -#define _LINEARALLOCATOR_H - -#include - -struct Chunk { - Chunk *prev; - volatile size_t offs; - // To avoid false sharing - char _padding[56]; -}; - -/** - * Holds detached chunks from a LinearAllocator for deferred deallocation. - * Used to keep trace memory alive during processor execution while allowing - * the allocator to be reset for new allocations. - */ -struct ChunkList { - Chunk *head; - size_t chunk_size; - - ChunkList() : head(nullptr), chunk_size(0) {} - ChunkList(Chunk *h, size_t sz) : head(h), chunk_size(sz) {} -}; - -class LinearAllocator { -private: - size_t _chunk_size; - Chunk *_tail; - Chunk *_reserve; - - Chunk *allocateChunk(Chunk *current); - void freeChunk(Chunk *current); - void reserveChunk(Chunk *current); - Chunk *getNextChunk(Chunk *current); - -public: - explicit LinearAllocator(size_t chunk_size); - ~LinearAllocator(); - - void clear(); - - /** - * Detaches all chunks from this allocator, returning them as a ChunkList. - * The allocator is reset to an empty state with a fresh chunk for new allocations. - * The detached chunks remain allocated and valid until freeChunks() is called. - * - * This enables deferred deallocation: reset the allocator immediately while - * keeping old data alive until it's no longer needed. - */ - ChunkList detachChunks(); - - /** - * Frees all chunks in a previously detached ChunkList. - * Call this after processing is complete to deallocate the memory. - */ - static void freeChunks(ChunkList& chunks); - - void *alloc(size_t size); -}; - -#endif // _LINEARALLOCATOR_H diff --git a/ddprof-lib/src/main/cpp/livenessTracker.cpp b/ddprof-lib/src/main/cpp/livenessTracker.cpp deleted file mode 100644 index efacdcda6..000000000 --- a/ddprof-lib/src/main/cpp/livenessTracker.cpp +++ /dev/null @@ -1,451 +0,0 @@ -/* - * Copyright 2021, 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include - -#include "arch.h" -#include "context.h" -#include "context_api.h" -#include "hotspot/vmStructs.h" -#include "incbin.h" -#include "jniHelper.h" -#include "livenessTracker.h" -#include "log.h" -#include "os.h" -#include "profiler.h" -#include "thread.h" -#include "threadLocal.h" -#include "tsc.h" -#include -#include - -constexpr int LivenessTracker::MAX_TRACKING_TABLE_SIZE; -constexpr int LivenessTracker::MIN_SAMPLING_INTERVAL; - -void LivenessTracker::cleanup_table(bool forced) { - u64 current = load(_last_gc_epoch); - u64 target_gc_epoch = load(_gc_epoch); - - if ((target_gc_epoch == _last_gc_epoch || - !__atomic_compare_exchange_n(&_last_gc_epoch, ¤t, - target_gc_epoch, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)) && - !forced) { - // if the last processed GC epoch hasn't changed, or if we failed to update - // it, there's nothing to do - return; - } - - JNIEnv *env = VM::jni(); - - int epoch_diff = (int)(target_gc_epoch - current); - - _table_lock.lock(); - u32 sz = _table_size; - if (sz > 0) { - u64 start = OS::nanotime(), end; - u32 newsz = 0; - std::set kept_classes; - for (u32 i = 0; i < sz; i++) { - if (_table[i].ref != nullptr && - !env->IsSameObject(_table[i].ref, nullptr)) { - // it survived one more GarbageCollectionFinish event - u32 target = newsz++; - if (target != i) { - _table[target] = _table[i]; // will clone TrackingEntry at 'i' - _table[i].ref = nullptr; // will nullify the original ref - _table[i].call_trace_id = 0; - } - _table[target].age += epoch_diff; - } else { - jweak tmpRef = _table[i].ref; - _table[i].ref = nullptr; - env->DeleteWeakGlobalRef(tmpRef); - _table[i].call_trace_id = 0; - } - } - - _table_size = newsz; - - end = OS::nanotime(); - Log::debug("Liveness tracker cleanup took %.2fms (%.2fus/element)", - 1.0f * (end - start) / 1000 / 1000, - 1.0f * (end - start) / 1000 / sz); - } - _table_lock.unlock(); -} - -void LivenessTracker::flush(std::set &tracked_thread_ids) { - if (!_enabled) { - // disabled - return; - } - flush_table(&tracked_thread_ids); -} - -void LivenessTracker::flush_table(std::set *tracked_thread_ids) { - JNIEnv *env = VM::jni(); - u64 start = OS::nanotime(), end; - - // make sure that the tracking table is cleaned up before we start flushing it - // this is to make sure we are including as few false 'live' objects as - // possible - cleanup_table(); - - _table_lock.lock(); - - u32 sz; - for (u32 i = 0; i < (sz = _table_size); i++) { - jobject ref = env->NewLocalRef(_table[i].ref); - if (ref != nullptr) { - if (tracked_thread_ids != nullptr) { - tracked_thread_ids->insert(_table[i].tid); - } - ObjectLivenessEvent event; - event._start_time = _table[i].time; - event._age = _table[i].age; - event._alloc = _table[i].alloc; - event._skipped = _table[i].skipped; - event._ctx = _table[i].ctx; - - jclass clz = env->GetObjectClass(ref); - jstring name_str = (jstring)env->CallObjectMethod(clz, _Class_getName); - env->DeleteLocalRef(clz); - jniExceptionCheck(env); - const char *name = env->GetStringUTFChars(name_str, nullptr); - event._id = name != nullptr - ? Profiler::instance()->lookupClass(name, strlen(name)) - : 0; - env->ReleaseStringUTFChars(name_str, name); - - Profiler::instance()->recordDeferredSample(_table[i].tid, _table[i].call_trace_id, BCI_LIVENESS, &event); - } - - env->DeleteLocalRef(ref); - } - - _table_lock.unlock(); - - if (_record_heap_usage) { - bool isLastGc = HeapUsage::isLastGCUsageSupported(); - size_t used = isLastGc ? HeapUsage::get()._used_at_last_gc - : loadAcquire(_used_after_last_gc); - if (used == 0) { - used = HeapUsage::get()._used; - isLastGc = false; - } - Profiler::instance()->writeHeapUsage(used, isLastGc); - } - - end = OS::nanotime(); - if (sz) { - Log::debug("Liveness tracker flush took %.2fms (%.2fus/element)", - 1.0f * (end - start) / 1000 / 1000, - 1.0f * (end - start) / 1000 / sz); - } -} - -Error LivenessTracker::initialize_table(JNIEnv *jni, int sampling_interval) { - _table_max_cap = 0; - jlong max_heap = HeapUsage::getMaxHeap(jni); - if (max_heap == -1) { - return Error("Can not track liveness for allocation samples without heap " - "size information."); - } - - int required_table_capacity = - sampling_interval > 0 ? max_heap / sampling_interval : max_heap; - - if (required_table_capacity > MAX_TRACKING_TABLE_SIZE) { - Log::warn("Tracking liveness for allocation samples with interval %d can " - "not cover full heap.", - sampling_interval); - } - _table_max_cap = std::min(MAX_TRACKING_TABLE_SIZE, required_table_capacity); - - _table_cap = std::max( - 2048, - _table_max_cap / - 8); // the table will grow at most 3 times before fully covering heap - - return Error::OK; -} - -Error LivenessTracker::start(Arguments &args) { - Error err = initialize(args); - if (err) { - return err; - } - if (!_enabled) { - // disabled - return Error::OK; - } - - // Self-register with the profiler for liveness checking - Profiler::instance()->registerLivenessChecker([this](std::unordered_set& buffer) { - this->getLiveTraceIds(buffer); - }); - - // Enable Java Object Sample events - jvmtiEnv *jvmti = VM::jvmti(); - jvmti->SetEventNotificationMode( - JVMTI_ENABLE, JVMTI_EVENT_GARBAGE_COLLECTION_FINISH, nullptr); - - return Error::OK; -} - -void LivenessTracker::stop() { - if (!_enabled) { - // disabled - return; - } - cleanup_table(); - flush_table(nullptr); - - // do not disable GC notifications here - the tracker is supposed to survive - // multiple recordings -} - -Error LivenessTracker::initialize(Arguments &args) { - _enabled = args._gc_generations || args._record_liveness; - - if (!_enabled) { - return Error::OK; - } - - // _record_heap_usage controls per-session JFR event emission only, not the - // tracking table. Update it before the _initialized guard so each profiler - // start gets the correct setting even when the table persists across recordings. - _record_heap_usage = args._record_heap_usage; - - if (_initialized) { - // if the tracker was previously initialized return the stored result for - // consistency this hack also means that if the profiler is started with - // different arguments for liveness tracking those will be ignored it is - // required in order to be able to track the object liveness across many - // recordings - return _stored_error; - } - _initialized = true; - - if (VM::hotspot_version() < 11) { - Log::warn("Liveness tracking requires Java 11+"); - // disable liveness tracking - _table_max_cap = 0; - return _stored_error = Error::OK; - } - - JNIEnv *env = VM::jni(); - - Error err = initialize_table(env, args._memory); - if (err) { - Log::warn("Liveness tracking requires heap size information"); - // disable liveness tracking - _table_max_cap = 0; - return _stored_error = Error::OK; - } - if (!(_Class = env->FindClass("java/lang/Class"))) { - jniExceptionCheck(env, true); - err = Error("Unable to find java/lang/Class"); - } else if (!(_Class_getName = env->GetMethodID(_Class, "getName", - "()Ljava/lang/String;"))) { - jniExceptionCheck(env, true); - err = Error("Unable to find java/lang/Class.getName"); - } - if (err) { - Log::warn("Liveness tracking requires access to java.lang.Class#getName()"); - // disable liveness tracking - _table_max_cap = 0; - return _stored_error = Error::OK; - } - - _subsample_ratio = args._live_samples_ratio; - - _table_size = 0; - _table_cap = - std::min(2048, _table_max_cap); // with default 512k sampling interval, it's - // enough for 1G of heap - _table = (TrackingEntry *)malloc(sizeof(TrackingEntry) * _table_cap); - - _gc_epoch = 0; - _last_gc_epoch = 0; - - return _stored_error = Error::OK; -} - -static void* create_mt19937() { - // std::mt19937 itself is noexcept, but std::random_device and `new` may throw. - // If that happens we let the failure terminate the process (same outcome as - // failing thread_local initialization previously). - return static_cast(new std::mt19937(std::random_device{}())); -} - -static void* create_uniform_real_distribution() { - // std::uniform_real_distribution<> construction is noexcept, but `new` may throw. - // If allocation fails the process is likely to abort anyway. - return static_cast(new std::uniform_real_distribution<>(0, 1.0)); -} - -static void free_mt19937(void* p) { - std::mt19937* mt = static_cast(p); - delete mt; -} - -static void free_uniform_real_distribution(void* p) { - std::uniform_real_distribution<>* urd = static_cast*>(p); - delete urd; -} - -void LivenessTracker::track(JNIEnv *env, AllocEvent &event, jint tid, - jobject object, u64 call_trace_id) { - if (!_enabled) { - // disabled - return; - } - if (_table_max_cap == 0) { - // we are not to store any objects - return; - } - - static ThreadLocal gen; - static ThreadLocal*, create_uniform_real_distribution, free_uniform_real_distribution> dis; - static ThreadLocal skipped; - - if (_subsample_ratio < 1.0) { - std::mt19937* genp = gen.get(); - std::uniform_real_distribution<>* disp = dis.get(); - if (disp->operator()(*genp) > _subsample_ratio) { - skipped.set(skipped.get() + static_cast(event._weight) * event._size); - return; - } - } - - jweak ref = env->NewWeakGlobalRef(object); - if (ref == nullptr) { - return; - } - bool retried = false; -retry: - if (!_table_lock.tryLockShared()) { - // we failed to add the weak reference to the table so it won't get cleaned - // up otherwise - env->DeleteWeakGlobalRef(ref); - return; - } - - // Increment _table_size in a thread-safe manner (CAS) and store the new value - // in idx It bails out if _table_size would overflow _table_cap - int idx; - do { - idx = __atomic_load_n(&_table_size, __ATOMIC_RELAXED); - } while (idx < _table_cap && - !__sync_bool_compare_and_swap(&_table_size, idx, idx + 1)); - - if (idx < _table_cap) { - _table[idx].tid = tid; - _table[idx].time = TSC::ticks(); - _table[idx].ref = ref; - _table[idx].alloc = event; - _table[idx].skipped = skipped.get(); - skipped.set(0); - _table[idx].age = 0; - _table[idx].call_trace_id = call_trace_id; - _table[idx].ctx = ContextApi::snapshot(); - } - - _table_lock.unlockShared(); - - if (idx == _table_cap) { - if (!retried) { - // guarantees we don't busy loop until memory exhaustion - retried = true; - - // try cleanup before resizing - there is a good chance it will free some - // space - cleanup_table(true); - - if (_table_cap < _table_max_cap) { - - // Let's increase the size of the table - // This should only ever happen when sampling interval * size of table - // is smaller than maximum heap size. So we only support increasing - // the size of the table, not decreasing it. - _table_lock.lock(); - - // Only increase the size of the table to _table_max_cap elements - int newcap = std::min(_table_cap * 2, _table_max_cap); - if (_table_cap != newcap) { - TrackingEntry *tmp = (TrackingEntry *)realloc( - _table, sizeof(TrackingEntry) * newcap); - if (tmp != nullptr) { - _table = tmp; - _table_cap = newcap; - Log::debug( - "Increased size of Liveness tracking table to %d entries", - _table_cap); - } else { - Log::debug("Cannot add sampled object to Liveness tracking table, " - "resize attempt failed, the table is overflowing"); - } - } - - _table_lock.unlock(); - - goto retry; - } else { - Log::debug("Cannot add sampled object to Liveness tracking table, it's " - "overflowing"); - env->DeleteWeakGlobalRef(ref); - } - } else { - env->DeleteWeakGlobalRef(ref); - } - skipped.set(0); // reset the subsampling skipped bytes - } -} - -void JNICALL LivenessTracker::GarbageCollectionFinish(jvmtiEnv *jvmti_env) { - LivenessTracker::instance()->onGC(); -} - -void LivenessTracker::onGC() { - if (!_initialized) { - return; - } - - // just increment the epoch - atomicIncRelaxed(_gc_epoch,u64(1)); - - if (!HeapUsage::isLastGCUsageSupported()) { - store(_used_after_last_gc, HeapUsage::get(false)._used); - } -} - -void LivenessTracker::getLiveTraceIds(std::unordered_set& out_buffer) { - out_buffer.clear(); - - if (!_enabled || !_initialized) { - return; - } - - // Lock the table to iterate over tracking entries - _table_lock.lockShared(); - - // Reserve space to avoid reallocations during filling - // Note: unordered_set uses rehash for capacity management - out_buffer.rehash(static_cast(_table_size / 0.75f)); - - // Collect call_trace_id values from all live tracking entries - for (int i = 0; i < _table_size; i++) { - TrackingEntry* entry = &_table[i]; - if (entry->ref != nullptr) { - out_buffer.insert(entry->call_trace_id); - } - } - - _table_lock.unlockShared(); -} diff --git a/ddprof-lib/src/main/cpp/livenessTracker.h b/ddprof-lib/src/main/cpp/livenessTracker.h deleted file mode 100644 index cbeb842a7..000000000 --- a/ddprof-lib/src/main/cpp/livenessTracker.h +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2021, 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _LIVENESSTRACKER_H -#define _LIVENESSTRACKER_H - -#include "arch.h" -#include "context.h" -#include "engine.h" -#include "event.h" -#include "spinLock.h" -#include -#include -#include -#include - -class Recording; - -typedef struct TrackingEntry { - jweak ref; - AllocEvent alloc; - double skipped; - u64 call_trace_id; - jint tid; - jlong time; - jlong age; - Context ctx; -} TrackingEntry; - -// Aligned to satisfy SpinLock member alignment requirement (64 bytes) -// Required because this class contains SpinLock _table_lock member -class alignas(alignof(SpinLock)) LivenessTracker { - friend Recording; - -private: - // pre-c++17 we should mark these inline(or out of class) - constexpr static int MAX_TRACKING_TABLE_SIZE = 262144; - constexpr static int MIN_SAMPLING_INTERVAL = 524288; // 512kiB - - bool _initialized; - bool _enabled; - Error _stored_error; - - SpinLock _table_lock; - volatile int _table_size; - int _table_cap; - int _table_max_cap; - TrackingEntry *_table; - - double _subsample_ratio; - - bool _record_heap_usage; - - jclass _Class; - jmethodID _Class_getName; - - volatile u64 _gc_epoch; - volatile u64 _last_gc_epoch; - - size_t _used_after_last_gc; - - Error initialize(Arguments &args); - Error initialize_table(JNIEnv *jni, int sampling_interval); - - void cleanup_table(bool force = false); - - void flush_table(std::set *tracked_thread_ids); - - void onGC(); - void runCleanup(); - - jlong getMaxMemory(JNIEnv *env); - -public: - static LivenessTracker *instance() { - static LivenessTracker instance; - return &instance; - } - // Delete copy constructor and assignment operator to prevent copies - LivenessTracker(const LivenessTracker&) = delete; - LivenessTracker& operator=(const LivenessTracker&) = delete; - - LivenessTracker() - : _initialized(false), _enabled(false), _stored_error(Error::OK), - _table_size(0), _table_cap(0), _table_max_cap(0), _table(NULL), - _subsample_ratio(0.1), _record_heap_usage(false), _Class(NULL), - _Class_getName(0), _gc_epoch(0), _last_gc_epoch(0), - _used_after_last_gc(0) {} - - Error start(Arguments &args); - void stop(); - void track(JNIEnv *env, AllocEvent &event, jint tid, jobject object, u64 call_trace_id); - void flush(std::set &tracked_thread_ids); - - static void JNICALL GarbageCollectionFinish(jvmtiEnv *jvmti_env); - -private: - void getLiveTraceIds(std::unordered_set& out_buffer); -}; - -#endif // _LIVENESSTRACKER_H diff --git a/ddprof-lib/src/main/cpp/log.cpp b/ddprof-lib/src/main/cpp/log.cpp deleted file mode 100644 index 14fd1c250..000000000 --- a/ddprof-lib/src/main/cpp/log.cpp +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2021 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "log.h" -#include "profiler.h" -#include - -const char *const Log::LEVEL_NAME[] = {"TRACE", "DEBUG", "INFO", - "WARN", "ERROR", "NONE"}; - -FILE *Log::_file = stdout; -LogLevel Log::_level = LOG_NONE; - -void Log::open(Arguments &args) { - open(args._log, args._loglevel); - - if (args._unknown_arg != NULL) { - warn("Unknown argument: %s", args._unknown_arg); - } -} - -void Log::open(const char *file_name, const char *level) { - if (_file != stdout && _file != stderr) { - fclose(_file); - } - - if (file_name == NULL || strcmp(file_name, "stdout") == 0) { - _file = stdout; - } else if (strcmp(file_name, "stderr") == 0) { - _file = stderr; - } else if ((_file = fopen(file_name, "w")) == NULL) { - _file = stdout; - warn("Could not open log file: %s", file_name); - } - - LogLevel l = LOG_NONE; - if (level != NULL) { - for (int i = LOG_TRACE; i <= LOG_NONE; i++) { - if (strcasecmp(LEVEL_NAME[i], level) == 0) { - l = (LogLevel)i; - break; - } - } - } - __atomic_store_n(&_level, l, __ATOMIC_RELEASE); -} - -void Log::close() { - if (_file != stdout && _file != stderr) { - fclose(_file); - _file = stdout; - } -} - -void Log::log(LogLevel level, const char *msg, va_list args) { - // DD specific: we don't want to spam stdout/stderr nor JFR with - // logs we don't want. - if (level < _level) { - return; - } - - char buf[1024]; - size_t len = vsnprintf(buf, sizeof(buf), msg, args); - if (len >= sizeof(buf)) { - len = sizeof(buf) - 1; - buf[len] = 0; - } - - // all warnings get logged to the JFR, logging anything else requires config - // override errors cannot be logged to the JFR because the JFR may not be - // ready this means all logging we want to be able to find in JFR files must - // be done at WARN level, and any logging done which prevents creation of the - // JFR should be done at ERROR level - if (level == LOG_WARN || (level >= _level && level < LOG_ERROR)) { -// Profiler::instance()->writeLog(level, buf, len); - } - - // always log errors, but only errors - if (level == LOG_ERROR) { - fprintf(_file, - "{\"@version\":\"1\",\"message\":\"%s\",\"logger_name\":\"java-" - "profiler\",\"level\":\"%s\"}\n", - buf, LEVEL_NAME[level]); - fflush(_file); - } -} - -void Log::trace(const char *msg, ...) { - va_list args; - va_start(args, msg); - log(LOG_TRACE, msg, args); - va_end(args); -} - -void Log::debug(const char *msg, ...) { - va_list args; - va_start(args, msg); - log(LOG_DEBUG, msg, args); - va_end(args); -} - -void Log::info(const char *msg, ...) { - va_list args; - va_start(args, msg); - log(LOG_INFO, msg, args); - va_end(args); -} - -void Log::warn(const char *msg, ...) { - va_list args; - va_start(args, msg); - log(LOG_WARN, msg, args); - va_end(args); -} - -void Log::error(const char *msg, ...) { - va_list args; - va_start(args, msg); - log(LOG_ERROR, msg, args); - va_end(args); -} diff --git a/ddprof-lib/src/main/cpp/log.h b/ddprof-lib/src/main/cpp/log.h deleted file mode 100644 index 9ae04b23a..000000000 --- a/ddprof-lib/src/main/cpp/log.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2021 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _LOG_H -#define _LOG_H - -#include -#include - -#ifdef __GNUC__ -#define ATTR_FORMAT __attribute__((format(printf, 1, 2))) -#else -#define ATTR_FORMAT -#endif - -enum LogLevel { LOG_TRACE, LOG_DEBUG, LOG_INFO, LOG_WARN, LOG_ERROR, LOG_NONE }; - -class Arguments; - -class Log { -private: - static FILE *_file; - static LogLevel _level; - -public: - static const char *const LEVEL_NAME[]; - - static void open(Arguments &args); - static void open(const char *file_name, const char *level); - static void close(); - - static void log(LogLevel level, const char *msg, va_list args); - - static void ATTR_FORMAT trace(const char *msg, ...); - static void ATTR_FORMAT debug(const char *msg, ...); - static void ATTR_FORMAT info(const char *msg, ...); - static void ATTR_FORMAT warn(const char *msg, ...); - static void ATTR_FORMAT error(const char *msg, ...); - - static LogLevel level() { return _level; } -}; - -#endif // _LOG_H diff --git a/ddprof-lib/src/main/cpp/mallocTracer.cpp b/ddprof-lib/src/main/cpp/mallocTracer.cpp deleted file mode 100644 index 1eaaa7260..000000000 --- a/ddprof-lib/src/main/cpp/mallocTracer.cpp +++ /dev/null @@ -1,390 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include -#include "codeCache.h" -#include "guards.h" -#include "libraries.h" -#include "mallocTracer.h" -#include "os.h" -#include "pidController.h" -#include "profiler.h" -#include "symbols.h" -#include "tsc.h" -#include "vmEntry.h" - -#define SAVE_IMPORT(FUNC) \ - do { \ - void** _entry = lib->findImport(im_##FUNC); \ - if (_entry != NULL) _orig_##FUNC = (decltype(_orig_##FUNC))*_entry; \ - } while (0) - -static void* (*_orig_malloc)(size_t); -static void* (*_orig_calloc)(size_t, size_t); -static void* (*_orig_realloc)(void*, size_t); -static int (*_orig_posix_memalign)(void**, size_t, size_t); -static void* (*_orig_aligned_alloc)(size_t, size_t); - -// Inline helper to avoid repeating the running+ret+size guard in each hook. -// CriticalSection prevents reentrancy: profiler-internal allocations triggered -// inside recordMalloc (e.g. sample buffer allocation) re-enter these hooks via -// the patched GOT; without the guard they would be double-counted. -// Acquiring the CS here also blocks concurrent same-thread timer samples -// (SIGPROF/SIGVTALRM) for the duration of recordMalloc; this is acceptable -// because the window is short. -static inline void maybeRecord(void* ret, size_t size) { - if (MallocTracer::running() && ret && size) { - CriticalSection cs; - if (cs.entered()) { - MallocTracer::recordMalloc(ret, size); - } - } -} - -extern "C" void* malloc_hook(size_t size) { - void* ret = _orig_malloc(size); - maybeRecord(ret, size); - return ret; -} - -extern "C" void* calloc_hook(size_t num, size_t size) { - void* ret = _orig_calloc(num, size); - // num * size may wrap on size_t, but the wrapped value is only forwarded - // to maybeRecord, which discards it when ret == NULL (the libc returns - // NULL on overflow per POSIX). - if (num && size) { - maybeRecord(ret, num * size); - } - return ret; -} - -extern "C" void* realloc_hook(void* addr, size_t size) { - void* ret = _orig_realloc(addr, size); - // Record every successful realloc, regardless of whether addr was NULL. - // Without a free hook we cannot subtract the prior allocation, so a - // realloc that grows an existing buffer is double-counted against the - // original malloc when both were sampled. This is benign for the leak - // detector: the prior sample (if any) ages out as the freed address is - // never seen again, and missing the realloc entirely would leave a - // phantom live allocation on the freed old address. - maybeRecord(ret, size); - return ret; -} - -extern "C" int posix_memalign_hook(void** memptr, size_t alignment, size_t size) { - int ret = _orig_posix_memalign(memptr, alignment, size); - if (ret == 0 && memptr) { - // POSIX guarantees *memptr is set to the allocated block when ret == 0; - // maybeRecord is the sole NULL/size gate for non-conforming libc. - maybeRecord(*memptr, size); - } - return ret; -} - -extern "C" void* aligned_alloc_hook(size_t alignment, size_t size) { - void* ret = _orig_aligned_alloc(alignment, size); - maybeRecord(ret, size); - return ret; -} - -volatile u64 MallocTracer::_interval; -volatile u64 MallocTracer::_bytes_until_sample; -u64 MallocTracer::_configured_interval; -volatile u64 MallocTracer::_sample_count; -volatile u64 MallocTracer::_last_config_update_ts; -volatile bool MallocTracer::_running = false; -PidController MallocTracer::_pid(MallocTracer::TARGET_SAMPLES_PER_WINDOW, - 31, 511, 3, MallocTracer::CONFIG_UPDATE_CHECK_PERIOD_SECS, 15); - -Mutex MallocHooker::_patch_lock; -int MallocHooker::_patched_libs = 0; -bool MallocHooker::_initialized = false; -// xoroshiro128+ PRNG state — shared, relaxed atomics. -// Benign races are acceptable: occasional duplicate output is harmless -// for a sampling PRNG and thread_local cannot be used on the malloc path. -static u64 _xo_state[2]; - -static pthread_t _current_thread; -static volatile bool _nested_malloc = false; -static volatile bool _nested_posix_memalign = false; - -// Test if calloc() implementation calls malloc() -static void* nested_malloc_hook(size_t size) { - if (pthread_self() == _current_thread) { - _nested_malloc = true; - } - return _orig_malloc(size); -} - -// Test if posix_memalign() implementation calls aligned_alloc() -static void* nested_aligned_alloc_hook(size_t alignment, size_t size) { - if (pthread_self() == _current_thread) { - _nested_posix_memalign = true; - } - return _orig_aligned_alloc(alignment, size); -} - -// In some implementations, specifically on musl, calloc() calls malloc() internally, -// and posix_memalign() calls aligned_alloc(). Detect such cases to prevent double-accounting. -void MallocHooker::detectNestedMalloc() { - if (_orig_malloc != NULL && _orig_calloc != NULL) { - CodeCache* libc = Libraries::instance()->findLibraryByAddress((void*)_orig_calloc); - if (libc != NULL) { - UnloadProtection handle(libc); - if (handle.isValid()) { - libc->patchImport(im_malloc, (void*)nested_malloc_hook); - - _current_thread = pthread_self(); - free(_orig_calloc(1, 1)); - _current_thread = pthread_t(0); - - // Restore original malloc so libc doesn't carry the probe hook until patchLibraries() runs. - libc->patchImport(im_malloc, (void*)_orig_malloc); - } - } - } - - if (_orig_posix_memalign != NULL && _orig_aligned_alloc != NULL) { - CodeCache* libc = Libraries::instance()->findLibraryByAddress((void*)_orig_posix_memalign); - if (libc != NULL) { - UnloadProtection handle(libc); - if (handle.isValid()) { - libc->patchImport(im_aligned_alloc, (void*)nested_aligned_alloc_hook); - - _current_thread = pthread_self(); - void* pm_probe = NULL; - _orig_posix_memalign(&pm_probe, sizeof(void*), sizeof(void*)); - _current_thread = pthread_t(0); - if (pm_probe != NULL) free(pm_probe); - - // Restore original aligned_alloc so libc doesn't carry the probe hook. - libc->patchImport(im_aligned_alloc, (void*)_orig_aligned_alloc); - } - } - } -} - -// Call each intercepted function at least once to ensure its GOT entry is updated -static void resolveMallocSymbols() { - static volatile intptr_t sink; - - void* p0 = malloc(1); - void* p1 = realloc(p0, 2); - if (p1 == NULL) { - // realloc failed; p0 is still valid and must be freed explicitly. - free(p0); - } - void* p2 = calloc(1, 1); - void* p3 = aligned_alloc(sizeof(void*), sizeof(void*)); - void* p4 = NULL; - if (posix_memalign(&p4, sizeof(void*), sizeof(void*)) == 0) free(p4); - free(p3); - free(p2); - free(p1); - - sink = (intptr_t)p1 + (intptr_t)p2 + (intptr_t)p3 + (intptr_t)p4; -} - -// Seed xoroshiro128+ state from a 64-bit value using splitmix64. -static void splitmix64_seed(u64 seed) { - seed += 0x9e3779b97f4a7c15ULL; - seed = (seed ^ (seed >> 30)) * 0xbf58476d1ce4e5b9ULL; - seed = (seed ^ (seed >> 27)) * 0x94d049bb133111ebULL; - __atomic_store_n(&_xo_state[0], seed ^ (seed >> 31), __ATOMIC_RELAXED); - seed += 0x9e3779b97f4a7c15ULL; - seed = (seed ^ (seed >> 30)) * 0xbf58476d1ce4e5b9ULL; - seed = (seed ^ (seed >> 27)) * 0x94d049bb133111ebULL; - __atomic_store_n(&_xo_state[1], seed ^ (seed >> 31), __ATOMIC_RELAXED); -} - -bool MallocHooker::initialize() { - if (_initialized) return _orig_malloc != NULL; - - CodeCache* lib = Libraries::instance()->findLibraryByAddress((void*)MallocTracer::recordMalloc); - if (lib == NULL) { - _initialized = true; - return false; - } - - resolveMallocSymbols(); - - SAVE_IMPORT(malloc); - SAVE_IMPORT(calloc); - SAVE_IMPORT(realloc); - SAVE_IMPORT(posix_memalign); - SAVE_IMPORT(aligned_alloc); - - detectNestedMalloc(); - - lib->mark( - [](const char* s) -> bool { - return strcmp(s, "malloc_hook") == 0 - || strcmp(s, "calloc_hook") == 0 - || strcmp(s, "realloc_hook") == 0 - || strcmp(s, "posix_memalign_hook") == 0 - || strcmp(s, "aligned_alloc_hook") == 0; - }, - MARK_ASYNC_PROFILER); - - splitmix64_seed(TSC::ticks()); - _initialized = true; - return _orig_malloc != NULL; -} - -void MallocHooker::patchLibraries() { - // Defensive guard: _orig_malloc is set by initialize(), which runs in - // MallocTracer::start() before _running is set and this path is reached. - // Guards against stale or unexpected direct calls. - if (_orig_malloc == NULL) return; - - MutexLocker ml(_patch_lock); - - const CodeCacheArray& native_libs = Libraries::instance()->native_libs(); - int native_lib_count = native_libs.count(); - - // _patched_libs is intentionally monotonic: hooks are permanent and cannot be - // uninstalled safely (library unloading races). On profiler restart, only - // newly-loaded libraries need patching. - TEST_LOG("MallocHooker::patchLibraries: _patched_libs=%d native_lib_count=%d _orig_malloc=%p", - _patched_libs, native_lib_count, (void*)_orig_malloc); - while (_patched_libs < native_lib_count) { - CodeCache* cc = native_libs[_patched_libs++]; - - UnloadProtection handle(cc); - if (!handle.isValid()) { - TEST_LOG("MallocHooker::patchLibraries: skipping (invalid handle) %s", cc->name()); - continue; - } - - TEST_LOG("MallocHooker::patchLibraries: patching %s has_malloc=%d", - cc->name(), cc->findImport(im_malloc) != nullptr); - if (_orig_malloc) cc->patchImport(im_malloc, (void*)malloc_hook); - if (_orig_realloc) cc->patchImport(im_realloc, (void*)realloc_hook); - if (_orig_aligned_alloc) cc->patchImport(im_aligned_alloc, (void*)aligned_alloc_hook); - // On musl, calloc/posix_memalign delegate to malloc/aligned_alloc internally; - // hooking them too would double-count. Leave the GOT entry untouched instead. - if (_orig_calloc && !_nested_malloc) cc->patchImport(im_calloc, (void*)calloc_hook); - if (_orig_posix_memalign && !_nested_posix_memalign) cc->patchImport(im_posix_memalign, (void*)posix_memalign_hook); - } -} - -void MallocHooker::installHooks() { - patchLibraries(); -} - -static inline u64 xo_rotl(u64 x, int k) { - return (x << k) | (x >> (64 - k)); -} - -u64 MallocTracer::nextPoissonInterval() { - // xoroshiro128+ — relaxed atomics tolerate benign races on the shared state. - u64 s0 = __atomic_load_n(&_xo_state[0], __ATOMIC_RELAXED); - u64 s1 = __atomic_load_n(&_xo_state[1], __ATOMIC_RELAXED); - u64 result = s0 + s1; - s1 ^= s0; - __atomic_store_n(&_xo_state[0], xo_rotl(s0, 55) ^ s1 ^ (s1 << 14), __ATOMIC_RELAXED); - __atomic_store_n(&_xo_state[1], xo_rotl(s1, 36), __ATOMIC_RELAXED); - double u = (double)(result >> 11) / (double)(1ULL << 53); - if (u < 1e-18) u = 1e-18; - return (u64)(__atomic_load_n(&_interval, __ATOMIC_ACQUIRE) * -log(u)); -} - -bool MallocTracer::shouldSample(size_t size) { - if (__atomic_load_n(&_interval, __ATOMIC_ACQUIRE) <= 1) return true; - while (true) { - u64 prev = __atomic_load_n(&_bytes_until_sample, __ATOMIC_RELAXED); - if (size < prev) { - if (__atomic_compare_exchange_n(&_bytes_until_sample, &prev, prev - size, - false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) - return false; - } else { - u64 next = nextPoissonInterval(); - if (__atomic_compare_exchange_n(&_bytes_until_sample, &prev, next, - false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) - return true; - } - } -} - -void MallocTracer::updateConfiguration(u64 events, double time_coefficient) { - double signal = _pid.compute(events, time_coefficient); - int64_t new_interval = (int64_t)__atomic_load_n(&_interval, __ATOMIC_ACQUIRE) - (int64_t)signal; - if (new_interval < (int64_t)_configured_interval) - new_interval = (int64_t)_configured_interval; - if (new_interval > (int64_t)(1ULL << 40)) - new_interval = (int64_t)(1ULL << 40); - __atomic_store_n(&_interval, (u64)new_interval, __ATOMIC_RELEASE); -} - -void MallocTracer::recordMalloc(void* address, size_t size) { - if (shouldSample(size)) { - u64 current_interval = __atomic_load_n(&_interval, __ATOMIC_ACQUIRE); - MallocEvent event; - event._start_time = TSC::ticks(); - event._address = (uintptr_t)address; - event._size = size; - // _interval == 0 means sample every allocation; weight is 1.0. - if (size == 0 || current_interval <= 1) { - event._weight = 1.0f; - } else { - event._weight = (float)(1.0 / (1.0 - exp(-(double)size / (double)current_interval))); - } - - Profiler::instance()->recordSample(NULL, size, OS::threadId(), BCI_NATIVE_MALLOC, 0, &event); - - u64 current_samples = __atomic_add_fetch(&_sample_count, 1, __ATOMIC_RELAXED); - if ((current_samples % TARGET_SAMPLES_PER_WINDOW) == 0) { - u64 now = OS::nanotime(); - u64 prev_ts = __atomic_load_n(&_last_config_update_ts, __ATOMIC_ACQUIRE); - u64 time_diff = now - prev_ts; - u64 check_period_ns = (u64)CONFIG_UPDATE_CHECK_PERIOD_SECS * 1000000000ULL; - if (time_diff > check_period_ns) { - if (__atomic_compare_exchange_n(&_last_config_update_ts, &prev_ts, now, - false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { - __atomic_fetch_sub(&_sample_count, current_samples, __ATOMIC_RELEASE); - updateConfiguration(current_samples, - (double)check_period_ns / time_diff); - } - } - } - } -} - -Error MallocTracer::start(Arguments& args) { - _configured_interval = args._nativemem > 0 ? args._nativemem : 0; - __atomic_store_n(&_interval, _configured_interval, __ATOMIC_RELEASE); - __atomic_store_n(&_bytes_until_sample, - _configured_interval > 1 ? nextPoissonInterval() : 0, - __ATOMIC_RELEASE); - __atomic_store_n(&_sample_count, (u64)0, __ATOMIC_RELEASE); - // Clear accumulated integral/derivative so a fresh session is not biased by - // state from a prior one (relevant for tests that stop and restart the profiler). - _pid.reset(); - __atomic_store_n(&_last_config_update_ts, OS::nanotime(), __ATOMIC_RELEASE); - - // initialize() is idempotent and returns false when symbol resolution fails. - if (!MallocHooker::initialize()) { - return Error("Failed to resolve malloc symbols; native memory profiling unavailable"); - } - - // Enable recording before patching so a concurrent dlopen() during patchLibraries() - // sees running()==true and patches the new library via installHooks(). - // _orig_* pointers are already resolved in initialize(), so this is safe. - __atomic_store_n(&_running, true, __ATOMIC_RELEASE); - MallocHooker::patchLibraries(); - - return Error::OK; -} - -void MallocTracer::stop() { - // Ideally, we should reset original malloc entries, but it's not currently safe - // in the view of library unloading. Consider using dl_iterate_phdr. - __atomic_store_n(&_running, false, __ATOMIC_RELEASE); -} diff --git a/ddprof-lib/src/main/cpp/mallocTracer.h b/ddprof-lib/src/main/cpp/mallocTracer.h deleted file mode 100644 index 46eef3386..000000000 --- a/ddprof-lib/src/main/cpp/mallocTracer.h +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _MALLOCTRACER_H -#define _MALLOCTRACER_H - -#include -#include "engine.h" -#include "event.h" -#include "mutex.h" -#include "pidController.h" - -// Manages GOT-patching for malloc interception across all loaded native libraries. -class MallocHooker { - private: - static Mutex _patch_lock; - static int _patched_libs; - static bool _initialized; - - static void detectNestedMalloc(); - - public: - // Returns true if symbols were successfully resolved. - static bool initialize(); - static void patchLibraries(); - static void installHooks(); -}; - -class MallocTracer : public Engine { - private: - static volatile u64 _interval; - static volatile u64 _bytes_until_sample; - - static u64 _configured_interval; - static volatile u64 _sample_count; - static volatile u64 _last_config_update_ts; - static const int CONFIG_UPDATE_CHECK_PERIOD_SECS = 1; - static const int TARGET_SAMPLES_PER_WINDOW = 100; - - static volatile bool _running; - static PidController _pid; - - static u64 nextPoissonInterval(); - static bool shouldSample(size_t size); - static void updateConfiguration(u64 events, double time_coefficient); - - public: - const char* name() { - return "MallocTracer"; - } - - Error start(Arguments& args); - void stop(); - - static inline bool running() { - return __atomic_load_n(&_running, __ATOMIC_ACQUIRE); - } - - static inline void installHooks() { - MallocHooker::installHooks(); - } - - static void recordMalloc(void* address, size_t size); -}; - -#endif // _MALLOCTRACER_H diff --git a/ddprof-lib/src/main/cpp/mutex.cpp b/ddprof-lib/src/main/cpp/mutex.cpp deleted file mode 100644 index 8b9c92b64..000000000 --- a/ddprof-lib/src/main/cpp/mutex.cpp +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "mutex.h" -#include "signalSafety.h" - - -Mutex::Mutex() { - pthread_mutexattr_t attr; - pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); - pthread_mutex_init(&_mutex, &attr); -} - -void Mutex::lock() { - DEBUG_ASSERT_NOT_IN_SIGNAL(); - pthread_mutex_lock(&_mutex); -} - -void Mutex::unlock() { - pthread_mutex_unlock(&_mutex); -} - -WaitableMutex::WaitableMutex() : Mutex() { - pthread_cond_init(&_cond, NULL); -} - -bool WaitableMutex::waitUntil(u64 wall_time) { - struct timespec ts = {(time_t)(wall_time / 1000000), (long)(wall_time % 1000000) * 1000}; - return pthread_cond_timedwait(&_cond, &_mutex, &ts) != 0; -} - -void WaitableMutex::notify() { - pthread_cond_signal(&_cond); -} diff --git a/ddprof-lib/src/main/cpp/mutex.h b/ddprof-lib/src/main/cpp/mutex.h deleted file mode 100644 index 7d017536d..000000000 --- a/ddprof-lib/src/main/cpp/mutex.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _MUTEX_H -#define _MUTEX_H - -#include -#include "arch.h" - - -class Mutex { - protected: - pthread_mutex_t _mutex; - - public: - Mutex(); - - void lock(); - void unlock(); -}; - -class WaitableMutex : public Mutex { - protected: - pthread_cond_t _cond; - - public: - WaitableMutex(); - - bool waitUntil(u64 wall_time); - void notify(); -}; - -class MutexLocker { - private: - Mutex* _mutex; - - public: - MutexLocker(Mutex& mutex) : _mutex(&mutex) { - _mutex->lock(); - } - - ~MutexLocker() { - _mutex->unlock(); - } -}; - -#endif // _MUTEX_H diff --git a/ddprof-lib/src/main/cpp/nativeSocketSampler.cpp b/ddprof-lib/src/main/cpp/nativeSocketSampler.cpp deleted file mode 100644 index 79591e7d5..000000000 --- a/ddprof-lib/src/main/cpp/nativeSocketSampler.cpp +++ /dev/null @@ -1,423 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "nativeSocketSampler.h" - -#if defined(__linux__) - -#include "common.h" -#include "flightRecorder.h" -#include "libraryPatcher.h" -#include "log.h" -#include "os.h" -#include "profiler.h" -#include "tsc.h" -#include "vmEntry.h" - -#include -#include -#include -#include -#include -#include -#include - -static thread_local PoissonSampler _send_sampler; -static thread_local PoissonSampler _recv_sampler; - -// Debug-only hook-fire counters, paired with TEST_LOG (common.h). Gated at -// compile time to keep release hot paths free of cross-thread atomic writes. -#ifdef DEBUG -static std::atomic _send_hook_calls{0}; -static std::atomic _recv_hook_calls{0}; -static std::atomic _write_hook_calls{0}; -static std::atomic _read_hook_calls{0}; -static std::atomic _record_accept_calls{0}; -static std::atomic _record_reject_calls{0}; -#endif - -// intentional process-lifetime singleton — matches MallocTracer pattern; no destructor needed -NativeSocketSampler* const NativeSocketSampler::_instance = new NativeSocketSampler(); -std::atomic NativeSocketSampler::_orig_send{nullptr}; -std::atomic NativeSocketSampler::_orig_recv{nullptr}; -std::atomic NativeSocketSampler::_orig_write{nullptr}; -std::atomic NativeSocketSampler::_orig_read{nullptr}; - -std::string NativeSocketSampler::resolveAddr(int fd) { - struct sockaddr_storage ss; - socklen_t len = sizeof(ss); - if (getpeername(fd, (struct sockaddr*)&ss, &len) != 0) { - TEST_LOG("NativeSocketSampler::resolveAddr getpeername fd=%d failed errno=%d", fd, errno); - return ""; - } - char host[INET6_ADDRSTRLEN]; - int port = 0; - if (ss.ss_family == AF_INET) { - struct sockaddr_in* s = (struct sockaddr_in*)&ss; - if (inet_ntop(AF_INET, &s->sin_addr, host, sizeof(host)) == nullptr) return ""; - port = ntohs(s->sin_port); - } else if (ss.ss_family == AF_INET6) { - struct sockaddr_in6* s = (struct sockaddr_in6*)&ss; - if (inet_ntop(AF_INET6, &s->sin6_addr, host, sizeof(host)) == nullptr) return ""; - port = ntohs(s->sin6_port); - } else { - return ""; - } - // [addr]:port — INET6_ADDRSTRLEN(46) + brackets(2) + colon(1) + port(5) + NUL(1) = 55; round to 64. - static const int FORMATTED_ADDR_BUF = 64; - char buf[FORMATTED_ADDR_BUF]; - int n; - if (ss.ss_family == AF_INET6) { - n = snprintf(buf, sizeof(buf), "[%s]:%d", host, port); - } else { - n = snprintf(buf, sizeof(buf), "%s:%d", host, port); - } - // Truncation is theoretical (buf is 64 bytes, max needed is 55), but snprintf - // already NUL-terminates on truncation; suppress unused-variable warning in release. - (void)n; - return std::string(buf); -} - -bool NativeSocketSampler::isSocket(int fd) { - // Accepts any SOCK_STREAM socket (including AF_UNIX); AF_INET/AF_INET6 filtering - // is deferred to resolveAddr() which is only called for sampled events. AF_UNIX - // will produce an empty remoteAddress field in the JFR event. - if (fd < 0) return false; - if ((size_t)fd >= (size_t)FD_TYPE_CACHE_SIZE) { - int so_type; - socklen_t solen = sizeof(so_type); - return getsockopt(fd, SOL_SOCKET, SO_TYPE, &so_type, &solen) == 0 - && so_type == SOCK_STREAM; - } - // Acquire on the gen load pairs with the release on the gen-bump in start() - // and on the cache cell store below; without it, on a weakly-ordered arch - // (aarch64) a thread could observe a freshly written cell without the matching - // gen bump (or vice versa), defeating the generation-tag invalidation contract. - uint8_t gen = _fd_cache_gen.load(std::memory_order_acquire); - uint8_t cached = _fd_type_cache[fd].load(std::memory_order_acquire); - // High nibble encodes generation; entry is valid only when it matches current gen mod 16. - if ((cached >> 4) == (gen & 0xF)) { - uint8_t type = cached & 0xF; - // A cached NON_SOCKET verdict is safe to trust: the worst case is that a - // newly-socketed fd reuse under-samples until the next gen reset, which is - // the documented accepted staleness tradeoff. - if (type == FD_TYPE_NON_SOCKET) return false; - // Cached SOCKET: trust the verdict on the hot path; revalidation is deferred - // to recordEvent() on sampled write/read events (see revalidateSocket()). - if (type == FD_TYPE_SOCKET) return true; - } - - int so_type; - socklen_t solen = sizeof(so_type); - int rc = getsockopt(fd, SOL_SOCKET, SO_TYPE, &so_type, &solen); - if (rc == 0) { - bool tcp = (so_type == SOCK_STREAM); - uint8_t type = tcp ? FD_TYPE_SOCKET : FD_TYPE_NON_SOCKET; - _fd_type_cache[fd].store((uint8_t)(((gen & 0xF) << 4) | type), - std::memory_order_release); - return tcp; - } - // Only cache the non-socket verdict when getsockopt definitively says - // "not a socket" (ENOTSOCK). Transient errors (EBADF on a racing close, - // EINTR, etc.) must NOT poison the cache: a sticky misclassification - // would survive fd reuse via dup2() and silently suppress sampling for - // the rest of the session. - if (errno == ENOTSOCK) { - _fd_type_cache[fd].store((uint8_t)(((gen & 0xF) << 4) | FD_TYPE_NON_SOCKET), - std::memory_order_release); - } - return false; -} - -void NativeSocketSampler::insertFdAddrLocked(int fd, std::string addr) { - auto it = _fd_cache.find(fd); - if (it != _fd_cache.end()) { - it->second->second = std::move(addr); - _fd_lru_list.splice(_fd_lru_list.begin(), _fd_lru_list, it->second); - } else { - if ((int)_fd_cache.size() >= MAX_FD_CACHE) { - _fd_cache.erase(_fd_lru_list.back().first); - _fd_lru_list.pop_back(); - } - _fd_lru_list.emplace_front(fd, std::move(addr)); - _fd_cache.emplace(fd, _fd_lru_list.begin()); - } -} - -bool NativeSocketSampler::revalidateSocket(int fd) { - int so_type; - socklen_t solen = sizeof(so_type); - int rc = getsockopt(fd, SOL_SOCKET, SO_TYPE, &so_type, &solen); - if (rc == 0 && so_type == SOCK_STREAM) return true; - // fd was reused for a non-socket or is already closed; update the type cache. - if (fd >= 0 && (size_t)fd < (size_t)FD_TYPE_CACHE_SIZE) { - uint8_t gen = _fd_cache_gen.load(std::memory_order_acquire); - _fd_type_cache[fd].store( - (uint8_t)(((gen & 0xF) << 4) | FD_TYPE_NON_SOCKET), - std::memory_order_release); - } - return false; -} - -bool NativeSocketSampler::shouldSample(u64 duration_ticks, int op, float &weight) { - // op 0 (send) and op 2 (write) are outbound → share _send_sampler. - // op 1 (recv) and op 3 (read) are inbound → share _recv_sampler. - PoissonSampler &sampler = (op == 0 || op == 2) ? _send_sampler : _recv_sampler; - return sampler.sample(duration_ticks, - (u64)_rate_limiter.interval(), - _rate_limiter.epoch(), - weight); -} - -void NativeSocketSampler::recordEvent(int fd, u64 t0, u64 t1, ssize_t bytes, u8 op) { - if (!Profiler::instance()->isRunning()) return; - // Clamp TSC inversion: a thread migrating cores between the two TSC reads can - // observe t1 < t0. Pass 0 to the sampler so the event is not force-sampled, - // and record duration 0 in the JFR event (consistent with safeDuration in flightRecorder). - u64 dur = t1 >= t0 ? t1 - t0 : 0; - float weight = 0.0f; - bool sampled = shouldSample(dur, op, weight); - if (!sampled) { -#ifdef DEBUG - uint64_t n = _record_reject_calls.fetch_add(1, std::memory_order_relaxed); - if (n < 3 || (n & 0x3FF) == 0) { - TEST_LOG("NativeSocketSampler::recordEvent REJECT #%llu fd=%d op=%u bytes=%zd dur_ticks=%llu", - (unsigned long long)(n + 1), fd, (unsigned)op, bytes, - (unsigned long long)dur); - } -#endif - return; - } - // write/read hooks (op 2/3) call isSocket() which trusts the cached SOCKET verdict - // without re-probing, to avoid a getsockopt syscall on every I/O. Revalidate here, - // on sampled events only, so a closed-and-reused fd is caught before we emit an event. - if ((op == 2 || op == 3) && !revalidateSocket(fd)) return; -#ifdef DEBUG - { - uint64_t n = _record_accept_calls.fetch_add(1, std::memory_order_relaxed); - if (n < 3 || (n & 0x3F) == 0) { - TEST_LOG("NativeSocketSampler::recordEvent ACCEPT #%llu fd=%d op=%u bytes=%zd dur_ticks=%llu weight=%f", - (unsigned long long)(n + 1), fd, (unsigned)op, bytes, - (unsigned long long)dur, (double)weight); - } - } -#endif - - NativeSocketEvent event; - event._start_time = t0; - event._end_time = t1; - event._operation = op; - event._remote_addr[0] = '\0'; - // Always re-probe the peer address on sampled events so that a closed-and-reused - // fd (new TCP connection on the same fd number) gets a fresh address rather than - // the previous peer's address from the LRU cache. The cost (one getpeername per - // sampled event) is bounded by the sampling rate and acceptable here. - { - std::string resolved = resolveAddr(fd); - if (!resolved.empty()) { - strncpy(event._remote_addr, resolved.c_str(), sizeof(event._remote_addr) - 1); - event._remote_addr[sizeof(event._remote_addr) - 1] = '\0'; - std::lock_guard lock(_fd_cache_mutex); - insertFdAddrLocked(fd, std::move(resolved)); - } else { - // resolveAddr returned empty (AF_UNIX, not yet connected, etc.); fall back - // to the cached value if one exists. - std::lock_guard lock(_fd_cache_mutex); - auto it = _fd_cache.find(fd); - if (it != _fd_cache.end()) { - _fd_lru_list.splice(_fd_lru_list.begin(), _fd_lru_list, it->second); - strncpy(event._remote_addr, it->second->second.c_str(), sizeof(event._remote_addr) - 1); - event._remote_addr[sizeof(event._remote_addr) - 1] = '\0'; - } - } - } - // ret > 0 checked above; cast is safe. - event._bytes = (u64)bytes; - event._weight = weight; - - Profiler::instance()->recordSample(NULL, (u64)bytes, OS::threadId(), - BCI_NATIVE_SOCKET, 0, &event); - - _rate_limiter.recordFire(); -} - -ssize_t NativeSocketSampler::send_hook(int fd, const void* buf, size_t len, int flags) { - // Defensive guard against direct invocation outside of PLT dispatch (e.g. tests - // that obtain the static symbol address before LibraryPatcher::patch_socket_functions - // has run). Production hooks are unreachable until setOriginalFunctions() has been - // called under _lock, so this branch is not exercised on the normal path. - send_fn fn = _orig_send.load(std::memory_order_acquire); - if (fn == nullptr) { errno = ENOSYS; return -1; } - if (!LibraryPatcher::_socket_active.load(std::memory_order_acquire)) return fn(fd, buf, len, flags); - NativeSocketSampler* self = _instance; - if (!self->isSocket(fd)) return fn(fd, buf, len, flags); -#ifdef DEBUG - { - uint64_t n = _send_hook_calls.fetch_add(1, std::memory_order_relaxed); - if (n < 3 || (n & 0x3FF) == 0) { - TEST_LOG("NativeSocketSampler::send_hook #%llu fd=%d len=%zu flags=0x%x", - (unsigned long long)(n + 1), fd, len, flags); - } - } -#endif - u64 t0 = TSC::ticks(); - return record_if_positive(fd, fn(fd, buf, len, flags), t0, TSC::ticks(), 0); -} - -ssize_t NativeSocketSampler::recv_hook(int fd, void* buf, size_t len, int flags) { - recv_fn fn = _orig_recv.load(std::memory_order_acquire); - if (fn == nullptr) { errno = ENOSYS; return -1; } - if (!LibraryPatcher::_socket_active.load(std::memory_order_acquire)) return fn(fd, buf, len, flags); - NativeSocketSampler* self = _instance; - if (!self->isSocket(fd)) return fn(fd, buf, len, flags); -#ifdef DEBUG - { - uint64_t n = _recv_hook_calls.fetch_add(1, std::memory_order_relaxed); - if (n < 3 || (n & 0x3FF) == 0) { - TEST_LOG("NativeSocketSampler::recv_hook #%llu fd=%d len=%zu flags=0x%x", - (unsigned long long)(n + 1), fd, len, flags); - } - } -#endif - u64 t0 = TSC::ticks(); - return record_if_positive(fd, fn(fd, buf, len, flags), t0, TSC::ticks(), 1); -} - -ssize_t NativeSocketSampler::write_hook(int fd, const void* buf, size_t len) { - write_fn fn = _orig_write.load(std::memory_order_acquire); - if (fn == nullptr) { errno = ENOSYS; return -1; } - if (!LibraryPatcher::_socket_active.load(std::memory_order_acquire)) return fn(fd, buf, len); - NativeSocketSampler* self = _instance; - bool is_socket = self->isSocket(fd); -#ifdef DEBUG - { - uint64_t n = _write_hook_calls.fetch_add(1, std::memory_order_relaxed); - if (n < 3 || (n & 0x3FF) == 0) { - TEST_LOG("NativeSocketSampler::write_hook #%llu fd=%d len=%zu is_socket=%d", - (unsigned long long)(n + 1), fd, len, (int)is_socket); - } - } -#endif - if (!is_socket) return fn(fd, buf, len); - u64 t0 = TSC::ticks(); - return record_if_positive(fd, fn(fd, buf, len), t0, TSC::ticks(), 2); -} - -ssize_t NativeSocketSampler::read_hook(int fd, void* buf, size_t len) { - read_fn fn = _orig_read.load(std::memory_order_acquire); - if (fn == nullptr) { errno = ENOSYS; return -1; } - if (!LibraryPatcher::_socket_active.load(std::memory_order_acquire)) return fn(fd, buf, len); - NativeSocketSampler* self = _instance; - bool is_socket = self->isSocket(fd); -#ifdef DEBUG - { - uint64_t n = _read_hook_calls.fetch_add(1, std::memory_order_relaxed); - if (n < 3 || (n & 0x3FF) == 0) { - TEST_LOG("NativeSocketSampler::read_hook #%llu fd=%d len=%zu is_socket=%d", - (unsigned long long)(n + 1), fd, len, (int)is_socket); - } - } -#endif - if (!is_socket) return fn(fd, buf, len); - u64 t0 = TSC::ticks(); - return record_if_positive(fd, fn(fd, buf, len), t0, TSC::ticks(), 3); -} - -Error NativeSocketSampler::check(Arguments &args) { - if (!args._nativesocket) { - return Error("natsock profiling not requested"); - } - return Error::OK; -} - -Error NativeSocketSampler::start(Arguments &args) { - // Initial sampling period: args._nativesocket_interval (ns) when > 0, - // otherwise 1 ms default. Converted to TSC ticks (time-weighted sampling). - // - // Overflow guard: the naive (interval_ns * tsc_freq) can wrap u64 when the - // configured interval is large. At a 3 GHz TSC the product overflows around - // interval_ns ≈ 6.1 s. Compute via divide-first to keep the intermediate - // bounded: ticks = interval_ns * (freq/1e9) ≈ ns_per_tick⁻¹ * interval_ns. - long init_interval; - if (args._nativesocket_interval > 0) { - u64 ns = (u64)args._nativesocket_interval; - u64 tsc_freq = TSC::frequency(); - u64 secs = ns / 1000000000ULL; - u64 sub_ns = ns % 1000000000ULL; - // Guard secs * tsc_freq against u64 overflow before the LONG_MAX clamp. - // At 3 GHz the product wraps at secs ≈ 6.1e9; clamp early to avoid UB. - u64 ticks; - if (tsc_freq > 0 && secs > (u64)LONG_MAX / tsc_freq) { - ticks = (u64)LONG_MAX; - } else { - ticks = secs * tsc_freq + (sub_ns * tsc_freq) / 1000000000ULL; - if (ticks > (u64)LONG_MAX) ticks = (u64)LONG_MAX; - } - init_interval = (long)ticks; - } else { - init_interval = (long)(TSC::frequency() / 1000); - } - if (init_interval < 1) { - init_interval = DEFAULT_INTERVAL_TICKS; - } - // One limiter for all four hooks (send/write and recv/read): ~83 events/s (~5000/min) total. - // A large interval driven by heavy write traffic does not suppress long blocking reads: - // time-weighted sampling gives P = 1 - exp(-duration/interval) → 1 when duration >> interval, - // so slow calls self-select regardless of interval magnitude. Only short-duration calls - // (which carry no latency signal) are suppressed when the interval is large. - _rate_limiter.start(init_interval, TARGET_EVENTS_PER_SECOND, - PID_WINDOW_SECS, PID_P_GAIN, PID_I_GAIN, PID_D_GAIN, PID_CUTOFF_S); - // Clear the fd->addr cache and reset the fd-type cache generation for the new - // session so stale entries from a prior run cannot produce misattributed events - // even if stop() was not called. clearFdCache() bumps _fd_cache_gen under the - // mutex so the clear and the gen bump are atomic with respect to concurrent - // isSocket() calls. A single call per start() keeps the mod-16 generation-wrap - // budget at the full 16 cycles documented in nativeSocketSampler.h. - clearFdCache(); -#ifdef DEBUG - _send_hook_calls.store(0, std::memory_order_relaxed); - _recv_hook_calls.store(0, std::memory_order_relaxed); - _write_hook_calls.store(0, std::memory_order_relaxed); - _read_hook_calls.store(0, std::memory_order_relaxed); - _record_accept_calls.store(0, std::memory_order_relaxed); - _record_reject_calls.store(0, std::memory_order_relaxed); - TEST_LOG("NativeSocketSampler::start interval_ticks=%ld tsc_freq=%llu", - init_interval, (unsigned long long)TSC::frequency()); -#endif - if (!LibraryPatcher::patch_socket_functions()) { - return Error("failed to install native socket hooks (dlsym returned NULL)"); - } - return Error::OK; -} - -void NativeSocketSampler::stop() { -#ifdef DEBUG - TEST_LOG("NativeSocketSampler::stop summary send=%llu recv=%llu write=%llu read=%llu accept=%llu reject=%llu", - (unsigned long long)_send_hook_calls.load(std::memory_order_relaxed), - (unsigned long long)_recv_hook_calls.load(std::memory_order_relaxed), - (unsigned long long)_write_hook_calls.load(std::memory_order_relaxed), - (unsigned long long)_read_hook_calls.load(std::memory_order_relaxed), - (unsigned long long)_record_accept_calls.load(std::memory_order_relaxed), - (unsigned long long)_record_reject_calls.load(std::memory_order_relaxed)); -#endif - LibraryPatcher::unpatch_socket_functions(); - clearFdCache(); -} - -void NativeSocketSampler::clearFdCache() { - std::lock_guard lock(_fd_cache_mutex); - _fd_cache.clear(); - _fd_lru_list.clear(); - // Bump the generation under the lock so the clear and the bump are atomic - // with respect to concurrent isSocket() calls: no thread can insert an - // entry tagged with the old generation after the map is cleared. - _fd_cache_gen.fetch_add(1, std::memory_order_release); -} - -#else // !__linux__ - -NativeSocketSampler* const NativeSocketSampler::_instance = new NativeSocketSampler(); - -#endif // __linux__ diff --git a/ddprof-lib/src/main/cpp/nativeSocketSampler.h b/ddprof-lib/src/main/cpp/nativeSocketSampler.h deleted file mode 100644 index e45bf6011..000000000 --- a/ddprof-lib/src/main/cpp/nativeSocketSampler.h +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _NATIVESOCKETSAMPLER_H -#define _NATIVESOCKETSAMPLER_H - -#include "arch.h" -#include "arguments.h" -#include "engine.h" -#include "event.h" - -#if defined(__linux__) - -#include "poissonSampler.h" -#include "rateLimiter.h" -#include -#include -#include -#include -#include - -class LibraryPatcher; - -// Synchronisation strategy -// ------------------------- -// Hook functions (send_hook / recv_hook / write_hook / read_hook) run on the -// calling Java thread, NOT in a signal handler. Therefore malloc and locking -// are safe inside hooks. -// -// fd-to-addr cache : guarded by _fd_cache_mutex (std::mutex). -// TOCTOU note: the cache is checked under lock, then -// released for resolveAddr(); a concurrent thread may -// emplace the same fd before re-acquisition. emplace() -// is idempotent in that case (first writer wins). -// Address staleness on fd reuse is accepted: worst case -// is one misattributed event per reuse. -// _fd_type_cache : std::atomic array, lock-free. Entry encoding: -// bits [7:4] = generation mod 16, bits [3:0] = type -// (0=unknown, 1=TCP socket, 2=non-TCP). Valid only when -// high nibble matches _fd_cache_gen mod 16. A cached SOCKET -// verdict is trusted on the hot path; revalidation via -// getsockopt() is deferred to recordEvent() for sampled -// write/read events (revalidateSocket()). A cached NON_SOCKET -// verdict is trusted (worst case: a reused fd under-samples -// until the next gen reset). -// _rate_limiter : RateLimiter — owns std::atomic interval, epoch, and -// event count. PID update races are resolved by CAS -// inside RateLimiter::maybeUpdateInterval(). -// Sampling state : thread_local PoissonSampler (in nativeSocketSampler.cpp). -// No cross-thread contention; each thread maintains its -// own independent Poisson process. The per-second PID -// window observes the aggregate fire count via the shared -// atomic inside RateLimiter. -// Hook install/remove : guarded by the profiler's main state lock (MutexLocker -// in Profiler::start / Profiler::stop). No deadlock -// risk because hook bodies do NOT acquire the profiler -// signal lock. - -class NativeSocketSampler : public Engine { -public: - // Typedefs for libc send/recv/write/read signatures. - typedef ssize_t (*send_fn)(int, const void*, size_t, int); - typedef ssize_t (*recv_fn)(int, void*, size_t, int); - typedef ssize_t (*write_fn)(int, const void*, size_t); - typedef ssize_t (*read_fn)(int, void*, size_t); - - static NativeSocketSampler* instance() { return _instance; } - - Error check(Arguments &args) override; - Error start(Arguments &args) override; - void stop() override; - - // Clears the fd-to-address cache and resets the fd-type cache. - // Called from both start() (to reset state on restart) and stop(). - // Intentionally NOT called on JFR chunk boundaries. - void clearFdCache(); - - // PLT hooks installed by LibraryPatcher::patch_socket_functions(). - static ssize_t send_hook(int fd, const void* buf, size_t len, int flags); - static ssize_t recv_hook(int fd, void* buf, size_t len, int flags); - static ssize_t write_hook(int fd, const void* buf, size_t len); - static ssize_t read_hook(int fd, void* buf, size_t len); - - // Called once by LibraryPatcher::patch_socket_functions() to install the - // real libc function pointers before any PLT entries are patched. - static void setOriginalFunctions(send_fn s, recv_fn r, write_fn w, read_fn rd) { - _orig_send.store(s, std::memory_order_release); - _orig_recv.store(r, std::memory_order_release); - _orig_write.store(w, std::memory_order_release); - _orig_read.store(rd, std::memory_order_release); - } - - // For testing only: retrieve the current original function pointers. - static void getOriginalFunctions(send_fn& s, recv_fn& r, write_fn& w, read_fn& rd) { - s = _orig_send.load(std::memory_order_acquire); - r = _orig_recv.load(std::memory_order_acquire); - w = _orig_write.load(std::memory_order_acquire); - rd = _orig_read.load(std::memory_order_acquire); - } - -private: - static NativeSocketSampler* const _instance; - - // Set by setOriginalFunctions() (called under _lock, before PLT patching) and - // read by the hooks on arbitrary application threads. Declared std::atomic with - // release/acquire pairing so a stop()→start() restart cycle, which rewrites these - // pointers while a stale-epoch hook may still be in flight, has no data race and no - // value tearing on any memory model. The acquire load in each hook also pairs with - // the release store here to publish the pointer before the hook observes it. - static std::atomic _orig_send; - static std::atomic _orig_recv; - static std::atomic _orig_write; - static std::atomic _orig_read; - - // Target aggregate event rate: ~83 events/s (~5000/min) across all four hooks - // (send/write and recv/read) combined. - static const int TARGET_EVENTS_PER_SECOND = 83; - static const int PID_WINDOW_SECS = 1; - - // PID controller gains. Tuned for a 1-second observation window targeting - // ~83 events/s. P=31 is proportional gain; I=511 accumulates steady-state - // error over the window; D=3 damps oscillation; cutoff=15s low-passes the - // derivative to suppress high-frequency noise. - static constexpr double PID_P_GAIN = 31.0; - static constexpr double PID_I_GAIN = 511.0; - static constexpr double PID_D_GAIN = 3.0; - static constexpr double PID_CUTOFF_S = 15.0; - - // Default sampling interval in TSC ticks (equals 1 ms only on a ~1 GHz TSC; - // serves as a numeric floor for pathologically low TSC frequencies). - static const long DEFAULT_INTERVAL_TICKS = 1000000; // fallback used in start() when the TSC-derived interval rounds to < 1 - - // Rate limiter: owns the PID controller, interval, epoch, and fire counter. - // NativeSocketSampler uses it directly (not via RateLimitedSampler) because - // it has two sampling channels (send + recv) that share one rate target but - // need independent per-thread PoissonSampler state. - RateLimiter _rate_limiter; - - // fd → "ip:port" LRU cache. Bounded to MAX_FD_CACHE entries; on overflow - // the least-recently-used entry is evicted. All access is under _fd_cache_mutex. - // Address is always re-probed on sampled events (see recordEvent) so fd reuse - // is detected within one sampling interval. - using FdAddrList = std::list>; - FdAddrList _fd_lru_list; - std::unordered_map _fd_cache; - std::mutex _fd_cache_mutex; - - // fd-type cache for write/read hooks. Lock-free: one atomic byte per fd number. - // Encoding: bits [7:4] = generation mod 16, bits [3:0] = type (0=unknown/invalid - // — implicit zero in fresh array, never written explicitly; 1=TCP socket; - // 2=non-TCP). An entry is valid only when its high nibble equals _fd_cache_gen - // mod 16. Incrementing _fd_cache_gen invalidates all entries in O(1) without - // touching the 65536-entry array. - // - // KNOWN LIMITATION (mod-16 generation wrap): _fd_cache_gen is only consulted via - // its low 4 bits. After 16 start() cycles the generation wraps and stale entries - // from a previous incarnation become indistinguishable from current ones until each - // fd is naturally re-probed. Profiler restarts are not exercised in production - // (only in tests), so the wrap is benign in practice. If restart-in-prod ever - // becomes a supported mode, widen _fd_cache_gen to uint32_t and store the full - // generation in a wider per-fd cell. - // Fds outside [0, FD_TYPE_CACHE_SIZE) are probed on every call. - static const int FD_TYPE_CACHE_SIZE = 65536; - // FD_TYPE_UNKNOWN is the implicit value-zero sentinel for never-written entries - // and gen-mismatch entries; it is decoded by the (cached >> 4) != gen path in - // isSocket(), not by an explicit comparison against this constant. - static const uint8_t FD_TYPE_UNKNOWN = 0; - static const uint8_t FD_TYPE_SOCKET = 1; - static const uint8_t FD_TYPE_NON_SOCKET = 2; - std::atomic _fd_cache_gen{0}; // incremented on each cache reset - std::atomic _fd_type_cache[FD_TYPE_CACHE_SIZE]; - - NativeSocketSampler() = default; - - // Resolve the peer address for fd; returns empty string on failure. - std::string resolveAddr(int fd); - - // Revalidates that fd is still a SOCK_STREAM socket; updates the type cache on - // mismatch. Called from recordEvent() for write/read ops on sampled events only. - bool revalidateSocket(int fd); - - // Inserts or updates fd→addr in the LRU cache, evicting the LRU entry if full. - // Must be called with _fd_cache_mutex held. - void insertFdAddrLocked(int fd, std::string addr); - -public: - // Test seams — not part of the production API. - static const int MAX_FD_CACHE = 65536; - - int fdAddrCacheSizeForTest() { - std::lock_guard lock(_fd_cache_mutex); - return (int)_fd_cache.size(); - } - void fdAddrCacheInsertForTest(int fd, const std::string& addr) { - std::lock_guard lock(_fd_cache_mutex); - insertFdAddrLocked(fd, addr); - } - -private: - - // Returns true if fd is a SOCK_STREAM socket (including AF_UNIX). - // Uses the fd-type cache; calls getsockopt on first encounter per fd and on - // every cached-SOCKET hit to revalidate against fd reuse (a closed socket fd - // reassigned to a regular file/pipe must not keep emitting socket events). - bool isSocket(int fd); - - // Decide whether to sample and compute weight. - // Returns true if the call should be recorded; sets weight out-param. - // Implements per-thread Poisson-process sampling: each thread maintains its - // own Exp-distributed countdown; when it expires the event is sampled and a - // new countdown is drawn. weight = 1 / (1 - exp(-duration/interval)). - // duration_ticks: wall time of the I/O call in TSC ticks. - // op: 0 = send, 1 = recv, 2 = write, 3 = read. - bool shouldSample(u64 duration_ticks, int op, float &weight); - - // Common recording logic shared by all four hooks. - void recordEvent(int fd, u64 t0, u64 t1, ssize_t bytes, u8 op); - - // Records the event if ret > 0; returns ret unchanged. Shared tail for all four hooks. - static inline ssize_t record_if_positive(int fd, ssize_t ret, u64 t0, u64 t1, u8 op) { - if (ret > 0) _instance->recordEvent(fd, t0, t1, ret, op); - return ret; - } -}; - -#else // !__linux__ - -class NativeSocketSampler : public Engine { -public: - static NativeSocketSampler* instance() { return _instance; } - Error check(Arguments &args) override { return Error::OK; } - Error start(Arguments &args) override { return Error::OK; } - void stop() override {} - void clearFdCache() {} -private: - static NativeSocketSampler* const _instance; - NativeSocketSampler() {} -}; - -#endif // __linux__ - -#endif // _NATIVESOCKETSAMPLER_H diff --git a/ddprof-lib/src/main/cpp/objectSampler.cpp b/ddprof-lib/src/main/cpp/objectSampler.cpp deleted file mode 100644 index ebcb3371d..000000000 --- a/ddprof-lib/src/main/cpp/objectSampler.cpp +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright 2022 Andrei Pangin - * Copyright 2022, 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include - -#include "context.h" -#include "objectSampler.h" -#include "pidController.h" -#include "profiler.h" -#include "thread.h" -#include -#include -#include -#include - -ObjectSampler *const ObjectSampler::_instance = new ObjectSampler(); - -bool ObjectSampler::normalizeClassSignature(const char *class_name, - const char **out_name, - size_t *out_len) { - if (out_name == NULL || out_len == NULL) { - return false; - } - if (class_name == NULL) { - return false; - } - size_t len = strlen(class_name); - if (len == 0) { - return false; - } - if (class_name[0] == 'L') { - // "Lname;" must have at least 3 chars (one body char) and a trailing ';'. - if (len < 3 || class_name[len - 1] != ';') { - return false; - } - *out_name = class_name + 1; - *out_len = len - 2; - } else { - *out_name = class_name; - *out_len = len; - } - return true; -} - -void ObjectSampler::SampledObjectAlloc(jvmtiEnv *jvmti, JNIEnv *jni, - jthread thread, jobject object, - jclass object_klass, jlong size) { - ObjectSampler::instance()->recordAllocation(jvmti, jni, thread, BCI_ALLOC, - object, object_klass, size); -} - -void ObjectSampler::recordAllocation(jvmtiEnv *jvmti, JNIEnv *jni, - jthread thread, int event_type, - jobject object, jclass object_klass, - jlong size) { - if (!__atomic_load_n(&_active, __ATOMIC_RELAXED)) { - return; - } - - if (jvmti == NULL) { - return; - } - - int tid = ProfiledThread::currentTid(); - - AllocEvent event; - - // Initialise so a JVMTI impl that returns JVMTI_ERROR_NONE without - // populating class_name cannot hand a stack-garbage pointer to Deallocate. - char *class_name = NULL; - if (jvmti->GetClassSignature(object_klass, &class_name, NULL) != 0 || - class_name == NULL) { - // Drop the sample: recording it under the default class id 0 - // would corrupt allocation attribution. - // NOTE: Do NOT call Deallocate here. The JVMTI spec does not guarantee - // output buffers are populated on a non-JVMTI_ERROR_NONE return; the - // pointer value is unspecified, so passing it to Deallocate is unsafe - // in practice and observed to crash with SIGSEGV. - return; - } - const char *name_slice = NULL; - size_t name_len = 0; - int id = -1; - if (normalizeClassSignature(class_name, &name_slice, &name_len)) { - id = Profiler::instance()->lookupClass(name_slice, name_len); - } - jvmti->Deallocate((unsigned char *)class_name); - if (id == -1) { - return; - } - event._id = id; - - u64 call_trace_id = 0; - // we do record the details and stacktraces only for when recording - // allocations or liveness - if (_record_allocations || _record_liveness) { - event._size = size; - event._weight = (float)((size == 0 || _interval == 0) - ? 1 - : 1 / (1 - exp(-size / (double)_interval))); - - call_trace_id = Profiler::instance()->recordJVMTISample(size, tid, thread, BCI_ALLOC, &event, !_record_allocations); - - if (call_trace_id == 0) { - return; - } - } - - if (_record_allocations && !_disable_rate_limiting) { - u64 current_samples = __sync_add_and_fetch(&_alloc_event_count, 1); - // in order to lower the number of atomic reads from the timestamp variable - // the check will be performed only each N samples - if ((current_samples % _target_samples_per_window) == 0) { - static u64 check_period_ns = - static_cast(CONFIG_UPDATE_CHECK_PERIOD_SECS) * 1000 * 1000 * - 1000; - u64 now = OS::nanotime(); - u64 prev = __atomic_load_n(&_last_config_update_ts, __ATOMIC_RELAXED); - u64 time_diff = now - prev; - // the config was last updated more than CONFIG_UPDATE_CHECK_PERIOD_SECS - // seconds ago - if (time_diff > check_period_ns) { - // this branch can be entered on multiple threads concurrently but only - // one will be able to make the config change - if (__atomic_compare_exchange(&_last_config_update_ts, &prev, &now, - false, __ATOMIC_ACQ_REL, - __ATOMIC_RELAXED)) { - __sync_fetch_and_add(&_alloc_event_count, -current_samples); - updateConfiguration(current_samples, - static_cast(check_period_ns) / time_diff); - } - } - } - } - - // Either we are recording liveness or tracking GC generations (lightweight - // liveness samples) - if (_gc_generations || _record_liveness) { - LivenessTracker::instance()->track(jni, event, tid, object, call_trace_id); - } -} - -Error ObjectSampler::check(Arguments &args) { - if (!VM::canSampleObjects()) { - return Error("Allocation Sampling is not supported on this JVM"); - } - - _interval = - std::max(args._memory, - static_cast( - 256 * 1024)); // do not allow shorter interval than 256kiB - _configured_interval = args._memory; - _record_allocations = args._record_allocations; - _record_liveness = args._record_liveness; - _gc_generations = args._gc_generations; - - // Test-only: Check environment variable to disable rate limiting - const char* disable_rate_limit_env = getenv("DDPROF_TEST_DISABLE_RATE_LIMIT"); - _disable_rate_limiting = (disable_rate_limit_env != nullptr && strcmp(disable_rate_limit_env, "1") == 0); - - _max_stack_depth = Profiler::instance()->max_stack_depth(); - - return Error::OK; -} - -Error ObjectSampler::start(Arguments &args) { - Error error = check(args); - if (error) { - return error; - } - if (_interval > 0) { - if (_record_liveness || _gc_generations) { - error = LivenessTracker::instance()->start(args); - if (error) { - return error; - } - } - - jvmtiEnv *jvmti = VM::jvmti(); - // JVMTI Object Sampler is a 'solo' feature, meaning that it can only be - // used by one JVMTI environment. Therefore, we can rely on the fact that if - // this agent gets hold of the sample it will be its exclusive owner. - jvmti->SetHeapSamplingInterval(_interval); - jvmti->SetEventNotificationMode(JVMTI_ENABLE, - JVMTI_EVENT_SAMPLED_OBJECT_ALLOC, NULL); - __atomic_store_n(&_active, true, __ATOMIC_RELEASE); - __atomic_store_n(&_last_config_update_ts, OS::nanotime(), __ATOMIC_RELEASE); - // need to reset the running sum in order for 'updateConfiguration' to be - // able to generate proper diffs - _alloc_event_count = 0; - } - - return Error::OK; -} - -void ObjectSampler::stop() { - __atomic_store_n(&_active, false, __ATOMIC_RELEASE); - jvmtiEnv *jvmti = VM::jvmti(); - jvmti->SetEventNotificationMode(JVMTI_DISABLE, - JVMTI_EVENT_SAMPLED_OBJECT_ALLOC, NULL); - - if (_record_liveness || _gc_generations) { - LivenessTracker::instance()->stop(); - } -} - -Error ObjectSampler::updateConfiguration(u64 events, double time_coefficient) { - static PidController pid_controller( - _target_samples_per_window, // target 6k events per minute or 1k per - // second - 31, // use a rather strong proportional gain in order to react quickly to - // bursts - 511, // emphasize the integration based gain to focus on long-term rate - // limiting rather than on fair distribution - 3, // the derivational gain is rather small because the allocation rate - // can change abruptly (low impact of the predicted allocation rate) - CONFIG_UPDATE_CHECK_PERIOD_SECS, 15); - - double signal = pid_controller.compute(events, time_coefficient); - int64_t signal_adjustment = static_cast(signal); - // use ints to avoid any wrap around - int64_t new_interval = static_cast(_interval) - signal_adjustment; - - // Clamp to never go below configured min - if (new_interval < static_cast(_configured_interval)) { - new_interval = static_cast(_configured_interval); - } - - // We actually need to consider the max interval from JVMTI api (max int32) - if (new_interval > INT32_MAX) { - new_interval = INT32_MAX; - } - - if (new_interval != _interval) { - // clamp the sampling interval to the max positive int value to avoid overflow - _interval = new_interval; - VM::jvmti()->SetHeapSamplingInterval(_interval); - } - - return Error::OK; -} diff --git a/ddprof-lib/src/main/cpp/objectSampler.h b/ddprof-lib/src/main/cpp/objectSampler.h deleted file mode 100644 index 88718087c..000000000 --- a/ddprof-lib/src/main/cpp/objectSampler.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2022 Andrei Pangin - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _OBJECTSAMPLER_H -#define _OBJECTSAMPLER_H - -#include "arch.h" -#include "engine.h" -#include "jfrMetadata.h" -#include "livenessTracker.h" -#include -#include -#include - -typedef int (*get_sampling_interval)(); - -class ObjectSampler : public Engine { - friend Recording; - friend class ObjectSamplerTestAccessor; - -private: - static ObjectSampler *const _instance; - - bool _active; - int _interval; - int _configured_interval; - bool _record_allocations; - bool _record_liveness; - bool _gc_generations; - int _max_stack_depth; - - u64 _last_config_update_ts; - u64 _alloc_event_count; - bool _disable_rate_limiting; - - const static int CONFIG_UPDATE_CHECK_PERIOD_SECS = 1; - int _target_samples_per_window = 100; // ~6k samples per minute by default - - Error updateConfiguration(u64 events, double time_coefficient); - - ObjectSampler() - : _active(false), _interval(0), _configured_interval(0), - _record_allocations(false), _record_liveness(false), - _gc_generations(false), _max_stack_depth(0), - _last_config_update_ts(0), _alloc_event_count(0), - _disable_rate_limiting(false) {} - -protected: - void recordAllocation(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread, - int event_type, jobject object, jclass object_klass, - jlong size); - -public: - static ObjectSampler *const instance() { return _instance; } - - Error check(Arguments &args); - Error start(Arguments &args); - void stop(); - - virtual long interval() const { return _interval; } - - static void JNICALL SampledObjectAlloc(jvmtiEnv *jvmti, JNIEnv *jni, - jthread thread, jobject object, - jclass object_klass, jlong size); - - // Strips the "L...;" wrapper from a JVMTI class signature for - // Profiler::lookupClass. Returns false (and leaves outputs untouched) - // when class_name is null/empty, the L-form is malformed, or either - // out pointer is null. Public for unit testing. - static bool normalizeClassSignature(const char *class_name, - const char **out_name, size_t *out_len); -}; - -#endif // _OBJECTSAMPLER_H diff --git a/ddprof-lib/src/main/cpp/os.h b/ddprof-lib/src/main/cpp/os.h deleted file mode 100644 index 6d50420ec..000000000 --- a/ddprof-lib/src/main/cpp/os.h +++ /dev/null @@ -1,249 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _OS_H -#define _OS_H - -#include -#include -#include -#include -#include -#include "arch.h" - - -typedef void (*SigAction)(int, siginfo_t*, void*); -typedef void (*SigHandler)(int); -typedef void (*TimerCallback)(void*); - -// Interrupt threads with this signal. The same signal is used inside JDK to interrupt I/O operations. -const int WAKEUP_SIGNAL = SIGIO; - -enum ThreadState { - THREAD_UNKNOWN, - THREAD_RUNNING, - THREAD_SLEEPING -}; - -struct ProcessInfo { - int pid = 0; - int ppid = 0; - char name[16]; // Process name from /proc/{pid}/stats - char cmdline[2048]; // Command line from /proc/{pid}/cmdline - unsigned int uid = 0; // User ID - unsigned char state = 0; // Process state (R, S, D, Z, T, etc.) - u64 start_time = 0; // Process start time (milliseconds since epoch) - - // CPU & thread stats - float cpu_user = 0; // User CPU time (seconds) - float cpu_system = 0; // System CPU time (seconds) - float cpu_percent = 0; // CPU utilization percentage - int threads = 0; // Number of threads - - // Memory stats (in bytes) - u64 vm_size = 0; // Total virtual memory size - u64 vm_rss = 0; // Resident memory size - u64 rss_anon = 0; // Resident anonymous memory - u64 rss_files = 0; // Resident file mappings - u64 rss_shmem = 0; // Resident shared memory - - // Page fault stats - u64 minor_faults = 0; // Minor page faults (no I/O required) - u64 major_faults = 0; // Major page faults (I/O required) - - // I/O stats - u64 io_read = 0; // KB read from storage - u64 io_write = 0; // KB written to storage -}; - - -class ThreadList { - protected: - u32 _index; - u32 _count; - - ThreadList() : _index(0), _count(0) { - } - - public: - virtual ~ThreadList() {} - - u32 index() const { return _index; } - u32 count() const { return _count; } - - bool hasNext() const { - return _index < _count; - } - - virtual int next() = 0; - virtual void update() = 0; -}; - - -// W^X memory support -class JitWriteProtection { - private: - u64 _prev; - bool _restore; - - public: - JitWriteProtection(bool enable); - ~JitWriteProtection(); -}; - - -class OS { - public: - static const size_t page_size; - static const size_t page_mask; - static const long clock_ticks_per_sec; - - static u64 nanotime(); - static u64 micros(); - static u64 processStartTime(); - static void sleep(u64 nanos); - - // Sleep for up to max_nanos, restarting against an absolute monotonic - // deadline on EINTR so unrelated signals (SIGCHLD, debugger - // SIGSTOP/SIGCONT, etc.) do not shorten the wait. Returns early when - // keep_sleeping becomes false — used by background threads (e.g. the - // Libraries refresher) to respond to a stop request without waiting - // out the full interval. Pair with pthread_kill(thread, SOMESIG) + - // keep_sleeping=false in the caller's stop path: the signal triggers - // EINTR, the loop re-checks the flag, and the function returns. - // - // On Linux this uses clock_nanosleep with TIMER_ABSTIME + CLOCK_MONOTONIC - // so the deadline is absolute and arithmetic-free. On macOS we fall - // back to nanosleep with a recomputed remainder (clock_nanosleep with - // TIMER_ABSTIME is unavailable there). - static void sleepWhile(u64 max_nanos, std::atomic& keep_sleeping); - - static u64 overrun(siginfo_t* siginfo); - - static u64 hton64(u64 x); - static u64 ntoh64(u64 x); - - static int getMaxThreadId(); - static int processId(); - static int threadId(); - static const char* schedPolicy(int thread_id); - static bool threadName(int thread_id, char* name_buf, size_t name_len); - static ThreadState threadState(int thread_id); - static u64 threadCpuTime(int thread_id); - static ThreadList* listThreads(); - - static bool isLinux(); - static bool isMusl(); - - // Returns nullptr on sigaction() failure; returns the previous sa_sigaction otherwise. - static SigAction installSignalHandler(int signo, SigAction action, SigHandler handler = NULL); - static SigAction replaceCrashHandler(SigAction action); - static int getProfilingSignal(int mode); - static bool sendSignalToThread(int thread_id, int signo); - - // Send a signal to a specific thread with a cookie payload (synchronous — - // the kernel queues it and returns). Uses rt_tgsigqueueinfo(2) on Linux; - // receiver sees si_code == SI_QUEUE with the cookie in si_value.sival_ptr. - // Engines use this when they need to discriminate their own signals from - // foreign ones in the handler. - static bool sendSignalWithCookie(int thread_id, int signo, void* cookie); - - // Accept-policy for a signal in a handler context. Async-signal-safe. - // - // Returns true iff the signal SHOULD be processed by the caller's engine: - // - origin check disabled (accept-all fallback, DDPROF_SIGNAL_ORIGIN_CHECK=0) - // → true - // - origin check enabled AND siginfo matches both expected_si_code and - // expected_cookie → true - // - otherwise → false - // - // Naming note: this is a policy predicate, NOT a pure "did this signal - // come from us?" identification. A `false` answer means "forward this to - // the chained handler"; a `true` answer means "process normally". - // `expected_si_code` is SI_TIMER (timer_create) or SI_QUEUE - // (sendSignalWithCookie). - static bool shouldProcessSignal(siginfo_t* siginfo, int expected_si_code, void* expected_cookie); - - // Forwards a signal we decided not to process to the previously-installed - // handler (as captured by installSignalHandler). When - // DDPROF_FORWARD_APPLY_SIGMASK=1 is set, also reproduces the previous - // handler's sa_mask so the chained handler sees the same signal-blocking - // environment it would under normal kernel delivery. - // Async-signal-safe on the fast path (no sa_mask applied). When - // DDPROF_FORWARD_APPLY_SIGMASK=1 is set, the slow path uses raw - // rt_sigprocmask syscalls which are async-signal-safe. The forwarded - // handler's safety is the caller's concern. - // - // Drop semantics (Linux): signals whose previous disposition was SIG_DFL or - // SIG_IGN are silently dropped — kernel-default termination is NOT reproduced - // (terminating the process on every forwarded foreign signal would be worse - // than ignoring it). Callers must not rely on default-action reproduction. - // - // macOS: always a no-op (rt_tgsigqueueinfo is unavailable so no cookie - // discrimination is possible; all signals are accepted by the engine handler - // and never forwarded). signalOriginCheckEnabled() returns false on macOS. - static void forwardForeignSignal(int signo, siginfo_t* siginfo, void* ucontext); - - // Runtime feature flag: is the signal origin check active? Reads - // DDPROF_SIGNAL_ORIGIN_CHECK env var and caches the result. Default on; - // set to "false"/"0"/"off"/"no" to disable (regression tests only). - // Callers running from signal-handler context must ensure - // primeSignalOriginCheck() was called earlier from non-signal context - // so the env-var getenv() cost is not paid during signal delivery. - static bool signalOriginCheckEnabled(); - - // Prime the signalOriginCheckEnabled() cache from non-signal context. - // Called by engine start() paths before installing handlers so later - // env-var overrides are picked up. Safe to call multiple times; no-op - // after the first call unless forceReload=true (used only by unit tests). - // - // Without priming, the cache holds its safe defaults (origin check - // enabled, sigmask chain disabled) — signals still classify correctly - // even if priming never runs. - static void primeSignalOriginCheck(bool forceReload = false); - - static void* safeAlloc(size_t size); - static void safeFree(void* addr, size_t size); - - static bool getCpuDescription(char* buf, size_t size); - static int getCpuCount(); - static u64 getProcessCpuTime(u64* utime, u64* stime); - static u64 getTotalCpuTime(u64* utime, u64* stime); - - static int createMemoryFile(const char* name); - static void copyFile(int src_fd, int dst_fd, off_t offset, size_t size); - static void freePageCache(int fd, off_t start_offset); - static int mprotect(void* addr, size_t size, int prot); - - static bool checkPreloaded(); - - static u64 getSystemBootTime(); - static u64 getRamSize(); - static int getProcessIds(int* pids, int max_pids); - static bool getBasicProcessInfo(int pid, ProcessInfo* info); - static bool getDetailedProcessInfo(ProcessInfo* info); - - // DataDog-specific extensions - static SigAction replaceSigsegvHandler(SigAction action); - static SigAction replaceSigbusHandler(SigAction action); - - // Signal handler protection - prevents other libraries from overwriting our handlers - static void protectSignalHandlers(SigAction segvHandler, SigAction busHandler); - static SigAction getSegvChainTarget(); - static SigAction getBusChainTarget(); - static void* getSigactionHook(); - static void resetSignalHandlersForTesting(); - - static int getMaxThreadId(int floor) { - int maxThreadId = getMaxThreadId(); - return maxThreadId < floor ? floor : maxThreadId; - } - - static int truncateFile(int fd); - static void mallocArenaMax(int arena_max); -}; - -#endif // _OS_H diff --git a/ddprof-lib/src/main/cpp/os_linux.cpp b/ddprof-lib/src/main/cpp/os_linux.cpp deleted file mode 100644 index ab59d8f19..000000000 --- a/ddprof-lib/src/main/cpp/os_linux.cpp +++ /dev/null @@ -1,1264 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifdef __linux__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" -#include "counters.h" -#include "log.h" -#include "os.h" - -#ifndef __musl__ -#include -#endif - - -#ifdef __LP64__ -# define MMAP_SYSCALL __NR_mmap -#else -# define MMAP_SYSCALL __NR_mmap2 -#endif - -#define COMM_LEN 16 - -class LinuxThreadList : public ThreadList { - private: - DIR* _dir; - int* _thread_array; - u32 _capacity; - - void addThread(int thread_id) { - if (_count >= _capacity) { - _capacity = _count * 2; - _thread_array = (int*)realloc(_thread_array, _capacity * sizeof(int)); - } - _thread_array[_count++] = thread_id; - } - - void fillThreadArray() { - if (_dir != NULL) { - rewinddir(_dir); - struct dirent* entry; - while ((entry = readdir(_dir)) != NULL) { - if (entry->d_name[0] != '.') { - addThread(atoi(entry->d_name)); - } - } - } - } - - public: - LinuxThreadList() : ThreadList() { - _dir = opendir("/proc/self/task"); - _capacity = 128; - _thread_array = (int*)malloc(_capacity * sizeof(int)); - fillThreadArray(); - } - - ~LinuxThreadList() { - free(_thread_array); - if (_dir != NULL) { - closedir(_dir); - } - } - - int next() { - return _thread_array[_index++]; - } - - void update() { - _index = _count = 0; - fillThreadArray(); - } -}; - - -JitWriteProtection::JitWriteProtection(bool enable) { - // Not used on Linux -} - -JitWriteProtection::~JitWriteProtection() { - // Not used on Linux -} - - -static constexpr int MAX_SIGNALS = 64; -static SigAction installed_sigaction[MAX_SIGNALS]; - -// Full previous sigaction per signal, used by forwardForeignSignal to chain -// signals that do not originate from ddprof to whatever handler was installed -// before us. -// -// Write protocol: installSignalHandler takes installed_oldaction_mutex -// around the sigaction() call and the subsequent publish. This serialises -// concurrent installers and ensures the struct write + _valid flag transition -// are atomic with respect to other installers. The release-store on _valid -// synchronises with the acquire-load in forwardForeignSignal so handlers on -// other CPUs only observe the struct after it is fully written. -// -// Store-exactly-once: once installed_oldaction_valid[signo] is true, the -// slot is frozen — subsequent installSignalHandler calls do NOT overwrite -// the captured previous action. This preserves the ORIGINAL chain target -// across profiler restarts or re-installs, even if a foreign library has -// since overwritten our handler and re-chained us. Losing the original -// would break the chain back to e.g. the JVM's SIGSEGV handler. -// See PR #494 for design discussion on the store-exactly-once pattern. -static struct sigaction installed_oldaction[MAX_SIGNALS]; -static bool installed_oldaction_valid[MAX_SIGNALS]; -static pthread_mutex_t installed_oldaction_mutex = PTHREAD_MUTEX_INITIALIZER; - -const size_t OS::page_size = sysconf(_SC_PAGESIZE); -const size_t OS::page_mask = OS::page_size - 1; -const long OS::clock_ticks_per_sec = sysconf(_SC_CLK_TCK); - - -u64 OS::nanotime() { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC, &ts); - return (u64)ts.tv_sec * 1000000000 + ts.tv_nsec; -} - -u64 OS::micros() { - struct timeval tv; - gettimeofday(&tv, NULL); - return (u64)tv.tv_sec * 1000000 + tv.tv_usec; -} - -u64 OS::processStartTime() { - static u64 start_time = 0; - - if (start_time == 0) { - char buf[64]; - snprintf(buf, sizeof(buf), "/proc/%d", processId()); - - struct stat st; - if (stat(buf, &st) == 0) { - start_time = (u64)st.st_mtim.tv_sec * 1000 + st.st_mtim.tv_nsec / 1000000; - } - } - - return start_time; -} - -void OS::sleep(u64 nanos) { - struct timespec ts = {(time_t)(nanos / 1000000000), (long)(nanos % 1000000000)}; - nanosleep(&ts, NULL); -} - -void OS::sleepWhile(u64 max_nanos, std::atomic& keep_sleeping) { - // Compute the deadline once and reuse it across EINTR retries so - // unrelated signals don't shorten the wait. - u64 deadline = OS::nanotime() + max_nanos; - struct timespec ts; - ts.tv_sec = (time_t)(deadline / 1000000000ULL); - ts.tv_nsec = (long)(deadline % 1000000000ULL); - while (keep_sleeping.load(std::memory_order_acquire) && - clock_nanosleep(CLOCK_MONOTONIC, TIMER_ABSTIME, &ts, nullptr) == EINTR) { - // Re-issue against the same absolute deadline. EINTR absorbs - // SIGCHLD, SIGSTOP/CONT, etc. without arithmetic on remaining time. - } -} - -u64 OS::overrun(siginfo_t* siginfo) { - return siginfo->si_overrun; -} - -u64 OS::hton64(u64 x) { - return htonl(1) == 1 ? x : bswap_64(x); -} - -u64 OS::ntoh64(u64 x) { - return ntohl(1) == 1 ? x : bswap_64(x); -} - -int OS::getMaxThreadId() { - char buf[16] = "65536"; - int fd = open("/proc/sys/kernel/pid_max", O_RDONLY); - if (fd != -1) { - ssize_t r = read(fd, buf, sizeof(buf) - 1); - (void) r; - close(fd); - } - return atoi(buf); -} - -int OS::processId() { - static const int self_pid = getpid(); - - return self_pid; -} - -int OS::threadId() { - return syscall(__NR_gettid); -} - -const char* OS::schedPolicy(int thread_id) { - int sched_policy = sched_getscheduler(thread_id); - if (sched_policy >= SCHED_BATCH) { - return sched_policy >= SCHED_IDLE ? "SCHED_IDLE" : "SCHED_BATCH"; - } - return "SCHED_OTHER"; -} - -bool OS::threadName(int thread_id, char* name_buf, size_t name_len) { - char buf[64]; - snprintf(buf, sizeof(buf), "/proc/self/task/%d/comm", thread_id); - int fd = open(buf, O_RDONLY); - if (fd == -1) { - return false; - } - - ssize_t r = read(fd, name_buf, name_len); - close(fd); - - if (r > 0) { - // /proc comm is newline-terminated; strip the trailing newline. - // Otherwise NUL-terminate after the last byte when there is room, - // only truncating the final byte if the buffer is completely full. - // This guarantees NUL-termination without silently dropping a real - // last character. - if (name_buf[r - 1] == '\n') { - name_buf[r - 1] = 0; - } else if ((size_t)r < name_len) { - name_buf[r] = 0; - } else { - name_buf[name_len - 1] = 0; - } - return true; - } - return false; -} - -ThreadState OS::threadState(int thread_id) { - char buf[512]; - snprintf(buf, sizeof(buf), "/proc/self/task/%d/stat", thread_id); - int fd = open(buf, O_RDONLY); - if (fd == -1) { - return THREAD_UNKNOWN; - } - - ThreadState state = THREAD_UNKNOWN; - if (read(fd, buf, sizeof(buf)) > 0) { - char* s = strchr(buf, ')'); - state = s != NULL && (s[2] == 'R' || s[2] == 'D') ? THREAD_RUNNING : THREAD_SLEEPING; - } - - close(fd); - return state; -} - -u64 OS::threadCpuTime(int thread_id) { - clockid_t thread_cpu_clock; - if (thread_id) { - thread_cpu_clock = ((~(unsigned int)(thread_id)) << 3) | 6; // CPUCLOCK_SCHED | CPUCLOCK_PERTHREAD_MASK - } else { - thread_cpu_clock = CLOCK_THREAD_CPUTIME_ID; - } - - struct timespec ts; - if (clock_gettime(thread_cpu_clock, &ts) == 0) { - return (u64)ts.tv_sec * 1000000000 + ts.tv_nsec; - } - return 0; -} - -ThreadList* OS::listThreads() { - return new LinuxThreadList(); -} - -bool OS::isLinux() { - return true; -} - -// _CS_GNU_LIBC_VERSION is not defined on musl -const static bool musl = confstr(_CS_GNU_LIBC_VERSION, NULL, 0) == 0 && errno != 0; - -bool OS::isMusl() { - return musl; -} - -SigAction OS::installSignalHandler(int signo, SigAction action, SigHandler handler) { - struct sigaction sa; - struct sigaction oldsa; - memset(&oldsa, 0, sizeof(oldsa)); - sigemptyset(&sa.sa_mask); - - if (handler != NULL) { - sa.sa_handler = handler; - sa.sa_flags = 0; - } else { - sa.sa_sigaction = action; - sa.sa_flags = SA_SIGINFO | SA_RESTART; - if (signo > 0 && signo < MAX_SIGNALS) { - installed_sigaction[signo] = action; - } - } - - // NOT async-signal-safe — uses pthread_mutex_lock. installSignalHandler - // must only be called from non-signal context (engine start/stop). Callers - // in CTimer::start and WallClockASGCT::initialize satisfy this contract. - // - // Take the mutex around sigaction() + publish so concurrent installers - // serialise, and a signal arriving mid-install always sees either the - // pre-install state (_valid=false → forward is a no-op) or the fully - // published post-install state. - pthread_mutex_lock(&installed_oldaction_mutex); - - int rc = sigaction(signo, &sa, &oldsa); - - // Cache the full previous action so forwardForeignSignal can chain. - // Skip on sigaction() failure — otherwise we would publish uninitialised - // garbage as "the previous handler" and the forwarder would jump into it. - // - // Only store in the sigaction (3-arg) path — 1-arg handlers are used for - // transient SIG_IGN / SIG_DFL setups (e.g. ITimer::check) and are never - // meant to be forwarded to. - // - // Store-exactly-once: once a slot is marked valid, preserve that capture. - // A foreign library may later install its own handler over ours; the - // next time we install (profiler restart) sigaction() would return the - // foreign handler as oldsa. Overwriting our stored oldaction with that - // would lose the ORIGINAL chain target (e.g. JVM's handler) and is never - // what we want — the original is what real chained delivery must reach. - // - // Publication is release-ordered: the struct write is visible before - // _valid flips to true, so any handler on another CPU that observes - // _valid==true sees a fully-initialised oldaction. See also the reader - // side in forwardForeignSignal. - if (rc == 0 && action != NULL && signo > 0 && signo < MAX_SIGNALS - && !__atomic_load_n(&installed_oldaction_valid[signo], __ATOMIC_RELAXED) // RELAXED: mutex provides ordering - && oldsa.sa_sigaction != action) { - installed_oldaction[signo] = oldsa; - __atomic_store_n(&installed_oldaction_valid[signo], true, __ATOMIC_RELEASE); - } - - pthread_mutex_unlock(&installed_oldaction_mutex); - - return rc == 0 ? oldsa.sa_sigaction : nullptr; -} - -static void restoreSignalHandler(int signo, siginfo_t* siginfo, void* ucontext) { - signal(signo, SIG_DFL); -} - -SigAction OS::replaceCrashHandler(SigAction action) { - struct sigaction sa; - sigaction(SIGSEGV, NULL, &sa); - SigAction old_action = sa.sa_handler == SIG_DFL ? restoreSignalHandler : sa.sa_sigaction; - sigemptyset(&sa.sa_mask); - sa.sa_sigaction = action; - sa.sa_flags |= SA_SIGINFO | SA_RESTART | SA_NODEFER; - sigaction(SIGSEGV, &sa, NULL); - return old_action; -} - -int OS::getProfilingSignal(int mode) { - static int preferred_signals[2] = {SIGPROF, SIGVTALRM}; - - const u64 allowed_signals = - 1ULL << SIGPROF | 1ULL << SIGVTALRM | 1ULL << SIGSTKFLT | 1ULL << SIGPWR | -(1ULL << SIGRTMIN); - - int& signo = preferred_signals[mode]; - int initial_signo = signo; - int other_signo = preferred_signals[1 - mode]; - - do { - struct sigaction sa; - if ((allowed_signals & (1ULL << signo)) != 0 && signo != other_signo && sigaction(signo, NULL, &sa) == 0) { - if (sa.sa_handler == SIG_DFL || sa.sa_handler == SIG_IGN || sa.sa_sigaction == installed_sigaction[signo]) { - return signo; - } - } - } while ((signo = (signo + 53) & 63) != initial_signo); - - return signo; -} - -bool OS::sendSignalToThread(int thread_id, int signo) { - return syscall(__NR_tgkill, processId(), thread_id, signo) == 0; -} - -#ifndef __NR_rt_tgsigqueueinfo -#error "__NR_rt_tgsigqueueinfo is not defined on this platform. \ -sendSignalWithCookie requires rt_tgsigqueueinfo(2). \ -Ensure your kernel headers define __NR_rt_tgsigqueueinfo." -#endif - -bool OS::sendSignalWithCookie(int thread_id, int signo, void* cookie) { - // Deliver a SIGQUEUE-style signal to a specific thread, carrying a cookie - // the handler can use to confirm the signal originated from the profiler. - // Uses rt_tgsigqueueinfo(2) directly so the same code path works on both - // glibc and musl (pthread_sigqueue is a glibc-only wrapper). The kernel - // requires si_code < 0 on user-submitted siginfo, so we set SI_QUEUE. - // The kernel rewrites si_pid/si_uid to the caller's real credentials for - // unprivileged senders, so setting them here is advisory at best. - siginfo_t si; - memset(&si, 0, sizeof(si)); - si.si_signo = signo; - si.si_code = SI_QUEUE; - si.si_value.sival_ptr = cookie; - return syscall(__NR_rt_tgsigqueueinfo, processId(), thread_id, signo, &si) == 0; -} - -// File-scope origin-check state. Written once in primeSignalOriginCheck() -// (non-signal context, before handlers are installed); read from signal handlers. -// Writers use __ATOMIC_RELEASE; readers use __ATOMIC_ACQUIRE so the release- -// acquire pair establishes happens-before and guarantees readers see the fully -// written state (including s_forward_apply_sigmask written before the release). -// -// Plain `volatile` is deliberately not used here: volatile suppresses compiler -// optimisations but does not imply any inter-thread memory ordering. Use -// explicit __atomic_* primitives when multi-thread visibility matters. -// -// Three-state enum eliminates the UNPRIMED vs ENABLED ambiguity: a signal that -// fires before primeSignalOriginCheck runs sees UNPRIMED and is accepted (safe -// default), not false-DISABLED. -enum class OriginCheckState : uint8_t { UNPRIMED = 0, ENABLED = 1, DISABLED = 2 }; -static OriginCheckState s_origin_check_state = OriginCheckState::UNPRIMED; - -// Opt-in sa_mask-respecting chain in forwardForeignSignal(). Off by default: -// avoids the extra signal-mask setup/restore work on each foreign signal -// (~1 µs on modern Linux, ~30% of the per-signal end-to-end cost). Set -// DDPROF_FORWARD_APPLY_SIGMASK=1 if the chained handler is known to require -// the kernel's normal sa_mask environment (rare — SIGSEGV/SIGBUS handlers are -// the main case and are out of scope here). -// Written with __ATOMIC_RELEASE so readers using __ATOMIC_ACQUIRE see a -// consistent value after the release-store in primeSignalOriginCheck. -static bool s_forward_apply_sigmask = false; - -bool OS::shouldProcessSignal(siginfo_t* siginfo, int expected_si_code, void* expected_cookie) { - // Acquire-load pairs with the release-store in primeSignalOriginCheck(), - // ensuring this thread observes the fully written state (including - // s_forward_apply_sigmask) after the prime completes. - auto state = static_cast( - __atomic_load_n(reinterpret_cast(&s_origin_check_state), __ATOMIC_ACQUIRE)); - if (state != OriginCheckState::ENABLED) { - // UNPRIMED (safe default: accept all) or DISABLED (feature off). - return true; - } - if (siginfo == nullptr || siginfo->si_code != expected_si_code) { - return false; - } - return siginfo->si_value.sival_ptr == expected_cookie; -} - -void OS::forwardForeignSignal(int signo, siginfo_t* siginfo, void* ucontext) { - // Preserve errno: syscall(rt_sigprocmask) on the slow path (and any - // chained handler) may set errno. Callers that save errno AFTER - // forwardForeignSignal (e.g. CTimer::signalHandler) would see a clobbered - // value without this guard. - int saved_errno = errno; - if (signo <= 0 || signo >= MAX_SIGNALS) { - errno = saved_errno; - return; - } - // Acquire-load the valid flag — synchronises with the release-store in - // installSignalHandler so we only touch the oldaction struct after it - // has been fully written. - if (!__atomic_load_n(&installed_oldaction_valid[signo], __ATOMIC_ACQUIRE)) { - errno = saved_errno; - return; - } - // ASYNC-SIGNAL-SAFE CONSTRAINT: forwardForeignSignal is called from signal - // context and cannot hold installed_oldaction_mutex. The release-acquire on - // installed_oldaction_valid synchronises the initial struct write (write-once), - // but does NOT prevent torn reads from a concurrent installSignalHandler - // mid-write. A seqlock would be the correct fix; instead, we enforce by - // design that installSignalHandler (profiler restart) cannot run concurrently - // with signal delivery. Callers must quiesce signal delivery before restarting. - struct sigaction prev = installed_oldaction[signo]; - - // By default we chain WITHOUT reproducing prev.sa_mask. Our own handler - // was installed without SA_NODEFER so the kernel has already blocked - // `signo` for the duration of this handler, which covers the common - // reentrancy concern. Benchmarks show that applying sa_mask via - // pthread_sigmask(SIG_BLOCK)+SIG_SETMASK costs ~1 µs per foreign signal - // (two rt_sigprocmask syscalls) — ~30% per-signal overhead in the - // slow-path micro-benchmark. In the trivyjni / Go-ITIMER_PROF scenario - // the plan targets, prev.sa_mask is empty anyway so the correctness - // cost is zero. - // - // Correctness escape hatch: set DDPROF_FORWARD_APPLY_SIGMASK=1 to - // enable the slow path. Use this when the chained handler is known - // to rely on the kernel's normal sa_mask environment (e.g. a JVM - // crash handler that expects other fatal signals blocked during its - // run). Default off keeps high-frequency profiling cheap. - sigset_t saved_mask; - bool need_mask = false; - // Acquire-load pairs with the release-store in primeSignalOriginCheck(). - if (__atomic_load_n(&s_forward_apply_sigmask, __ATOMIC_ACQUIRE)) { - // Probe prev.sa_mask for any set signal. POSIX offers no constant-time - // "is empty?" primitive, and looping sigismember over all signals - // inside a hot signal handler would cost more than the pthread_sigmask - // syscalls it is meant to avoid. We scan the raw bytes of sa_mask for - // any non-zero word, which exploits the fact that: - // - glibc defines sigset_t as a fixed-size bitmap (__sigset_t - // containing an unsigned-long array); zero bits == empty set. - // - musl defines sigset_t identically (unsigned long array). - // - Both target platforms (linux-x86_64, linux-aarch64) initialise - // sigset_t via sigemptyset to all-zero bytes. - // This is not POSIX-guaranteed but is reliable on every Linux libc - // we ship against. A future port to a libc that encodes "empty" as - // non-zero bytes would need a sigismember loop here. - // - // We use an inline word-sized loop instead of memcmp: memcmp is not on - // the POSIX async-signal-safe list (signal-safety(7)), and - // forwardForeignSignal is documented as async-signal-safe in os.h. - // The word-sized loop compiles to a handful of loads and has no libc dependency. - const unsigned long* p = reinterpret_cast(&prev.sa_mask); - for (size_t i = 0; i < sizeof(prev.sa_mask) / sizeof(unsigned long); ++i) { - if (p[i] != 0) { - need_mask = true; - break; - } - } - if (need_mask) { - // Use _NSIG/8 (kernel sigset size), NOT sizeof(sigset_t): the glibc - // sigset_t is 128 bytes but rt_sigprocmask(2) expects the kernel ABI - // size of _NSIG/8 == 8 bytes. Passing sizeof(sigset_t) returns EINVAL. - need_mask = syscall(__NR_rt_sigprocmask, SIG_BLOCK, &prev.sa_mask, &saved_mask, - _NSIG / 8) == 0; - } - } - - if (prev.sa_flags & SA_SIGINFO) { - // SIG_DFL (== 0) is caught by the nullptr check; SIG_IGN (== 1) is stored - // as sa_sigaction == (void*)1 when SA_SIGINFO happens to be set — calling it - // would jump to address 0x1. - if (prev.sa_sigaction != nullptr - && prev.sa_sigaction != reinterpret_cast(SIG_IGN)) { - prev.sa_sigaction(signo, siginfo, ucontext); - } - } else if (prev.sa_handler != SIG_DFL && prev.sa_handler != SIG_IGN - && prev.sa_handler != nullptr) { - // Chain to a 1-arg handler. Note: nullptr check is a safety net for - // uninitialised previous action (not a POSIX value). - // - // SIG_DFL / SIG_IGN fall through with no chain call: - // - SIG_IGN: no handler to run — skipping matches the kernel's - // ignore semantics. - // - SIG_DFL: we do NOT reproduce the kernel's default action - // (which for SIGPROF/SIGVTALRM is termination). A foreign - // SIGPROF arriving when the prior state was SIG_DFL is simply - // dropped. Reproducing the default would kill the process on - // every foreign signal — strictly worse for the Go-ITIMER_PROF - // scenario the classifier is designed to handle. - prev.sa_handler(signo); - } - - if (need_mask) { - syscall(__NR_rt_sigprocmask, SIG_SETMASK, &saved_mask, nullptr, _NSIG / 8); - } - errno = saved_errno; -} - -bool OS::signalOriginCheckEnabled() { - return static_cast( - __atomic_load_n(reinterpret_cast(&s_origin_check_state), __ATOMIC_ACQUIRE)) - == OriginCheckState::ENABLED; -} - -// Parse a boolean env-var value. Returns `default_value` for unset (null) and -// for values that trim to empty. Logs a warning and returns `default_value` -// when the value is set but not one of the documented spellings so operators -// don't silently get the default. -// Expected spellings are case-insensitive and trimmed of surrounding whitespace. -static bool parseBoolEnv(const char* name, const char* v, bool default_value) { - if (v == nullptr) { - return default_value; - } - // Trim leading and trailing whitespace (users who typed "=1 " or "\t1\n"). - const char* start = v; - while (*start == ' ' || *start == '\t' || *start == '\n' || - *start == '\r' || *start == '\f' || *start == '\v') { - ++start; - } - if (*start == '\0') { - return default_value; - } - const char* end = start + strlen(start); - while (end > start && (end[-1] == ' ' || end[-1] == '\t' || end[-1] == '\n' || - end[-1] == '\r' || end[-1] == '\f' || end[-1] == '\v')) { - --end; - } - size_t len = (size_t)(end - start); - if ((len == 5 && strncasecmp(start, "false", 5) == 0) || - (len == 1 && strncasecmp(start, "0", 1) == 0) || - (len == 3 && strncasecmp(start, "off", 3) == 0) || - (len == 2 && strncasecmp(start, "no", 2) == 0)) { - return false; - } - if ((len == 4 && strncasecmp(start, "true", 4) == 0) || - (len == 1 && strncasecmp(start, "1", 1) == 0) || - (len == 2 && strncasecmp(start, "on", 2) == 0) || - (len == 3 && strncasecmp(start, "yes", 3) == 0)) { - return true; - } - Log::warn("%s has unrecognised value %s; using default (%s). " - "Expected one of: true/false/1/0/on/off/yes/no.", - name, v, default_value ? "enabled" : "disabled"); - return default_value; -} - -void OS::primeSignalOriginCheck(bool forceReload) { - // Acquire-load on state pairs with the release-store below. This - // matters for forceReload=true paths in tests that concurrently write - // the flag from non-signal context. - if (static_cast( - __atomic_load_n(reinterpret_cast(&s_origin_check_state), __ATOMIC_ACQUIRE)) - != OriginCheckState::UNPRIMED && !forceReload) { - return; - } - // Default ON — reject foreign signals. - bool enabled = parseBoolEnv("DDPROF_SIGNAL_ORIGIN_CHECK", - getenv("DDPROF_SIGNAL_ORIGIN_CHECK"), - /*default=*/true); - // Default OFF — slow chain path is opt-in. See s_forward_apply_sigmask. - bool apply_mask = parseBoolEnv("DDPROF_FORWARD_APPLY_SIGMASK", - getenv("DDPROF_FORWARD_APPLY_SIGMASK"), - /*default=*/false); - - // Both stores use __ATOMIC_RELEASE; readers use __ATOMIC_ACQUIRE. - // This release-acquire pairing guarantees readers that observe the - // published state see the associated flag values. The ordering also covers - // the forceReload=true test path — callers must still quiesce signal - // delivery before calling with forceReload=true to avoid a torn read. - __atomic_store_n(&s_forward_apply_sigmask, apply_mask, __ATOMIC_RELEASE); - __atomic_store_n(reinterpret_cast(&s_origin_check_state), - static_cast(enabled ? OriginCheckState::ENABLED : OriginCheckState::DISABLED), - __ATOMIC_RELEASE); -} - -void* OS::safeAlloc(size_t size) { - // Naked syscall can be used inside a signal handler. - // Also, we don't want to catch our own calls when profiling mmap. - intptr_t result = syscall(MMAP_SYSCALL, NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (result < 0 && result > -4096) { - return NULL; - } - return (void*)result; -} - -void OS::safeFree(void* addr, size_t size) { - syscall(__NR_munmap, addr, size); -} - -bool OS::getCpuDescription(char* buf, size_t size) { - int fd = open("/proc/cpuinfo", O_RDONLY); - if (fd == -1) { - return false; - } - - ssize_t r = read(fd, buf, size); - close(fd); - if (r <= 0) { - return false; - } - buf[static_cast(r) < size ? r : size - 1] = 0; - - char* c; - do { - c = strchr(buf, '\n'); - } while (c != NULL && *(buf = c + 1) != '\n'); - - *buf = 0; - return true; -} - -int OS::getCpuCount() { - return sysconf(_SC_NPROCESSORS_ONLN); -} - -u64 OS::getProcessCpuTime(u64* utime, u64* stime) { - struct tms buf; - clock_t real = times(&buf); - *utime = buf.tms_utime; - *stime = buf.tms_stime; - return real; -} - -u64 OS::getTotalCpuTime(u64* utime, u64* stime) { - int fd = open("/proc/stat", O_RDONLY); - if (fd == -1) { - return (u64)-1; - } - - u64 real = (u64)-1; - char buf[128] = {0}; - if (read(fd, buf, sizeof(buf)) >= 12) { - u64 user, nice, system, idle; - if (sscanf(buf + 4, "%llu %llu %llu %llu", &user, &nice, &system, &idle) == 4) { - *utime = user + nice; - *stime = system; - real = user + nice + system + idle; - } - } - - close(fd); - return real; -} - -int OS::createMemoryFile(const char* name) { - return syscall(__NR_memfd_create, name, 0); -} - -void OS::copyFile(int src_fd, int dst_fd, off_t offset, size_t size) { - // copy_file_range() is probably better, but not supported on all kernels - while (size > 0) { - ssize_t bytes = sendfile(dst_fd, src_fd, &offset, size); - if (bytes <= 0) { - break; - } - size -= (size_t)bytes; - } -} - -void OS::freePageCache(int fd, off_t start_offset) { - posix_fadvise(fd, start_offset & ~page_mask, 0, POSIX_FADV_DONTNEED); -} - -int OS::mprotect(void* addr, size_t size, int prot) { - return ::mprotect(addr, size, prot); -} - -static int checkPreloadedCallback(dl_phdr_info* info, size_t size, void* data) { - Dl_info* dl_info = (Dl_info*)data; - - Dl_info libprofiler = dl_info[0]; - Dl_info libc = dl_info[1]; - - if ((void*)info->dlpi_addr == libprofiler.dli_fbase) { - // async-profiler found first - return 1; - } else if ((void*)info->dlpi_addr == libc.dli_fbase) { - // libc found first - return -1; - } - - return 0; -} - -// Checks if async-profiler is preloaded through the LD_PRELOAD mechanism. -// This is done by analyzing the order of loaded dynamic libraries. -bool OS::checkPreloaded() { - if (getenv("LD_PRELOAD") == NULL) { - return false; - } - - // Find async-profiler shared object - Dl_info libprofiler; - if (dladdr((const void*)OS::checkPreloaded, &libprofiler) == 0) { - return false; - } - - // Find libc shared object - Dl_info libc; - if (dladdr((const void*)exit, &libc) == 0) { - return false; - } - - Dl_info info[2] = {libprofiler, libc}; - return dl_iterate_phdr(checkPreloadedCallback, (void*)info) == 1; -} - -u64 OS::getRamSize() { - static u64 mem_total = 0; - - if (mem_total == 0) { - FILE* file = fopen("/proc/meminfo", "r"); - if (!file) return 0; - - char line[1024]; - while (fgets(line, sizeof(line), file)) { - if (strncmp(line, "MemTotal:", 9) == 0) { - mem_total = strtoull(line + 9, NULL, 10) * 1024; - break; - } - } - - fclose(file); - } - - return mem_total; -} - -u64 OS::getSystemBootTime() { - static u64 system_boot_time = 0; - - if (system_boot_time == 0) { - FILE* file = fopen("/proc/stat", "r"); - if (!file) return 0; - - char line[1024]; - while (fgets(line, sizeof(line), file)) { - if (strncmp(line, "btime", 5) == 0) { - system_boot_time = strtoull(line + 5, NULL, 10); - break; - } - } - - fclose(file); - } - - return system_boot_time; -} - -int OS::getProcessIds(int* pids, int max_pids) { - int count = 0; - DIR* proc = opendir("/proc"); - if (!proc) return 0; - - for (dirent* de; (de = readdir(proc)) && count < max_pids;) { - int pid = atoi(de->d_name); - if (pid > 0) { - pids[count++] = pid; - } - } - - closedir(proc); - return count; -} - -static bool readProcessCmdline(int pid, ProcessInfo* info) { - char path[64]; - snprintf(path, sizeof(path), "/proc/%d/cmdline", pid); - - int fd = open(path, O_RDONLY); - if (fd == -1) { - return false; - } - - const size_t max_read = sizeof(info->cmdline) - 1; - size_t len = 0; - - ssize_t r; - while ((r = read(fd, info->cmdline + len, max_read - len))) { - if (r > 0) { - len += (size_t)r; - if (len == max_read) break; - } else { - if (errno == EINTR) continue; - close(fd); - return false; - } - } - - close(fd); - - // Replace null bytes with spaces (arguments are separated by null bytes) - for (size_t i = 0; i < len; i++) { - if (info->cmdline[i] == '\0') { - info->cmdline[i] = ' '; - } - } - - // Ensure null termination - info->cmdline[len] = '\0'; - - // Remove trailing space if present - while (len > 0 && info->cmdline[len - 1] == ' ') { - info->cmdline[--len] = '\0'; - } - - return true; -} - -static bool readProcessStats(int pid, ProcessInfo* info) { - char path[64]; - snprintf(path, sizeof(path), "/proc/%d/stat", pid); - - int fd = open(path, O_RDONLY); - if (fd == -1) return false; - - char buffer[4096]; - size_t len = 0; - - ssize_t r; - while ((r = read(fd, buffer + len, sizeof(buffer) - 1 - len))) { - if (r > 0) { - len += (size_t)r; - if (len == sizeof(buffer) - 1) break; - } else { - if (errno == EINTR) continue; - close(fd); - return false; - } - } - close(fd); - - if (len == 0) return false; - buffer[len] = '\0'; - - int parsed_pid, ppid; - char comm[COMM_LEN] = {0}; - char state; - u64 minflt, majflt, utime, stime; - u64 starttime; - u64 vsize, rss; - int threads; - int parsed = - sscanf(buffer, - "%d " /* 1 pid */ - "(%15[^)]) " /* 2 comm (read until ')') */ - "%c %d " /* 3 state, 4 ppid */ - "%*d %*d %*d %*d %*u " /* 5-9 skip */ - "%llu %*u %llu %*u " /* 10-13 minflt,-,majflt,- */ - "%llu %llu " /* 14-15 utime, stime */ - "%*d %*d %*d %*d " /* 16-19 skip */ - "%d " /* 20 threads */ - "%*d " /* 21 skip */ - "%llu " /* 22 starttime */ - "%llu " /* 23 vsize */ - "%llu", /* 24 rss */ - &parsed_pid, comm, &state, &ppid, &minflt, &majflt, &utime, &stime, &threads, &starttime, &vsize, &rss); - - if (parsed < 12) return false; - - memcpy(info->name, comm, COMM_LEN); - info->pid = parsed_pid; - info->ppid = ppid; - info->state = (unsigned char)state; - info->minor_faults = minflt; - info->major_faults = majflt; - info->cpu_user = (float)utime / OS::clock_ticks_per_sec; - info->cpu_system = (float)stime / OS::clock_ticks_per_sec; - info->threads = threads; - info->vm_size = vsize; - // (24) rss - convert from number of pages to bytes - info->vm_rss = rss * OS::page_size; - info->start_time = (OS::getSystemBootTime() + starttime / OS::clock_ticks_per_sec) * 1000; - return true; -} - -static bool readProcessStatus(int pid, ProcessInfo* info) { - char path[64]; - snprintf(path, sizeof(path), "/proc/%d/status", pid); - FILE* file = fopen(path, "r"); - if (!file) { - return false; - } - - int read_count = 0; - char line[1024]; - char key[32]; - u64 value; - while (fgets(line, sizeof(line), file) && read_count < 6) { - if (sscanf(line, "%31s %llu", key, &value) != 2) { - continue; - } - - if (strncmp(key, "Uid", 3) == 0) { - read_count++; - info->uid = (unsigned int)value; - } else if (strncmp(key, "RssAnon", 7) == 0) { - read_count++; - info->rss_anon = value * 1024; - } else if (strncmp(key, "RssFile", 7) == 0) { - read_count++; - info->rss_files = value * 1024; - } else if (strncmp(key, "RssShmem", 8) == 0) { - read_count++; - info->rss_shmem = value * 1024; - } else if (strncmp(key, "VmSize", 6) == 0) { - read_count++; - info->vm_size = value * 1024; - } else if (strncmp(key, "VmRSS", 5) == 0) { - read_count++; - info->vm_rss = value * 1024; - } - } - - fclose(file); - return true; -} - -static bool readProcessIO(int pid, ProcessInfo* info) { - char path[64]; - snprintf(path, sizeof(path), "/proc/%d/io", pid); - FILE* file = fopen(path, "r"); - if (!file) return false; - - int read_count = 0; - char line[1024]; - while (fgets(line, sizeof(line), file) && read_count < 2) { - if (strncmp(line, "read_bytes:", 11) == 0) { - u64 read_bytes = strtoull(line + 11, NULL, 10); - info->io_read = read_bytes >> 10; - read_count++; - } else if (strncmp(line, "write_bytes:", 12) == 0) { - u64 write_bytes = strtoull(line + 12, NULL, 10); - info->io_write = write_bytes >> 10; - read_count++; - } - } - - fclose(file); - return true; -} - -bool OS::getBasicProcessInfo(int pid, ProcessInfo* info) { - return readProcessStats(pid, info); -} - -bool OS::getDetailedProcessInfo(ProcessInfo* info) { - readProcessStatus(info->pid, info); - readProcessIO(info->pid, info); - readProcessCmdline(info->pid, info); - return true; -} - -// DataDog-specific implementations - -int OS::truncateFile(int fd) { - int rslt = ftruncate(fd, 0); - if (rslt == 0) { - return lseek(fd, 0, SEEK_SET); - } - return rslt; -} - -void OS::mallocArenaMax(int arena_max) { -#ifndef __musl__ - mallopt(M_ARENA_MAX, arena_max); -#endif -} - -SigAction OS::replaceSigsegvHandler(SigAction action) { - return OS::replaceCrashHandler(action); -} - -SigAction OS::replaceSigbusHandler(SigAction action) { - struct sigaction sa; - sigaction(SIGBUS, NULL, &sa); - SigAction old_action = sa.sa_sigaction; - sa.sa_sigaction = action; - sigaction(SIGBUS, &sa, NULL); - return old_action; -} - -// ============================================================================ -// sigaction interposition to prevent other libraries from overwriting our -// SIGSEGV/SIGBUS handlers. This is needed because libraries like wasmtime -// install broken signal handlers that call malloc() (not async-signal-safe). -// ============================================================================ - -// Our protected handlers and their chain targets -static SigAction _protected_segv_handler = nullptr; -static SigAction _protected_bus_handler = nullptr; -static volatile SigAction _segv_chain_target = nullptr; -static volatile SigAction _bus_chain_target = nullptr; - -// Original handlers (JVM's) saved before we install ours - used for oldact in sigaction hook -static struct sigaction _orig_segv_sigaction; -static struct sigaction _orig_bus_sigaction; -static bool _orig_handlers_saved = false; - -// Real sigaction function pointer (resolved via dlsym) -typedef int (*real_sigaction_t)(int, const struct sigaction*, struct sigaction*); -static real_sigaction_t _real_sigaction = nullptr; - -void OS::protectSignalHandlers(SigAction segvHandler, SigAction busHandler) { - // Resolve real sigaction BEFORE enabling protection, while we can still use RTLD_DEFAULT - if (_real_sigaction == nullptr) { - _real_sigaction = (real_sigaction_t)dlsym(RTLD_DEFAULT, "sigaction"); - } - // Save the current (JVM's) signal handlers BEFORE we install ours. - // These will be returned as oldact when we intercept other libraries' sigaction calls, - // so they chain to JVM instead of back to us (which would cause infinite loops). - if (!__atomic_load_n(&_orig_handlers_saved, __ATOMIC_ACQUIRE) && _real_sigaction != nullptr) { - _real_sigaction(SIGSEGV, nullptr, &_orig_segv_sigaction); - _real_sigaction(SIGBUS, nullptr, &_orig_bus_sigaction); - __atomic_store_n(&_orig_handlers_saved, true, __ATOMIC_RELEASE); - } - _protected_segv_handler = segvHandler; - _protected_bus_handler = busHandler; -} - -SigAction OS::getSegvChainTarget() { - return __atomic_load_n(&_segv_chain_target, __ATOMIC_ACQUIRE); -} - -SigAction OS::getBusChainTarget() { - return __atomic_load_n(&_bus_chain_target, __ATOMIC_ACQUIRE); -} - -// sigaction_hook - intercepts sigaction(2) calls from any library via GOT patching. -// -// PROBLEM SOLVED -// ============== -// Without interception, a library (e.g. wasmtime) can overwrite our SIGSEGV handler: -// -// Before: kernel --> our_handler --> JVM_handler -// After lib calls sigaction(SIGSEGV, lib_handler, &oldact): -// kernel --> lib_handler -// lib_handler stores oldact = our_handler as its chain target -// => when lib chains on unhandled fault: lib_handler --> our_handler --> lib_handler --> ... -// INFINITE LOOP -// -// HANDLER CHAIN AFTER SETUP -// ========================== -// -// protectSignalHandlers() replaceSigsegvHandler() LibraryPatcher::patch_sigaction() -// | | | -// v v v -// save JVM handler install our_handler GOT-patch sigaction -// into _orig_segv_sigaction as real OS handler => all future sigaction() -// calls go through us -// -// Signal delivery chain: -// -// kernel -// | -// v -// our_handler (installed via replaceSigsegvHandler, never displaced) -// | -// +-- handled by us? --> done -// | -// v (not handled) -// _segv_chain_target (lib_handler, set when we intercepted lib's sigaction call) -// | -// +-- handled by lib? --> done -// | -// v (lib chains to its saved oldact) -// _orig_segv_sigaction (JVM's original handler, what we returned as oldact to lib) -// | -// v -// JVM handles or terminates -// -// INTERCEPTION LOGIC (this function) -// =================================== -// Case 1 - Install call [act != nullptr, SA_SIGINFO]: -// - Save lib's handler as _segv_chain_target (we'll call it if we can't handle) -// - Return _orig_segv_sigaction as oldact (NOT our handler, to break the loop) -// - Do NOT actually install lib's handler (keep ours on top) -// -// Case 2 - Query-only call [act == nullptr, oldact != nullptr]: -// - Return _orig_segv_sigaction as oldact (same reason: lib must not see our handler) -// - A lib that queries, stores the result, then uses it as a chain target would -// loop if we returned our handler here. -// -// Case 3 - 1-arg handler [act != nullptr, no SA_SIGINFO]: -// - Pass through: we cannot safely chain 1-arg handlers (different calling convention) -// -// Case 4 - Any other signal, or protection not yet active: -// - Pass through to real sigaction unchanged. -// -static int sigaction_hook(int signum, const struct sigaction* act, struct sigaction* oldact) { - // _real_sigaction must be resolved before any GOT patching happens - if (_real_sigaction == nullptr) { - errno = EFAULT; - return -1; - } - - // If this is SIGSEGV or SIGBUS and we have protected handlers installed, - // intercept the call to keep our handler on top. - // We intercept both install calls (act != nullptr) and query-only calls (act == nullptr) - // to ensure callers always see the JVM's original handler, never ours. - // A caller that gets our handler as oldact and later chains to it would cause an - // infinite loop: us -> them -> us -> ... - if (signum == SIGSEGV && _protected_segv_handler != nullptr) { - if (act != nullptr) { - // Install call: only intercept SA_SIGINFO handlers (3-arg form) for safe chaining - if (act->sa_flags & SA_SIGINFO) { - SigAction new_handler = act->sa_sigaction; - // Don't intercept if it's our own handler being installed - if (new_handler != _protected_segv_handler) { - // Save their handler as our chain target - __atomic_exchange_n(&_segv_chain_target, new_handler, __ATOMIC_ACQ_REL); - if (oldact != nullptr) { - // Return the original (JVM's) handler, not ours, to prevent - // the caller from chaining back to us. - *oldact = _orig_segv_sigaction; - } - Counters::increment(SIGACTION_INTERCEPTED); - // Don't actually install their handler - keep ours on top - return 0; - } - } - // Let 1-arg handlers (without SA_SIGINFO) pass through - we can't safely chain them - } else if (oldact != nullptr) { - // Query-only call: return the JVM's original handler, not ours. - // Same reason: a caller that stores our handler and later chains to it causes loops. - *oldact = _orig_segv_sigaction; - return 0; - } - } else if (signum == SIGBUS && _protected_bus_handler != nullptr) { - if (act != nullptr) { - if (act->sa_flags & SA_SIGINFO) { - SigAction new_handler = act->sa_sigaction; - if (new_handler != _protected_bus_handler) { - __atomic_exchange_n(&_bus_chain_target, new_handler, __ATOMIC_ACQ_REL); - if (oldact != nullptr) { - *oldact = _orig_bus_sigaction; - } - Counters::increment(SIGACTION_INTERCEPTED); - return 0; - } - } - } else if (oldact != nullptr) { - *oldact = _orig_bus_sigaction; - return 0; - } - } - - // For all other cases, pass through to real sigaction - return _real_sigaction(signum, act, oldact); -} - -void* OS::getSigactionHook() { - return (void*)sigaction_hook; -} - -void OS::resetSignalHandlersForTesting() { - __atomic_store_n(&_orig_handlers_saved, false, __ATOMIC_RELEASE); - memset(&_orig_segv_sigaction, 0, sizeof(_orig_segv_sigaction)); - memset(&_orig_bus_sigaction, 0, sizeof(_orig_bus_sigaction)); - _protected_segv_handler = nullptr; - _protected_bus_handler = nullptr; - __atomic_store_n(&_segv_chain_target, (SigAction)nullptr, __ATOMIC_RELEASE); - __atomic_store_n(&_bus_chain_target, (SigAction)nullptr, __ATOMIC_RELEASE); - - // Clear the foreign-signal forwarding cache so tests that install and - // verify chaining start from a clean slate. Hold the publish mutex so a - // concurrent installSignalHandler cannot race with us during the reset. - pthread_mutex_lock(&installed_oldaction_mutex); - for (int i = 0; i < MAX_SIGNALS; ++i) { - __atomic_store_n(&installed_oldaction_valid[i], false, __ATOMIC_RELEASE); - memset(&installed_oldaction[i], 0, sizeof(installed_oldaction[i])); - installed_sigaction[i] = nullptr; - } - pthread_mutex_unlock(&installed_oldaction_mutex); - // Full memory barrier: ensure any in-flight signal handlers on other CPUs - // observe the cleared valid flags before resetSignalHandlersForTesting returns. - __atomic_thread_fence(__ATOMIC_SEQ_CST); - - // _real_sigaction is intentionally not reset: safe to reuse across tests -} - -#endif // __linux__ diff --git a/ddprof-lib/src/main/cpp/os_macos.cpp b/ddprof-lib/src/main/cpp/os_macos.cpp deleted file mode 100644 index 0aee02704..000000000 --- a/ddprof-lib/src/main/cpp/os_macos.cpp +++ /dev/null @@ -1,549 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifdef __APPLE__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" -#include "os.h" - - -class MacThreadList : public ThreadList { - private: - task_t _task; - thread_array_t _thread_array; - - void deallocate() { - if (_thread_array != NULL) { - for (u32 i = 0; i < _count; i++) { - mach_port_deallocate(_task, _thread_array[i]); - } - vm_deallocate(_task, (vm_address_t)_thread_array, _count * sizeof(thread_t)); - _thread_array = NULL; - } - } - - public: - MacThreadList() { - _task = mach_task_self(); - _thread_array = NULL; - task_threads(_task, &_thread_array, &_count); - } - - ~MacThreadList() { - deallocate(); - } - - int next() { - return (int)_thread_array[_index++]; - } - - void update() { - deallocate(); - _index = _count = 0; - task_threads(_task, &_thread_array, &_count); - } -}; - - -JitWriteProtection::JitWriteProtection(bool enable) { -#ifdef __aarch64__ - // Mimic pthread_jit_write_protect_np(), but save the previous state - if (*(volatile char*)0xfffffc10c) { - u64 val = enable ? *(volatile u64*)0xfffffc118 : *(volatile u64*)0xfffffc110; - u64 prev; - asm volatile("mrs %0, s3_6_c15_c1_5" : "=r" (prev) : : ); - if (prev != val) { - _prev = prev; - _restore = true; - asm volatile("msr s3_6_c15_c1_5, %0\n" - "isb" - : "+r" (val) : : "memory"); - return; - } - } - // Already in the required mode, or write protection is not supported - _restore = false; -#endif -} - -JitWriteProtection::~JitWriteProtection() { -#ifdef __aarch64__ - if (_restore) { - u64 prev = _prev; - asm volatile("msr s3_6_c15_c1_5, %0\n" - "isb" - : "+r" (prev) : : "memory"); - } -#endif -} - - -static SigAction installed_sigaction[32]; -static SigAction orig_sigbus_handler; -static SigAction orig_sigsegv_handler; - -const size_t OS::page_size = sysconf(_SC_PAGESIZE); -const size_t OS::page_mask = OS::page_size - 1; -const long OS::clock_ticks_per_sec = sysconf(_SC_CLK_TCK); - -static mach_timebase_info_data_t timebase = {0, 0}; - -u64 OS::nanotime() { - if (timebase.denom == 0) { - mach_timebase_info(&timebase); - } - return (u64)mach_absolute_time() * timebase.numer / timebase.denom; -} - -u64 OS::micros() { - struct timeval tv; - gettimeofday(&tv, NULL); - return (u64)tv.tv_sec * 1000000 + tv.tv_usec; -} - -void OS::sleep(u64 nanos) { - struct timespec ts = {(time_t)(nanos / 1000000000), (long)(nanos % 1000000000)}; - nanosleep(&ts, NULL); -} - -void OS::sleepWhile(u64 max_nanos, std::atomic& keep_sleeping) { - // macOS does not expose clock_nanosleep(TIMER_ABSTIME). Recompute the - // remaining interval against OS::nanotime() each iteration so spurious - // wake-ups don't shorten the wait, mirroring the Linux semantics. - u64 deadline = OS::nanotime() + max_nanos; - while (keep_sleeping.load(std::memory_order_acquire)) { - u64 now = OS::nanotime(); - if (now >= deadline) { - return; - } - u64 remaining = deadline - now; - struct timespec ts = {(time_t)(remaining / 1000000000ULL), (long)(remaining % 1000000000ULL)}; - nanosleep(&ts, nullptr); - } -} - -u64 OS::overrun(siginfo_t* siginfo) { - return 0; -} - -u64 OS::processStartTime() { - static u64 start_time = 0; - - if (start_time == 0) { - struct proc_bsdinfo info; - if (proc_pidinfo(processId(), PROC_PIDTBSDINFO, 0, &info, sizeof(info)) > 0) { - start_time = (u64)info.pbi_start_tvsec * 1000 + info.pbi_start_tvusec / 1000; - } - } - - return start_time; -} - -u64 OS::hton64(u64 x) { - return OSSwapHostToBigInt64(x); -} - -u64 OS::ntoh64(u64 x) { - return OSSwapBigToHostInt64(x); -} - -int OS::getMaxThreadId() { - return 0x7fffffff; -} - -int OS::processId() { - static const int self_pid = getpid(); - - return self_pid; -} - -int OS::threadId() { - // Used to be pthread_mach_thread_np(pthread_self()), - // but pthread_mach_thread_np is not async signal safe - mach_port_t port = mach_thread_self(); - mach_port_deallocate(mach_task_self(), port); - return (int)port; -} - -const char* OS::schedPolicy(int thread_id) { - // Not used on macOS - return "SCHED_OTHER"; -} - -bool OS::threadName(int thread_id, char* name_buf, size_t name_len) { - pthread_t thread = pthread_from_mach_thread_np(thread_id); - return thread && pthread_getname_np(thread, name_buf, name_len) == 0 && name_buf[0] != 0; -} - -ThreadState OS::threadState(int thread_id) { - struct thread_basic_info info; - mach_msg_type_number_t size = sizeof(info); - if (thread_info((thread_act_t)thread_id, THREAD_BASIC_INFO, (thread_info_t)&info, &size) != 0) { - return THREAD_UNKNOWN; - } - return info.run_state == TH_STATE_RUNNING ? THREAD_RUNNING : THREAD_SLEEPING; -} - -u64 OS::threadCpuTime(int thread_id) { - if (thread_id == 0) thread_id = threadId(); - - struct thread_basic_info info; - mach_msg_type_number_t size = sizeof(info); - if (thread_info((thread_act_t)thread_id, THREAD_BASIC_INFO, (thread_info_t)&info, &size) != 0) { - return 0; - } - return u64(info.user_time.seconds + info.system_time.seconds) * 1000000000 + - u64(info.user_time.microseconds + info.system_time.microseconds) * 1000; -} - -ThreadList* OS::listThreads() { - return new MacThreadList(); -} - -bool OS::isLinux() { - return false; -} - -bool OS::isMusl() { - return false; -} - -SigAction OS::installSignalHandler(int signo, SigAction action, SigHandler handler) { - struct sigaction sa; - struct sigaction oldsa; - sigemptyset(&sa.sa_mask); - - if (handler != NULL) { - sa.sa_handler = handler; - sa.sa_flags = 0; - } else { - sa.sa_sigaction = action; - sa.sa_flags = SA_SIGINFO | SA_RESTART; - if (signo > 0 && signo < sizeof(installed_sigaction) / sizeof(installed_sigaction[0])) { - installed_sigaction[signo] = action; - } - } - - sigaction(signo, &sa, &oldsa); - return oldsa.sa_sigaction; -} - -static void restoreSignalHandler(int signo, siginfo_t* siginfo, void* ucontext) { - signal(signo, SIG_DFL); -} - -SigAction OS::replaceCrashHandler(SigAction action) { - // It is not well specified when macOS raises SIGBUS and when SIGSEGV. - // HotSpot handles both similarly, so do we. - struct sigaction sa; - - sigaction(SIGBUS, NULL, &sa); - orig_sigbus_handler = sa.sa_handler == SIG_DFL ? restoreSignalHandler : sa.sa_sigaction; - sigemptyset(&sa.sa_mask); - sa.sa_sigaction = action; - sa.sa_flags |= SA_SIGINFO | SA_RESTART | SA_NODEFER; - sigaction(SIGBUS, &sa, NULL); - - sigaction(SIGSEGV, NULL, &sa); - orig_sigsegv_handler = sa.sa_handler == SIG_DFL ? restoreSignalHandler : sa.sa_sigaction; - sigemptyset(&sa.sa_mask); - sa.sa_sigaction = action; - sa.sa_flags |= SA_SIGINFO | SA_RESTART| SA_NODEFER; - sigaction(SIGSEGV, &sa, NULL); - - // Return an action that dispatches to one of the original handlers depending on signo, - // so that the caller does not need to deal with multiple handlers - return [](int signo, siginfo_t* siginfo, void* ucontext) { - (signo == SIGBUS ? orig_sigbus_handler : orig_sigsegv_handler)(signo, siginfo, ucontext); - }; -} - -int OS::getProfilingSignal(int mode) { - static int preferred_signals[2] = {SIGPROF, SIGVTALRM}; - - const u64 allowed_signals = - 1ULL << SIGPROF | 1ULL << SIGVTALRM | 1ULL << SIGEMT | 1ULL << SIGSYS; - - int& signo = preferred_signals[mode]; - int initial_signo = signo; - int other_signo = preferred_signals[1 - mode]; - - do { - struct sigaction sa; - if ((allowed_signals & (1ULL << signo)) != 0 && signo != other_signo && sigaction(signo, NULL, &sa) == 0) { - if (sa.sa_handler == SIG_DFL || sa.sa_handler == SIG_IGN || sa.sa_sigaction == installed_sigaction[signo]) { - return signo; - } - } - } while ((signo = (signo + 1) & 31) != initial_signo); - - return signo; -} - -bool OS::sendSignalToThread(int thread_id, int signo) { -#ifdef __aarch64__ - register long x0 asm("x0") = thread_id; - register long x1 asm("x1") = signo; - register long x16 asm("x16") = 328; - asm volatile("svc #0x80" - : "+r" (x0) - : "r" (x1), "r" (x16) - : "memory"); - return x0 == 0; -#else - int result; - asm volatile("syscall" - : "=a" (result) - : "a" (0x2000148), "D" (thread_id), "S" (signo) - : "rcx", "r11", "memory"); - return result == 0; -#endif -} - -// macOS does not expose rt_tgsigqueueinfo. Fall back to plain per-thread -// delivery without a payload. The Go-setitimer deadlock the origin check -// defends against is Linux-specific, so macOS can keep the pre-fix behaviour. -bool OS::sendSignalWithCookie(int thread_id, int signo, void* /*cookie*/) { - return sendSignalToThread(thread_id, signo); -} - -// On macOS, signalOriginCheckEnabled() returns false (feature off) and -// shouldProcessSignal() returns true (accept-all). This is consistent: -// no rt_tgsigqueueinfo means no cookie discrimination is possible, so -// the handler degrades to pre-fix accept-all behaviour. callers must not -// use signalOriginCheckEnabled() to infer rejection statistics on macOS. -// On macOS the origin-check helper degrades to "accept everything" since the -// fallback sender cannot carry a cookie. Handlers that call this helper -// therefore behave exactly as before on macOS. -bool OS::shouldProcessSignal(siginfo_t* /*siginfo*/, int /*expected_si_code*/, void* /*expected_cookie*/) { - return true; -} - -void OS::forwardForeignSignal(int /*signo*/, siginfo_t* /*siginfo*/, void* /*ucontext*/) { - // No-op on macOS — see comment above. -} - -bool OS::signalOriginCheckEnabled() { - return false; -} - -void OS::primeSignalOriginCheck(bool /*forceReload*/) { - // No-op on macOS. -} - -void* OS::safeAlloc(size_t size) { - // mmap() is not guaranteed to be async signal safe, but in practice, it is. - // There is no a reasonable alternative anyway. - void* result = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - if (result == MAP_FAILED) { - return NULL; - } - return result; -} - -void OS::safeFree(void* addr, size_t size) { - munmap(addr, size); -} - -bool OS::getCpuDescription(char* buf, size_t size) { - return sysctlbyname("machdep.cpu.brand_string", buf, &size, NULL, 0) == 0; -} - -int OS::getCpuCount() { - int cpu_count; - size_t size = sizeof(cpu_count); - return sysctlbyname("hw.logicalcpu", &cpu_count, &size, NULL, 0) == 0 ? cpu_count : 1; -} - -u64 OS::getProcessCpuTime(u64* utime, u64* stime) { - struct tms buf; - clock_t real = times(&buf); - *utime = buf.tms_utime; - *stime = buf.tms_stime; - return real; -} - -u64 OS::getTotalCpuTime(u64* utime, u64* stime) { - natural_t cpu_count; - processor_info_array_t cpu_info_array; - mach_msg_type_number_t cpu_info_count; - - host_name_port_t host = mach_host_self(); - kern_return_t ret = host_processor_info(host, PROCESSOR_CPU_LOAD_INFO, &cpu_count, &cpu_info_array, &cpu_info_count); - mach_port_deallocate(mach_task_self(), host); - if (ret != 0) { - return (u64)-1; - } - - processor_cpu_load_info_data_t* cpu_load = (processor_cpu_load_info_data_t*)cpu_info_array; - u64 user = 0; - u64 system = 0; - u64 idle = 0; - for (natural_t i = 0; i < cpu_count; i++) { - user += cpu_load[i].cpu_ticks[CPU_STATE_USER] + cpu_load[i].cpu_ticks[CPU_STATE_NICE]; - system += cpu_load[i].cpu_ticks[CPU_STATE_SYSTEM]; - idle += cpu_load[i].cpu_ticks[CPU_STATE_IDLE]; - } - vm_deallocate(mach_task_self(), (vm_address_t)cpu_info_array, cpu_info_count * sizeof(int)); - - *utime = user; - *stime = system; - return user + system + idle; -} - -int OS::createMemoryFile(const char* name) { - // Not supported on macOS - return -1; -} - -void OS::copyFile(int src_fd, int dst_fd, off_t offset, size_t size) { - char* buf = (char*)mmap(NULL, size + offset, PROT_READ, MAP_PRIVATE, src_fd, 0); - if (buf == NULL) { - return; - } - - while (size > 0) { - ssize_t bytes = write(dst_fd, buf + offset, size < 262144 ? size : 262144); - if (bytes <= 0) { - break; - } - offset += (size_t)bytes; - size -= (size_t)bytes; - } - - munmap(buf, offset); -} - -void OS::freePageCache(int fd, off_t start_offset) { - // Not supported on macOS -} - -int OS::mprotect(void* addr, size_t size, int prot) { - if (prot & PROT_WRITE) prot |= VM_PROT_COPY; - return vm_protect(mach_task_self(), (vm_address_t)addr, size, 0, prot); -} - -// Checks if async-profiler is preloaded through the DYLD_INSERT_LIBRARIES mechanism. -// This is done by analyzing the order of loaded dynamic libraries. -bool OS::checkPreloaded() { - if (getenv("DYLD_INSERT_LIBRARIES") == NULL) { - return false; - } - - // Find async-profiler shared object - Dl_info libprofiler; - if (dladdr((const void*)OS::checkPreloaded, &libprofiler) == 0) { - return false; - } - - // Find libc shared object - Dl_info libc; - if (dladdr((const void*)exit, &libc) == 0) { - return false; - } - - uint32_t images = _dyld_image_count(); - for (uint32_t i = 0; i < images; i++) { - void* image_base = (void*)_dyld_get_image_header(i); - - if (image_base == libprofiler.dli_fbase) { - // async-profiler found first - return true; - } else if (image_base == libc.dli_fbase) { - // libc found first - return false; - } - } - - return false; -} - -u64 OS::getSystemBootTime() { - return 0; -} - -u64 OS::getRamSize() { - return 0; -} - -int OS::getProcessIds(int* pids, int max_pids) { - return 0; -} - -bool OS::getBasicProcessInfo(int pid, ProcessInfo* info) { - return false; -} - -bool OS::getDetailedProcessInfo(ProcessInfo* info) { - return false; -} - -// DataDog-specific implementations - -int OS::truncateFile(int fd) { - int rslt = ftruncate(fd, 0); - if (rslt == 0) { - return lseek(fd, 0, SEEK_SET); - } - return rslt; -} - -void OS::mallocArenaMax(int arena_max) { - // Not supported on macOS -} - -SigAction OS::replaceSigbusHandler(SigAction action) { - return OS::replaceCrashHandler(action); -} - -SigAction OS::replaceSigsegvHandler(SigAction action) { - struct sigaction sa; - sigaction(SIGSEGV, NULL, &sa); - SigAction old_action = sa.sa_sigaction; - sa.sa_sigaction = action; - sigaction(SIGSEGV, &sa, NULL); - return old_action; -} - -// No GOT-based sigaction interception on macOS — these are no-ops. -void OS::protectSignalHandlers(SigAction segvHandler, SigAction busHandler) { -} - -SigAction OS::getSegvChainTarget() { - return nullptr; -} - -SigAction OS::getBusChainTarget() { - return nullptr; -} - -void* OS::getSigactionHook() { - return nullptr; // No sigaction interception on macOS -} - -void OS::resetSignalHandlersForTesting() { - // No-op: no sigaction interception state on macOS -} - -#endif // __APPLE__ diff --git a/ddprof-lib/src/main/cpp/otel_context.cpp b/ddprof-lib/src/main/cpp/otel_context.cpp deleted file mode 100644 index a5085a84c..000000000 --- a/ddprof-lib/src/main/cpp/otel_context.cpp +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "otel_context.h" - -// OTEP #4947 TLS pointer — visible in dynsym for external profiler discovery -DLLEXPORT thread_local OtelThreadContextRecord* otel_thread_ctx_v1 = nullptr; diff --git a/ddprof-lib/src/main/cpp/otel_context.h b/ddprof-lib/src/main/cpp/otel_context.h deleted file mode 100644 index 82a6701b2..000000000 --- a/ddprof-lib/src/main/cpp/otel_context.h +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _OTEL_CONTEXT_H -#define _OTEL_CONTEXT_H - -#include "arch.h" -#include "asprof.h" -#include -#include - -// Max total record size including header -static const int OTEL_MAX_RECORD_SIZE = 640; -// Header: trace_id(16) + span_id(8) + valid(1) + reserved(1) + attrs_data_size(2) = 28 -static const int OTEL_HEADER_SIZE = 28; -// Max space for attribute data -static const int OTEL_MAX_ATTRS_DATA_SIZE = OTEL_MAX_RECORD_SIZE - OTEL_HEADER_SIZE; - -/** - * OTEP #4947-compliant Thread Local Context Record. - * - * 640-byte packed structure matching the OTEP specification layout: - * offset 0x00: trace_id[16] — W3C 128-bit trace ID (big-endian) - * offset 0x10: span_id[8] — 64-bit span ID (big-endian) - * offset 0x18: valid — 1 = record ready for reading - * offset 0x19: _reserved — OTEP spec reserved, must be 0 - * offset 0x1A: attrs_data_size — number of valid bytes in attrs_data - * offset 0x1C: attrs_data[612] — encoded key/value attribute pairs - * - * Each attribute in attrs_data: - * key_index: uint8 — index into process context's attribute_key_map - * length: uint8 — length of value string (max 255) - * value: uint8[length] — UTF-8 value bytes - * - * Discovery: external profilers find the TLS pointer - * otel_thread_ctx_v1 via ELF dynsym table. - */ -struct __attribute__((packed)) OtelThreadContextRecord { - uint8_t trace_id[16]; - uint8_t span_id[8]; - uint8_t valid; - uint8_t _reserved; // OTEP spec: reserved for future use, must be 0 - uint16_t attrs_data_size; - uint8_t attrs_data[OTEL_MAX_ATTRS_DATA_SIZE]; -}; -static_assert(sizeof(OtelThreadContextRecord) == OTEL_MAX_RECORD_SIZE, - "OtelThreadContextRecord size must match OTEL_MAX_RECORD_SIZE (640 bytes); " - "update the Java constant ThreadContext.OTEL_MAX_RECORD_SIZE if the struct changes"); - -// OTEP #4947 TLS pointer — MUST appear in dynsym for external profiler discovery -DLLEXPORT extern thread_local OtelThreadContextRecord* otel_thread_ctx_v1; - -/** - * OTEL context storage manager (OTEP #4947 TLS pointer model). - * - * Each thread gets a pre-allocated OtelThreadContextRecord cached in - * ProfiledThread. The TLS pointer otel_thread_ctx_v1 is set permanently - * to the record during thread initialization; detach/attach (context writes) - * never touch it. Readers must not assume the TLS pointer is cleared during - * teardown; record liveness is determined by the owning thread lifetime and - * the valid flag in the record. - * - * Signal safety: signal handlers must never access - * otel_thread_ctx_v1 directly (TLS lazy init can deadlock - * in musl). Instead they read via ProfiledThread::getOtelContextRecord(). - */ - -#endif /* _OTEL_CONTEXT_H */ diff --git a/ddprof-lib/src/main/cpp/otel_process_ctx.cpp b/ddprof-lib/src/main/cpp/otel_process_ctx.cpp deleted file mode 100644 index 00642e679..000000000 --- a/ddprof-lib/src/main/cpp/otel_process_ctx.cpp +++ /dev/null @@ -1,913 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License (Version 2.0). -// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. - -#include "otel_process_ctx.h" - -#ifndef _GNU_SOURCE - #define _GNU_SOURCE -#endif - -// Note: Things here are needed for NOOP. Things that are only for non-NOOP get added further below. - -#include - -#define ADD_QUOTES_HELPER(x) #x -#define ADD_QUOTES(x) ADD_QUOTES_HELPER(x) - -static const otel_process_ctx_data empty_data = { - .deployment_environment_name = NULL, - .service_instance_id = NULL, - .service_name = NULL, - .service_version = NULL, - .telemetry_sdk_language = NULL, - .telemetry_sdk_version = NULL, - .telemetry_sdk_name = NULL, - .resource_attributes = NULL, - .extra_attributes = NULL, - .thread_ctx_config = NULL -}; - -#if (defined(OTEL_PROCESS_CTX_NOOP) && OTEL_PROCESS_CTX_NOOP) || !defined(__linux__) - // NOOP implementations when OTEL_PROCESS_CTX_NOOP is defined or not on Linux - - otel_process_ctx_result otel_process_ctx_publish(const otel_process_ctx_data *data) { - (void) data; // Suppress unused parameter warning - return (otel_process_ctx_result) {.success = false, .error_message = "OTEL_PROCESS_CTX_NOOP mode is enabled - no-op implementation (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - - bool otel_process_ctx_drop_current(void) { - return true; // Nothing to do, this always succeeds - } - - #ifndef OTEL_PROCESS_CTX_NO_READ - otel_process_ctx_read_result otel_process_ctx_read(void) { - return (otel_process_ctx_read_result) {.success = false, .error_message = "OTEL_PROCESS_CTX_NOOP mode is enabled - no-op implementation (" __FILE__ ":" ADD_QUOTES(__LINE__) ")", .data = empty_data}; - } - - bool otel_process_ctx_read_drop(otel_process_ctx_read_result *result) { - (void) result; // Suppress unused parameter warning - return false; - } - #endif // OTEL_PROCESS_CTX_NO_READ -#else // OTEL_PROCESS_CTX_NOOP - -#ifdef __cplusplus - #include - using std::atomic_thread_fence; - using std::memory_order_seq_cst; -#else - #include -#endif -#include -#include -#include -#include -#include -#include -#include -#include - -#define KEY_VALUE_LIMIT 4096 -#define UINT14_MAX 16383 -#define OTEL_CTX_SIGNATURE "OTEL_CTX" - -#ifndef PR_SET_VMA - #define PR_SET_VMA 0x53564d41 - #define PR_SET_VMA_ANON_NAME 0 -#endif - -#ifndef MFD_CLOEXEC - #define MFD_CLOEXEC 1U -#endif - -#ifndef MFD_ALLOW_SEALING - #define MFD_ALLOW_SEALING 2U -#endif - -#ifndef MFD_NOEXEC_SEAL - #define MFD_NOEXEC_SEAL 8U -#endif - -// memfd_create is not declared in glibc < 2.27; use syscall() directly. -// Provide __NR_memfd_create for architectures where old kernel headers omit it. -#ifndef __NR_memfd_create - #if defined(__x86_64__) - #define __NR_memfd_create 319 - #elif defined(__aarch64__) - #define __NR_memfd_create 279 - #elif defined(__arm__) - #define __NR_memfd_create 385 - #elif defined(__i386__) - #define __NR_memfd_create 356 - #endif -#endif - -#ifdef __NR_memfd_create - static int _otel_memfd_create(const char *name, unsigned int flags) { - return (int)syscall(__NR_memfd_create, name, flags); - } -#endif - -/** - * The process context data that's written into the published anonymous mapping. - * - * An outside-of-process reader will read this struct + otel_process_payload to get the data. - */ -typedef struct __attribute__((packed, aligned(8))) { - char otel_process_ctx_signature[8]; // Always "OTEL_CTX" - uint32_t otel_process_ctx_version; // Always > 0, incremented when the data structure changes, currently v2 - uint32_t otel_process_payload_size; // Always > 0, size of storage - uint64_t otel_process_monotonic_published_at_ns; // Timestamp from when the context was published in nanoseconds from CLOCK_BOOTTIME. 0 during updates. - char *otel_process_payload; // Always non-null, points to the storage for the data; expected to be a protobuf map of string key/value pairs, null-terminated -} otel_process_ctx_mapping; - -/** - * The full state of a published process context. - * - * It is used to store the all data for the process context and that needs to be kept around while the context is published. - */ -typedef struct { - // The pid of the process that published the context. - pid_t publisher_pid; - // The actual mapping of the process context. Note that because we `madvise(..., MADV_DONTFORK)` this mapping is not - // propagated to child processes and thus `mapping` is only valid on the process that published the context. - otel_process_ctx_mapping *mapping; - // The process context payload. - char *payload; -} otel_process_ctx_state; - -/** - * Only one context is active, so we keep its state as a global. - */ -static otel_process_ctx_state published_state; - -static otel_process_ctx_result otel_process_ctx_update(uint64_t monotonic_published_at_ns, const otel_process_ctx_data *data); -static otel_process_ctx_result otel_process_ctx_encode_protobuf_payload(char **out, uint32_t *out_size, otel_process_ctx_data data); - -static uint64_t monotonic_time_now_ns(void) { - struct timespec ts; - if (clock_gettime(CLOCK_BOOTTIME, &ts) == -1) return 0; - return ts.tv_sec * 1000000000ULL + ts.tv_nsec; -} - -static bool ctx_is_published(otel_process_ctx_state state) { - return state.mapping != NULL && state.mapping != MAP_FAILED && getpid() == state.publisher_pid; -} - -// The process context is designed to be read by an outside-of-process reader. Thus, for concurrency purposes the steps -// on this method are ordered in a way to avoid races, or if not possible to avoid, to allow the reader to detect if there was a race. -otel_process_ctx_result otel_process_ctx_publish(const otel_process_ctx_data *data) { - if (!data) return (otel_process_ctx_result) {.success = false, .error_message = "otel_process_ctx_data is NULL (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - - uint64_t monotonic_published_at_ns = monotonic_time_now_ns(); - if (monotonic_published_at_ns == 0) { - return (otel_process_ctx_result) {.success = false, .error_message = "Failed to get current time (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - - // Step: If the context has been published by this process, update it in place - if (ctx_is_published(published_state)) return otel_process_ctx_update(monotonic_published_at_ns, data); - - // Step: Drop any previous context state if it exists - // No state should be around anywhere after this step. - if (!otel_process_ctx_drop_current()) { - return (otel_process_ctx_result) {.success = false, .error_message = "Failed to drop previous context (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - - // Step: Prepare the payload to be published - // The payload SHOULD be ready and valid before trying to actually create the mapping. - uint32_t payload_size = 0; - otel_process_ctx_result result = otel_process_ctx_encode_protobuf_payload(&published_state.payload, &payload_size, *data); - if (!result.success) return result; - - // Step: Create the mapping - const ssize_t mapping_size = sizeof(otel_process_ctx_mapping); - published_state.publisher_pid = getpid(); // This allows us to detect in forks that we shouldn't touch the mapping -#ifdef __NR_memfd_create - int fd = _otel_memfd_create("OTEL_CTX", MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_NOEXEC_SEAL); - if (fd < 0) { - // MFD_NOEXEC_SEAL is a newer flag; older kernels reject unknown flags, so let's retry without it - fd = _otel_memfd_create("OTEL_CTX", MFD_CLOEXEC | MFD_ALLOW_SEALING); - } -#else - int fd = -1; // memfd_create unavailable; fall through to anonymous mmap below -#endif - bool failed_to_close_fd = false; - if (fd >= 0) { - // Try to create mapping from memfd - if (ftruncate(fd, mapping_size) == -1) { - close(fd); // Swallow errors here, truncation already failed anyway - otel_process_ctx_drop_current(); - return (otel_process_ctx_result) {.success = false, .error_message = "Failed to truncate memfd (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - published_state.mapping = (otel_process_ctx_mapping *) mmap(NULL, mapping_size, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0); - failed_to_close_fd = (close(fd) == -1); - } else { - // Fallback: Use an anonymous mapping instead - published_state.mapping = (otel_process_ctx_mapping *) mmap(NULL, mapping_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - } - if (published_state.mapping == MAP_FAILED || failed_to_close_fd) { - otel_process_ctx_drop_current(); - - if (failed_to_close_fd) { - return (otel_process_ctx_result) {.success = false, .error_message = "Failed to close memfd (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } else { - return (otel_process_ctx_result) {.success = false, .error_message = "Failed to allocate mapping (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - } - - // Step: Setup MADV_DONTFORK - // This ensures that the mapping is not propagated to child processes (they should call update/publish again). - if (madvise(published_state.mapping, mapping_size, MADV_DONTFORK) == -1) { - if (otel_process_ctx_drop_current()) { - return (otel_process_ctx_result) {.success = false, .error_message = "Failed to setup MADV_DONTFORK (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } else { - return (otel_process_ctx_result) {.success = false, .error_message = "Failed to drop context (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - } - - // Step: Populate the mapping - // The payload and any extra fields must come first and not be reordered with the monotonic_published_at_ns by the compiler. - *published_state.mapping = (otel_process_ctx_mapping) { - .otel_process_ctx_signature = { 'O', 'T', 'E', 'L', '_', 'C', 'T', 'X' }, - .otel_process_ctx_version = 2, - .otel_process_payload_size = payload_size, - .otel_process_monotonic_published_at_ns = 0, // Set in "Step: Populate the monotonic_published_at_ns into the mapping" below - .otel_process_payload = published_state.payload - }; - - // Step: Synchronization - Mapping has been filled and is missing monotonic_published_at_ns - // Make sure the initialization of the mapping + payload above does not get reordered with setting the monotonic_published_at_ns below. Setting - // the monotonic_published_at_ns is what tells an outside reader that the context is fully published. - atomic_thread_fence(memory_order_seq_cst); - - // Step: Populate the monotonic_published_at_ns into the mapping - // The monotonic_published_at_ns must come last and not be reordered with the fields above by the compiler. After this step, external readers - // can read the monotonic_published_at_ns and know that the payload is ready to be read. - published_state.mapping->otel_process_monotonic_published_at_ns = monotonic_published_at_ns; - - // Step: Attempt to name the mapping so outside readers can: - // * Find it by name - // * Hook on prctl to detect when new mappings are published - if (prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, published_state.mapping, mapping_size, OTEL_CTX_SIGNATURE) == -1) { - // Naming an anonymous mapping is an optional Linux 5.17+ feature (`CONFIG_ANON_VMA_NAME`). - // Many distros, such as Ubuntu and Arch enable it. On earlier kernel versions or kernels without the feature, this call can fail. - // - // It's OK for this to fail because (per-usecase): - // 1. "Find it by name" => As a fallback, it's possible to scan the mappings and for the memfd name. - // 2. "Hook on prctl" => When hooking on prctl via eBPF it's still possible to see this call, even when it's not supported/enabled. - // This works even on older kernels! For this reason we unconditionally make this call even on older kernels -- to - // still allow detection via hooking onto prctl. - } - - // All done! - - return (otel_process_ctx_result) {.success = true, .error_message = NULL}; -} - -bool otel_process_ctx_drop_current(void) { - otel_process_ctx_state state = published_state; - - // Zero out the state and make sure no operations below are reordered with zeroing - published_state = (otel_process_ctx_state) {.publisher_pid = 0, .mapping = NULL, .payload = NULL}; - atomic_thread_fence(memory_order_seq_cst); - - bool success = true; - - // The mapping only exists if it was created by the current process; if it was inherited by a fork it doesn't exist anymore - // (due to the MADV_DONTFORK) and we don't need to do anything to it. - if (ctx_is_published(state)) { - success = munmap(state.mapping, sizeof(otel_process_ctx_mapping)) == 0; - } - - // The payload may have been inherited from a parent. This is a regular malloc so we need to free it so we don't leak. - free(state.payload); - - return success; -} - -static otel_process_ctx_result otel_process_ctx_update(uint64_t monotonic_published_at_ns, const otel_process_ctx_data *data) { - if (data == NULL || !ctx_is_published(published_state)) { - return (otel_process_ctx_result) {.success = false, .error_message = "Unexpected: otel_process_ctx_data is NULL or context is not published (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - - if (monotonic_published_at_ns == published_state.mapping->otel_process_monotonic_published_at_ns) { - // Advance published_at_ns to allow readers to detect the update - monotonic_published_at_ns++; - } - - // Step: Prepare the new payload to be published - // The payload SHOULD be ready and valid before trying to actually update the mapping. - uint32_t payload_size = 0; - char *payload = nullptr; - otel_process_ctx_result result = otel_process_ctx_encode_protobuf_payload(&payload, &payload_size, *data); - if (!result.success) return result; - - // Step: Zero out monotonic_published_at_ns in the mapping - // This enables readers to detect that an update is in-progress - published_state.mapping->otel_process_monotonic_published_at_ns = 0; - - // Step: Synchronization - Make sure readers observe the zeroing above before anything else below - atomic_thread_fence(memory_order_seq_cst); - - // Step: Install updated data - published_state.mapping->otel_process_payload_size = payload_size; - published_state.mapping->otel_process_payload = payload; - - // Step: Synchronization - Make sure readers observe the updated data before anything else below - atomic_thread_fence(memory_order_seq_cst); - - // Step: Install new monotonic_published_at_ns - // The update is now complete -- readers that observe the new timestamp will observe the updated payload - published_state.mapping->otel_process_monotonic_published_at_ns = monotonic_published_at_ns; - - // Step: Attempt to name the mapping so outside readers can detect the update - if (prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, published_state.mapping, sizeof(otel_process_ctx_mapping), OTEL_CTX_SIGNATURE) == -1) { - // It's OK for this to fail -- see otel_process_ctx_publish for why - } - - // Step: Update bookkeeping - free(published_state.payload); // This was still pointing to the old payload - published_state.payload = payload; - - // All done! - - return (otel_process_ctx_result) {.success = true, .error_message = NULL}; -} - -// The caller is responsible for enforcing that value fits within UINT14_MAX -static size_t protobuf_varint_size(uint16_t value) { return value >= 128 ? 2 : 1; } - -// Field tag for record + varint len + data -static size_t protobuf_record_size(size_t len) { return 1 + protobuf_varint_size(len) + len; } - -static size_t protobuf_string_size(const char *str) { return protobuf_record_size(strlen(str)); } - -static size_t protobuf_otel_keyvalue_string_size(const char *key, const char *value) { - size_t key_field_size = protobuf_string_size(key); // String - size_t value_field_size = protobuf_record_size(protobuf_string_size(value)); // Nested AnyValue message with a string inside - return key_field_size + value_field_size; // Does not include the keyvalue record tag + size, only its payload -} - -static size_t protobuf_otel_array_value_content_size(const char **strings) { - size_t total = 0; - for (size_t i = 0; strings[i] != NULL; i++) { - total += protobuf_record_size(protobuf_string_size(strings[i])); // ArrayValue.values[i]: AnyValue{string_value} - } - return total; -} - -// As a simplification, we enforce that keys and values are <= 4096 (KEY_VALUE_LIMIT) so that their size + extra bytes always fits within UINT14_MAX -static otel_process_ctx_result validate_and_calculate_protobuf_payload_size(size_t *out_pairs_size, const char **pairs) { - size_t num_entries = 0; - for (size_t i = 0; pairs[i] != NULL; i++) num_entries++; - if (num_entries % 2 != 0) { - return (otel_process_ctx_result) {.success = false, .error_message = "Value in otel_process_ctx_data is NULL (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - - *out_pairs_size = 0; - for (size_t i = 0; pairs[i * 2] != NULL; i++) { - const char *key = pairs[i * 2]; - const char *value = pairs[i * 2 + 1]; - - if (strlen(key) > KEY_VALUE_LIMIT) { - return (otel_process_ctx_result) {.success = false, .error_message = "Length of key in otel_process_ctx_data exceeds 4096 limit (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - if (strlen(value) > KEY_VALUE_LIMIT) { - return (otel_process_ctx_result) {.success = false, .error_message = "Length of value in otel_process_ctx_data exceeds 4096 limit (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - - *out_pairs_size += protobuf_record_size(protobuf_otel_keyvalue_string_size(key, value)); // KeyValue message - } - return (otel_process_ctx_result) {.success = true, .error_message = NULL}; -} - -/** - * Writes a protobuf varint encoding for the given value. - * As a simplification, only supports values that fit in 1 or 2 bytes (0-16383 UINT14_MAX). - */ -static void write_protobuf_varint(char **ptr, uint16_t value) { - if (protobuf_varint_size(value) == 1) { - *(*ptr)++ = (char)value; - } else { - // Two bytes: first byte has MSB set, second byte has value - *(*ptr)++ = (char)((value & 0x7F) | 0x80); // Low 7 bits + continuation bit - *(*ptr)++ = (char)(value >> 7); // High 7 bits - } -} - -static void write_protobuf_string(char **ptr, const char *str) { - size_t len = strlen(str); - write_protobuf_varint(ptr, len); - memcpy(*ptr, str, len); - *ptr += len; -} - -static void write_protobuf_tag(char **ptr, uint8_t field_number) { - *(*ptr)++ = (char)((field_number << 3) | 2); // Field type is always 2 (LEN) -} - -static void write_attribute(char **ptr, uint8_t field_number, const char *key, const char *value) { - write_protobuf_tag(ptr, field_number); - write_protobuf_varint(ptr, protobuf_otel_keyvalue_string_size(key, value)); - - // KeyValue - write_protobuf_tag(ptr, 1); // KeyValue.key (field 1) - write_protobuf_string(ptr, key); - write_protobuf_tag(ptr, 2); // KeyValue.value (field 2) - write_protobuf_varint(ptr, protobuf_string_size(value)); - - // AnyValue - write_protobuf_tag(ptr, 1); // AnyValue.string_value (field 1) - write_protobuf_string(ptr, value); -} - -static void write_array_attribute(char **ptr, uint8_t field_number, const char *key, const char **strings) { - size_t array_value_content_size = protobuf_otel_array_value_content_size(strings); - size_t any_value_content_size = protobuf_record_size(array_value_content_size); - size_t kv_content_size = protobuf_string_size(key) + protobuf_record_size(any_value_content_size); - - write_protobuf_tag(ptr, field_number); - write_protobuf_varint(ptr, kv_content_size); - - write_protobuf_tag(ptr, 1); // KeyValue.key (field 1) - write_protobuf_string(ptr, key); - - write_protobuf_tag(ptr, 2); // KeyValue.value (field 2) = AnyValue message - write_protobuf_varint(ptr, any_value_content_size); - - write_protobuf_tag(ptr, 5); // AnyValue.array_value (field 5) = ArrayValue message - write_protobuf_varint(ptr, array_value_content_size); - - for (size_t i = 0; strings[i] != NULL; i++) { // ArrayValue.values (field 1) - repeated AnyValue entries - write_protobuf_tag(ptr, 1); // ArrayValue.values[i] - write_protobuf_varint(ptr, protobuf_string_size(strings[i])); // Inner AnyValue size - write_protobuf_tag(ptr, 1); // AnyValue.string_value (field 1) - write_protobuf_string(ptr, strings[i]); - } -} - -// Encode the payload as protobuf bytes. -// -// This method implements an extremely compact but limited protobuf encoder for the ProcessContext message. -// It encodes all fields as Resource attributes (KeyValue pairs). -// For extra compact code, it fixes strings at up to 4096 bytes. -static otel_process_ctx_result otel_process_ctx_encode_protobuf_payload(char **out, uint32_t *out_size, otel_process_ctx_data data) { - const char *pairs[] = { - "deployment.environment.name", data.deployment_environment_name, - "service.instance.id", data.service_instance_id, - "service.name", data.service_name, - "service.version", data.service_version, - "telemetry.sdk.language", data.telemetry_sdk_language, - "telemetry.sdk.version", data.telemetry_sdk_version, - "telemetry.sdk.name", data.telemetry_sdk_name, - NULL - }; - - size_t pairs_size = 0; - otel_process_ctx_result validation_result = validate_and_calculate_protobuf_payload_size(&pairs_size, (const char **) pairs); - if (!validation_result.success) return validation_result; - - size_t resource_attributes_pairs_size = 0; - if (data.resource_attributes != NULL) { - validation_result = validate_and_calculate_protobuf_payload_size(&resource_attributes_pairs_size, data.resource_attributes); - if (!validation_result.success) return validation_result; - } - - size_t extra_attributes_pairs_size = 0; - if (data.extra_attributes != NULL) { - validation_result = validate_and_calculate_protobuf_payload_size(&extra_attributes_pairs_size, data.extra_attributes); - if (!validation_result.success) return validation_result; - } - - size_t thread_ctx_pairs_size = 0; - if (data.thread_ctx_config != NULL) { - if (data.thread_ctx_config->schema_version != NULL) { - const char *thread_ctx_pairs[] = {"threadlocal.schema_version", data.thread_ctx_config->schema_version, NULL}; - validation_result = validate_and_calculate_protobuf_payload_size(&thread_ctx_pairs_size, thread_ctx_pairs); - if (!validation_result.success) return validation_result; - } - if (data.thread_ctx_config->attribute_key_map != NULL) { - if (data.thread_ctx_config->schema_version == NULL) { - return (otel_process_ctx_result) {.success = false, .error_message = "attribute_key_map requires schema_version to be set (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - for (size_t i = 0; data.thread_ctx_config->attribute_key_map[i] != NULL; i++) { - if (strlen(data.thread_ctx_config->attribute_key_map[i]) > KEY_VALUE_LIMIT) { - return (otel_process_ctx_result) {.success = false, .error_message = "Length of attribute_key_map entry exceeds 4096 limit (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - } - size_t array_value_content_size = protobuf_otel_array_value_content_size(data.thread_ctx_config->attribute_key_map); - size_t any_value_content_size = protobuf_record_size(array_value_content_size); - size_t kv_content_size = protobuf_string_size("threadlocal.attribute_key_map") + protobuf_record_size(any_value_content_size); - if (kv_content_size > UINT14_MAX) { - return (otel_process_ctx_result) {.success = false, .error_message = "Encoded size of attribute_key_map exceeds UINT14_MAX limit (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - thread_ctx_pairs_size += protobuf_record_size(kv_content_size); - } - } - - size_t resource_size = pairs_size + resource_attributes_pairs_size; - if (resource_size > UINT14_MAX) { - return (otel_process_ctx_result) {.success = false, .error_message = "Encoded size of resource attributes exceeds UINT14_MAX limit (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - size_t total_size = protobuf_record_size(resource_size) + extra_attributes_pairs_size + thread_ctx_pairs_size; - - char *encoded = (char *) calloc(total_size, 1); - if (!encoded) { - return (otel_process_ctx_result) {.success = false, .error_message = "Failed to allocate memory for payload (" __FILE__ ":" ADD_QUOTES(__LINE__) ")"}; - } - char *ptr = encoded; - - // ProcessContext.resource (field 1) - write_protobuf_tag(&ptr, 1); - write_protobuf_varint(&ptr, resource_size); - - for (size_t i = 0; pairs[i * 2] != NULL; i++) { - write_attribute(&ptr, 1, pairs[i * 2], pairs[i * 2 + 1]); - } - - for (size_t i = 0; data.resource_attributes != NULL && data.resource_attributes[i * 2] != NULL; i++) { - write_attribute(&ptr, 1, data.resource_attributes[i * 2], data.resource_attributes[i * 2 + 1]); - } - - // ProcessContext.extra_attributes (field 2) - for (size_t i = 0; data.extra_attributes != NULL && data.extra_attributes[i * 2] != NULL; i++) { - write_attribute(&ptr, 2, data.extra_attributes[i * 2], data.extra_attributes[i * 2 + 1]); - } - - if (data.thread_ctx_config != NULL) { - if (data.thread_ctx_config->schema_version != NULL) { - write_attribute(&ptr, 2, "threadlocal.schema_version", data.thread_ctx_config->schema_version); - } - if (data.thread_ctx_config->attribute_key_map != NULL) { - write_array_attribute(&ptr, 2, "threadlocal.attribute_key_map", data.thread_ctx_config->attribute_key_map); - } - } - - *out = encoded; - *out_size = (uint32_t) total_size; - - return (otel_process_ctx_result) {.success = true, .error_message = NULL}; -} - -#ifndef OTEL_PROCESS_CTX_NO_READ - #include - #include - #include - #include - - // Note: The below parsing code is only for otel_process_ctx_read and is only provided for debugging - // and testing purposes. - - static void *parse_mapping_start(char *line) { - char *endptr = NULL; - unsigned long long start = strtoull(line, &endptr, 16); - if (start == 0 || start == ULLONG_MAX) return NULL; - return (void *)(uintptr_t) start; - } - - static otel_process_ctx_mapping *try_finding_mapping(void) { - char line[8192]; - otel_process_ctx_mapping *result = NULL; - - FILE *fp = fopen("/proc/self/maps", "r"); - if (!fp) return result; - - while (fgets(line, sizeof(line), fp)) { - bool is_process_ctx = strstr(line, "[anon_shmem:OTEL_CTX]") != NULL || strstr(line, "[anon:OTEL_CTX]") != NULL || strstr(line, "/memfd:OTEL_CTX") != NULL; - if (is_process_ctx) { - result = (otel_process_ctx_mapping *)parse_mapping_start(line); - break; - } - } - - fclose(fp); - return result; - } - - // Helper function to read a protobuf varint (limited to 1-2 bytes, max value UINT14_MAX, matching write_protobuf_varint above) - static bool read_protobuf_varint(char **ptr, char *end_ptr, uint16_t *value) { - if (*ptr >= end_ptr) return false; - - unsigned char first_byte = (unsigned char)**ptr; - (*ptr)++; - - if (first_byte < 128) { - *value = first_byte; - return true; - } else { - if (*ptr >= end_ptr) return false; - unsigned char second_byte = (unsigned char)**ptr; - (*ptr)++; - - *value = (first_byte & 0x7F) | (second_byte << 7); - return *value <= UINT14_MAX; - } - } - - // Helper function to read a protobuf string into a buffer, within the same limits as the encoder imposes - static bool read_protobuf_string(char **ptr, char *end_ptr, char *buffer) { - uint16_t len; - if (!read_protobuf_varint(ptr, end_ptr, &len) || len >= KEY_VALUE_LIMIT + 1 || *ptr + len > end_ptr) return false; - - memcpy(buffer, *ptr, len); - buffer[len] = '\0'; - *ptr += len; - - return true; - } - - // Reads field name and validates the fixed LEN wire type - static bool read_protobuf_tag(char **ptr, char *end_ptr, uint8_t *field_number) { - if (*ptr >= end_ptr) return false; - - unsigned char tag = (unsigned char)**ptr; - (*ptr)++; - - uint8_t wire_type = tag & 0x07; - *field_number = tag >> 3; - - return wire_type == 2; // We only need the LEN wire type for now - } - - // Peeks at the key of an OTel KeyValue message without advancing the pointer. - static bool peek_protobuf_key(char *ptr, char *end_ptr, char *key_buffer) { - char *p = ptr; - uint8_t kv_field; - if (!read_protobuf_tag(&p, end_ptr, &kv_field)) return false; - if (kv_field != 1) return false; // KeyValue.key is field 1 - return read_protobuf_string(&p, end_ptr, key_buffer); - } - - // Reads an OTel KeyValue message (key string + AnyValue-wrapped string) into the provided buffers. - static bool read_protobuf_keyvalue(char **ptr, char *end_ptr, char *key_buffer, char *value_buffer) { - bool key_found = false; - bool value_found = false; - - while (*ptr < end_ptr) { - uint8_t kv_field; - if (!read_protobuf_tag(ptr, end_ptr, &kv_field)) return false; - - if (kv_field == 1) { // KeyValue.key - if (!read_protobuf_string(ptr, end_ptr, key_buffer)) return false; - key_found = true; - } else if (kv_field == 2) { // KeyValue.value (AnyValue) - uint16_t _any_len; // Unused, but we still need to consume + validate the varint - if (!read_protobuf_varint(ptr, end_ptr, &_any_len)) return false; - uint8_t any_field; - if (!read_protobuf_tag(ptr, end_ptr, &any_field)) return false; - - if (any_field == 1) { // AnyValue.string_value - if (!read_protobuf_string(ptr, end_ptr, value_buffer)) return false; - value_found = true; - } - } - } - - return key_found && value_found; - } - - // Reads an AnyValue.array_value (field 5) from ptr; ptr must be at KeyValue.value (tag 2). - // Allocates a NULL-terminated array of strings and sets *out_array immediately. On error the caller must free it. - static bool read_protobuf_array_value_strings(char **ptr, char *end_ptr, char *value_buffer, const char ***out_array) { - // Reject duplicate fields — if the output pointer already holds a value, the protobuf data is malformed - if (*out_array) return false; - uint8_t field; - if (!read_protobuf_tag(ptr, end_ptr, &field) || field != 2) return false; - uint16_t any_len; - if (!read_protobuf_varint(ptr, end_ptr, &any_len)) return false; - char *any_end = *ptr + any_len; - if (any_end > end_ptr) return false; - - if (!read_protobuf_tag(ptr, any_end, &field) || field != 5) return false; - uint16_t array_len; - if (!read_protobuf_varint(ptr, any_end, &array_len)) return false; - char *array_end = *ptr + array_len; - if (array_end > any_end) return false; - - size_t max = 100; - size_t capacity = max + 1; - const char **arr = (const char **) calloc(capacity, sizeof(char *)); - if (!arr) return false; - *out_array = arr; - size_t count = 0; - - while (*ptr < array_end) { - if (count >= max) return false; - if (!read_protobuf_tag(ptr, array_end, &field) || field != 1) return false; - uint16_t item_len; - if (!read_protobuf_varint(ptr, array_end, &item_len)) return false; - char *item_end = *ptr + item_len; - if (item_end > array_end) return false; - if (!read_protobuf_tag(ptr, item_end, &field) || field != 1) return false; - if (!read_protobuf_string(ptr, item_end, value_buffer)) return false; - char *dup = strdup(value_buffer); - if (!dup) return false; - arr[count++] = dup; - } - - return true; - } - - // Simplified protobuf decoder to match the exact encoder above. If the protobuf data doesn't match the encoder, this will - // return false. - static bool otel_process_ctx_decode_payload(char *payload, uint32_t payload_size, otel_process_ctx_data *data_out, char *key_buffer, char *value_buffer) { - char *ptr = payload; - char *end_ptr = payload + payload_size; - - *data_out = empty_data; - - // Parse ProcessContext wrapper - expect field 1 (resource) - uint8_t process_ctx_field; - if (!read_protobuf_tag(&ptr, end_ptr, &process_ctx_field) || process_ctx_field != 1) return false; - - uint16_t resource_len; - if (!read_protobuf_varint(&ptr, end_ptr, &resource_len)) return false; - char *resource_end = ptr + resource_len; - if (resource_end > end_ptr) return false; - - size_t resource_index = 0; - size_t resource_capacity = 201; // Allocate space for 100 pairs + NULL terminator entry - data_out->resource_attributes = (const char **) calloc(resource_capacity, sizeof(char *)); - if (data_out->resource_attributes == NULL) return false; - - size_t extra_attributes_index = 0; - size_t extra_attributes_capacity = 201; // Allocate space for 100 pairs + NULL terminator entry - data_out->extra_attributes = (const char **) calloc(extra_attributes_capacity, sizeof(char *)); - if (data_out->extra_attributes == NULL) return false; - - // Parse resource attributes (field 1) - while (ptr < resource_end) { - uint8_t field_number; - if (!read_protobuf_tag(&ptr, resource_end, &field_number) || field_number != 1) return false; - - uint16_t kv_len; - if (!read_protobuf_varint(&ptr, resource_end, &kv_len)) return false; - char *kv_end = ptr + kv_len; - if (kv_end > resource_end) return false; - - if (!read_protobuf_keyvalue(&ptr, kv_end, key_buffer, value_buffer)) return false; - - char *value = strdup(value_buffer); - if (!value) return false; - - // Dispatch based on key - const char **field = NULL; - if (strcmp(key_buffer, "deployment.environment.name") == 0) { field = &data_out->deployment_environment_name; } - else if (strcmp(key_buffer, "service.instance.id") == 0) { field = &data_out->service_instance_id; } - else if (strcmp(key_buffer, "service.name") == 0) { field = &data_out->service_name; } - else if (strcmp(key_buffer, "service.version") == 0) { field = &data_out->service_version; } - else if (strcmp(key_buffer, "telemetry.sdk.language") == 0) { field = &data_out->telemetry_sdk_language; } - else if (strcmp(key_buffer, "telemetry.sdk.version") == 0) { field = &data_out->telemetry_sdk_version; } - else if (strcmp(key_buffer, "telemetry.sdk.name") == 0) { field = &data_out->telemetry_sdk_name; } - - if (field != NULL) { - if (*field != NULL) { free(value); return false; } - *field = value; - } else { - char *key = strdup(key_buffer); - - if (!key || resource_index + 2 >= resource_capacity) { - free(key); - free(value); - return false; - } - data_out->resource_attributes[resource_index] = key; - data_out->resource_attributes[resource_index + 1] = value; - resource_index += 2; - } - } - - // Parse extra attributes (field 2) - while (ptr < end_ptr) { - uint8_t extra_ctx_field; - if (!read_protobuf_tag(&ptr, end_ptr, &extra_ctx_field) || extra_ctx_field != 2) return false; - - uint16_t kv_len; - if (!read_protobuf_varint(&ptr, end_ptr, &kv_len)) return false; - char *kv_end = ptr + kv_len; - if (kv_end > end_ptr) return false; - - if (!peek_protobuf_key(ptr, kv_end, key_buffer)) return false; - - if (strcmp(key_buffer, "threadlocal.attribute_key_map") == 0) { - // Consume key to advance ptr - uint8_t kv_field; - if (!read_protobuf_tag(&ptr, kv_end, &kv_field) || kv_field != 1) return false; - if (!read_protobuf_string(&ptr, kv_end, key_buffer)) return false; - if (!data_out->thread_ctx_config) { - otel_thread_ctx_config_data *setup = (otel_thread_ctx_config_data *) calloc(1, sizeof(otel_thread_ctx_config_data)); - if (!setup) return false; - data_out->thread_ctx_config = setup; - } - if (!read_protobuf_array_value_strings(&ptr, kv_end, value_buffer, &((otel_thread_ctx_config_data *)data_out->thread_ctx_config)->attribute_key_map)) return false; - } else { - if (!read_protobuf_keyvalue(&ptr, kv_end, key_buffer, value_buffer)) return false; - - char *value = strdup(value_buffer); - if (!value) return false; - - // Dispatch based on key - if (strcmp(key_buffer, "threadlocal.schema_version") == 0) { - if (!data_out->thread_ctx_config) { - otel_thread_ctx_config_data *setup = (otel_thread_ctx_config_data *) calloc(1, sizeof(otel_thread_ctx_config_data)); - if (!setup) { free(value); return false; } - data_out->thread_ctx_config = setup; - } else { - if (((otel_thread_ctx_config_data *)data_out->thread_ctx_config)->schema_version) { - free((void *)((otel_thread_ctx_config_data *)data_out->thread_ctx_config)->schema_version); - } - } - ((otel_thread_ctx_config_data *)data_out->thread_ctx_config)->schema_version = value; - } else { - char *key = strdup(key_buffer); - if (!key || extra_attributes_index + 2 >= extra_attributes_capacity) { - free(key); - free(value); - return false; - } - data_out->extra_attributes[extra_attributes_index] = key; - data_out->extra_attributes[extra_attributes_index + 1] = value; - extra_attributes_index += 2; - } - } - } - - // Validate all required fields were found - return data_out->deployment_environment_name != NULL && - data_out->service_instance_id != NULL && - data_out->service_name != NULL && - data_out->service_version != NULL && - data_out->telemetry_sdk_language != NULL && - data_out->telemetry_sdk_version != NULL && - data_out->telemetry_sdk_name != NULL; - } - - void otel_process_ctx_read_data_drop(otel_process_ctx_data data) { - if (data.deployment_environment_name) free((void *)data.deployment_environment_name); - if (data.service_instance_id) free((void *)data.service_instance_id); - if (data.service_name) free((void *)data.service_name); - if (data.service_version) free((void *)data.service_version); - if (data.telemetry_sdk_language) free((void *)data.telemetry_sdk_language); - if (data.telemetry_sdk_version) free((void *)data.telemetry_sdk_version); - if (data.telemetry_sdk_name) free((void *)data.telemetry_sdk_name); - if (data.resource_attributes) { - for (int i = 0; data.resource_attributes[i] != NULL; i++) free((void *)data.resource_attributes[i]); - free((void *)data.resource_attributes); - } - if (data.extra_attributes) { - for (int i = 0; data.extra_attributes[i] != NULL; i++) free((void *)data.extra_attributes[i]); - free((void *)data.extra_attributes); - } - if (data.thread_ctx_config) { - if (data.thread_ctx_config->schema_version) free((void *)data.thread_ctx_config->schema_version); - if (data.thread_ctx_config->attribute_key_map) { - for (int i = 0; data.thread_ctx_config->attribute_key_map[i] != NULL; i++) { - free((void *)data.thread_ctx_config->attribute_key_map[i]); - } - free((void *)data.thread_ctx_config->attribute_key_map); - } - free((void *)data.thread_ctx_config); - } - } - - otel_process_ctx_read_result otel_process_ctx_read(void) { - otel_process_ctx_mapping *mapping = try_finding_mapping(); - if (!mapping) { - return (otel_process_ctx_read_result) {.success = false, .error_message = "No OTEL_CTX mapping found (" __FILE__ ":" ADD_QUOTES(__LINE__) ")", .data = empty_data}; - } - - if (strncmp(mapping->otel_process_ctx_signature, OTEL_CTX_SIGNATURE, sizeof(mapping->otel_process_ctx_signature)) != 0 || mapping->otel_process_ctx_version != 2) { - return (otel_process_ctx_read_result) {.success = false, .error_message = "Invalid OTEL_CTX signature or version (" __FILE__ ":" ADD_QUOTES(__LINE__) ")", .data = empty_data}; - } - - otel_process_ctx_data data = empty_data; - - char *key_buffer = (char *) calloc(KEY_VALUE_LIMIT + 1, 1); - char *value_buffer = (char *) calloc(KEY_VALUE_LIMIT + 1, 1); - if (!key_buffer || !value_buffer) { - free(key_buffer); - free(value_buffer); - return (otel_process_ctx_read_result) {.success = false, .error_message = "Failed to allocate decode buffers (" __FILE__ ":" ADD_QUOTES(__LINE__) ")", .data = empty_data}; - } - - bool success = otel_process_ctx_decode_payload(mapping->otel_process_payload, mapping->otel_process_payload_size, &data, key_buffer, value_buffer); - free(key_buffer); - free(value_buffer); - - if (!success) { - otel_process_ctx_read_data_drop(data); - return (otel_process_ctx_read_result) {.success = false, .error_message = "Failed to decode payload (" __FILE__ ":" ADD_QUOTES(__LINE__) ")", .data = empty_data}; - } - - return (otel_process_ctx_read_result) {.success = true, .error_message = NULL, .data = data}; - } - - bool otel_process_ctx_read_drop(otel_process_ctx_read_result *result) { - if (!result || !result->success) return false; - otel_process_ctx_read_data_drop(result->data); - *result = (otel_process_ctx_read_result) {.success = false, .error_message = "Data dropped", .data = empty_data}; - return true; - } -#endif // OTEL_PROCESS_CTX_NO_READ - -#endif // OTEL_PROCESS_CTX_NOOP diff --git a/ddprof-lib/src/main/cpp/otel_process_ctx.h b/ddprof-lib/src/main/cpp/otel_process_ctx.h deleted file mode 100644 index 4e83a4805..000000000 --- a/ddprof-lib/src/main/cpp/otel_process_ctx.h +++ /dev/null @@ -1,153 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License (Version 2.0). -// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. - -#pragma once - -#define OTEL_PROCESS_CTX_VERSION_MAJOR 0 -#define OTEL_PROCESS_CTX_VERSION_MINOR 1 -#define OTEL_PROCESS_CTX_VERSION_PATCH 0 -#define OTEL_PROCESS_CTX_VERSION_STRING "0.1.0" - -#ifdef __cplusplus -extern "C" { -#endif - -#include - -/** - * # OpenTelemetry Process Context reference implementation - * - * `otel_process_ctx.h` and `otel_process_ctx.c` provide a reference implementation for the OpenTelemetry - * process-level context sharing specification. - * (https://github.com/open-telemetry/opentelemetry-specification/pull/4719/) - * - * This reference implementation is Linux-only, as the specification currently only covers Linux. - * On non-Linux OS's (or when OTEL_PROCESS_CTX_NOOP is defined) no-op versions of functions are supplied. - */ - - /** - * Config for the experimental thread context sharing mechanism, see - * https://docs.google.com/document/d/1eatbHpEXXhWZEPrXZpfR58-5RIx-81mUgF69Zpn3Rz4/edit?tab=t.bmgoq3yor67o for usage - * details. - */ -typedef struct { - const char *schema_version; - // NULL-terminated array of attribute key strings to be used in thread context. - // Can be NULL if not needed. - const char **attribute_key_map; -} otel_thread_ctx_config_data; - -/** - * Data that can be published as a process context. - * - * Every string MUST be valid for the duration of the call to `otel_process_ctx_publish`. - * Strings will be copied into the context. - * - * Strings MUST be: - * * Non-NULL - * * UTF-8 encoded - * * Not longer than INT16_MAX bytes - * - * Strings MAY be: - * * Empty - */ -typedef struct { - // https://opentelemetry.io/docs/specs/semconv/registry/attributes/deployment/#deployment-environment-name - const char *deployment_environment_name; - // https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/#service-instance-id - const char *service_instance_id; - // https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/#service-name - const char *service_name; - // https://opentelemetry.io/docs/specs/semconv/registry/attributes/service/#service-version - const char *service_version; - // https://opentelemetry.io/docs/specs/semconv/registry/attributes/telemetry/#telemetry-sdk-language - const char *telemetry_sdk_language; - // https://opentelemetry.io/docs/specs/semconv/registry/attributes/telemetry/#telemetry-sdk-version - const char *telemetry_sdk_version; - // https://opentelemetry.io/docs/specs/semconv/registry/attributes/telemetry/#telemetry-sdk-name - const char *telemetry_sdk_name; - // Additional key/value pairs as resource attributes https://opentelemetry.io/docs/specs/otel/resource/sdk/ - // Can be NULL if no resource attributes are needed; if non-NULL, this array MUST be terminated with a NULL entry. - // Every even entry is a key, every odd entry is a value (E.g. "key1", "value1", "key2", "value2", NULL). - const char **resource_attributes; - // Additional key/value pairs as extra attributes (ProcessContext.extra_attributes in process_context.proto) - // Can be NULL if no extra attributes are needed; if non-NULL, this array MUST be terminated with a NULL entry. - // Every even entry is a key, every odd entry is a value (E.g. "key1", "value1", "key2", "value2", NULL). - const char **extra_attributes; - // Experimental thread context sharing mechanism configuration. See struct definition for details. Can be NULL. - const otel_thread_ctx_config_data *thread_ctx_config; -} otel_process_ctx_data; - -/** Number of entries in the `otel_process_ctx_data` struct. Can be used to easily detect when the struct is updated. */ -#define OTEL_PROCESS_CTX_DATA_ENTRIES sizeof(otel_process_ctx_data) / sizeof(char *) - -typedef struct { - bool success; - const char *error_message; // Static strings only, non-NULL if success is false -} otel_process_ctx_result; - -/** - * Publishes a OpenTelemetry process context with the given data. - * - * The context should remain alive until the application exits (or is just about to exit). - * This method is NOT thread-safe. - * - * Calling `publish` multiple times is supported and will replace a previous context (only one is published at any given - * time). Calling `publish` multiple times usually happens when: - * * Some of the `otel_process_ctx_data` changes due to a live system reconfiguration for the same process - * * The process is forked (to provide a new `service_instance_id`) - * - * This API can be called in a fork of the process that published the previous context, even though - * the context is not carried over into forked processes (although part of its memory allocations are). - * - * @param data Pointer to the data to publish. This data is copied into the context and only needs to be valid for the duration of - * the call. Must not be `NULL`. - * @return The result of the operation. - */ -otel_process_ctx_result otel_process_ctx_publish(const otel_process_ctx_data *data); - -/** - * Drops the current OpenTelemetry process context, if any. - * - * This method is safe to call even there's no current context. - * This method is NOT thread-safe. - * - * This API can be called in a fork of the process that published the current context to clean memory allocations - * related to the parent's context (even though the context itself is not carried over into forked processes). - * - * @return `true` if the context was successfully dropped or no context existed, `false` otherwise. - */ -bool otel_process_ctx_drop_current(void); - -/** This can be disabled if no read support is required. */ -#ifndef OTEL_PROCESS_CTX_NO_READ - typedef struct { - bool success; - const char *error_message; // Static strings only, non-NULL if success is false - otel_process_ctx_data data; // Strings are allocated using `malloc` and the caller is responsible for `free`ing them - } otel_process_ctx_read_result; - - /** - * Reads the current OpenTelemetry process context, if any. - * - * Useful for debugging and testing purposes. Underlying returned strings in `data` are dynamically allocated using - * `malloc` and `otel_process_ctx_read_drop` must be called to free them. - * - * Thread-safety: This function assumes there is no concurrent mutation of the process context. - * - * @return The result of the operation. If successful, `data` contains the retrieved context data. - */ - otel_process_ctx_read_result otel_process_ctx_read(void); - - /** - * Drops the data resulting from a previous call to `otel_process_ctx_read`. - * - * @param result The result of a previous call to `otel_process_ctx_read`. Must not be `NULL`. - * @return `true` if the data was successfully dropped, `false` otherwise. - */ - bool otel_process_ctx_read_drop(otel_process_ctx_read_result *result); -#endif - -#ifdef __cplusplus -} -#endif diff --git a/ddprof-lib/src/main/cpp/perfEvents.h b/ddprof-lib/src/main/cpp/perfEvents.h deleted file mode 100644 index 4caeffb6d..000000000 --- a/ddprof-lib/src/main/cpp/perfEvents.h +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2017 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _PERFEVENTS_H -#define _PERFEVENTS_H - -#ifdef __linux__ - -#include "arch.h" -#include "arguments.h" -#include "engine.h" -#include - -class PerfEvent; -class PerfEventType; -class StackContext; - -class PerfEvents : public Engine { -private: - static volatile bool _enabled; - static int _max_events; - static PerfEvent *_events; - static PerfEventType *_event_type; - static long _interval; - static Ring _ring; - static CStack _cstack; - static bool _use_mmap_page; - - // cppcheck-suppress unusedPrivateFunction - static u64 readCounter(siginfo_t *siginfo, void *ucontext); - // cppcheck-suppress unusedPrivateFunction - static void signalHandler(int signo, siginfo_t *siginfo, void *ucontext); - -public: - Error check(Arguments &args); - Error start(Arguments &args); - void stop(); - - virtual int registerThread(int tid); - virtual void unregisterThread(int tid); - long interval() const { return _interval; } - - const char *name() { return "PerfEvents"; } - - static int walkKernel(int tid, const void **callchain, int max_depth, - StackContext *java_ctx); - - static void resetBuffer(int tid); - - static const char *getEventName(int event_id); - - inline void enableEvents(bool enabled) { _enabled = enabled; } -}; - -#else - -#include "engine.h" - -class StackContext; - -class PerfEvents : public Engine { -public: - Error check(Arguments &args) { - return Error("PerfEvents are unsupported on this platform"); - } - - Error start(Arguments &args) { - return Error("PerfEvents are unsupported on this platform"); - } - - static int walkKernel(int tid, const void **callchain, int max_depth, - StackContext *java_ctx) { - return 0; - } - - inline void enableEvents(bool enabled) {} - - static void resetBuffer(int tid) {} - - static const char *getEventName(int event_id) { return NULL; } - - virtual int registerThread(int tid) { return -1; } - virtual void unregisterThread(int tid) {} - - long interval() const { return -1; } -}; - -#endif // __linux__ - -#endif // _PERFEVENTS_H diff --git a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp b/ddprof-lib/src/main/cpp/perfEvents_linux.cpp deleted file mode 100644 index 85ec23d5e..000000000 --- a/ddprof-lib/src/main/cpp/perfEvents_linux.cpp +++ /dev/null @@ -1,1075 +0,0 @@ -/* - * Copyright 2017 Andrei Pangin - * Copyright 2025, 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifdef __linux__ - -#include "arch.h" -#include "arguments.h" -#include "context.h" -#include "guards.h" -#include "debugSupport.h" -#include "jvmSupport.inline.h" -#include "jvmThread.h" -#include "libraries.h" -#include "log.h" -#include "os.h" -#include "perfEvents.h" -#include "profiler.h" -#include "spinLock.h" -#include "stackFrame.h" -#include "stackWalker.h" -#include "symbols.h" -#include "thread.h" -#include "threadState.inline.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// Ancient fcntl.h does not define F_SETOWN_EX constants and structures -#ifndef F_SETOWN_EX -#define F_SETOWN_EX 15 -#define F_OWNER_TID 0 - -struct f_owner_ex { - int type; - pid_t pid; -}; -#endif // F_SETOWN_EX - -enum { - HW_BREAKPOINT_R = 1, - HW_BREAKPOINT_W = 2, - HW_BREAKPOINT_RW = 3, - HW_BREAKPOINT_X = 4 -}; - -static int fetchInt(const char *file_name) { - int fd = open(file_name, O_RDONLY); - if (fd == -1) { - return 0; - } - - char num[16] = "0"; - ssize_t r = read(fd, num, sizeof(num) - 1); - (void)r; - close(fd); - return atoi(num); -} - -// Get perf_event_attr.config numeric value of the given tracepoint name -// by reading /sys/kernel/debug/tracing/events//id file -static int findTracepointId(const char *name) { - char buf[256]; - if ((size_t)snprintf(buf, sizeof(buf), - "/sys/kernel/debug/tracing/events/%s/id", - name) >= sizeof(buf)) { - return 0; - } - - *strchr(buf, ':') = '/'; // make path from event name - - return fetchInt(buf); -} - -// Get perf_event_attr.type for the given event source -// by reading /sys/bus/event_source/devices//type -static int findDeviceType(const char *name) { - char buf[256]; - if ((size_t)snprintf(buf, sizeof(buf), - "/sys/bus/event_source/devices/%s/type", - name) >= sizeof(buf)) { - return 0; - } - return fetchInt(buf); -} - -// Convert pmu/event-name/ to pmu/param1=N,param2=M/ -static void resolvePmuEventName(const char *device, char *event, size_t size) { - char buf[256]; - if ((size_t)snprintf(buf, sizeof(buf), - "/sys/bus/event_source/devices/%s/events/%s", device, - event) >= sizeof(buf)) { - return; - } - - int fd = open(buf, O_RDONLY); - if (fd == -1) { - return; - } - - ssize_t r = read(fd, event, size); - if (r > 0 && (size_t(r) == size || event[r - 1] == '\n')) { - event[r - 1] = 0; - } - close(fd); -} - -// Set a PMU parameter (such as umask) to the corresponding config field -static bool setPmuConfig(const char *device, const char *param, __u64 *config, - __u64 val) { - char buf[256]; - if ((size_t)snprintf(buf, sizeof(buf), - "/sys/bus/event_source/devices/%s/format/%s", device, - param) >= sizeof(buf)) { - return false; - } - - int fd = open(buf, O_RDONLY); - if (fd == -1) { - return false; - } - - ssize_t r = read(fd, buf, sizeof(buf)); - close(fd); - - if (r > 0 && r < (int)sizeof(buf)) { - if (strncmp(buf, "config:", 7) == 0) { - config[0] |= val << atoi(buf + 7); - return true; - } else if (strncmp(buf, "config1:", 8) == 0) { - config[1] |= val << atoi(buf + 8); - return true; - } else if (strncmp(buf, "config2:", 8) == 0) { - config[2] |= val << atoi(buf + 8); - return true; - } - } - return false; -} - -static void **_pthread_entry = NULL; - -// Intercept thread creation/termination by patching libjvm's GOT entry for -// pthread_setspecific(). HotSpot puts VMThread into TLS on thread start, and -// resets on thread end. -static int pthread_setspecific_hook(pthread_key_t key, const void *value) { - assert(JVMThread::isInitialized()); - if (JVMThread::key() != key) { - return pthread_setspecific(key, value); - } - if (pthread_getspecific(key) == value) { - return 0; - } - - if (value != NULL) { - ProfiledThread::initCurrentThread(); - int result = pthread_setspecific(key, value); - Profiler::registerThread(ProfiledThread::currentTid()); - return result; - } else { - int tid = ProfiledThread::currentTid(); - { - SignalBlocker blocker; - Profiler::unregisterThread(tid); - ProfiledThread::release(); - } - return pthread_setspecific(key, value); - } -} - -static void **lookupThreadEntry() { - // Depending on Zing version, pthread_setspecific is called either from - // libazsys.so or from libjvm.so - if (VM::isZing()) { - CodeCache *libazsys = Libraries::instance()->findLibraryByName("libazsys"); - if (libazsys != NULL) { - void **entry = libazsys->findImport(im_pthread_setspecific); - if (entry != NULL) { - return entry; - } - } - } - - CodeCache *lib = Libraries::instance()->findJvmLibrary("libj9thr"); - return lib != NULL ? lib->findImport(im_pthread_setspecific) : NULL; -} - -struct FunctionWithCounter { - const char *name; - int counter_arg; -}; - -struct PerfEventType { - const char *name; - long default_interval; - __u32 type; - __u64 config; - __u64 config1; - __u64 config2; - int counter_arg; - - enum { - IDX_PREDEFINED = 12, - IDX_RAW, - IDX_PMU, - IDX_BREAKPOINT, - IDX_TRACEPOINT, - IDX_KPROBE, - IDX_UPROBE, - }; - - static PerfEventType AVAILABLE_EVENTS[]; - static FunctionWithCounter KNOWN_FUNCTIONS[]; - - static char probe_func[256]; - - // Find which argument of a known function serves as a profiling counter, - // e.g. the first argument of malloc() is allocation size - static int findCounterArg(const char *name) { - for (FunctionWithCounter *func = KNOWN_FUNCTIONS; func->name != NULL; - func++) { - if (strcmp(name, func->name) == 0) { - return func->counter_arg; - } - } - return 0; - } - - // Breakpoint format: func[+offset][/len][:rwx][{arg}] - static PerfEventType *getBreakpoint(const char *name, __u32 bp_type, - __u32 bp_len) { - char buf[256]; - strncpy(buf, name, sizeof(buf) - 1); - buf[sizeof(buf) - 1] = 0; - - // Parse counter arg [{arg}] - int counter_arg = 0; - char *c = strrchr(buf, '{'); - if (c != NULL && c[1] >= '1' && c[1] <= '9') { - *c++ = 0; - counter_arg = atoi(c); - } - - // Parse access type [:rwx] - c = strrchr(buf, ':'); - if (c != NULL && c != name && c[-1] != ':') { - *c++ = 0; - if (strcmp(c, "r") == 0) { - bp_type = HW_BREAKPOINT_R; - } else if (strcmp(c, "w") == 0) { - bp_type = HW_BREAKPOINT_W; - } else if (strcmp(c, "x") == 0) { - bp_type = HW_BREAKPOINT_X; - bp_len = sizeof(long); - } else { - bp_type = HW_BREAKPOINT_RW; - } - } - - // Parse length [/8] - c = strrchr(buf, '/'); - if (c != NULL) { - *c++ = 0; - bp_len = (__u32)strtol(c, NULL, 0); - } - - // Parse offset [+0x1234] - long long offset = 0; - c = strrchr(buf, '+'); - if (c != NULL) { - *c++ = 0; - offset = strtoll(c, NULL, 0); - } - - // Parse symbol or absolute address - __u64 addr; - if (strncmp(buf, "0x", 2) == 0) { - addr = (__u64)strtoll(buf, NULL, 0); - } else { - addr = (__u64)(uintptr_t)dlsym(RTLD_DEFAULT, buf); - if (addr == 0) { - addr = (__u64)(uintptr_t)Libraries::instance()->resolveSymbol(buf); - } - if (c == NULL) { - // If offset is not specified explicitly, add the default breakpoint - // offset - offset = BREAKPOINT_OFFSET; - } - } - - if (addr == 0) { - return NULL; - } - - PerfEventType *breakpoint = &AVAILABLE_EVENTS[IDX_BREAKPOINT]; - breakpoint->config = bp_type; - breakpoint->config1 = addr + offset; - breakpoint->config2 = bp_len; - breakpoint->counter_arg = bp_type == HW_BREAKPOINT_X && counter_arg == 0 - ? findCounterArg(buf) - : counter_arg; - return breakpoint; - } - - static PerfEventType *getTracepoint(int tracepoint_id) { - PerfEventType *tracepoint = &AVAILABLE_EVENTS[IDX_TRACEPOINT]; - tracepoint->config = tracepoint_id; - return tracepoint; - } - - static PerfEventType *getProbe(PerfEventType *probe, const char *type, - const char *name, __u64 ret) { - strncpy(probe_func, name, sizeof(probe_func) - 1); - probe_func[sizeof(probe_func) - 1] = 0; - - if (probe->type == 0 && (probe->type = findDeviceType(type)) == 0) { - return NULL; - } - - long long offset = 0; - char *c = strrchr(probe_func, '+'); - if (c != NULL) { - *c++ = 0; - offset = strtoll(c, NULL, 0); - } - - probe->config = ret; - probe->config1 = (__u64)(uintptr_t)probe_func; - probe->config2 = offset; - return probe; - } - - static PerfEventType *getRawEvent(__u64 config) { - PerfEventType *raw = &AVAILABLE_EVENTS[IDX_RAW]; - raw->config = config; - return raw; - } - - static PerfEventType *getPmuEvent(const char *name) { - char buf[256]; - strncpy(buf, name, sizeof(buf) - 1); - buf[sizeof(buf) - 1] = 0; - - char *descriptor = strchr(buf, '/'); - *descriptor++ = 0; - descriptor[strlen(descriptor) - 1] = 0; - - PerfEventType *raw = &AVAILABLE_EVENTS[IDX_PMU]; - if ((raw->type = findDeviceType(buf)) == 0) { - return NULL; - } - - // pmu/rNNN/ - if (descriptor[0] == 'r' && descriptor[1] >= '0') { - char *end; - raw->config = strtoull(descriptor + 1, &end, 16); - if (*end == 0) { - return raw; - } - } - - // Resolve event name to the list of parameters - resolvePmuEventName(buf, descriptor, sizeof(buf) - (descriptor - buf)); - - raw->config = 0; - raw->config1 = 0; - raw->config2 = 0; - - // Parse parameters - while (descriptor != NULL && descriptor[0]) { - char *p = descriptor; - if ((descriptor = strchr(p, ',')) != NULL || - (descriptor = strchr(p, ':')) != NULL) { - *descriptor++ = 0; - } - - __u64 val = 1; - char *eq = strchr(p, '='); - if (eq != NULL) { - *eq++ = 0; - val = strtoull(eq, NULL, 0); - } - - if (strcmp(p, "config") == 0) { - raw->config = val; - } else if (strcmp(p, "config1") == 0) { - raw->config1 = val; - } else if (strcmp(p, "config2") == 0) { - raw->config2 = val; - } else if (!setPmuConfig(buf, p, &raw->config, val)) { - return NULL; - } - } - - return raw; - } - - static PerfEventType *forName(const char *name) { - // Look through the table of predefined perf events - for (int i = 0; i <= IDX_PREDEFINED; i++) { - if (strcmp(name, AVAILABLE_EVENTS[i].name) == 0) { - return &AVAILABLE_EVENTS[i]; - } - } - - // Hardware breakpoint - if (strncmp(name, "mem:", 4) == 0) { - return getBreakpoint(name + 4, HW_BREAKPOINT_RW, 1); - } - - // Raw tracepoint ID - if (strncmp(name, "trace:", 6) == 0) { - int tracepoint_id = atoi(name + 6); - return tracepoint_id > 0 ? getTracepoint(tracepoint_id) : NULL; - } - - // kprobe or uprobe - if (strncmp(name, "kprobe:", 7) == 0) { - return getProbe(&AVAILABLE_EVENTS[IDX_KPROBE], "kprobe", name + 7, 0); - } - if (strncmp(name, "uprobe:", 7) == 0) { - return getProbe(&AVAILABLE_EVENTS[IDX_UPROBE], "uprobe", name + 7, 0); - } - if (strncmp(name, "kretprobe:", 10) == 0) { - return getProbe(&AVAILABLE_EVENTS[IDX_KPROBE], "kprobe", name + 10, 1); - } - if (strncmp(name, "uretprobe:", 10) == 0) { - return getProbe(&AVAILABLE_EVENTS[IDX_UPROBE], "uprobe", name + 10, 1); - } - - // Raw PMU register: rNNN - if (name[0] == 'r' && name[1] >= '0') { - char *end; - __u64 reg = strtoull(name + 1, &end, 16); - if (*end == 0) { - return getRawEvent(reg); - } - } - - // Raw perf event descriptor: pmu/event-descriptor/ - const char *s = strchr(name, '/'); - if (s > name && s[1] != 0 && s[strlen(s) - 1] == '/') { - return getPmuEvent(name); - } - - // Kernel tracepoints defined in debugfs - s = strchr(name, ':'); - if (s != NULL && s[1] != ':') { - int tracepoint_id = findTracepointId(name); - if (tracepoint_id > 0) { - return getTracepoint(tracepoint_id); - } - } - - // Finally, treat event as a function name and return an execution - // breakpoint - return getBreakpoint(name, HW_BREAKPOINT_X, sizeof(long)); - } -}; - -// See perf_event_open(2) -#define LOAD_MISS(perf_hw_cache_id) \ - ((perf_hw_cache_id) | PERF_COUNT_HW_CACHE_OP_READ << 8 | \ - PERF_COUNT_HW_CACHE_RESULT_MISS << 16) - -PerfEventType PerfEventType::AVAILABLE_EVENTS[] = { - {"cpu", DEFAULT_CPU_INTERVAL, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_TASK_CLOCK}, - {"page-faults", 1, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_PAGE_FAULTS}, - {"context-switches", 1, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CONTEXT_SWITCHES}, - - {"cycles", 1000000, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CPU_CYCLES}, - {"instructions", 1000000, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS}, - {"cache-references", 1000000, PERF_TYPE_HARDWARE, - PERF_COUNT_HW_CACHE_REFERENCES}, - {"cache-misses", 1000, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES}, - {"branch-instructions", 1000000, PERF_TYPE_HARDWARE, - PERF_COUNT_HW_BRANCH_INSTRUCTIONS}, - {"branch-misses", 1000, PERF_TYPE_HARDWARE, PERF_COUNT_HW_BRANCH_MISSES}, - {"bus-cycles", 1000000, PERF_TYPE_HARDWARE, PERF_COUNT_HW_BUS_CYCLES}, - - {"L1-dcache-load-misses", 1000000, PERF_TYPE_HW_CACHE, - LOAD_MISS(PERF_COUNT_HW_CACHE_L1D)}, - {"LLC-load-misses", 1000, PERF_TYPE_HW_CACHE, - LOAD_MISS(PERF_COUNT_HW_CACHE_LL)}, - {"dTLB-load-misses", 1000, PERF_TYPE_HW_CACHE, - LOAD_MISS(PERF_COUNT_HW_CACHE_DTLB)}, - - {"rNNN", 1000, PERF_TYPE_RAW, 0}, /* IDX_RAW */ - {"pmu/event-descriptor/", 1000, PERF_TYPE_RAW, 0}, /* IDX_PMU */ - - {"mem:breakpoint", 1, PERF_TYPE_BREAKPOINT, 0}, /* IDX_BREAKPOINT */ - {"trace:tracepoint", 1, PERF_TYPE_TRACEPOINT, 0}, /* IDX_TRACEPOINT */ - - {"kprobe:func", 1, 0, 0}, /* IDX_KPROBE */ - {"uprobe:path", 1, 0, 0}, /* IDX_UPROBE */ -}; - -FunctionWithCounter PerfEventType::KNOWN_FUNCTIONS[] = { - {"malloc", 1}, {"mmap", 2}, {"munmap", 2}, {"read", 3}, {"write", 3}, - {"send", 3}, {"recv", 3}, {"sendto", 3}, {"recvfrom", 3}, {NULL}}; - -char PerfEventType::probe_func[256]; - -class RingBuffer { -private: - const char *_start; - unsigned long _offset; - -public: - RingBuffer(struct perf_event_mmap_page *page) { - _start = (const char *)page + OS::page_size; - } - - struct perf_event_header *seek(u64 offset) { - _offset = (unsigned long)offset & OS::page_mask; - return (struct perf_event_header *)(_start + _offset); - } - - u64 next() { - _offset = (_offset + sizeof(u64)) & OS::page_mask; - return *(u64 *)(_start + _offset); - } - - u64 peek(unsigned long words) { - unsigned long peek_offset = (_offset + words * sizeof(u64)) & OS::page_mask; - return *(u64 *)(_start + peek_offset); - } -}; - -class PerfEvent : public SpinLock { -private: - int _fd; - struct perf_event_mmap_page *_page; - - friend class PerfEvents; -}; - -volatile bool PerfEvents::_enabled = false; -int PerfEvents::_max_events = -1; -PerfEvent *PerfEvents::_events = NULL; -PerfEventType *PerfEvents::_event_type = NULL; -long PerfEvents::_interval; -Ring PerfEvents::_ring; -CStack PerfEvents::_cstack; -bool PerfEvents::_use_mmap_page; - -static int __intsort(const void *a, const void *b) { - return *(const int *)a > *(const int *)b; -} - -int PerfEvents::registerThread(int tid) { - if (_max_events == -1) { - // It hasn't been started - return 0; - } - if (tid >= _max_events) { - Log::warn("tid[%d] > pid_max[%d]. Restart profiler after changing pid_max", - tid, _max_events); - return -1; - } - - if (__atomic_load_n(&_events[tid]._fd, __ATOMIC_ACQUIRE) > 0) { - Log::debug("Thread %d is already registered for perf_event_open", tid); - return 0; - } - - PerfEventType *event_type = _event_type; - if (event_type == NULL) { - return -1; - } - - // Mark _events[tid] early to prevent duplicates. Real fd will be put later. - if (!__sync_bool_compare_and_swap(&_events[tid]._fd, 0, -1)) { - // Lost race. The event is created either from PerfEvents::start() or from - // pthread hook. - return 0; - } - - struct perf_event_attr attr = {0}; - attr.size = sizeof(attr); - attr.type = event_type->type; - - if (attr.type == PERF_TYPE_BREAKPOINT) { - attr.bp_type = event_type->config; - } else { - attr.config = event_type->config; - } - attr.config1 = event_type->config1; - attr.config2 = event_type->config2; - - // Hardware events may not always support zero skid - if (attr.type == PERF_TYPE_SOFTWARE) { - attr.precise_ip = 2; - } - - attr.sample_period = _interval; - attr.sample_type = PERF_SAMPLE_CALLCHAIN; - attr.disabled = 1; - attr.wakeup_events = 1; - attr.exclude_callchain_user = 1; - - if (!(_ring & RING_KERNEL)) { - attr.exclude_kernel = 1; - } - if (!(_ring & RING_USER)) { - attr.exclude_user = 1; - } - - if (_cstack >= CSTACK_FP) { - attr.exclude_callchain_user = 1; - } - -#ifdef PERF_ATTR_SIZE_VER5 - if (_cstack == CSTACK_LBR) { - attr.sample_type |= PERF_SAMPLE_BRANCH_STACK | PERF_SAMPLE_REGS_USER; - attr.branch_sample_type = - PERF_SAMPLE_BRANCH_USER | PERF_SAMPLE_BRANCH_CALL_STACK; - attr.sample_regs_user = 1ULL << PERF_REG_PC; - } -#else -#warning "Compiling without LBR support. Kernel headers 4.1+ required" -#endif - - int fd = syscall(__NR_perf_event_open, &attr, tid, -1, -1, 0); - - if (fd == -1) { - int err = errno; - Log::warn("perf_event_open for TID %d failed: %s", tid, strerror(err)); - _events[tid]._fd = 0; - return err; - } - - if (!__sync_bool_compare_and_swap(&_events[tid]._fd, -1, fd)) { - // Lost race. The event is created either from start() or from - // onThreadStart() - close(fd); - return 0; - } - - void *page = NULL; - if ((_ring & RING_KERNEL)) { - page = _use_mmap_page ? mmap(NULL, 2 * OS::page_size, - PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0) - : NULL; - if (page == MAP_FAILED) { - Log::info("perf_event mmap failed: %s", strerror(errno)); - page = NULL; - } - } - - _events[tid].reset(); - _events[tid]._fd = fd; - _events[tid]._page = (struct perf_event_mmap_page *)page; - - struct f_owner_ex ex; - ex.type = F_OWNER_TID; - ex.pid = tid; - - fcntl(fd, F_SETFL, O_ASYNC); - fcntl(fd, F_SETSIG, SIGPROF); - fcntl(fd, F_SETOWN_EX, &ex); - - ioctl(fd, PERF_EVENT_IOC_RESET, 0); - ioctl(fd, PERF_EVENT_IOC_REFRESH, 1); - - return 0; -} - -void PerfEvents::unregisterThread(int tid) { - if (tid >= _max_events) { - return; - } - - PerfEvent *event = &_events[tid]; - int fd = event->_fd; - if (fd > 0 && __sync_bool_compare_and_swap(&event->_fd, fd, 0)) { - ioctl(fd, PERF_EVENT_IOC_DISABLE, 0); - close(fd); - } - if (event->_page != NULL) { - event->lock(); - munmap(event->_page, 2 * OS::page_size); - event->_page = NULL; - event->unlock(); - } -} - -u64 PerfEvents::readCounter(siginfo_t *siginfo, void *ucontext) { - switch (_event_type->counter_arg) { - case 1: - return StackFrame(ucontext).arg0(); - case 2: - return StackFrame(ucontext).arg1(); - case 3: - return StackFrame(ucontext).arg2(); - case 4: - return StackFrame(ucontext).arg3(); - default: { - u64 counter; - return read(siginfo->si_fd, &counter, sizeof(counter)) == sizeof(counter) - ? counter - : 1; - } - } -} - -void PerfEvents::signalHandler(int signo, siginfo_t *siginfo, void *ucontext) { - SIGNAL_HANDLER_GUARD(); - if (siginfo->si_code <= 0) { - // Looks like an external signal; don't treat as a profiling event - return; - } - // Atomically try to enter critical section - prevents all reentrancy races - CriticalSection cs; - if (!cs.entered()) { - return; // Another critical section is active, defer profiling - } - ProfiledThread *current = ProfiledThread::currentSignalSafe(); - if (current != NULL) { - current->noteCPUSample(Profiler::instance()->recordingEpoch()); - } - int tid = current != NULL ? current->tid() : OS::threadId(); - if (_enabled) { - Shims::instance().setSighandlerTid(tid); - - u64 counter = readCounter(siginfo, ucontext); - ExecutionEvent event; - event._execution_mode = getThreadExecutionMode(); - Profiler::instance()->recordSample(ucontext, counter, tid, BCI_CPU, 0, - &event); - Shims::instance().setSighandlerTid(-1); - } else { - resetBuffer(tid); - } - - ioctl(siginfo->si_fd, PERF_EVENT_IOC_RESET, 0); - ioctl(siginfo->si_fd, PERF_EVENT_IOC_REFRESH, 1); -} - -Error PerfEvents::check(Arguments &args) { - // The official way of knowing if perf_event_open() support is enabled - // is checking for the existence of the file - // /proc/sys/kernel/perf_event_paranoid - struct stat statbuf; - if (stat("/proc/sys/kernel/perf_event_paranoid", &statbuf) != 0) { - return Error("/proc/sys/kernel/perf_event_paranoid doesn't exist"); - } - - PerfEventType *event_type = - PerfEventType::forName(args._event == NULL ? EVENT_CPU : args._event); - if (event_type == NULL) { - return Error("Unsupported event type"); - } else if (event_type->counter_arg > 4) { - return Error("Only arguments 1-4 can be counted"); - } - - if (_pthread_entry == NULL && - (_pthread_entry = lookupThreadEntry()) == NULL) { - return Error("Could not set pthread hook"); - } - - struct perf_event_attr attr = {0}; - attr.size = sizeof(attr); - attr.type = event_type->type; - - if (attr.type == PERF_TYPE_BREAKPOINT) { - attr.bp_type = event_type->config; - } else { - attr.config = event_type->config; - } - attr.config1 = event_type->config1; - attr.config2 = event_type->config2; - - attr.sample_period = event_type->default_interval; - attr.sample_type = PERF_SAMPLE_CALLCHAIN; - attr.disabled = 1; - - if (!(_ring & RING_KERNEL)) { - attr.exclude_kernel = 1; - } else if (!Symbols::haveKernelSymbols()) { - Libraries::instance()->updateSymbols(true); - attr.exclude_kernel = Symbols::haveKernelSymbols() ? 0 : 1; - } - if (!(_ring & RING_USER)) { - attr.exclude_user = 1; - } - - if (_cstack == CSTACK_FP || _cstack == CSTACK_DWARF) { - attr.exclude_callchain_user = 1; - } - - if (args._cstack >= CSTACK_FP) { - attr.exclude_callchain_user = 1; - } - -#ifdef PERF_ATTR_SIZE_VER5 - if (args._cstack == CSTACK_LBR) { - attr.sample_type |= PERF_SAMPLE_BRANCH_STACK | PERF_SAMPLE_REGS_USER; - attr.branch_sample_type = - PERF_SAMPLE_BRANCH_USER | PERF_SAMPLE_BRANCH_CALL_STACK; - attr.sample_regs_user = 1ULL << PERF_REG_PC; - } -#endif - - int fd = syscall(__NR_perf_event_open, &attr, 0, -1, -1, 0); - if (fd == -1) { - return Error(strerror(errno)); - } - - close(fd); - return Error::OK; -} - -Error PerfEvents::start(Arguments &args) { - _event_type = - PerfEventType::forName(args._event == NULL ? EVENT_CPU : args._event); - ; - if (_event_type == NULL) { - return Error("Unsupported event type"); - } else if (_event_type->counter_arg > 4) { - return Error("Only arguments 1-4 can be counted"); - } - - // if an arbitrary perf event type is specified pick the interval from - // args._interval directly otherwise ask for the effective CPU sampler - // interval - int interval = args._event != NULL && args._event != EVENT_CPU - ? args._interval - : args.cpuSamplerInterval(); - if (interval < 0) { - return Error("interval must be positive"); - } - - if (_pthread_entry == NULL && - (_pthread_entry = lookupThreadEntry()) == NULL) { - return Error("Could not set pthread hook"); - } - - _interval = interval ? interval : _event_type->default_interval; - - _ring = args._ring; - if ((_ring & RING_KERNEL) && !Symbols::haveKernelSymbols()) { - static bool logged = false; - if (!logged) { - Log::info("Kernel symbols are unavailable due to restrictions. Try\n" - " sysctl kernel.kptr_restrict=0\n" - " sysctl kernel.perf_event_paranoid=1"); - logged = true; - } - _ring = RING_USER; - } - _cstack = args._cstack; - _use_mmap_page = _cstack != CSTACK_NO && - (_ring != RING_USER || _cstack == CSTACK_DEFAULT || - _cstack == CSTACK_LBR); - - int max_events = OS::getMaxThreadId(); - if (max_events != _max_events) { - free(_events); - _events = (PerfEvent *)calloc(max_events, sizeof(PerfEvent)); - _max_events = max_events; - } - - OS::installSignalHandler(SIGPROF, signalHandler); - - // Enable pthread hook before traversing currently running threads - __atomic_store_n(_pthread_entry, (void *)pthread_setspecific_hook, - __ATOMIC_RELEASE); - - // Create perf_events for all existing threads - int threads_sz = 0, threads_cap; - int *threads = (int *)malloc((threads_cap = 1024) * sizeof(int)); - ThreadList *thread_list = OS::listThreads(); - // get a fixed list of all the threads - while (thread_list->hasNext()) { - int tid = thread_list->next(); - if (threads_sz == threads_cap) { - threads = (int *)realloc(threads, (threads_cap += 1024) * sizeof(int)); - } - threads[threads_sz++] = tid; - } - delete thread_list; - - qsort(threads, threads_sz, sizeof(int), __intsort); - - int err = 0; - int threads_idx = 0; - for (int tid = 0; tid < _max_events; tid++) { - // if we didn't go over all existing threads and the tid matches an existing - // thread - if (threads_idx < threads_sz && tid == threads[threads_idx]) { - // create the timer - if ((err = registerThread(tid)) != 0) { - if (err == ESRCH) { - // ignore because the thread doesn't exist anymore - err = 0; - } else { - // if we fail, stop creating more perf_events - break; - } - } - // finally, increase threads_idx - threads_idx += 1; - } - // if the tid doesn't matches an existing thread - else { - // destroy the timer - unregisterThread(tid); - } - } - - free(threads); - - if (err != 0) { - *_pthread_entry = (void *)pthread_setspecific; - Profiler::instance()->switchThreadEvents(JVMTI_DISABLE); - if (err == EACCES || err == EPERM) { - return Error("No access to perf events. Try --all-user option or 'sysctl " - "kernel.perf_event_paranoid=1'"); - } else { - return Error("Perf events unavailable"); - } - } - - if (threads_idx != threads_sz) { - Log::error("perfEvents: we didn't go over all events, threads_idx = %d, " - "threads_sz = %d", - threads_idx, threads_sz); - } - - return Error::OK; -} - -void PerfEvents::stop() { - // As we don't have snapshot feature, it's wasteful to unregister all the - // threads to re-register them right after when doing a stop+start to capture - // the data. Instead, since we know we are continuously profiling and we know - // the interval doesn't change, simply don't unregister threads on stop, and - // check whether the thread has been registered already on start. -} - -int PerfEvents::walkKernel(int tid, const void **callchain, int max_depth, - StackContext *java_ctx) { - if (!(_ring & RING_KERNEL)) { - // we are not capturing kernel stacktraces - return 0; - } - - PerfEvent *event = &_events[tid]; - if (!event->tryLock()) { - return 0; // the event is being destroyed - } - - int depth = 0; - - struct perf_event_mmap_page *page = event->_page; - if (page != NULL) { - u64 tail = page->data_tail; - u64 head = page->data_head; - rmb(); - - RingBuffer ring(page); - - while (tail < head) { - struct perf_event_header *hdr = ring.seek(tail); - if (hdr->type == PERF_RECORD_SAMPLE) { - u64 nr = ring.next(); - while (nr-- > 0) { - u64 ip = ring.next(); - if (ip < PERF_CONTEXT_MAX) { - const void *iptr = (const void *)ip; - if (JVMSupport::isJitCode(iptr) || depth >= max_depth) { - // Stop at the first Java frame - java_ctx->pc = iptr; - goto stack_complete; - } - callchain[depth++] = iptr; - } - } - - if (_cstack == CSTACK_LBR) { - u64 bnr = ring.next(); - - // Last userspace PC is stored right after branch stack - const void *pc = (const void *)ring.peek(bnr * 3 + 2); - if (JVMSupport::isJitCode(pc) || depth >= max_depth) { - java_ctx->pc = pc; - goto stack_complete; - } - callchain[depth++] = pc; - - while (bnr-- > 0) { - const void *from = (const void *)ring.next(); - const void *to = (const void *)ring.next(); - ring.next(); - - if (JVMSupport::isJitCode(to) || depth >= max_depth) { - java_ctx->pc = to; - goto stack_complete; - } - callchain[depth++] = to; - - if (JVMSupport::isJitCode(from) || depth >= max_depth) { - java_ctx->pc = from; - goto stack_complete; - } - callchain[depth++] = from; - } - } - - break; - } - tail += hdr->size; - } - - stack_complete: - page->data_tail = head; - } - - event->unlock(); - - return depth; -} - -void PerfEvents::resetBuffer(int tid) { - PerfEvent *event = &_events[tid]; - if (!event->tryLock()) { - return; // the event is being destroyed - } - - struct perf_event_mmap_page *page = event->_page; - if (page != NULL) { - u64 head = page->data_head; - rmb(); - page->data_tail = head; - } - - event->unlock(); -} - -const char *PerfEvents::getEventName(int event_id) { - if (event_id >= 0 && - (size_t)event_id < - sizeof(PerfEventType::AVAILABLE_EVENTS) / sizeof(PerfEventType)) { - return PerfEventType::AVAILABLE_EVENTS[event_id].name; - } - return NULL; -} - -#endif // __linux__ diff --git a/ddprof-lib/src/main/cpp/pidController.cpp b/ddprof-lib/src/main/cpp/pidController.cpp deleted file mode 100644 index a6851d1b6..000000000 --- a/ddprof-lib/src/main/cpp/pidController.cpp +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2023 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "pidController.h" - -double PidController::compute(u64 input, double time_delta_coefficient) { - // time_delta_coefficient allows variable sampling window - // the values are linearly scaled using that coefficient to reinterpret the - // given value within the expected sampling window - double absolute_error = - static_cast(_target) - input * time_delta_coefficient; - - double avg_error = (_alpha * absolute_error) + ((1 - _alpha) * _avg_error); - double derivative = avg_error - _avg_error; - - // PID formula: - // u[k] = Kp e[k] + Ki e_i[k] + Kd e_d[k], control signal - double signal = _proportional_gain * absolute_error + - _integral_gain * _integral_value + - _derivative_gain * derivative; - - _integral_value += absolute_error; - _avg_error = avg_error; - - return signal; -} diff --git a/ddprof-lib/src/main/cpp/pidController.h b/ddprof-lib/src/main/cpp/pidController.h deleted file mode 100644 index 051356c4a..000000000 --- a/ddprof-lib/src/main/cpp/pidController.h +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2023 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _PIDCONTROLLER_H -#define _PIDCONTROLLER_H - -#include "arch.h" -#include - -/* - * A simple implementation of a PID controller. - * Heavily influenced by - * https://tttapa.github.io/Pages/Arduino/Control-Theory/Motor-Fader/PID-Cpp-Implementation.html - */ -class PidController { -private: - u64 _target; - double _proportional_gain; - double _derivative_gain; - double _integral_gain; - double _alpha; - - double _avg_error; - double _integral_value; - - inline static double computeAlpha(float cutoff) { - if (cutoff <= 0) - return 1; - // α(fₙ) = cos(2πfₙ) - 1 + √( cos(2πfₙ)² - 4 cos(2πfₙ) + 3 ) - const double c = std::cos(2 * double(M_PI) * cutoff); - return c - 1 + std::sqrt(c * c - 4 * c + 3); - } - -public: - PidController(u64 target_per_second, double proportional_gain, - double integral_gain, double derivative_gain, - int sampling_window, double cutoff_secs) - : _target(target_per_second * sampling_window), - _proportional_gain(proportional_gain), - _derivative_gain(derivative_gain / sampling_window), - _integral_gain(integral_gain * sampling_window), - _alpha(computeAlpha(sampling_window / cutoff_secs)), - _avg_error(0), - _integral_value(0) {} - - double compute(u64 input, double time_delta_seconds); - - // Reset integrator/derivative state. Intended for tests and for cases where the - // controller must start from a clean slate (e.g. a new profiling session on a - // different workload). - inline void reset() { - _avg_error = 0; - _integral_value = 0; - } -}; - -#endif \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/poissonSampler.h b/ddprof-lib/src/main/cpp/poissonSampler.h deleted file mode 100644 index 68e264a30..000000000 --- a/ddprof-lib/src/main/cpp/poissonSampler.h +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _POISSONSAMPLER_H -#define _POISSONSAMPLER_H - -#include "arch.h" -#include - -/** - * Thread-local, dimension-agnostic Poisson-process sampler. - * - * ## Concept - * - * Many profiler engines need to sub-sample a high-frequency stream of - * measurements and produce an unbiased aggregate estimate from the surviving - * samples. The classic approach is a deterministic threshold counter (fire - * every N units), but that creates phase-locking artifacts and handles - * measurements that span multiple intervals poorly. - * - * PoissonSampler models the stream as a Poisson process: between consecutive - * sample points the gap is drawn independently from Exp(mean = @p interval). - * This guarantees memoryless inter-arrival times and correct behaviour for - * measurements of any size relative to the interval. - * - * ## Sampling decision - * - * The sampler maintains a monotonically growing accumulator @c _used and a - * Exp-distributed threshold @c _threshold. On each call: - * - * 1. @p value is added to @c _used. - * 2. If @c _used < @c _threshold: no sample, return false. - * 3. Otherwise: the threshold has been crossed. Set @c _threshold to - * @c _used plus a fresh draw from Exp(@p interval) so the next gap is - * independent and any surplus from a measurement that overshot the old - * threshold is discarded (no burst sampling after a large @p value). - * Compute and return the weight (see below), return true. - * - * ## Weight formula and estimator invariant - * - * The probability that a Poisson process with rate 1/@p interval produces - * at least one event during an interval of length @p value is: - * - * P = 1 - exp(-value / interval) - * - * The inverse-transform weight is: - * - * weight = 1 / P = 1 / (1 - exp(-value / interval)) - * - * This satisfies the unbiasedness invariant for every measurement: - * - * E[weight * value | sampled] * P(sampled) = (1/P) * value * P = value - * - * Therefore sum(weight_i * value_i) over all sampled events is an unbiased - * estimator of the total accumulated dimension over the recording window, - * regardless of the distribution of individual measurement sizes: - * - * - value << interval → P ≈ value/interval, weight ≈ interval/value, - * weight * value ≈ interval (one interval per event) - * - value >> interval → P ≈ 1, weight ≈ 1, - * weight * value ≈ value (full measurement credited) - * - * ## Dimension agnosticism - * - * The class is generic over the accumulated dimension. The caller chooses - * what @p value represents; @p interval must be in the same units: - * - * - TSC ticks → latency / time-in-I/O profiling - * - Bytes → throughput / allocation profiling - * - Event count → frequency profiling (pass value = 1 per event) - * - * ## Thread safety and epoch-based reset - * - * Instances must be declared @c thread_local. All state is private to the - * owning thread; no locks or atomics are used in the hot path. - * - * To support profiler restart, the engine exposes a shared - * @c std::atomic epoch counter that it increments on @c start(). - * Each PoissonSampler caches the last seen epoch; when it differs the - * sampler reinitialises lazily on the next @c sample() call, re-seeding - * the PRNG and resetting the accumulator and threshold. - * - * ## Usage - * - * @code - * // Engine header: - * std::atomic _epoch{0}; // bumped in start() - * std::atomic _interval; // PID-controlled mean gap - * - * // Engine translation unit: - * static thread_local PoissonSampler _send_sampler; - * static thread_local PoissonSampler _recv_sampler; - * - * // In the hook / measurement path: - * float weight; - * if (_send_sampler.sample(value, (u64)_interval.load(), _epoch.load(), weight)) { - * // record event; sum(weight * value) estimates total accumulated value - * } - * @endcode - */ -class PoissonSampler { -public: - /** - * Decide whether to sample this measurement and compute its weight. - * - * @param value The measurement for this call, in the chosen unit. - * Must be in the same units as @p interval. - * A value of 0 is never sampled. - * @param interval Mean inter-sample gap in the same units as @p value. - * Controls the average number of events recorded per - * unit of accumulated @p value. Must be > 0. - * @param epoch_now Current profiler epoch from the owning engine's - * shared atomic. When it differs from the cached epoch - * the sampler resets all state before evaluating the - * current measurement. - * @param weight [out] Set on true return to 1/(1-exp(-value/interval)). - * Multiply by @p value to get this event's contribution - * to the total-accumulated-value estimate. - * @return true if this measurement should be recorded. - */ - bool sample(u64 value, u64 interval, u64 epoch_now, float &weight) { - if (value == 0 || interval == 0) { - return false; - } - if (_epoch != epoch_now) { - reset(interval, epoch_now); - } - _used += value; - if (_used < _threshold) { - return false; - } - // Threshold crossed: rebase the next threshold on the current accumulator - // plus a fresh Exp draw. A single large measurement can push _used far - // past _threshold (e.g. one long blocking read/write spanning many - // intervals); advancing relative to _threshold (+=) would leave the next - // threshold below _used and fire on every subsequent call until the gap is - // worked off — a burst-sampling bias. Rebasing on _used drops the crossed - // surplus so the next inter-arrival gap starts fresh and independent. - _threshold = _used + nextExp(interval); - // Float precision: when value >> interval, expf(-value/interval) rounds to 0.0f, - // so weight = 1.0f / (1.0f - 0.0f) = 1.0f. This is a conservative lower bound — - // large events count as weight >= 1.0. Intentional; avoids the cost of double arithmetic. - float p = 1.0f - expf(-(float)value / (float)interval); - weight = (p > 0.0f) ? 1.0f / p : 1.0f; - return true; - } - -private: - u64 _epoch{0}; // last seen profiler epoch; 0 = not yet initialised - u64 _used{0}; // accumulated value since the last threshold crossing - u64 _threshold{0}; // next Exp-distributed threshold to cross - u64 _rng{0}; // xorshift64 PRNG state; must never be 0 - - /** - * Reinitialise all state for a new profiling session. - * - * The PRNG is seeded from the instance's own address XOR'd with a - * multiple of the new epoch. Because @c thread_local instances live at - * fixed but thread-specific addresses, and because distinct samplers - * within the same thread occupy different addresses, each (thread, - * sampler, session) triple receives an independent random stream. - */ - void reset(u64 interval, u64 epoch_now) { - // Fibonacci hashing of epoch_now spreads low-entropy epoch values - // across the full 64-bit range before XOR-ing with the address. - _rng = (u64)(uintptr_t)this ^ (epoch_now * 0x9e3779b97f4a7c15ULL); - if (_rng == 0) _rng = 1; // xorshift64 must not start at 0 - _used = 0; - _threshold = nextExp(interval); - _epoch = epoch_now; - } - - /** - * Draw one sample from Exp(mean = @p interval). - * - * Uses xorshift64 to produce a uniform pseudo-random value, then applies - * the inverse CDF of the exponential distribution: - * - * X = -interval * ln(U), U ~ Uniform(0, 1] - * - * U > 0 because the +0.5 offset in `((double)_rng + 0.5) * 2^-64` ensures the - * product is strictly positive even if _rng were 0. The xorshift64 invariant - * (_rng != 0) is independently required by the recurrence (0 is a fixed point). - * The magic constant 5.421010862427522e-20 ≈ 1/2^64 converts a u64 to [0, 1]. - * - * Use `double` (53 mantissa bits) rather than `float` (24): with the xorshift64 - * invariant (_rng != 0), the minimum _rng is 1, giving U_min = 1.5 × 2^-64 ≈ - * 8.13e-20 for both precisions. The real advantage of double is quantisation - * density: float maps 2^64 rng values to only 2^24 distinct U values near the - * high end, clustering inter-arrival draws; double maps them to 2^53 distinct - * values, producing a far smoother distribution. The maximum Exp draw is - * -ln(U_min) * interval ≈ 44 * interval for both precisions. - * - * ### Why xorshift64 instead of a C++ standard generator? - * - * The C++ facility (std::mt19937, std::minstd_rand, …) is - * unsuitable here for several reasons: - * - * 1. **Hot-path overhead.** nextExp() is called only once per fired - * event (~83 times/second at the default rate), so raw throughput - * is not the primary concern. The concern is code-size and - * instruction-cache pressure: std::mt19937 carries ~2.5 KB of - * state and its generate step touches all of it. xorshift64 fits - * in a single 8-byte field already present in the struct. - * - * 2. **Seeding.** std::random_device — the canonical seed source — - * may block, throw, or return low-entropy values on some Linux - * configurations (e.g., early boot, containers without /dev/urandom - * entropy). Our seed (instance address XOR epoch hash) is always - * available, zero-cost, and produces independent streams per thread - * and per profiling session without any OS interaction. - * - * 3. **No allocation, no exceptions.** std::random_device and the - * distribution wrappers (std::uniform_real_distribution, etc.) may - * allocate and may throw. This code runs inside PLT hooks that - * intercept arbitrary application threads; allocation and exception - * handling in that context would be unsafe. - * - * 4. **Statistical sufficiency.** xorshift64 (Marsaglia 2003) passes the - * Diehard battery; it fails some BigCrush tests for linear-algebra-based - * statistics (MatrixRank, LinearComp), but those failure modes are - * irrelevant to inverse-CDF Exp sampling for aggregate weight estimates. - * The inverse-CDF transform amplifies non-uniformity only near U ≈ 0 - * (i.e., extremely large Exp draws), which correspond to very long - * inter-sample gaps — a rare tail that has negligible effect on aggregate - * estimates. - */ - u64 nextExp(u64 interval) { - _rng ^= _rng << 13; - _rng ^= _rng >> 7; - _rng ^= _rng << 17; - double u = ((double)_rng + 0.5) * 5.421010862427522e-20; - double raw = -(double)interval * log(u); - // Cap before narrowing cast: raw can exceed UINT64_MAX when interval is - // near LONG_MAX (the LONG_MAX sentinel used by start() for huge inputs). - // Casting an out-of-range double to u64 is UB in C++ (ISO §8.4 / §7.8). - return (raw >= (double)(u64)-1) ? (u64)-1 : (u64)raw; - } -}; - -#endif // _POISSONSAMPLER_H diff --git a/ddprof-lib/src/main/cpp/primeProbing.h b/ddprof-lib/src/main/cpp/primeProbing.h deleted file mode 100644 index 4272b0482..000000000 --- a/ddprof-lib/src/main/cpp/primeProbing.h +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef PRIME_PROBING_H -#define PRIME_PROBING_H - -#include "arch.h" -#include "common.h" - -// Hash table probe iterator for efficient collision resolution -// This class is not thread-safe - the caller must ensure that a single instance is used in a single thread only -class HashProbe { -private: - // Prime probing constants for better hash distribution and collision resolution - - // Prime numbers for semi-random probing, selected to provide good distribution - // These primes are carefully chosen to avoid patterns and clustering - // We are using prime numbers that will allow an arbitrary capacity value, up to - // 2224132796298468927597810244428305585566171739231 (LCM of all numbers in the list) - static constexpr u32 PRIME_STEPS[] = { - 1009, 1013, 1019, 1021, 1031, 1033, 1039, 1049, - 1051, 1061, 1063, 1069, 1087, 1091, 1093, 1097 - }; - static constexpr int PRIME_STEP_COUNT = sizeof(PRIME_STEPS) / sizeof(u32); - - u32 _slot; - u32 _step; - u32 _capacity; - u32 _step_count; - - inline static u32 baseSlot(u64 seed, u32 capacity) { - // Apply Knuth multiplicative hash directly to the seed - size_t hash = (static_cast(seed) * KNUTH_MULTIPLICATIVE_CONSTANT); - // Use high bits for better distribution (shift right to get top bits) - return static_cast((hash >> (sizeof(size_t) * 8 - 13)) % capacity); - } - - inline static u32 getStep(u64 seed, u32 capacity) { - int idx = (seed >> 4) % PRIME_STEP_COUNT; - int orig_idx = idx; - while ((capacity % PRIME_STEPS[idx]) == 0) { - // not a co-prime, try next one - idx = (idx + 1) % PRIME_STEP_COUNT; - if (idx == orig_idx) { - // unlikely case as the smallest failing capacity is much bigger than u32 - // we will fall-back to step of '1' anyway - return 1; - } - } - return PRIME_STEPS[idx]; - } - - inline static u32 advanceSlot(u32 slot, u32 step, u32 capacity) { - return (slot + step) & (capacity - 1); - } - -public: - HashProbe(u64 seed, u32 capacity) - : _slot(baseSlot(seed, capacity)) - , _step(getStep(seed, capacity)) - , _capacity(capacity) - , _step_count(0) { - } - - u32 slot() const { return _slot; } - u32 stepCount() const { return _step_count; } - - bool hasNext() const { return _step_count < _capacity; } - - void updateCapacity(u32 capacity) { - _capacity = capacity; - } - - u32 next() { - _slot = advanceSlot(_slot, _step, _capacity); - _step_count++; - return _slot; - } -}; - - -#endif // PRIME_PROBING_H diff --git a/ddprof-lib/src/main/cpp/profiler.cpp b/ddprof-lib/src/main/cpp/profiler.cpp deleted file mode 100644 index 777f65017..000000000 --- a/ddprof-lib/src/main/cpp/profiler.cpp +++ /dev/null @@ -1,1813 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2024, 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "profiler.h" -#include "asyncSampleMutex.h" -#include "mallocTracer.h" -#include "nativeSocketSampler.h" -#include "context.h" -#include "guards.h" -#include "common.h" -#include "counters.h" -#include "ctimer.h" -#include "dwarf.h" -#include "flightRecorder.h" -#include "itimer.h" -#include "hotspot/vmStructs.inline.h" -#include "hotspot/hotspotSupport.h" -#include "j9/j9Support.h" -#include "j9/j9WallClock.h" -#include "jvmSupport.h" -#include "jvmThread.h" -#include "libraryPatcher.h" -#include "objectSampler.h" -#include "os.h" -#include "perfEvents.h" -#include "safeAccess.h" -#include "stackFrame.h" -#include "stackWalker.h" -#include "symbols.h" -#include "thread.h" -#include "tsc.h" -#include "utils.h" -#include "wallClock.h" -#include "wallClockCounters.h" -#include "frames.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// The instance is not deleted on purpose, since profiler structures -// can be still accessed concurrently during VM termination -Profiler *const Profiler::_instance = new Profiler(); -volatile bool Profiler::_signals_initialized = false; -volatile bool Profiler::_need_JDK_8313796_workaround = true; - -static void (*orig_segvHandler)(int signo, siginfo_t *siginfo, void *ucontext); -static void (*orig_busHandler)(int signo, siginfo_t *siginfo, void *ucontext); - -static Engine noop_engine; -static MallocTracer malloc_tracer; -static PerfEvents perf_events; -static WallClockASGCT wall_asgct_engine; -static WallClockJvmti wall_jvmti_engine; -static J9WallClock j9_engine; -static ITimer itimer; -static ITimerJvmti itimer_jvmti; -static CTimer ctimer; -static CTimerJvmti ctimer_jvmti; - -void Profiler::onThreadStart(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { - ProfiledThread::initCurrentThread(); - ProfiledThread *current = ProfiledThread::current(); - current->setJavaThread(true); - int tid = current->tid(); - if (_thread_filter.enabled()) { - int slot_id = _thread_filter.registerThread(); - current->setFilterSlotId(slot_id); - _thread_filter.resetSlotRunState(slot_id); - _thread_filter.remove(slot_id); // Remove from filtering initially - } - if (thread != NULL) { - updateThreadName(jvmti, jni, thread, true); - } - - _cpu_engine->registerThread(tid); - _wall_engine->registerThread(tid); -} - -void Profiler::onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { - ProfiledThread *current = ProfiledThread::currentSignalSafe(); - int tid = -1; - - if (current != nullptr) { - // ProfiledThread is alive - do full cleanup and use efficient tid access - int slot_id = current->filterSlotId(); - tid = current->tid(); - - if (_thread_filter.enabled()) { - _thread_filter.unregisterThread(slot_id); - current->setFilterSlotId(-1); - } - - updateThreadName(jvmti, jni, thread, false); - // Block profiling signals around engine unregistration + TLS release to - // close the window where a wall-clock/CPU signal could sample a - // partially-torn-down thread (PROF-14674). - { - SignalBlocker blocker; - _cpu_engine->unregisterThread(tid); - _wall_engine->unregisterThread(tid); - ProfiledThread::release(); - } - return; - } - - // ProfiledThread already cleaned up - try to get tid from JVMTI as fallback - tid = JVMThread::nativeThreadId(jni, thread); - if (tid < 0) { - // No ProfiledThread AND can't get tid from JVMTI - nothing we can do - return; - } - - updateThreadName(jvmti, jni, thread, false); - _cpu_engine->unregisterThread(tid); - _wall_engine->unregisterThread(tid); -} - -int Profiler::registerThread(int tid) { - return _instance->_cpu_engine->registerThread(tid) | - _instance->_wall_engine->registerThread(tid); -} -#ifdef UNIT_TEST -static std::atomic g_test_last_unregistered_tid{-1}; - -int Profiler::lastUnregisteredTidForTest() { - return g_test_last_unregistered_tid.load(std::memory_order_relaxed); -} -void Profiler::resetUnregisterObservableForTest() { - g_test_last_unregistered_tid.store(-1, std::memory_order_relaxed); -} -#endif - -void Profiler::unregisterThread(int tid) { -#ifdef UNIT_TEST - // In gtest, _cpu_engine/_wall_engine are null (profiler not started). - // Record the tid so integration tests can verify the call happened without - // crashing on the null engine dereference. This bypasses the real engine - // unregister path entirely, so that path is covered only by JVM-level tests, - // not by these gtests. UNIT_TEST is defined solely for the gtest binaries - // (see GtestTaskBuilder); the shipped library never compiles this branch. - g_test_last_unregistered_tid.store(tid, std::memory_order_relaxed); - return; -#endif - _instance->_cpu_engine->unregisterThread(tid); - _instance->_wall_engine->unregisterThread(tid); -} - -const char *Profiler::asgctError(int code) { - switch (code) { - case ticks_no_Java_frame: - case ticks_unknown_not_Java: - // Not in Java context at all; this is not an error - return NULL; - case ticks_thread_exit: - // The last Java frame has been popped off, only native frames left - return NULL; - case ticks_GC_active: - return "GC_active"; - case ticks_unknown_Java: - return "unknown_Java"; - case ticks_not_walkable_Java: - return "not_walkable_Java"; - case ticks_not_walkable_not_Java: - return "not_walkable_not_Java"; - case ticks_deopt: - return "deoptimization"; - case ticks_safepoint: - return "safepoint"; - case ticks_skipped: - return "skipped"; - case ticks_unknown_state: - // Zing sometimes returns it - return "unknown_state"; - default: - // Should not happen - return "unexpected_state"; - } -} - -inline u32 Profiler::getLockIndex(int tid) { - u32 lock_index = tid; - lock_index ^= lock_index >> 8; - lock_index ^= lock_index >> 4; - return lock_index % CONCURRENCY_LEVEL; -} - -void Profiler::mangle(const char *name, char *buf, size_t size) { - char *buf_end = buf + size; - strcpy(buf, "_ZN"); - buf += 3; - - const char *c; - while ((c = strstr(name, "::")) != NULL && buf + (c - name) + 4 < buf_end) { - int n = snprintf(buf, buf_end - buf, "%d", (int)(c - name)); - if (n < 0 || n >= buf_end - buf) { - if (n < 0) { - Log::debug("Error in snprintf."); - } - goto end; - } - buf += n; - memcpy(buf, name, c - name); - buf += c - name; - name = c + 2; - } - if (buf < buf_end) { - snprintf(buf, buf_end - buf, "%d%sE*", (int)strlen(name), name); - } - -end: - buf_end[-1] = '\0'; -} - -const void *Profiler::resolveSymbol(const char *name) { - char mangled_name[256]; - if (strstr(name, "::") != NULL) { - mangle(name, mangled_name, sizeof(mangled_name)); - name = mangled_name; - } - - size_t len = strlen(name); - const CodeCacheArray& native_libs = _libs->native_libs(); - int native_lib_count = native_libs.count(); - if (len > 0 && name[len - 1] == '*') { - for (int i = 0; i < native_lib_count; i++) { - CodeCache *lib = native_libs[i]; - if (lib != NULL) { - const void *address = lib->findSymbolByPrefix(name, len - 1); - if (address != NULL) { - return address; - } - } - } - } else { - for (int i = 0; i < native_lib_count; i++) { - CodeCache *lib = native_libs[i]; - if (lib != NULL) { - const void *address = lib->findSymbol(name); - if (address != NULL) { - return address; - } - } - } - } - - return NULL; -} - -// For BCI_NATIVE_FRAME, library index is encoded ahead of the symbol name -const char *Profiler::getLibraryName(const char *native_symbol) { - short lib_index = NativeFunc::libIndex(native_symbol); - const CodeCacheArray& native_libs = _libs->native_libs(); - if (lib_index >= 0 && lib_index < native_libs.count()) { - CodeCache *lib = native_libs[lib_index]; - if (lib != NULL) { - const char *s = lib->name(); - if (s != NULL) { - const char *p = strrchr(s, '/'); - return p != NULL ? p + 1 : s; - } - } - } - return NULL; -} - -const char *Profiler::findNativeMethod(const void *address) { - CodeCache *lib = _libs->findLibraryByAddress(address); - const char *name = NULL; - if (lib != NULL) { - lib->binarySearch(address, &name); - } - return name; -} - -int Profiler::getNativeTrace(void *ucontext, ASGCT_CallFrame *frames, - int event_type, int tid, StackContext *java_ctx, - bool *truncated, int lock_index) { - if (_cstack == CSTACK_NO || - (event_type == BCI_ALLOC || event_type == BCI_ALLOC_OUTSIDE_TLAB) || - (event_type != BCI_CPU && event_type != BCI_WALL && - _cstack == CSTACK_DEFAULT)) { - return 0; - } - int max_depth = min(_max_stack_depth, MAX_NATIVE_FRAMES); - const void *callchain[max_depth + 1]; // we can read one frame past when trying to figure out whether the result is truncated - int native_frames = 0; - - if (event_type == BCI_CPU && _cpu_engine == &perf_events) { - native_frames += - PerfEvents::walkKernel(tid, callchain + native_frames, - max_depth - native_frames, java_ctx); - } - if (_cstack >= CSTACK_VM) { - return 0; - } else if (_cstack == CSTACK_DWARF) { - native_frames += StackWalker::walkDwarf(ucontext, callchain + native_frames, - max_depth - native_frames, - java_ctx, truncated); - } else { - native_frames += StackWalker::walkFP(ucontext, callchain + native_frames, - max_depth - native_frames, - java_ctx, truncated); - } - - return convertNativeTrace(native_frames, callchain, frames, lock_index); -} - -/** - * Populates an ASGCT_CallFrame with remote symbolication data. - * - * Packs pc_offset, mark, and lib_index into the jmethodID field for deferred - * symbol resolution during JFR serialization. This approach defers expensive - * symbol lookups to post-processing while still capturing marks needed for - * correct stack walk termination. - * - * @param frame The ASGCT_CallFrame to populate - * @param pc The program counter address - * @param lib The CodeCache library containing build-ID information - * @param mark The mark value (0 for regular frames, non-zero for JVM internals) - */ -void Profiler::populateRemoteFrame(ASGCT_CallFrame* frame, uintptr_t pc, CodeCache* lib, char mark) { - uintptr_t pc_offset = pc - (uintptr_t)lib->imageBase(); - uint32_t lib_index = (uint32_t)lib->libIndex(); - - unsigned long packed = RemoteFramePacker::pack(pc_offset, mark, lib_index); - - TEST_LOG("populateRemoteFrame: lib=%s, build_id=%s, pc=0x%lx, pc_offset=0x%lx, mark=%d, lib_index=%u, packed=0x%zx", - lib->name(), lib->buildId(), pc, pc_offset, (int)mark, lib_index, packed); - - frame->bci = BCI_NATIVE_FRAME_REMOTE; - frame->packed_remote_frame = packed; - - // Track remote symbolication usage - Counters::increment(REMOTE_SYMBOLICATION_FRAMES); -} - -/** - * Resolve native frame for upstream StackWalker (called from signal handler). - * Returns resolution decision without writing to ASGCT_CallFrame. - * - * For remote symbolication with build-ID: - * - Does symbol resolution to check marks (O(log n) binarySearch + O(1) mark check) - * - Packs mark + lib_index + pc_offset into method_id field - * - Returns is_marked=true if mark indicates termination (MARK_INTERPRETER, etc.) - * - * For traditional symbolication: - * - Does symbol resolution via binarySearch() on the found library - * - Checks marks after symbol resolution (same O(log n) + O(1) cost) - * - If no symbol found but PC is in a known library, packs as - * BCI_NATIVE_FRAME_REMOTE for library-relative rendering ([lib+0xoffset]) - */ -Profiler::NativeFrameResolution Profiler::resolveNativeFrameForWalkVM(uintptr_t pc, int lock_index) { - CodeCache* lib = _libs->findLibraryByAddress((void*)pc); - - if (_remote_symbolication && lib != nullptr && lib->hasBuildId()) { - // Get symbol name and check mark - const char *method_name = nullptr; - lib->binarySearch((void*)pc, &method_name); - char mark = (method_name != nullptr) ? NativeFunc::read_mark(method_name) : 0; - - if (mark != 0) { - return {nullptr, BCI_NATIVE_FRAME, true}; // Marked - stop processing - } - - // Pack remote symbolication data using utility struct - uintptr_t pc_offset = pc - (uintptr_t)lib->imageBase(); - uint32_t lib_index = (uint32_t)lib->libIndex(); - unsigned long packed = RemoteFramePacker::pack(pc_offset, mark, lib_index); - - return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE, false); - } - - // Traditional symbol resolution - const char *method_name = nullptr; - if (lib != nullptr) { - lib->binarySearch((void*)pc, &method_name); - } - if (method_name != nullptr && NativeFunc::is_marked(method_name)) { - return NativeFrameResolution(nullptr, BCI_NATIVE_FRAME, true); - } - - // No symbol but known library: pack for library-relative identification. - // Reuses BCI_NATIVE_FRAME_REMOTE encoding; resolveMethod() in flightRecorder.cpp - // distinguishes remote vs local rendering via hasBuildId() && isRemoteSymbolication(). - if (method_name == nullptr && lib != nullptr) { - uintptr_t pc_offset = pc - (uintptr_t)lib->imageBase(); - uint32_t lib_index = (uint32_t)lib->libIndex(); - unsigned long packed = RemoteFramePacker::pack(pc_offset, 0, lib_index); - return NativeFrameResolution(packed, BCI_NATIVE_FRAME_REMOTE, false); - } - - return NativeFrameResolution(method_name, BCI_NATIVE_FRAME, false); -} - -/** - * Converts a native callchain to ASGCT_CallFrame array. - * - * For libraries with build-IDs, uses remote symbolication (deferred resolution). - * For libraries without build-IDs, performs traditional symbol resolution. - * - * In both cases, performs symbol resolution via binarySearch() to check for - * marked frames (JVM internals) that should terminate the stack walk. - */ -int Profiler::convertNativeTrace(int native_frames, const void **callchain, - ASGCT_CallFrame *frames, int lock_index) { - int depth = 0; - void* prev_identifier = NULL; // Can be jmethodID or frame pointer for remote - - for (int i = 0; i < native_frames; i++) { - uintptr_t pc = (uintptr_t)callchain[i]; - - // Try remote symbolication first if enabled - if (_remote_symbolication) { - CodeCache* lib = _libs->findLibraryByAddress((void*)pc); - if (lib != nullptr && lib->hasBuildId()) { - // Check for marked frames via symbol resolution - // binarySearch() returns symbol name, then we check mark (O(1)) - const char *method_name = nullptr; - lib->binarySearch((void*)pc, &method_name); - char mark = (method_name != nullptr) ? NativeFunc::read_mark(method_name) : 0; - - if (mark != 0) { - // Terminate scan at marked frame - return depth; - } - - // Populate remote frame inline - no allocation needed! - // Pass the mark we already retrieved to avoid duplicate binarySearch - populateRemoteFrame(&frames[depth], pc, lib, mark); - - // Use frame address as identifier for duplicate detection - void* current_identifier = (void*)&frames[depth]; - if (current_identifier != prev_identifier || _cstack != CSTACK_LBR) { - prev_identifier = current_identifier; - depth++; - } - continue; - } - } - - // Fallback: Traditional symbol resolution - const char *method_name = findNativeMethod((void*)pc); - if (method_name != nullptr && NativeFunc::is_marked(method_name)) { - // Terminate scan at marked frame - return depth; - } - - // Store standard frame - jmethodID current_method = (jmethodID)method_name; - if (current_method == prev_identifier && _cstack == CSTACK_LBR) { - prev_identifier = NULL; - } else if (current_method != NULL) { - frames[depth].bci = BCI_NATIVE_FRAME; - frames[depth].method_id = current_method; - prev_identifier = current_method; - depth++; - } - } - - return depth; -} - -u64 Profiler::recordJVMTISample(u64 counter, int tid, jthread thread, jint event_type, Event *event, bool deferred) { - // Protect JVMTI sampling operations to prevent signal handler interference - CriticalSection cs; - atomicIncRelaxed(_total_samples); - - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - // Too many concurrent signals already - atomicIncRelaxed(_failures[-ticks_skipped]); - - return 0; - } - u64 call_trace_id = 0; - if (!_omit_stacktraces) { - // Defensive: the buffer slot can be observed as nullptr in pathological - // start sequences (e.g. a calloc failure path in Profiler::start before - // engines are enabled). Drop the sample rather than dereferencing. - CallTraceBuffer *buf = _calltrace_buffer[lock_index]; - if (buf == nullptr) { - atomicIncRelaxed(_failures[-ticks_skipped]); - _locks[lock_index].unlock(); - return 0; - } -#ifdef COUNTERS - u64 startTime = TSC::ticks(); -#endif // COUNTERS - ASGCT_CallFrame *frames = buf->_asgct_frames; - jvmtiFrameInfo *jvmti_frames = buf->_jvmti_frames; - - int num_frames = 0; - - if (VM::jvmti()->GetStackTrace(thread, 0, _max_stack_depth, jvmti_frames, &num_frames) == JVMTI_ERROR_NONE && num_frames > 0) { - // Convert to AsyncGetCallTrace format. - // Note: jvmti_frames and frames may overlap. - for (int i = 0; i < num_frames; i++) { - jint bci = jvmti_frames[i].location; - jmethodID mid = jvmti_frames[i].method; - frames[i].method_id = mid; - frames[i].bci = bci; - // see https://github.com/async-profiler/async-profiler/pull/1090 - LP64_ONLY(frames[i].padding = 0;) - } - // On JDK 21+, GetStackTrace on a virtual thread returns only the VT's - // logical stack; it stops at the continuation boundary and never includes - // carrier-thread frames. Without a synthetic root the trace appears - // truncated to the UI backend, which attributes it to "Missing Frames". - // Detect the VT case via JavaThread::_cont_entry being non-null on the - // carrier. This field is in gHotSpotVMStructs on all JDK 21+ builds so - // isCarryingVirtualThread() works regardless of JDK version. Append a - // synthetic "JVM Continuation" root frame to mark the boundary - // explicitly, matching the behaviour of walkVM without carrier_frames. - if (VM::isHotspot() && VM::hotspot_version() >= 21 && - num_frames < _max_stack_depth) { - VMThread* carrier = VMThread::current(); - if (carrier != nullptr && carrier->isCarryingVirtualThread()) { - frames[num_frames].bci = BCI_NATIVE_FRAME; - frames[num_frames].method_id = (jmethodID) "JVM Continuation"; - LP64_ONLY(frames[num_frames].padding = 0;) - num_frames++; - } - } - } - - call_trace_id = _call_trace_storage.put(num_frames, frames, false, counter); -#ifdef COUNTERS - u64 duration = TSC::ticks() - startTime; - if (duration > 0) { - Counters::increment(UNWINDING_TIME_JVMTI, duration); - } -#endif // COUNTERS - } - if (!deferred) { - _jfr.recordEvent(lock_index, tid, call_trace_id, event_type, event); - } - - _locks[lock_index].unlock(); - return call_trace_id; -} - -void Profiler::recordDeferredSample(int tid, u64 call_trace_id, jint event_type, Event *event) { - atomicIncRelaxed(_total_samples); - - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - // Too many concurrent signals already - atomicIncRelaxed(_failures[-ticks_skipped]); - return; - } - - _jfr.recordEvent(lock_index, tid, call_trace_id, event_type, event); - - _locks[lock_index].unlock(); -} - -bool Profiler::recordSample(void *ucontext, u64 counter, int tid, - jint event_type, u64 call_trace_id, Event *event, - u64 *recorded_call_trace_id) { - atomicIncRelaxed(_total_samples); - - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - // Too many concurrent signals already - atomicIncRelaxed(_failures[-ticks_skipped]); - - if (event_type == BCI_CPU && _cpu_engine == &perf_events) { - // Need to reset PerfEvents ring buffer, even though we discard the - // collected trace - PerfEvents::resetBuffer(tid); - } - return false; - } - - bool truncated = false; - // in lightweight mode we're just sampling the the context associated with the - // passage of CPU or wall time, we use the same event definitions but we - // record a null stacktrace we can skip the unwind if we've got a - // call_trace_id determined to be reusable at a higher level - - if (!_omit_stacktraces && call_trace_id == 0) { -#ifdef COUNTERS - u64 startTime = TSC::ticks(); -#endif // COUNTERS - ASGCT_CallFrame *frames = _calltrace_buffer[lock_index]->_asgct_frames; - - int num_frames = 0; - - StackContext java_ctx = {0}; - ASGCT_CallFrame *native_stop = frames + num_frames; - num_frames += getNativeTrace(ucontext, native_stop, event_type, tid, - &java_ctx, &truncated, lock_index); - assert(num_frames >= 0); - - int max_remaining = _max_stack_depth - num_frames; - if (max_remaining > 0) { - StackWalkRequest request = {event_type, lock_index, ucontext, frames + num_frames, max_remaining, &java_ctx, &truncated}; - num_frames += JVMSupport::walkJavaStack(request); - } - - assert(num_frames >= 0); - if (num_frames == 0) { - num_frames += makeFrame(frames + num_frames, BCI_ERROR, "no_Java_frame"); - } - - call_trace_id = - _call_trace_storage.put(num_frames, frames, truncated, counter); - ProfiledThread *thread = ProfiledThread::currentSignalSafe(); - if (thread != nullptr) { - thread->recordCallTraceId(call_trace_id); - } -#ifdef COUNTERS - u64 duration = TSC::ticks() - startTime; - if (duration > 0) { - Counters::increment(UNWINDING_TIME_ASYNC, duration); - } -#endif // COUNTERS - } - bool recorded = _jfr.recordEvent(lock_index, tid, call_trace_id, event_type, event); - if (recorded && recorded_call_trace_id != nullptr) { - *recorded_call_trace_id = call_trace_id; - } - - _locks[lock_index].unlock(); - return recorded; -} - -bool Profiler::recordSampleDelegated(void *ucontext, u64 weight, int tid, - jint event_type, Event *event) { - if (!VM::canRequestStackTrace()) { - return false; - } - - // Reserve the correlation ID up-front so we can pass the same value to the - // JVM (as user_data) and to our own event. - u64 correlation_id = atomicIncRelaxed(_sample_seq); - - Counters::increment(JVMTI_STACKS_REQUESTED); - jvmtiError rc = VM::requestStackTrace(ucontext, (jlong)correlation_id); - if (rc != JVMTI_ERROR_NONE) { - if (rc == JVMTI_ERROR_WRONG_PHASE) { - Counters::increment(JVMTI_STACKS_FAILED_WRONG_PHASE); - } else { - Counters::increment(JVMTI_STACKS_FAILED_OTHER); - } - return false; - } - - atomicIncRelaxed(_total_samples); - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - atomicIncRelaxed(_failures[-ticks_skipped]); - Counters::increment(JVMTI_STACKS_DROPPED_LOCK); - // The JVM-side stack trace request is already in flight; we just drop our - // sample event. The dangling StackTraceRequest entry in the JVM recording - // will simply have no matching datadog event, which is harmless. - return false; - } - - bool recorded = - _jfr.recordEventDelegated(lock_index, tid, correlation_id, event_type, event); - _locks[lock_index].unlock(); - return recorded; -} - -void Profiler::recordWallClockEpoch(int tid, WallClockEpochEvent *event) { - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - return; - } - _jfr.wallClockEpoch(lock_index, event); - _locks[lock_index].unlock(); -} - -void Profiler::recordTraceRoot(int tid, TraceRootEvent *event) { - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - return; - } - _jfr.recordTraceRoot(lock_index, tid, event); - _locks[lock_index].unlock(); -} - -void Profiler::recordQueueTime(int tid, QueueTimeEvent *event) { - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - return; - } - _jfr.recordQueueTime(lock_index, tid, event); - _locks[lock_index].unlock(); -} - -void Profiler::recordExternalSample(u64 weight, int tid, int num_frames, - ASGCT_CallFrame *frames, bool truncated, - jint event_type, Event *event) { - // Protect external sampling operations to prevent signal handler interference - CriticalSection cs; - atomicIncRelaxed(_total_samples); - - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - atomicIncRelaxed(_failures[-ticks_skipped]); - return; - } - - CallTraceBuffer *buf = _calltrace_buffer[lock_index]; - if (buf == nullptr) { - atomicIncRelaxed(_failures[-ticks_skipped]); - _locks[lock_index].unlock(); - return; - } - // External samplers (like ObjectSampler) provide standard frames only - ASGCT_CallFrame *extended_frames = buf->_asgct_frames; - for (int i = 0; i < num_frames; i++) { - extended_frames[i] = frames[i]; - } - - u64 call_trace_id = - _call_trace_storage.put(num_frames, extended_frames, truncated, weight); - _jfr.recordEvent(lock_index, tid, call_trace_id, event_type, event); - - _locks[lock_index].unlock(); -} - -void Profiler::writeLog(LogLevel level, const char *message) { - _jfr.recordLog(level, message, strlen(message)); -} - -void Profiler::writeLog(LogLevel level, const char *message, size_t len) { - _jfr.recordLog(level, message, len); -} - -void Profiler::writeDatadogProfilerSetting(int tid, int length, - const char *name, const char *value, - const char *unit) { - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - return; - } - _jfr.recordDatadogSetting(lock_index, length, name, value, unit); - _locks[lock_index].unlock(); -} - -void Profiler::writeHeapUsage(long value, bool live) { - int tid = ProfiledThread::currentTid(); - if (tid < 0) { - return; - } - u32 lock_index = getLockIndex(tid); - if (!_locks[lock_index].tryLock() && - !_locks[lock_index = (lock_index + 1) % CONCURRENCY_LEVEL].tryLock() && - !_locks[lock_index = (lock_index + 2) % CONCURRENCY_LEVEL].tryLock()) { - return; - } - _jfr.recordHeapUsage(lock_index, value, live); - _locks[lock_index].unlock(); -} - -void Profiler::prewarmUnwinder() { -#ifdef __linux__ - // J9 on aarch64 (and other JVMs) lazily loads libgcc_s.so.1 from its DWARF - // unwinder during stack walks. When that happens inside a signal handler - // frame, our dlopen_hook fires from signal context and tries to refresh the - // library list — Mutex::lock and malloc on a signal stack. By forcing the - // load here, before any signal handler is installed, subsequent calls find - // libgcc_s already mapped and the lazy-load path never runs. - // - // The handle is intentionally leaked: keeping the refcount > 0 prevents the - // library from being unmapped for the remainder of the process lifetime. - // - // SONAME note: "libgcc_s.so.1" is hardcoded deliberately. Referencing - // _Unwind_Backtrace from C++ would normally let the linker resolve the - // SONAME implicitly, but our release build uses -static-libgcc - // (ConfigurationPresets.kt: "-static-libgcc"), which embeds the unwinder - // into libjavaProfiler.so and removes libgcc_s.so from our NEEDED entries - // — so a symbol reference would not trigger the shared-library load. - // dlopen by SONAME is the only mechanism that works under static-libgcc. - // libgcc_s.so.1 has been the stable SONAME since 2002; a bump would - // constitute a glibc/GCC C++ ABI break and is treated as a fixed contract. - (void)dlopen("libgcc_s.so.1", RTLD_LAZY | RTLD_GLOBAL); -#endif -} - -void *Profiler::dlopen_hook(const char *filename, int flags) { - void *result = dlopen(filename, flags); - - if (result != NULL) { - if (!isInTrackedSignalContext()) { - // Either confirmed non-signal context, or no ProfiledThread on this - // thread (uninstrumented JVM internals — VM Thread, JIT, GC). We - // prefer synchronous refresh for the null-PT case too because (a) - // those threads call dlopen synchronously during normal JVM - // operation, and (b) wasmtime's broken sigaction patching depends - // on switchLibraryTrap running its work inline (the original reason - // for the trap). The residual risk — an uninstrumented thread - // calling dlopen from inside a foreign signal handler — is small: - // prewarmUnwinder() closes the known libgcc_s lazy-load case, and - // mainstream JVM signal handlers are AS-safe by design. - Libraries::instance()->refresh(); - } else { - // Confirmed signal context (one of our SignalHandlerScopes is on - // the stack). refresh() must NOT run here — parseLibraries - // acquires a Mutex and calls malloc, both AS-unsafe. Mark the - // library set dirty; the Libraries refresher thread picks it up - // within REFRESH_INTERVAL_NS (500 ms). - Libraries::instance()->markDirty(); - } - } - - return result; -} - -const char* Profiler::cstack() const { - switch (_cstack) { - case CSTACK_DEFAULT: return "default"; - case CSTACK_NO: return "no"; - case CSTACK_FP: return "fp"; - case CSTACK_DWARF: return "dwarf"; - case CSTACK_LBR: return "lbr"; - case CSTACK_VM: { - return _features.mixed ? "vmx" : "vm"; - } - default: return "default"; - } -} - -void Profiler::switchLibraryTrap(bool enable) { - if (_dlopen_entry == NULL) { - return; // Not initialized yet, nothing to do - } - void *impl = enable ? (void *)dlopen_hook : (void *)dlopen; - __atomic_store_n(_dlopen_entry, impl, __ATOMIC_RELEASE); -} - -void Profiler::enableEngines() { - _cpu_engine->enableEvents(true); - _wall_engine->enableEvents(true); -} - -void Profiler::disableEngines() { - _cpu_engine->enableEvents(false); - _wall_engine->enableEvents(false); -} - -void Profiler::segvHandler(int signo, siginfo_t *siginfo, void *ucontext) { - // J9 installs a SIGSEGV handler that uses siglongjmp() to recover from - // null-pointer-check faults during normal Java execution. When we chain to - // it, that longjmp unwinds past our stack frame and skips the RAII - // destructor, permanently leaking depth on the thread. Release the guard - // before chaining so depth is correct whether the chained handler returns - // or longjmps. - // - // Sanitizer-coverage note: this also means depth == 0 inside the chained - // handler, so DEBUG_ASSERT_NOT_IN_SIGNAL() will NOT fire for AS-unsafe - // code reachable from a chained handler that returns normally. This is - // the lesser of two evils — leaking depth on longjmp would silently - // break the production deferred-refresh gate, while the sanitizer gap - // is bounded to third-party signal handler code we don't own. - SIGNAL_HANDLER_GUARD(); - if (crashHandlerInternal(signo, siginfo, ucontext)) { - return; // Handled — destructor decrements depth - } - SIGNAL_HANDLER_GUARD_RELEASE(); - // Not handled, chain to next handler (may longjmp; never return through us) - SigAction chain = OS::getSegvChainTarget(); - if (chain != nullptr) { - chain(signo, siginfo, ucontext); - } else if (orig_segvHandler != nullptr) { - orig_segvHandler(signo, siginfo, ucontext); - } -} - -void Profiler::busHandler(int signo, siginfo_t *siginfo, void *ucontext) { - // See segvHandler: release before chaining in case the chained handler - // longjmps through us. - SIGNAL_HANDLER_GUARD(); - if (crashHandlerInternal(signo, siginfo, ucontext)) { - return; // Handled — destructor decrements depth - } - SIGNAL_HANDLER_GUARD_RELEASE(); - // Not handled, chain to next handler - SigAction chain = OS::getBusChainTarget(); - if (chain != nullptr) { - chain(signo, siginfo, ucontext); - } else if (orig_busHandler != nullptr) { - orig_busHandler(signo, siginfo, ucontext); - } -} - -// Returns: 0 = not handled (chain to next handler), non-zero = handled -int Profiler::crashHandlerInternal(int signo, siginfo_t *siginfo, void *ucontext) { - ProfiledThread* thrd = ProfiledThread::currentSignalSafe(); - - // First, try to handle safefetch - this doesn't need TLS or any protection - // because it directly checks the PC and modifies ucontext to skip the fault. - // This must be checked first before any reentrancy checks. - if (SafeAccess::handle_safefetch(signo, ucontext)) { - return 1; // handled - } - - // Reentrancy protection: use TLS-based tracking if available. - // If TLS is not available, we can only safely handle faults that we can - // prove are from our protected code paths (checked via sameStack heuristic - // in HotspotSupport::checkFault). For anything else, we must chain immediately - // to avoid claiming faults that aren't ours. - bool have_tls_protection = false; - if (thrd != nullptr) { - if (!thrd->enterCrashHandler()) { - // we are already in a crash handler; don't recurse! - return 0; // not handled, safe to chain - } - have_tls_protection = true; - } - // If thrd == nullptr, we proceed but with limited handling capability. - // Only HotspotSupport::checkFault (which has its own sameStack fallback) - // and the JDK-8313796 workaround can safely handle faults without TLS. - - StackFrame frame(ucontext); - uintptr_t pc = frame.pc(); - - uintptr_t fault_address = (uintptr_t)siginfo->si_addr; - if (pc == fault_address) { - // it is 'pc' that is causing the fault; can not access it safely - if (have_tls_protection) { - thrd->exitCrashHandler(); - } - return 0; // not handled, safe to chain - } - - if (WX_MEMORY && Trap::isFaultInstruction(pc)) { - if (have_tls_protection) { - thrd->exitCrashHandler(); - } - return 1; // handled - } - - if (VM::isHotspot()) { - // the following checks require vmstructs and therefore HotSpot - - // HotspotSupport::checkFault has its own fallback for when TLS is unavailable: - // it uses sameStack() heuristic to check if we're in a protected stack walk. - // If the fault is from our protected walk, it will longjmp and never return. - // If it returns, the fault wasn't from our code. - HotspotSupport::checkFault(thrd); - - // Workaround for JDK-8313796 if needed. Setting cstack=dwarf also helps - if (_need_JDK_8313796_workaround && - VMStructs::isInterpretedFrameValidFunc((const void *)pc) && - frame.skipFaultInstruction()) { - if (have_tls_protection) { - thrd->exitCrashHandler(); - } - return 1; // handled - } - } - - if (have_tls_protection) { - thrd->exitCrashHandler(); - } - return 0; // not handled, safe to chain -} - -void Profiler::setupSignalHandlers() { - // Do not re-run the signal setup (run only when VM has not been loaded yet) - if (__sync_bool_compare_and_swap(&_signals_initialized, false, true)) { - if (VM::isHotspot() || VM::isOpenJ9()) { - // HotSpot and J9 tolerate interposed SIGSEGV/SIGBUS handler; other JVMs probably not - // IMPORTANT: protectSignalHandlers must be called BEFORE replaceSigsegvHandler so that - // the original (JVM's) handlers are saved before we install ours. This way, when we - // intercept other libraries' sigaction calls and return oldact, we return the JVM's - // handler (not ours), preventing infinite chaining loops. - OS::protectSignalHandlers(segvHandler, busHandler); - orig_segvHandler = OS::replaceSigsegvHandler(segvHandler); - orig_busHandler = OS::replaceSigbusHandler(busHandler); - // Patch sigaction GOT in libraries with broken signal handlers (already loaded) - LibraryPatcher::patch_sigaction(); - } - } -} - - -/** - * Update thread name for the given thread - */ -void Profiler::updateThreadName(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread, - bool self) { - JitWriteProtection jit(true); // workaround for JDK-8262896 - jvmtiThreadInfo thread_info; - int native_thread_id = -1; - - if (self) { - // if updating the current thread, use the native thread id from the - // ProfilerThread, it is faster and safer. - native_thread_id = ProfiledThread::currentTid(); - assert(native_thread_id != -1); - } else { - native_thread_id = JVMThread::nativeThreadId(jni, thread); - if (jni->ExceptionCheck()) { - jni->ExceptionClear(); - } - } - - if (native_thread_id >= 0 && - jvmti->GetThreadInfo(thread, &thread_info) == 0) { - jlong java_thread_id = JVMThread::javaThreadId(jni, thread); - _thread_info.set(native_thread_id, thread_info.name, java_thread_id); - jvmti->Deallocate((unsigned char *)thread_info.name); - } -} - -void Profiler::updateJavaThreadNames() { - jvmtiEnv *jvmti = VM::jvmti(); - jint thread_count; - jthread *thread_objects; - if (jvmti->GetAllThreads(&thread_count, &thread_objects) != 0) { - return; - } - - JNIEnv *jni = VM::jni(); - for (int i = 0; i < thread_count; i++) { - if (thread_objects[i] == nullptr) { - continue; - } - updateThreadName(jvmti, jni, thread_objects[i]); - jni->DeleteLocalRef(thread_objects[i]); - } - - jvmti->Deallocate((unsigned char *)thread_objects); -} - -void Profiler::updateNativeThreadNames(bool defer_initializing) { - ThreadList *thread_list = OS::listThreads(); - constexpr size_t buffer_size = 64; - char name_buf[buffer_size]; // Stack-allocated buffer - - // A freshly cloned thread inherits the creating thread's comm until it sets - // its own name; for the threads we want here that creator is typically the - // main thread, so the inherited name is the process name. When deferring, we - // skip recording it and let a later pass capture the final name. - char proc_name[buffer_size]; - bool have_proc_name = - defer_initializing && OS::threadName(OS::processId(), proc_name, buffer_size); - - while (thread_list->hasNext()) { - int tid = thread_list->next(); - _thread_info.updateThreadName( - tid, [&](int tid) -> std::string { - if (OS::threadName(tid, name_buf, buffer_size)) { - // Skip a thread still showing the inherited process name: it is - // probably mid-initialization. Recording it would latch a - // provisional name (updateThreadName is first-writer-wins). - if (have_proc_name && strcmp(name_buf, proc_name) == 0) { - return std::string(); - } - // name_buf is NUL-terminated by OS::threadName; let - // std::string find the length rather than storing the - // full 64-byte buffer (NUL + trailing garbage). - return std::string(name_buf); - } - return std::string(); - }); - } - - delete thread_list; -} - -Engine *Profiler::selectCpuEngine(Arguments &args) { - if (args._cpu < 0 && - (args._event == NULL || strcmp(args._event, EVENT_NOOP) == 0)) { - return &noop_engine; - } else if (args._cpu >= 0 || strcmp(args._event, EVENT_CPU) == 0) { - if (VM::isOpenJ9()) { - if (!J9Support::shouldUseAsgct() || !J9Support::can_use_ASGCT()) { - if (!J9Support::is_jvmti_jmethodid_safe()) { - LOG_WARN("Safe jmethodID access is not available on this JVM. Using " - "CPU profiler on your own risk. Use -XX:+KeepJNIIDs=true JVM " - "flag to make access to jmethodIDs safe, if your JVM supports it"); - } - TEST_LOG("J9[cpu]=jvmti"); - return &j9_engine; - } - TEST_LOG("J9[cpu]=asgct"); - } - // Prefer the JVMTI JFR-delegated engine when the HotSpot extension is - // available and the user opted into jvmtistacks. On Linux, CTimerJvmti - // uses per-thread CPU timers. On other platforms (e.g. macOS) it is not - // supported, so fall back to ITimerJvmti which uses setitimer(ITIMER_PROF). - if (args._jvmtistacks) { - if (!ctimer_jvmti.check(args)) { - TEST_LOG("HS[cpu]=ctimer_jvmti"); - return &ctimer_jvmti; - } - if (!itimer_jvmti.check(args)) { - TEST_LOG("HS[cpu]=itimer_jvmti"); - return &itimer_jvmti; - } - Log::warn("jvmtistacks requested but no JVMTI CPU engine is available; falling back to ASGCT"); - } - return !ctimer.check(args) - ? (Engine *)&ctimer - : (!perf_events.check(args) ? (Engine *)&perf_events - : (Engine *)&itimer); - } else if (strcmp(args._event, EVENT_WALL) == 0) { - return &noop_engine; - } else if (strcmp(args._event, EVENT_ITIMER) == 0) { - return &itimer; - } else if (strcmp(args._event, EVENT_CTIMER) == 0) { - return &ctimer; - } else { - return &perf_events; - } -} - -Engine *Profiler::selectWallEngine(Arguments &args) { - if (args._wall < 0 && - (args._event == NULL || strcmp(args._event, EVENT_WALL) != 0)) { - return &noop_engine; - } - if (VM::isOpenJ9()) { - if (args._wallclock_sampler == JVMTI || !J9Support::shouldUseAsgct() || !J9Support::can_use_ASGCT()) { - if (!J9Support::is_jvmti_jmethodid_safe()) { - LOG_WARN("Safe jmethodID access is not available on this JVM. Using " - "wallclock profiler on your own risk. Use -XX:+KeepJNIIDs=true JVM " - "flag to make access to jmethodIDs safe, if your JVM supports it"); - } - j9_engine.sampleIdleThreads(); - TEST_LOG("J9[wall]=jvmti"); - return (Engine *)&j9_engine; - } else { - TEST_LOG("J9[wall]=asgct"); - return (Engine *)&wall_asgct_engine; - } - } - // jvmtistacks overrides _wallclock_sampler when the HotSpot extension is available. - if (args._jvmtistacks && VM::canRequestStackTrace()) { - TEST_LOG("HS[wall]=jvmti"); - return (Engine *)&wall_jvmti_engine; - } - switch (args._wallclock_sampler) { - case JVMTI: - fprintf(stderr, "[ddprof] [WARN] JVMTI wallclock is not available on this JVM, fallback to ASGCT wallclock\n"); - [[fallthrough]]; - case ASGCT: - default: - return (Engine*)&wall_asgct_engine; - } -} - -Engine *Profiler::selectAllocEngine(Arguments &args) { - if (VM::canSampleObjects()) { - return static_cast(ObjectSampler::instance()); - } else { - Log::info("Not enabling the alloc profiler, SampledObjectAlloc is not " - "supported on this JVM"); - return &noop_engine; - } -} - -Error Profiler::checkJvmCapabilities() { - if (!JVMThread::isInitialized()) { - return Error("Could not find JVMThread bridge. Unsupported JVM?"); - } - - if (!JVMThread::hasJavaThreadId()) { - return Error("Could not find Thread ID field. Unsupported JVM?"); - } - - if (VM::isUseAdaptiveGCBoundarySet()) { - return Error( - "The user has explicitly set -XX:+UseAdaptiveGCBoundary so the " - "profiler has been disabled to avoid the risk of crashing."); - } - - if (_dlopen_entry == NULL) { - CodeCache *lib = _libs->findJvmLibrary("libj9prt"); - if (lib == NULL || (_dlopen_entry = lib->findImport(im_dlopen)) == NULL) { - return Error("Could not set dlopen hook. Unsupported JVM?"); - } - } - - if (!VMStructs::libjvm()->hasDebugSymbols() && !VM::isOpenJ9()) { - Log::warn("Install JVM debug symbols to improve profile accuracy"); - } - - return Error::OK; -} - -void Profiler::check_JDK_8313796_workaround() { - int java_version = VM::java_version(); - int java_update_version = VM::java_update_version(); - - // JDK-8313796 has been fixed in JDK 22 and backported to - // JDK versions 11.0.21, 17.0.9 and 21.0.1 - bool fixed_version = java_version >= 22 || - (java_version == 11 && java_update_version >= 21) || - (java_version == 17 && java_update_version >= 9) || - (java_version == 21 && java_update_version >= 1); - _need_JDK_8313796_workaround = !fixed_version; -} - - -Error Profiler::start(Arguments &args, bool reset) { - MutexLocker ml(_state_lock); - if (state() > IDLE) { - return Error("Profiler already started"); - } - - // Force libgcc_s to load now (idempotent dlopen) so the JVM's DWARF - // unwinder cannot lazy-load it later from signal context. - prewarmUnwinder(); - - Error error = checkJvmCapabilities(); - if (error) { - return error; - } - - _omit_stacktraces = args._lightweight; - _remote_symbolication = args._remote_symbolication; - _libs->setRemoteSymbolication(_remote_symbolication); - _wall_precheck = args._wall_precheck; - _event_mask = - ((args._event != NULL && strcmp(args._event, EVENT_NOOP) != 0) ? EM_CPU - : 0) | - (args._cpu >= 0 ? EM_CPU : 0) | (args._wall >= 0 ? EM_WALL : 0) | - (args._record_allocations || args._record_liveness || args._gc_generations - ? EM_ALLOC - : 0) | - (args._nativemem >= 0 ? EM_NATIVEMEM : 0) | - (args._nativesocket ? EM_NATIVESOCKET : 0); - - if (_event_mask == 0) { - return Error("No profiling events specified"); - } - - // Commit _features before the reset block so any signal-handler code that - // reads _features.* observes the correct enabled state once profiling - // engines start. - _features = args._features; - if (VM::hotspot_version() < 8) { - _features.java_anchor = 0; - _features.gc_traces = 0; - } - if (!VMStructs::hasClassNames()) { - _features.vtable_target = 0; - } - if (!VMStructs::hasCompilerStructs()) { - _features.comp_task = 0; - } - - if (reset || _start_time == 0) { - // Reset counters. _sample_seq is intentionally not reset: it is a - // monotonically increasing uniqueness generator for correlation IDs and - // must not repeat values across recording sessions. - _total_samples = 0; - memset(_failures, 0, sizeof(_failures)); - - // Reset dictionaries. StringDictionary::clearAll() manages its own - // synchronisation (RefCountGuard drain). The exclusive _class_map_lock - // additionally fences out shared-lock readers introduced by #527 - // (deferred vtable receiver resolution) so they cannot observe a - // half-cleared class map. - { - ExclusiveLockGuard guard(&_class_map_lock); - _class_map.clearAll(); - } - _string_label_map.clearAll(); - _context_value_map.clearAll(); - - // Reset call trace storage - if (!_omit_stacktraces) { - lockAll(); - _call_trace_storage.clear(); - unlockAll(); - } - Counters::reset(); - WallClockCounters::reset(); - - // Reset thread names and IDs - _thread_info.clearAll(); - } - - // (Re-)allocate calltrace buffers - if (_max_stack_depth != args._jstackdepth) { - _max_stack_depth = args._jstackdepth; - size_t nelem = _max_stack_depth + RESERVED_FRAMES; - - for (int i = 0; i < CONCURRENCY_LEVEL; i++) { - // Allocate the replacement before touching the slot so a calloc failure - // does not leave the slot pointing at freed memory. - CallTraceBuffer *fresh = - (CallTraceBuffer*)calloc(nelem, sizeof(CallTraceBuffer)); - if (fresh == NULL) { - _max_stack_depth = 0; - return Error("Not enough memory to allocate stack trace buffers (try " - "smaller jstackdepth)"); - } - // Swap under the per-shard lock: all readers (recordJVMTISample, - // recordExternalSample) acquire this lock via tryLock before reading - // _calltrace_buffer, so no reader can observe a freed pointer mid-replacement. - _locks[i].lock(); - CallTraceBuffer *prev = _calltrace_buffer[i]; - _calltrace_buffer[i] = fresh; - _locks[i].unlock(); - free(prev); - } - } - - // Remote symbolication is now inline in ASGCT_CallFrame - // No separate pool allocation needed! - - _safe_mode = args._safe_mode; - if (VM::hotspot_version() < 8 || VM::isZing()) { - _safe_mode |= GC_TRACES | LAST_JAVA_PC; - } - - // TODO: Current way of setting filter is weird with the recent changes - _thread_filter.init(args._filter ? args._filter : "0"); - - // Minor optim: Register the current thread (start thread won't be called) - if (_thread_filter.enabled()) { - _thread_filter.clearActive(); - ProfiledThread *current = ProfiledThread::current(); - assert(current != nullptr); - int slot_id = current->filterSlotId(); - if (slot_id < 0) { - slot_id = _thread_filter.registerThread(); - current->setFilterSlotId(slot_id); - } - _thread_filter.remove(slot_id); // Remove from filtering initially (matches onThreadStart behavior) - } - - _cpu_engine = selectCpuEngine(args); - _wall_engine = selectWallEngine(args); - _cstack = args._cstack; - if (_cstack == CSTACK_DEFAULT) { - if (VMStructs::hasStackStructs() && OS::isLinux()) { - _cstack = CSTACK_VM; - } else if (DWARF_SUPPORTED) { - _cstack = CSTACK_DWARF; - } - } - if (_cstack == CSTACK_DWARF && !DWARF_SUPPORTED) { - _cstack = CSTACK_NO; - Log::warn("DWARF unwinding is not supported on this platform. Defaulting " - "to no native call stack unwinding."); - } else if (_cstack == CSTACK_LBR && _cpu_engine != &perf_events) { - _cstack = CSTACK_NO; - Log::warn("Branch stack is supported only with PMU events"); - } else if (_cstack == CSTACK_VM) { - if (!VMStructs::hasStackStructs()) { - _cstack = DWARF_SUPPORTED ? CSTACK_DWARF : CSTACK_NO; - Log::error("VMStructs stack walking is not supported on this JVM/platform, defaulting to the default native call stack unwinding mode."); - } - } - - LibraryPatcher::initialize(); - - // Kernel symbols are useful only for perf_events without --all-user - _libs->updateSymbols(_cpu_engine == &perf_events && (args._ring & RING_KERNEL)); - - // Extract build-ids for remote symbolication if enabled - if (_remote_symbolication) { - _libs->updateBuildIds(); - } - - enableEngines(); - - // Refresher must be running before the trap fires: dlopen_hook's - // signal-context branch only marks dirty and relies on the refresher - // to call refresh() within REFRESH_INTERVAL_NS (500 ms). - _libs->startRefresher(); - - // Always enable library trap to catch wasmtime loading and patch its broken sigaction - switchLibraryTrap(true); - - JfrMetadata::reset(); - JfrMetadata::initialize(args._context_attributes); - _num_context_attributes = args._context_attributes.size(); - error = _jfr.start(args, reset); - if (error) { - disableEngines(); - switchLibraryTrap(false); - _libs->stopRefresher(); - return error; - } - - int activated = 0; - if ((_event_mask & EM_CPU) && _cpu_engine != &noop_engine) { - error = _cpu_engine->start(args); - if (error) { - Log::warn("%s", error.message()); - error = Error::OK; // recoverable - } else { - activated |= EM_CPU; - } - } - if ((_event_mask & EM_WALL) && _wall_engine != &noop_engine) { - error = _wall_engine->start(args); - if (error) { - Log::warn("%s", error.message()); - error = Error::OK; // recoverable - } else { - activated |= EM_WALL; - } - } - if (_event_mask & EM_ALLOC) { - _alloc_engine = selectAllocEngine(args); - if (_alloc_engine != &noop_engine) { - error = _alloc_engine->start(args); - if (error) { - Log::warn("%s", error.message()); - error = Error::OK; // recoverable - } else { - activated |= EM_ALLOC; - } - } - } - if (_event_mask & EM_NATIVEMEM) { - error = malloc_tracer.start(args); - if (error) { - Log::warn("%s", error.message()); - if (_event_mask == EM_NATIVEMEM) { - // nativemem is the only requested mode: propagate the real error - disableEngines(); - switchLibraryTrap(false); - _libs->stopRefresher(); - lockAll(); - _jfr.stop(); - unlockAll(); - return error; - } - error = Error::OK; // recoverable when other modes are also active - } else { - activated |= EM_NATIVEMEM; - } - } - if (_event_mask & EM_NATIVESOCKET) { - error = NativeSocketSampler::instance()->start(args); - if (error) { - Log::warn("%s", error.message()); - error = Error::OK; // recoverable - } else { - activated |= EM_NATIVESOCKET; - } - } - - if (activated) { - switchThreadEvents(JVMTI_ENABLE); - - // Initialize this thread - // Note: passing all nullptrs results in not able to resolve the thread name here. - // However, the thread name will be updated later in updateJavaThreadNames(). - // TODO: find a better way to resolve the thread name. - onThreadStart(nullptr, nullptr, nullptr); - - _state.store(RUNNING, std::memory_order_release); - _start_time = time(NULL); - __atomic_add_fetch(&_epoch, 1, __ATOMIC_RELAXED); - return Error::OK; - } - // no engine was activated; perform cleanup - disableEngines(); - switchLibraryTrap(false); - _libs->stopRefresher(); - - lockAll(); - _jfr.stop(); - unlockAll(); - - return Error( - "Neither CPU, wallclock nor allocation profiling could be started"); -} - -Error Profiler::stop() { - MutexLocker ml(_state_lock); - if (state() != RUNNING) { - return Error("Profiler is not active"); - } - - disableEngines(); - - if (_event_mask & EM_ALLOC) - _alloc_engine->stop(); - if (_event_mask & EM_NATIVEMEM) - malloc_tracer.stop(); - // Stop the refresher BEFORE socket unpatch: the refresher calls - // install_socket_hooks() which re-reads _socket_active before acquiring the - // patch lock. If the refresher runs concurrently with unpatch_socket_functions() - // it can see _socket_active=true, wait for the lock, then re-patch PLT slots - // that unpatch just restored. Stopping the refresher here closes that window. - _libs->stopRefresher(); - if (_event_mask & EM_NATIVESOCKET) - NativeSocketSampler::instance()->stop(); - if (_event_mask & EM_WALL) - _wall_engine->stop(); - if (_event_mask & EM_CPU) - _cpu_engine->stop(); - - switchLibraryTrap(false); - switchThreadEvents(JVMTI_DISABLE); - Libraries::instance()->refresh(); - updateJavaThreadNames(); - updateNativeThreadNames(); - - // If jvmtistacks delegation was used this recording, surface likely - // misconfigurations. The JVM returns WRONG_PHASE when JFR is not recording - // and NOT_AVAILABLE when JFR is recording but the StackTraceRequest event is - // disabled. If the request was accepted the JVM will have written the - // stack trace, so no warning is needed. - if (VM::canRequestStackTrace()) { - long long requested = - Counters::getCounter(JVMTI_STACKS_REQUESTED); - long long wrong_phase = - Counters::getCounter(JVMTI_STACKS_FAILED_WRONG_PHASE); - long long other = - Counters::getCounter(JVMTI_STACKS_FAILED_OTHER); - long long dropped_lock = - Counters::getCounter(JVMTI_STACKS_DROPPED_LOCK); - if (requested > 0 && wrong_phase * 2 >= requested) { - fprintf(stderr, - "[java-profiler] jvmtistacks: %lld of %lld stack-trace requests " - "were rejected with WRONG_PHASE, so no async stack traces were " - "emitted by the JVM. Start JFR (e.g. " - "-XX:StartFlightRecording=...) before or as the profiler starts.\n", - wrong_phase, requested); - } else if (requested > 0 && other * 2 >= requested) { - fprintf(stderr, - "[java-profiler] jvmtistacks: %lld of %lld stack-trace requests " - "were rejected with NOT_AVAILABLE. The jdk.StackTraceRequest event " - "is likely disabled; enable it in the JFR configuration, e.g. " - "-XX:StartFlightRecording=...,+jdk.StackTraceRequest#enabled=true.\n", - other, requested); - } - if (dropped_lock > 0) { - fprintf(stderr, - "[java-profiler] jvmtistacks: %lld of %lld stack-trace requests " - "were dropped due to lock contention; the corresponding " - "jdk.StackTraceRequest events will have no matching profiler event.\n", - dropped_lock, requested); - } - } - - // writing these out before stopping the JFR recording allows to report the - // correct counts in the recording - _thread_info.reportCounters(); - - rotateDictsAndRun([&]{ _jfr.stop(); }); - - // Unpatch libraries AFTER JFR serialization completes - // Remote symbolication RemoteFrameInfo structs contain pointers to build-ID strings - // owned by library metadata, so we must keep library patches active until after serialization - LibraryPatcher::unpatch_libraries(); - - _state.store(IDLE, std::memory_order_release); - return Error::OK; -} - -Error Profiler::check(Arguments &args) { - MutexLocker ml(_state_lock); - if (state() > IDLE) { - return Error("Profiler already started"); - } - - Error error = checkJvmCapabilities(); - - if (!error && (args._event != NULL || args._cpu >= 0)) { - _cpu_engine = selectCpuEngine(args); - error = _cpu_engine->check(args); - } - if (!error && args._wall >= 0) { - _wall_engine = selectWallEngine(args); - error = _wall_engine->check(args); - } - if (!error && args._memory >= 0) { - _alloc_engine = selectAllocEngine(args); - error = _alloc_engine->check(args); - } - if (!error && args._nativemem >= 0) { - error = malloc_tracer.check(args); - } - if (!error && args._nativesocket) { - error = NativeSocketSampler::instance()->check(args); - } - if (!error) { - if (args._cstack == CSTACK_DWARF && !DWARF_SUPPORTED) { - return Error("DWARF unwinding is not supported on this platform"); - } else if (args._cstack == CSTACK_LBR && _cpu_engine != &perf_events) { - return Error("Branch stack is supported only with PMU events"); - } else if (_cstack >= CSTACK_VM && !(VMStructs::hasStackStructs() && OS::isLinux())) { - return Error( - "VMStructs stack walking is not supported on this JVM/platform"); - } - } - - return error; -} - -Error Profiler::dump(const char *path, const int length) { - MutexLocker ml(_state_lock); - State cur_state = state(); - if (cur_state != IDLE && cur_state != RUNNING) { - return Error("Profiler has not started"); - } - - if (cur_state == RUNNING) { - std::set thread_ids; - // flush the liveness tracker instance and note all the threads referenced - // by the live objects - LivenessTracker::instance()->flush(thread_ids); - - Libraries::instance()->refresh(); - updateJavaThreadNames(); - updateNativeThreadNames(); - - const CodeCacheArray& native_libs = _libs->native_libs(); - Counters::set(CODECACHE_NATIVE_COUNT, native_libs.count()); - Counters::set(CODECACHE_NATIVE_SIZE_BYTES, native_libs.memoryUsage()); - Counters::set(CODECACHE_RUNTIME_STUBS_SIZE_BYTES, - native_libs.memoryUsage()); - - Error err = Error::OK; - // rotateDictsAndRun rotates the dictionaries, takes lockAll() around the - // dump (fences ASGCT/JNI writers to CallTraceStorage), then clearStandby()s - // the rotated buffers. StringDictionary's RefCountGuard protocol handles - // its own writer/reader coordination; #527's classMapSharedGuard readers - // (deferred vtable receiver resolution) are coordinated through - // _class_map_lock. - rotateDictsAndRun([&]{ - err = _jfr.dump(path, length); - __atomic_add_fetch(&_epoch, 1, __ATOMIC_SEQ_CST); - }); - - _thread_info.clearAll(thread_ids); - _thread_info.reportCounters(); - - // reset unwinding counters - Counters::set(UNWINDING_TIME_ASYNC, 0); - Counters::set(UNWINDING_TIME_JVMTI, 0); - - return err; - } - - return Error::OK; -} - -void Profiler::lockAll() { - for (int i = 0; i < CONCURRENCY_LEVEL; i++) - _locks[i].lock(); -} - -void Profiler::unlockAll() { - for (int i = 0; i < CONCURRENCY_LEVEL; i++) - _locks[i].unlock(); -} - -void Profiler::switchThreadEvents(jvmtiEventMode mode) { - if (_thread_events_state != mode) { - jvmtiEnv *jvmti = VM::jvmti(); - jvmti->SetEventNotificationMode(mode, JVMTI_EVENT_THREAD_START, NULL); - jvmti->SetEventNotificationMode(mode, JVMTI_EVENT_THREAD_END, NULL); - _thread_events_state = mode; - } -} - -Error Profiler::runInternal(Arguments &args, std::ostream &out) { - switch (args._action) { - case ACTION_START: - case ACTION_RESUME: { - Error error = start(args, args._action == ACTION_START); - if (error) { - return error; - } - out << "Profiling started\n"; - break; - } - case ACTION_STOP: { - Error error = stop(); - if (error) { - return error; - } - break; - } - - case ACTION_CHECK: { - Error error = check(args); - if (error) { - return error; - } - out << "OK\n"; - break; - } - case ACTION_STATUS: { - MutexLocker ml(_state_lock); - if (state() == RUNNING) { - out << "Profiling is running for " << uptime() << " seconds\n"; - } else { - out << "Profiler is not active\n"; - } - break; - } - case ACTION_LIST: { - out << "Basic events:" << std::endl; - out << " " << EVENT_CPU << std::endl; - out << " " << EVENT_ALLOC << std::endl; - out << " " << EVENT_WALL << std::endl; - out << " " << EVENT_ITIMER << std::endl; - - out << "Java method calls:\n"; - out << " ClassName.methodName\n"; - - if (perf_events.check(args)) { - out << "Perf events:\n"; - // The first perf event is "cpu" which is already printed - for (int event_id = 1;; event_id++) { - const char *event_name = PerfEvents::getEventName(event_id); - if (event_name == NULL) - break; - out << " " << event_name << "\n"; - } - } - break; - } - case ACTION_VERSION: - out << PROFILER_VERSION; - out.flush(); - break; - default: - break; - } - return Error::OK; -} - -Error Profiler::run(Arguments &args) { return runInternal(args, std::cout); } - -Error Profiler::restart(Arguments &args) { - MutexLocker ml(_state_lock); - - Error error = stop(); - if (error) { - return error; - } - - return Error::OK; -} - -void Profiler::shutdown(Arguments &args) { - MutexLocker ml(_state_lock); - - // The last chance to dump profile before VM terminates - if (state() == RUNNING) { - args._action = ACTION_STOP; - Error error = run(args); - if (error) { - Log::error("%s", error.message()); - } - } - - _state.store(TERMINATED, std::memory_order_release); -} - -int Profiler::lookupClass(const char *key, size_t length) { - // StringDictionary::lookup() is internally thread-safe via _accepting + - // RefCountGuard; no external lock required (unlike the old Dictionary). - u32 id = _class_map.lookup(key, length); - return id != 0 ? static_cast(id) : -1; -} - -int Profiler::status(char* status, int max_len) { - return snprintf(status, max_len, - "== Java-Profiler Status ===\n" - " Running : %s\n" - " CPU Engine : %s\n" - " WallClock Engine : %s\n" - " Allocations : %s\n", - state() == RUNNING ? "true" : "false", - _cpu_engine != nullptr ? _cpu_engine->name() : "None", - _wall_engine != nullptr ? _wall_engine->name() : "None", - _alloc_engine != nullptr ? _alloc_engine->name() : "None"); -} diff --git a/ddprof-lib/src/main/cpp/profiler.h b/ddprof-lib/src/main/cpp/profiler.h deleted file mode 100644 index 9990e0f1c..000000000 --- a/ddprof-lib/src/main/cpp/profiler.h +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _PROFILER_H -#define _PROFILER_H - -#include "arch.h" -#include "arguments.h" -#include "callTraceStorage.h" -#include "codeCache.h" -#include "common.h" -#include "dictionary.h" -#include "stringDictionary.h" -#include "engine.h" -#include "event.h" -#include "flightRecorder.h" -#include "guards.h" -#include "libraries.h" -#include "log.h" -#include "mutex.h" -#include "objectSampler.h" -#include "spinLock.h" -#include "thread.h" -#include "threadFilter.h" -#include "threadInfo.h" -#include "trap.h" -#include "vmEntry.h" -#include -#include -#include -#include - -// avoid linking against newer symbols here for wide compatibility -#ifdef __GLIBC__ -#ifdef __aarch64__ -__asm__(".symver log,log@GLIBC_2.17"); -__asm__(".symver exp,exp@GLIBC_2.17"); -#endif -#endif - -#ifdef DEBUG -#include -static const char* force_stackwalk_crash_env = getenv("DDPROF_FORCE_STACKWALK_CRASH"); -#endif - -const int MAX_NATIVE_FRAMES = 128; -const int RESERVED_FRAMES = 10; // for synthetic frames - -union CallTraceBuffer { - ASGCT_CallFrame _asgct_frames[1]; - jvmtiFrameInfo _jvmti_frames[1]; -}; - -class FrameName; -class StackContext; -class VM; - -enum State { NEW, IDLE, RUNNING, TERMINATED }; - -// Aligned to satisfy SpinLock member alignment requirement (64 bytes) -// Required because this class contains the _locks[] SpinLock array. -class alignas(alignof(SpinLock)) Profiler { - friend VM; - -private: - // signal handlers - static volatile bool _signals_initialized; - - // JDK_8313796 workaround for unfixed versions - static volatile bool _need_JDK_8313796_workaround; - - Mutex _state_lock; - std::atomic _state; - // class unload hook - Trap _class_unload_hook_trap; - typedef void (*NotifyClassUnloadedFunc)(void *); - NotifyClassUnloadedFunc _notify_class_unloaded_func; - // -- - - ThreadInfo _thread_info; - StringDictionary _class_map{1}; - StringDictionary _string_label_map{2}; - StringDictionary _context_value_map{3}; - ThreadFilter _thread_filter; - CallTraceStorage _call_trace_storage; - FlightRecorder _jfr; - Engine *_cpu_engine; - Engine *_wall_engine = NULL; - Engine *_alloc_engine; - int _event_mask; - - time_t _start_time; - time_t _stop_time; - u32 _epoch; - WaitableMutex _timer_lock; - void *_timer_id; - - volatile u64 _total_samples; - // On a separate cache line: incremented from every signal handler via - // recordSampleDelegated; must not share a line with _failures (written by - // ASGCT paths) or _total_samples (written by every recording path). - alignas(DEFAULT_CACHE_LINE_SIZE) volatile u64 _sample_seq; - alignas(DEFAULT_CACHE_LINE_SIZE) u64 _failures[ASGCT_FAILURE_TYPES]; - bool _wall_precheck = false; - - SpinLock _class_map_lock; - SpinLock _locks[CONCURRENCY_LEVEL]; - CallTraceBuffer *_calltrace_buffer[CONCURRENCY_LEVEL]; - int _max_stack_depth; - StackWalkFeatures _features; - int _safe_mode; - CStack _cstack; - - volatile jvmtiEventMode _thread_events_state; - - Libraries* _libs; - u32 _num_context_attributes; - bool _omit_stacktraces; - bool _remote_symbolication; // Enable remote symbolication for native frames - - // dlopen() hook support - void **_dlopen_entry; - static void *dlopen_hook(const char *filename, int flags); - void switchLibraryTrap(bool enable); - static void prewarmUnwinder(); - - void enableEngines(); - void disableEngines(); - - void onThreadStart(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread); - void onThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread); - - u32 getLockIndex(int tid); - int getNativeTrace(void *ucontext, ASGCT_CallFrame *frames, int event_type, - int tid, StackContext *java_ctx, bool *truncated, int lock_index); - void updateThreadName(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread, - bool self = false); - void updateJavaThreadNames(); - void mangle(const char *name, char *buf, size_t size); - - Engine *selectCpuEngine(Arguments &args); - Engine *selectWallEngine(Arguments &args); - Engine *selectAllocEngine(Arguments &args); - Error checkJvmCapabilities(); - - void lockAll(); - void unlockAll(); - - // Rotate all three dictionaries, then run jfr_op under lockAll(). - // - // rotate() is self-contained: it uses _accepting + RefCountGuard to drain - // concurrent JNI readers, and SignalBlocker prevents profiling signals on - // this thread from inserting into old_active between Phase 1 and Phase 2. - // No external lock is required for rotation. - // - // lockAll() wraps jfr_op only — to gate call-trace writers (signal handlers - // and JNI paths that write to CallTraceStorage) from racing with the dump. - // Dictionary writers that bypass lockAll() (e.g. recordTrace0) are handled - // by the dictionary's own RefCountGuard protocol, not by lockAll(). - template - void rotateDictsAndRun(F jfr_op) { - SignalBlocker blocker; - _class_map.rotate(); - _string_label_map.rotate(); - _context_value_map.rotate(); - lockAll(); - jfr_op(); - unlockAll(); - _class_map.clearStandby(); - _string_label_map.clearStandby(); - _context_value_map.clearStandby(); - } - - static int crashHandlerInternal(int signo, siginfo_t *siginfo, void *ucontext); - static void check_JDK_8313796_workaround(); - - static Profiler *const _instance; - - inline State state() const { - return _state.load(std::memory_order_relaxed); - } - -public: - Profiler() - : _state_lock(), _state(State::NEW), _class_unload_hook_trap(2), - _notify_class_unloaded_func(NULL), _thread_info(), _class_map(1), - _string_label_map(2), _context_value_map(3), _thread_filter(), - _call_trace_storage(), _jfr(), _cpu_engine(NULL), _wall_engine(NULL), - _alloc_engine(NULL), _event_mask(0), - _start_time(0), _stop_time(0), _epoch(0), _timer_id(NULL), - _total_samples(0), _sample_seq(0), _failures(), _class_map_lock(), - _max_stack_depth(0), _features(), _safe_mode(0), _cstack(CSTACK_NO), - _thread_events_state(JVMTI_DISABLE), _libs(Libraries::instance()), - _num_context_attributes(0), _omit_stacktraces(false), - _remote_symbolication(false), _dlopen_entry(NULL) { - - for (int i = 0; i < CONCURRENCY_LEVEL; i++) { - _calltrace_buffer[i] = NULL; - } - } - - static inline Profiler *instance() { - return _instance; - } - - // Resolve names of native (non-Java) threads from /proc. Idempotent and - // allocation-light (no-op for already-named tids), so it is safe to call - // periodically from the Libraries refresher thread to capture transient - // compiler/GC threads before they exit. Must NOT be called from a signal - // handler: thread enumeration uses opendir/readdir/malloc. - // - // When defer_initializing is true (periodic refresher), a thread whose comm - // still equals the process's own (inherited) name is skipped: it is most - // likely still initializing and has not yet set its final pthread name. - // Recording it now would latch that provisional name permanently - // (ThreadInfo::updateThreadName is first-writer-wins). A later scan, or the - // dump-time pass (which passes false), records the final name instead. - void updateNativeThreadNames(bool defer_initializing = false); - - - inline void incFailure(int type) { - if (type < ASGCT_FAILURE_TYPES) { - atomicIncRelaxed(_failures[type]); - } - } - - int status(char* status, int max_len); - static const char *asgctError(int code); - - - inline int safe_mode() const { - return _safe_mode; - } - - inline const Libraries* libraries() const { - return _libs; - } - - inline CStack cstackMode() const { - return _cstack; - } - - inline const StackWalkFeatures& stackWalkFeatures() const { - return _features; - } - - inline bool isRunning() { - return _state.load(std::memory_order_acquire) == RUNNING; - } - - u64 total_samples() { return _total_samples; } - int max_stack_depth() { return _max_stack_depth; } - time_t uptime() { return time(NULL) - _start_time; } - Engine *cpuEngine() { return _cpu_engine; } - Engine *wallEngine() { return _wall_engine; } - - StringDictionary *classMap() { return &_class_map; } - SharedLockGuard classMapSharedGuard() { return SharedLockGuard(&_class_map_lock); } - StringDictionary *stringLabelMap() { return &_string_label_map; } - StringDictionary *contextValueMap() { return &_context_value_map; } - u32 numContextAttributes() { return _num_context_attributes; } - ThreadFilter *threadFilter() { return &_thread_filter; } - - const char* cstack() const; - int lookupClass(const char *key, size_t length); - void processCallTraces(std::function&)> processor) { - if (!_omit_stacktraces) { - _call_trace_storage.processTraces(processor); - } else { - // If stack traces are omitted, call processor with empty set - static std::unordered_set empty_traces; - processor(empty_traces); - } - } - - void registerLivenessChecker(LivenessChecker checker) { - _call_trace_storage.registerLivenessChecker(checker); - } - - inline u32 recordingEpoch() { - // no thread reordering constraints - return __atomic_load_n(&_epoch, __ATOMIC_RELAXED); - } - - Error run(Arguments &args); - Error runInternal(Arguments &args, std::ostream &out); - Error restart(Arguments &args); - void shutdown(Arguments &args); - Error check(Arguments &args); - Error start(Arguments &args, bool reset); - Error stop(); - Error dump(const char *path, const int length); - void logStats(); - void switchThreadEvents(jvmtiEventMode mode); - - /** - * Remote symbolication packed data layout (BCI_NATIVE_FRAME_REMOTE): - * - * When remote symbolication is enabled and a library has a build-ID, we defer - * full symbol resolution to post-processing and pack essential data into the - * 64-bit jmethodID field: - * - * Bits 0-43: pc_offset (44 bits, 16 TB range) - * Bits 44-46: mark (3 bits, 0-7 values) - * Bits 47-61: lib_index (15 bits, 32K libraries) - *. Bits 62-63: reserved - * - * Mark values indicate JVM internal frames that should terminate stack walks: - * 0 = no mark (regular native frame) - * MARK_VM_RUNTIME = 1 - * MARK_INTERPRETER = 2 - * MARK_COMPILER_ENTRY = 3 - * MARK_ASYNC_PROFILER = 4 - * - * During stack walking, we perform symbol resolution (binarySearch) to check - * marks and pack the mark value for later use. The performance is O(log n) for - * binarySearch + O(1) for mark extraction, same as traditional symbolication - * but simpler than maintaining a separate marked ranges index. - */ - struct RemoteFramePacker { - static const int PC_OFFSET_BITS = 44; - static const int MARK_BITS = 3; - static const int LIB_INDEX_BITS = 15; - - static const unsigned long PC_OFFSET_MASK = (1ULL << PC_OFFSET_BITS) - 1; // 0xFFFFFFFFFFF (44 bits) - static const unsigned long MARK_MASK = (1ULL << MARK_BITS) - 1; // 0x7 (3 bits) - static const unsigned long LIB_INDEX_MASK = (1ULL << LIB_INDEX_BITS) - 1; // 0x7FFF (15 bits) - - /** - * Pack remote symbolication data into a 64-bit jmethodID. - * Layout: pc_offset (44 bits) | mark (3 bits) | lib_index (15 bits) - */ - static inline unsigned long pack(uintptr_t pc_offset, char mark, uint32_t lib_index) { - return (unsigned long)( - (pc_offset & PC_OFFSET_MASK) | // Bits 0-43 - (((unsigned long)mark & MARK_MASK) << PC_OFFSET_BITS) | // Bits 44-46 - (((unsigned long)lib_index & LIB_INDEX_MASK) << (PC_OFFSET_BITS + MARK_BITS)) // Bits 47-62 - ); - } - - /** - * Unpack pc_offset from packed data. - */ - static inline uintptr_t unpackPcOffset(unsigned long packed) { - return (uintptr_t)(packed & PC_OFFSET_MASK); - } - - /** - * Unpack mark from packed data. - */ - static inline char unpackMark(unsigned long packed) { - return (char)((packed >> PC_OFFSET_BITS) & MARK_MASK); - } - - /** - * Unpack lib_index from packed data. - */ - static inline uint32_t unpackLibIndex(unsigned long packed) { - return (uint32_t)((packed >> (PC_OFFSET_BITS + MARK_BITS)) & LIB_INDEX_MASK); - } - }; - - // Result of resolving a native frame for symbolication - struct NativeFrameResolution { - union { - unsigned long packed_remote_frame; // Packed remote frame data (pc_offset|mark|lib_index) - const char* method_name; // Resolved method name - }; - int bci; // BCI_NATIVE_FRAME_REMOTE or BCI_NATIVE_FRAME - bool is_marked; // true if this is a marked C++ interpreter frame (stop processing) - NativeFrameResolution(const char* name, int bci_type, bool marked) - : method_name(name), bci(bci_type), is_marked(marked) {} - NativeFrameResolution(unsigned long packed, int bci_type, bool marked) - : packed_remote_frame(packed), bci(bci_type), is_marked(marked) {} - }; - - void populateRemoteFrame(ASGCT_CallFrame* frame, uintptr_t pc, CodeCache* lib, char mark); - NativeFrameResolution resolveNativeFrameForWalkVM(uintptr_t pc, int lock_index); - int convertNativeTrace(int native_frames, const void **callchain, - ASGCT_CallFrame *frames, int lock_index); - bool recordSample(void *ucontext, u64 weight, int tid, jint event_type, - u64 call_trace_id, Event *event, - u64 *recorded_call_trace_id = nullptr); - // Delegated sample path: stack-walking is performed by the HotSpot JFR - // RequestStackTrace extension (the JVM emits the stack trace into its own - // JFR recording). We only emit the CPU/wall sample event with no - // stack-trace reference, tagged by the correlation ID we passed to - // RequestStackTrace as user_data. - bool recordSampleDelegated(void *ucontext, u64 weight, int tid, - jint event_type, Event *event); - u64 recordJVMTISample(u64 weight, int tid, jthread thread, jint event_type, Event *event, bool deferred); - void recordDeferredSample(int tid, u64 call_trace_id, jint event_type, Event *event); - void recordExternalSample(u64 weight, int tid, int num_frames, - ASGCT_CallFrame *frames, bool truncated, - jint event_type, Event *event); - void recordWallClockEpoch(int tid, WallClockEpochEvent *event); - void recordTraceRoot(int tid, TraceRootEvent *event); - void recordQueueTime(int tid, QueueTimeEvent *event); - void writeLog(LogLevel level, const char *message); - void writeLog(LogLevel level, const char *message, size_t len); - void writeDatadogProfilerSetting(int tid, int length, const char *name, - const char *value, const char *unit); - void writeHeapUsage(long value, bool live); - int eventMask() const { return _event_mask; } - bool isRemoteSymbolication() const { return _remote_symbolication; } - - const void *resolveSymbol(const char *name); - const char *getLibraryName(const char *native_symbol); - const char *findNativeMethod(const void *address); - - static void segvHandler(int signo, siginfo_t *siginfo, void *ucontext); - static void busHandler(int signo, siginfo_t *siginfo, void *ucontext); - static void setupSignalHandlers(); - - static int registerThread(int tid); - static void unregisterThread(int tid); - -#ifdef UNIT_TEST - // Returns the tid most recently passed to unregisterThread(), or -1 if it - // has never been called (or since the last resetUnregisterObservableForTest). - // Used by integration tests to assert that cleanup_unregister wired - // Profiler::unregisterThread correctly without needing live engine instances. - static int lastUnregisteredTidForTest(); - static void resetUnregisterObservableForTest(); - - // Reads back the name recorded for a tid in _thread_info, or an empty string - // if none was recorded. Lets integration tests observe the result of - // updateNativeThreadNames() (notably the defer_initializing skip) without - // exposing the private _thread_info. Compiled only into gtest binaries. - std::string threadNameForTest(int tid) { - std::pair, u64> info = _thread_info.get(tid); - return info.first != nullptr ? *info.first : std::string(); - } -#endif - - - static void JNICALL ThreadStart(jvmtiEnv *jvmti, JNIEnv *jni, - jthread thread) { - instance()->onThreadStart(jvmti, jni, thread); - } - - static void JNICALL ThreadEnd(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread) { - instance()->onThreadEnd(jvmti, jni, thread); - } - - // Keep backward compatibility with the upstream async-profiler - inline CodeCache* findLibraryByAddress(const void *address) { - #ifdef DEBUG - // we need this code to simulate segfault during stackwalking - // this is a safe place to do it since this wrapper is used solely from the 'vm' stackwalker implementation - if (force_stackwalk_crash_env) { - TEST_LOG("FORCE_SIGSEGV"); - raise(SIGSEGV); - } - #endif - return Libraries::instance()->findLibraryByAddress(address); - } - - friend class Recording; -}; - -#endif // _PROFILER_H diff --git a/ddprof-lib/src/main/cpp/rateLimiter.h b/ddprof-lib/src/main/cpp/rateLimiter.h deleted file mode 100644 index e8c27d5bb..000000000 --- a/ddprof-lib/src/main/cpp/rateLimiter.h +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _RATELIMITER_H -#define _RATELIMITER_H - -#include "arch.h" -#include "os.h" -#include "pidController.h" -#include -#include - -/** - * Thread-safe rate limiter based on a PID controller. - * - * Maintains a shared target event rate by adjusting a sampling interval - * (in arbitrary units — TSC ticks, bytes, counts, …) via a PID feedback - * loop driven by the aggregate observed fire count across all threads. - * - * ## Typical use - * - * One RateLimiter instance is shared across threads. Per-thread sampling - * decisions are made by a companion sampler (e.g. PoissonSampler) that - * reads interval() and epoch() from this object. After each sampled event - * the thread calls recordFire() to feed back into the rate controller. - * - * ## Epoch-based lazy reset - * - * start() bumps an epoch counter. Per-thread samplers compare their cached - * epoch against epoch() on every sample call; a mismatch triggers a lazy - * reinitialisation, so no explicit iteration over threads is needed at start. - */ -class RateLimiter { -public: - RateLimiter() = default; - - /** - * Initialise for a new profiling session. - * - * @param init_interval_units Initial sampling interval in the chosen unit - * (e.g. TSC ticks for ~1 ms). Must be >= 1. - * @param target_per_second Target aggregate fire rate (events / second). - * @param pid_window_secs PID observation window in seconds. - * @param p_gain PID proportional gain. - * @param i_gain PID integral gain. - * @param d_gain PID derivative gain. - * @param cutoff_secs PID derivative low-pass cutoff in seconds. - */ - void start(long init_interval_units, - u64 target_per_second, - int pid_window_secs, - double p_gain, double i_gain, double d_gain, - double cutoff_secs) { - _interval.store(init_interval_units, std::memory_order_release); - _event_count.store(0, std::memory_order_relaxed); - _last_update_ns.store(OS::nanotime(), std::memory_order_release); - _epoch.fetch_add(1, std::memory_order_release); - _pid = PidController(target_per_second, p_gain, i_gain, d_gain, - pid_window_secs, cutoff_secs); - } - - /** Current sampling interval in the chosen unit. */ - long interval() const { - return _interval.load(std::memory_order_relaxed); - } - - /** Current epoch; bumped on every start(). */ - u64 epoch() const { - return _epoch.load(std::memory_order_relaxed); - } - - /** - * Record one sampled event and update the PID controller at most once - * per second. Safe to call from any thread concurrently. - */ - void recordFire() { - _event_count.fetch_add(1, std::memory_order_relaxed); - maybeUpdateInterval(); - } - -private: - static const u64 ONE_SECOND_NS = 1000000000ULL; - - std::atomic _interval{1}; - std::atomic _epoch{0}; - std::atomic _event_count{0}; - std::atomic _last_update_ns{0}; - PidController _pid{1, 1.0, 1.0, 1.0, 1, 1.0}; - - void maybeUpdateInterval() { - u64 now = OS::nanotime(); - u64 prev = _last_update_ns.load(std::memory_order_relaxed); - if (now - prev < ONE_SECOND_NS) { - return; - } - if (!_last_update_ns.compare_exchange_strong(prev, now, - std::memory_order_acq_rel, - std::memory_order_relaxed)) { - return; - } - // One-event-per-window imprecision: a concurrent recordFire() after this exchange - // loses its count for this window. Accepted: the PID controller tolerates this level - // of measurement noise without instability. - long count = _event_count.exchange(0, std::memory_order_relaxed); - double signal = _pid.compute(static_cast(count), 1.0); - long delta = (signal > (double)LONG_MAX) ? LONG_MAX - : (signal < (double)LONG_MIN) ? LONG_MIN - : (long)signal; - long new_interval = _interval.load(std::memory_order_relaxed) - delta; - if (new_interval < 1) { - new_interval = 1; - } else if (new_interval > (1L << 40)) { - new_interval = 1L << 40; - } - // Relaxed store: eventual consistency is acceptable. Threads reading _interval - // with relaxed loads will see the update within at most one additional window. - // Forcing release ordering here would add unnecessary cost on weak-ordering architectures. - _interval.store(new_interval, std::memory_order_relaxed); - } -}; - -#endif // _RATELIMITER_H diff --git a/ddprof-lib/src/main/cpp/refCountGuard.cpp b/ddprof-lib/src/main/cpp/refCountGuard.cpp deleted file mode 100644 index 583c3d8d4..000000000 --- a/ddprof-lib/src/main/cpp/refCountGuard.cpp +++ /dev/null @@ -1,244 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "refCountGuard.h" -#include "arch.h" -#include "counters.h" -#include "log.h" -#include "os.h" -#include "primeProbing.h" -#include "thread.h" -#include -#include - -// Static member definitions -RefCountSlot RefCountGuard::refcount_slots[RefCountGuard::MAX_THREADS]; -int RefCountGuard::slot_owners[RefCountGuard::MAX_THREADS]; - -// One-time warning latch: emit at most one Log::warn per process when -// reentrant nesting exceeds OUTER_STACK_DEPTH and the scanner can no longer -// see every displaced resource on the slot. -static std::atomic s_outer_stack_overflow_warned{false}; - -int RefCountGuard::getThreadRefCountSlot() { - ProfiledThread* thrd = ProfiledThread::currentSignalSafe(); - int tid = thrd != nullptr ? thrd->tid() : OS::threadId(); - - HashProbe probe(static_cast(tid), MAX_THREADS); - - int slot = probe.slot(); - for (int i = 0; i < MAX_PROBE_DISTANCE; i++) { - int expected = 0; - if (__atomic_compare_exchange_n(&slot_owners[slot], &expected, tid, false, __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)) { - return slot; - } - - if (__atomic_load_n(&slot_owners[slot], __ATOMIC_ACQUIRE) == tid) { - // Only treat as reentrant if the outer guard is still active. - // When count==0 the outer guard has already decremented and is - // just clearing slot_owners; creating a "reentrant" guard on a - // dying slot would publish active_ptr while the outer destructor - // is about to overwrite it, causing waitForRefCountToClear to - // miss the new resource. - if (__atomic_load_n(&refcount_slots[slot].count, __ATOMIC_ACQUIRE) > 0) { - return slot + MAX_THREADS; - } - // Fall through: probe for a fresh slot instead. - } - - if (probe.hasNext()) { - slot = probe.next(); - } - } - - return -1; -} - -RefCountGuard::RefCountGuard(void* resource) : _active(true), _is_reentrant(false), _outer_slot(-1), _my_slot(-1), _saved_ptr(nullptr) { - int raw = getThreadRefCountSlot(); - - if (raw == -1) { - _active = false; - return; - } - - _is_reentrant = (raw >= MAX_THREADS); - _my_slot = _is_reentrant ? (raw - MAX_THREADS) : raw; - - if (_is_reentrant) { - _saved_ptr = __atomic_load_n(&refcount_slots[_my_slot].active_ptr, __ATOMIC_ACQUIRE); - // Reentrant: increment count first so the scanner always sees the outer - // resource while active_ptr is being updated. fetch_add returns the - // PRE-increment count, which is the reentrancy depth this guard is - // about to occupy (depth 1 = first nested signal, depth 2 = second...). - uint32_t prev_count = __atomic_fetch_add(&refcount_slots[_my_slot].count, 1, __ATOMIC_RELEASE); - // Park the displaced active_ptr in outer_stack[prev_count - 1] so - // waitForRefCountToClear() can see every resource currently in use - // on this slot. outer_stack[0] holds the outermost (root) resource. - int idx = static_cast(prev_count) - 1; - if (idx >= 0 && idx < OUTER_STACK_DEPTH) { - _outer_slot = idx; - __atomic_store_n(&refcount_slots[_my_slot].outer_stack[idx], _saved_ptr, __ATOMIC_RELEASE); - } else { - // Reentrant nesting deeper than OUTER_STACK_DEPTH; the displaced - // resource lives only in this guard's _saved_ptr and is invisible - // to the scanner. Latch a single warning per process. - bool expected = false; - if (s_outer_stack_overflow_warned.compare_exchange_strong(expected, true, std::memory_order_relaxed)) { - Log::warn("RefCountGuard reentrancy depth %u exceeds OUTER_STACK_DEPTH=%d; scanner may miss intermediate resources", - static_cast(prev_count) + 1, OUTER_STACK_DEPTH); - } - } - __atomic_store_n(&refcount_slots[_my_slot].active_ptr, resource, __ATOMIC_RELEASE); - } else { - // Non-reentrant (count was 0): store pointer first so the scanner skips - // this slot during the activation window (count=0 → treated as inactive). - __atomic_store_n(&refcount_slots[_my_slot].active_ptr, resource, __ATOMIC_RELEASE); - __atomic_fetch_add(&refcount_slots[_my_slot].count, 1, __ATOMIC_RELEASE); - } -} - -RefCountGuard::~RefCountGuard() { - if (_active && _my_slot >= 0) { - if (_is_reentrant) { - // Restore outer active_ptr first, then (if we parked one) clear our - // outer_stack slot, then decrement count. Scanner always observes - // the outer resource while count > 0. - __atomic_store_n(&refcount_slots[_my_slot].active_ptr, _saved_ptr, __ATOMIC_RELEASE); - if (_outer_slot >= 0) { - __atomic_store_n(&refcount_slots[_my_slot].outer_stack[_outer_slot], nullptr, __ATOMIC_RELEASE); - } - __atomic_fetch_sub(&refcount_slots[_my_slot].count, 1, __ATOMIC_RELEASE); - } else { - __atomic_fetch_sub(&refcount_slots[_my_slot].count, 1, __ATOMIC_RELEASE); - __atomic_store_n(&refcount_slots[_my_slot].active_ptr, nullptr, __ATOMIC_RELEASE); - __atomic_store_n(&slot_owners[_my_slot], 0, __ATOMIC_RELEASE); - } - } -} - -RefCountGuard::RefCountGuard(RefCountGuard&& other) noexcept - : _active(other._active), _is_reentrant(other._is_reentrant), - _outer_slot(other._outer_slot), - _my_slot(other._my_slot), _saved_ptr(other._saved_ptr) { - other._active = false; -} - -RefCountGuard& RefCountGuard::operator=(RefCountGuard&& other) noexcept { - if (this != &other) { - if (_active && _my_slot >= 0) { - if (_is_reentrant) { - __atomic_store_n(&refcount_slots[_my_slot].active_ptr, _saved_ptr, __ATOMIC_RELEASE); - if (_outer_slot >= 0) { - __atomic_store_n(&refcount_slots[_my_slot].outer_stack[_outer_slot], nullptr, __ATOMIC_RELEASE); - } - __atomic_fetch_sub(&refcount_slots[_my_slot].count, 1, __ATOMIC_RELEASE); - } else { - __atomic_fetch_sub(&refcount_slots[_my_slot].count, 1, __ATOMIC_RELEASE); - __atomic_store_n(&refcount_slots[_my_slot].active_ptr, nullptr, __ATOMIC_RELEASE); - __atomic_store_n(&slot_owners[_my_slot], 0, __ATOMIC_RELEASE); - } - } - _active = other._active; - _is_reentrant = other._is_reentrant; - _outer_slot = other._outer_slot; - _my_slot = other._my_slot; - _saved_ptr = other._saved_ptr; - other._active = false; - } - return *this; -} - -// Returns true iff the slot currently references the resource we want to delete, -// either as active_ptr or as any non-null entry in outer_stack. -static inline bool slotReferences(const RefCountSlot& s, void* target) { - void* table = __atomic_load_n(&s.active_ptr, __ATOMIC_ACQUIRE); - if (table == target) return true; - for (int j = 0; j < RefCountSlot::OUTER_STACK_DEPTH; ++j) { - void* o = __atomic_load_n(&s.outer_stack[j], __ATOMIC_ACQUIRE); - if (o == target) return true; - } - return false; -} - -void RefCountGuard::waitForRefCountToClear(void* table_to_delete) { - const int SPIN_ITERATIONS = 100; - for (int spin = 0; spin < SPIN_ITERATIONS; ++spin) { - bool all_clear = true; - for (int i = 0; i < MAX_THREADS; ++i) { - uint32_t count = __atomic_load_n(&refcount_slots[i].count, __ATOMIC_ACQUIRE); - if (count == 0) { - // Check active_ptr to cover the non-reentrant constructor's activation window: - // active_ptr is stored (RELEASE) before count++ (RELEASE), so count==0 with - // active_ptr set means the thread is in the window and must be waited for. - void* aptr = __atomic_load_n(&refcount_slots[i].active_ptr, __ATOMIC_ACQUIRE); - if (aptr == table_to_delete) { all_clear = false; break; } - continue; - } - if (slotReferences(refcount_slots[i], table_to_delete)) { all_clear = false; break; } - } - if (all_clear) return; - spinPause(); - } - - const int MAX_WAIT_ITERATIONS = 5000; - struct timespec sleep_time = {0, 100000}; - for (int wait_count = 0; wait_count < MAX_WAIT_ITERATIONS; ++wait_count) { - bool all_clear = true; - for (int i = 0; i < MAX_THREADS; ++i) { - uint32_t count = __atomic_load_n(&refcount_slots[i].count, __ATOMIC_ACQUIRE); - if (count == 0) { - void* aptr = __atomic_load_n(&refcount_slots[i].active_ptr, __ATOMIC_ACQUIRE); - if (aptr == table_to_delete) { all_clear = false; break; } - continue; - } - if (slotReferences(refcount_slots[i], table_to_delete)) { all_clear = false; break; } - } - if (all_clear) return; - nanosleep(&sleep_time, nullptr); - } - - Counters::increment(DICTIONARY_DRAIN_TIMEOUTS, 1); - Log::warn("waitForRefCountToClear: timeout after ~500ms waiting for %p; " - "drain incomplete, proceeding (dictionary snapshot may miss late inserts)", - table_to_delete); -#ifndef NDEBUG - // Under DEBUG builds, treat the timeout as a fatal bug — keeping the abort - // out of release avoids turning a survivable rotation glitch into a crash - // in production. - abort(); -#endif -} - -void RefCountGuard::waitForAllRefCountsToClear() { - const int SPIN_ITERATIONS = 100; - for (int spin = 0; spin < SPIN_ITERATIONS; ++spin) { - bool any = false; - for (int i = 0; i < MAX_THREADS; ++i) { - if (__atomic_load_n(&refcount_slots[i].count, __ATOMIC_ACQUIRE) > 0) { any = true; break; } - // Also check active_ptr: non-reentrant constructor stores it (RELEASE) before - // count++ (RELEASE), so count==0 with active_ptr!=null means the thread is in - // the activation window and must be waited for. - if (__atomic_load_n(&refcount_slots[i].active_ptr, __ATOMIC_ACQUIRE) != nullptr) { any = true; break; } - } - if (!any) return; - spinPause(); - } - - const int MAX_WAIT_ITERATIONS = 5000; - struct timespec sleep_time = {0, 100000}; - int last_nonzero_slot = -1; - for (int wait_count = 0; wait_count < MAX_WAIT_ITERATIONS; ++wait_count) { - bool any = false; - for (int i = 0; i < MAX_THREADS; ++i) { - if (__atomic_load_n(&refcount_slots[i].count, __ATOMIC_ACQUIRE) > 0) { any = true; last_nonzero_slot = i; break; } - if (__atomic_load_n(&refcount_slots[i].active_ptr, __ATOMIC_ACQUIRE) != nullptr) { any = true; last_nonzero_slot = i; break; } - } - if (!any) return; - nanosleep(&sleep_time, nullptr); - } - Log::warn("waitForAllRefCountsToClear: timeout after ~500ms; slot %d last seen non-zero, proceeding", last_nonzero_slot); -} diff --git a/ddprof-lib/src/main/cpp/refCountGuard.h b/ddprof-lib/src/main/cpp/refCountGuard.h deleted file mode 100644 index 5f5f25ad5..000000000 --- a/ddprof-lib/src/main/cpp/refCountGuard.h +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _REFCOUNTGUARD_H -#define _REFCOUNTGUARD_H - -#include "arch.h" -#include - -/** - * Cache-aligned reference counting slot for thread-local reference counting. - * Each slot occupies a full cache line (64 bytes) to eliminate false sharing. - * - * ACTIVATION PROTOCOL (pointer-first): - * - Constructor: store active_ptr (RELEASE) first, then increment count (RELEASE) - * - Destructor: decrement count (RELEASE) first, then clear active_ptr (RELEASE) - * - Scanner: load count (ACQUIRE); if 0, also load active_ptr (ACQUIRE); - * treat the slot as active if either count > 0 or active_ptr != null - * - * The scanner checks active_ptr even when count==0 to cover the non-reentrant - * constructor's activation window (active_ptr stored but count not yet incremented) - * and the destructor's deactivation window (count decremented but active_ptr not yet - * cleared). The RELEASE/ACQUIRE pairing on active_ptr itself guarantees the scanner - * sees the stored value if it was written before the load in program order. - * - * REENTRANT NESTING: when a signal fires inside an outer guard (same thread), - * the displaced active_ptr is parked in outer_stack[] so the scanner can still - * see every resource currently in use on the slot. outer_stack is sized to - * OUTER_STACK_DEPTH; deeper nesting emits a one-time warning and the deepest - * displaced resource becomes invisible to the scanner (rare: it requires - * OUTER_STACK_DEPTH+1 nested signal deliveries on the same thread). - * Ordering: outer_stack[i] must be stored AFTER count++ but BEFORE active_ptr - * is overwritten; cleared AFTER active_ptr is restored but BEFORE count--. - */ -struct alignas(DEFAULT_CACHE_LINE_SIZE) RefCountSlot { - static constexpr int OUTER_STACK_DEPTH = 3; - - volatile uint32_t count; // Reference count (0 = inactive) - alignas(alignof(void*)) void* active_ptr; // Which resource is being referenced - void* outer_stack[OUTER_STACK_DEPTH]; // Displaced resources on reentrant nesting - // Trailing padding fills the cache line. - // Layout on 64-bit: count(4) + 4-byte gap + active_ptr(8) + OUTER_STACK_DEPTH * 8. - char padding[DEFAULT_CACHE_LINE_SIZE - alignof(void*) - (1 + OUTER_STACK_DEPTH) * sizeof(void*)]; - - RefCountSlot() : count(0), active_ptr(nullptr), outer_stack{}, padding{} { - static_assert(sizeof(RefCountSlot) == DEFAULT_CACHE_LINE_SIZE, - "RefCountSlot must be exactly one cache line"); - } -}; - -/** - * RAII guard for thread-local reference counting. - * - * Provides lock-free memory reclamation for any heap-allocated resource that - * may be accessed from signal handlers concurrently with deallocation. - * Uses the pointer-first protocol to avoid race conditions. - * - * Performance: ~44-94 cycles hot-path; thread-local cache line, zero contention. - * - * Correctness: - * - Pointer stored BEFORE count increment (activation) - * - Count decremented BEFORE pointer cleared (deactivation) - * - Scanner checks both count and active_ptr to close the activation window - * - * Reentrancy: - * - A signal handler may create a RefCountGuard while a JNI thread already - * holds one on the same slot (same tid). getThreadRefCountSlot() returns - * slot + MAX_THREADS to signal this case. The inner guard saves and restores - * the outer guard's active_ptr instead of clearing it, so the scanner never - * sees a null pointer for an active outer guard. - * - Ordering invariants differ for the reentrant case: - * Constructor: count incremented BEFORE overwriting active_ptr (outer resource - * stays visible to the scanner until the new pointer is installed). - * Destructor: active_ptr restored to saved outer pointer BEFORE decrementing - * count (scanner always sees outer resource while count is still elevated). - */ -class RefCountGuard { -public: - static constexpr int MAX_THREADS = 8192; - static constexpr int MAX_PROBE_DISTANCE = 32; - static constexpr int OUTER_STACK_DEPTH = RefCountSlot::OUTER_STACK_DEPTH; - - static RefCountSlot refcount_slots[MAX_THREADS]; - static int slot_owners[MAX_THREADS]; - -private: - bool _active; - bool _is_reentrant; - int _outer_slot; // index into RefCountSlot::outer_stack, or -1 if not parked - int _my_slot; - void* _saved_ptr; - - // Returns slot index in [0, MAX_THREADS) on fresh claim. - // Returns slot + MAX_THREADS when the calling thread already owns that slot - // (reentrant signal delivery); the caller must save/restore active_ptr. - static int getThreadRefCountSlot(); - -public: - explicit RefCountGuard(void* resource); - ~RefCountGuard(); - - RefCountGuard(const RefCountGuard&) = delete; - RefCountGuard& operator=(const RefCountGuard&) = delete; - - RefCountGuard(RefCountGuard&& other) noexcept; - RefCountGuard& operator=(RefCountGuard&& other) noexcept; - - bool isActive() const { return _active; } - - // Wait for all in-flight guards protecting ptr_to_delete to be released. - static void waitForRefCountToClear(void* ptr_to_delete); - - // Wait for ALL reference counts to clear. - static void waitForAllRefCountsToClear(); -}; - -#endif // _REFCOUNTGUARD_H diff --git a/ddprof-lib/src/main/cpp/reservoirSampler.h b/ddprof-lib/src/main/cpp/reservoirSampler.h deleted file mode 100644 index 5963c6036..000000000 --- a/ddprof-lib/src/main/cpp/reservoirSampler.h +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2024 Datadog - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef RESERVOIR_SAMPLER_H -#define RESERVOIR_SAMPLER_H - -#include -#include -#include - - -template -class ReservoirSampler { -private: - const int _size; - std::mt19937 _generator; - std::uniform_real_distribution _uniform; - std::uniform_int_distribution _random_index; - std::vector _reservoir; - -public: - ReservoirSampler(const int size) : - _size(size), - _generator([]() { - std::random_device rd; - std::seed_seq seed_seq{rd(), rd(), rd(), rd()}; - return std::mt19937(seed_seq); - }()), - _uniform(1e-16, 1.0), - _random_index(0, size - 1) { - _reservoir.reserve(size); - } - - std::vector& sample(const std::vector &input) { - _reservoir.clear(); - for (int i = 0; i < _size && i < (int)input.size(); i++) { - _reservoir.push_back(input[i]); - } - double weight = exp(log(_uniform(_generator)) / _size); - int target = _size + (int) (log(_uniform(_generator)) / log(1 - weight)); - assert(target >= 0); - while (target < (int)input.size()) { - _reservoir[_random_index(_generator)] = input[target]; - weight *= exp(log(_uniform(_generator)) / _size); - target += (int) (log(_uniform(_generator)) / log(1 - weight)); - } - return _reservoir; - } -}; - -#endif //RESERVOIR_SAMPLER_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/reverse_bits.h b/ddprof-lib/src/main/cpp/reverse_bits.h deleted file mode 100644 index 01cecc01c..000000000 --- a/ddprof-lib/src/main/cpp/reverse_bits.h +++ /dev/null @@ -1,23 +0,0 @@ -// -// Borrow the implementation from openjdk -// https://github.com/openjdk/jdk/blob/master/src/hotspot/share/utilities/reverse_bits.hpp -// - -#ifndef REVERSE_BITS_H -#define REVERSE_BITS_H -#include "arch.h" -#include - -static constexpr u32 rep_5555 = static_cast(UINT64_C(0x5555555555555555)); -static constexpr u32 rep_3333 = static_cast(UINT64_C(0x3333333333333333)); -static constexpr u32 rep_0F0F = static_cast(UINT64_C(0x0F0F0F0F0F0F0F0F)); - -inline u16 reverse16(u16 v) { - u32 x = static_cast(v); - x = ((x & rep_5555) << 1) | ((x >> 1) & rep_5555); - x = ((x & rep_3333) << 2) | ((x >> 2) & rep_3333); - x = ((x & rep_0F0F) << 4) | ((x >> 4) & rep_0F0F); - return __builtin_bswap16(static_cast(x)); -} - -#endif //REVERSE_BITS_H diff --git a/ddprof-lib/src/main/cpp/rustDemangler.cpp b/ddprof-lib/src/main/cpp/rustDemangler.cpp deleted file mode 100644 index 9866515e1..000000000 --- a/ddprof-lib/src/main/cpp/rustDemangler.cpp +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2021, 2023 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "rustDemangler.h" - -#include - -namespace RustDemangler { - -// With some exceptions we don't handle here, v0 Rust symbols can end in a -// prefix followed by a 16-hexdigit hash, which must be removed -const std::string hash_pre = "::h"; -const std::string hash_eg = "0123456789abcdef"; - -const std::array, 9> patterns = {{ - {"..", "::"}, - {"$C$", ","}, - {"$BP$", "*"}, - {"$GT$", ">"}, - {"$LT$", "<"}, - {"$LP$", "("}, - {"$RP$", ")"}, - {"$RF$", "&"}, - {"$SP$", "@"}, -}}; - -inline bool is_hexdig(const char c) { - return (c >= 'a' && c <= 'f') || (c >= '0' && c <= '9'); -} - -// Simple conversion from hex digit to integer -inline int hex_to_int(char dig) { - constexpr int k_a_hex_value = 0xa; - if (dig >= '0' && dig <= '9') { - return dig - '0'; - } - if (dig >= 'a' && dig <= 'f') { - return dig - 'a' + k_a_hex_value; - } - if (dig >= 'A' && dig <= 'F') { - return dig - 'A' + k_a_hex_value; - } - return -1; -} - -// Minimal check that a string can end, and does end, in a hashlike substring -inline bool has_hash(const std::string &str) { - // If the size can't conform, then the string is invalid - if (str.size() <= hash_pre.size() + hash_eg.size()) { - return false; - } - - // Check that the string contains the hash prefix in the right position - if (str.compare(str.size() - hash_eg.size() - hash_pre.size(), - hash_pre.size(), hash_pre)) { - return false; - } - - // Check that the string ends in lowercase hex digits - for (size_t i = str.size() - hash_eg.size(); i < str.size(); ++i) { - if (!is_hexdig(str[i])) { - return false; - } - } - return true; -} - -bool is_probably_rust_legacy(const std::string &str) { - // Is the string too short to have a hash part in thefirst place? - if (!has_hash(str)) { - return false; - } - - // Throw out `$$` and `$????$`, but not in-between - const char *ptr = str.data(); - const char *end = ptr + str.size() - hash_pre.size() - hash_eg.size(); - for (; ptr <= end; ++ptr) { - if (*ptr == '$') { - if (ptr[1] == '$') { - return false; - } - return ptr[2] == '$' || ptr[3] == '$' || ptr[4] == '$'; - } - if (*ptr == '.') { - return '.' != ptr[1] || - '.' != ptr[2]; // '.' and '..' are fine, '...' is not - } - } - return true; -} - -// Demangles a Rust string by building a copy piece-by-piece -std::string demangle(const std::string &str) { - std::string ret; - ret.reserve(str.size() - hash_eg.size() - hash_pre.size()); - - size_t i = 0; - - // Special-case for repairing C++ demangling defect for Rust - if (str[0] == '_' && str[1] == '$') { - ++i; - } - - for (; i < str.size() - hash_pre.size() - hash_eg.size(); ++i) { - - // Fast sieve for pattern-matching, since we know first chars - if (str[i] == '.' || str[i] == '$') { - bool replaced = false; - - // Try to replace one of the patterns - for (const auto &pair : patterns) { - const std::string &pattern = pair.first; - const std::string &replacement = pair.second; - if (!str.compare(i, pattern.size(), pattern)) { - ret += replacement; - i += pattern.size() - 1; // -1 because iterator inc - replaced = true; - break; - } - } - - // If we failed to replace, try a few failovers. Notably, we recognize - // that Rust may insert Unicode code points in the function name (other - // implementations treat many individual points as patterns to search on) - if (!replaced && str[i] == '.') { - // Special-case for '.' - ret += '-'; - } else if (!replaced && !str.compare(i, 2, "$u") && str[i + 4] == '$') { - const size_t k_nb_read_chars = 5; - const int hexa_base = 16; - const int hi = hex_to_int(str[i + 2]); - const int lo = hex_to_int(str[i + 3]); - if (hi != -1 && lo != -1) { - ret += static_cast(lo + hexa_base * hi); - i += k_nb_read_chars - 1; // - 1 because iterator inc - } else { - // We didn't have valid unicode values. No further processing is - // done, reinsert the `$u...$` sequence into the output string. - ret += str.substr(i, k_nb_read_chars); - i += k_nb_read_chars - 1; // -1 because iterator inc - } - } else if (!replaced) { - ret += str[i]; - } - } else { - ret += str[i]; - } - } - - return ret; -} -} // namespace RustDemangler diff --git a/ddprof-lib/src/main/cpp/rustDemangler.h b/ddprof-lib/src/main/cpp/rustDemangler.h deleted file mode 100644 index ab0b63cad..000000000 --- a/ddprof-lib/src/main/cpp/rustDemangler.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright 2021, 2023 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#pragma once -#include - -namespace RustDemangler { -bool is_probably_rust_legacy(const std::string &str); -std::string demangle(const std::string &str); -}; // namespace RustDemangler diff --git a/ddprof-lib/src/main/cpp/safeAccess.cpp b/ddprof-lib/src/main/cpp/safeAccess.cpp deleted file mode 100644 index ce650f3c2..000000000 --- a/ddprof-lib/src/main/cpp/safeAccess.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright 2025 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -#include "safeAccess.h" -#include -#include - -extern "C" int safefetch32_cont(int* adr, int errValue); -extern "C" int64_t safefetch64_cont(int64_t* adr, int64_t errValue); - -#ifdef __APPLE__ - #if defined(__x86_64__) - #define current_pc context_rip - #elif defined(__aarch64__) - #define DU3_PREFIX(s, m) __ ## s.__ ## m - #define current_pc uc_mcontext->DU3_PREFIX(ss,pc) - #endif -#else - #if defined(__x86_64__) - #define current_pc uc_mcontext.gregs[REG_RIP] - #elif defined(__aarch64__) - #define current_pc uc_mcontext.pc - #endif -#endif - -/** - Loading a 32-bit/64-bit value from specific address, the errValue will be returned - if the address is invalid. - The load is protected by the 'handle_safefetch` signal handler, who sets next `pc` - to `safefetch32_cont/safefetch64_cont`, upon returning from signal handler, - `safefetch32_cont/safefetch64_cont` returns `errValue` - **/ -#if defined(__x86_64__) - #ifdef __APPLE__ - asm(R"( - .globl _safefetch32_impl - .private_extern _safefetch32_impl - _safefetch32_impl: - movl (%rdi), %eax - ret - .globl _safefetch32_cont - .private_extern _safefetch32_cont - _safefetch32_cont: - movl %esi, %eax - ret - .globl _safefetch64_impl - .private_extern _safefetch64_impl - _safefetch64_impl: - movq (%rdi), %rax - ret - .globl _safefetch64_cont - .private_extern _safefetch64_cont - _safefetch64_cont: - movq %rsi, %rax - ret - )"); - #else - asm(R"( - .text - .globl safefetch32_impl - .hidden safefetch32_impl - .type safefetch32_impl, %function - safefetch32_impl: - movl (%rdi), %eax - ret - .globl safefetch32_cont - .hidden safefetch32_cont - .type safefetch32_cont, %function - safefetch32_cont: - movl %esi, %eax - ret - .globl safefetch64_impl - .hidden safefetch64_impl - .type safefetch64_impl, %function - safefetch64_impl: - movq (%rdi), %rax - ret - .globl safefetch64_cont - .hidden safefetch64_cont - .type safefetch64_cont, %function - safefetch64_cont: - movq %rsi, %rax - ret - )"); - #endif // __APPLE__ -#elif defined(__aarch64__) - #ifdef __APPLE__ - asm(R"( - .globl _safefetch32_impl - .private_extern _safefetch32_impl - _safefetch32_impl: - ldr w0, [x0] - ret - .globl _safefetch32_cont - .private_extern _safefetch32_cont - _safefetch32_cont: - mov w0, w1 - ret - .globl _safefetch64_impl - .private_extern _safefetch64_impl - _safefetch64_impl: - ldr x0, [x0] - ret - .globl _safefetch64_cont - .private_extern _safefetch64_cont - _safefetch64_cont: - mov x0, x1 - ret - )"); - #else - asm(R"( - .text - .globl safefetch32_impl - .hidden safefetch32_impl - .type safefetch32_impl, %function - safefetch32_impl: - ldr w0, [x0] - ret - .globl safefetch32_cont - .hidden safefetch32_cont - .type safefetch32_cont, %function - safefetch32_cont: - mov w0, w1 - ret - .globl safefetch64_impl - .hidden safefetch64_impl - .type safefetch64_impl, %function - safefetch64_impl: - ldr x0, [x0] - ret - .globl safefetch64_cont - .hidden safefetch64_cont - .type safefetch64_cont, %function - safefetch64_cont: - mov x0, x1 - ret - )"); - #endif -#endif - -bool SafeAccess::safeCopy(void* dst, const void* src, size_t len) { - // Two-sentinel pattern (same as isReadable): a real-data word may equal - // one sentinel by chance, but not both — if both fetches return their - // sentinel, the access truly faulted. - // - // All safefetch32 loads issued here use 4-byte-aligned addresses. Pages - // are 4 KiB (or 16 KiB on Apple Silicon), both divisible by 4, so an - // aligned 4-byte load never spans a page boundary. The only fault - // possible is when the aligned address itself lies in an unmapped page; - // we never spuriously fault on an over-read past `src + len`. - static const int32_t SENT_A = (int32_t)0x55AA55AA; - static const int32_t SENT_B = (int32_t)0xAA55AA55; - uint8_t* d = (uint8_t*)dst; - const uint8_t* s = (const uint8_t*)src; - size_t i = 0; - - // Front fixup: if `src` is not 4-byte aligned, fetch at the previous - // aligned address (1..3 bytes before src). That address lies in the - // same 4-byte word as src — and since pages are 4-byte aligned, in - // the same page as src. The leading k bytes of the fetched word lie - // before the caller's range and are discarded via the +k offset; they - // never reach `dst`. - size_t k = (uintptr_t)s & 3u; - if (k != 0 && i < len) { - int32_t* aligned = (int32_t*)(s - k); - int32_t v1 = safefetch32_impl(aligned, SENT_A); - int32_t v2 = safefetch32_impl(aligned, SENT_B); - if (v1 == SENT_A && v2 == SENT_B) { - return false; - } - size_t take = (4 - k < len) ? (4 - k) : len; - memcpy(d, ((const uint8_t*)&v1) + k, take); - i = take; - } - - // Middle + tail: (s + i) is now 4-byte aligned. The final iteration may - // load up to 3 over-read bytes past `src + len`, but those bytes sit in - // the same 4-byte-aligned word and therefore the same page as the bytes - // we actually wanted — never a fault from the over-read alone. - while (i < len) { - int32_t* aligned = (int32_t*)(s + i); - int32_t v1 = safefetch32_impl(aligned, SENT_A); - int32_t v2 = safefetch32_impl(aligned, SENT_B); - if (v1 == SENT_A && v2 == SENT_B) { - return false; - } - size_t chunk = (len - i >= 4) ? 4 : (len - i); - memcpy(d + i, &v1, chunk); // memcpy from local — no UAF risk - i += chunk; - } - return true; -} - -bool SafeAccess::handle_safefetch(int sig, void* context) { - ucontext_t* uc = (ucontext_t*)context; - uintptr_t pc = uc->current_pc; - if ((sig == SIGSEGV || sig == SIGBUS) && uc != nullptr) { - if (pc == (uintptr_t)safefetch32_impl) { - uc->current_pc = (uintptr_t)safefetch32_cont; - return true; - } else if (pc == (uintptr_t)safefetch64_impl) { - uc->current_pc = (uintptr_t)safefetch64_cont; - return true; - } - } - return false; -} - -// NOINLINE implementations using safefetch infrastructure -// These provide stable function addresses for JVM patching in vmStructs.cpp -void* SafeAccess::load(void** ptr, void* default_value) { - return loadPtr(ptr, default_value); -} - -int32_t SafeAccess::load32(int32_t* ptr, int32_t default_value) { - int res = safefetch32_impl((int*)ptr, (int)default_value); - return static_cast(res); -} - -void* SafeAccess::loadPtr(void** ptr, void* default_value) { -#if defined(__x86_64__) || defined(__aarch64__) - int64_t res = safefetch64_impl((int64_t*)ptr, (int64_t)reinterpret_cast(default_value)); - return (void*)static_cast(res); -#elif defined(__i386__) || defined(__arm__) || defined(__thumb__) - int res = safefetch32_impl((int*)ptr, (int)default_value); - return (void*)res; -#endif - return *ptr; -} diff --git a/ddprof-lib/src/main/cpp/safeAccess.h b/ddprof-lib/src/main/cpp/safeAccess.h deleted file mode 100644 index 43ed9ce3a..000000000 --- a/ddprof-lib/src/main/cpp/safeAccess.h +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2021 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _SAFEACCESS_H -#define _SAFEACCESS_H - -#include "arch.h" -#include "codeCache.h" -#include "os.h" -#include -#include - -extern "C" int safefetch32_impl(int* adr, int errValue); -extern "C" int64_t safefetch64_impl(int64_t* adr, int64_t errValue); - -#ifdef __clang__ -#define NOINLINE __attribute__((noinline)) -#else -#define NOINLINE __attribute__((noinline, noclone)) -#endif -#define NOADDRSANITIZE __attribute__((no_sanitize("address"))) -#define NOSANALIGSANITIZE __attribute__((no_sanitize("alignment"))) - -class SafeAccess { -public: - - /** - * Safely reads a 32-bit value from the given address. - * - *

CRITICAL: This function MUST NOT be inlined. The safefetch mechanism relies on - * faults occurring at the exact address of safefetch32_impl. If this function is - * inlined, the compiler may optimize the load into the caller's code, bypassing - * the fault protection in handle_safefetch(). - * - * @param ptr Address to read from (may be invalid) - * @param errorValue Value to return if the read faults - * @return The value at ptr, or errorValue if the read faults - */ - NOINLINE - static int safeFetch32(int* ptr, int errorValue) { - return safefetch32_impl(ptr, errorValue); - } - - /** - * Safely reads a 64-bit value from the given address. - * - *

CRITICAL: This function MUST NOT be inlined. See safeFetch32 for details. - */ - NOINLINE - static int64_t safeFetch64(int64_t* ptr, int64_t errorValue) { - return safefetch64_impl(ptr, errorValue); - } - - // Copies up to len bytes from src to dst using safefetch32_impl so that a - // page-unmap or repurpose of src memory during the copy does not crash the - // process. Returns true on full success, false if any read faulted. dst must - // have at least len bytes capacity; reads from src may over-read up to 3 - // bytes past src+len (over-read is also safefetch-protected). - NOINLINE - static bool safeCopy(void* dst, const void* src, size_t len); - - static bool handle_safefetch(int sig, void* context); - - // NOINLINE functions with stable addresses for JVM patching (vmStructs.cpp) - NOINLINE __attribute__((aligned(16))) - static void *load(void **ptr, void* default_value = nullptr); - - NOINLINE __attribute__((aligned(16))) - static int32_t load32(int32_t *ptr, int32_t default_value = 0); - - NOINLINE __attribute__((aligned(16))) - static void *loadPtr(void** ptr, void* default_value); - - static inline bool isReadable(const void* ptr) { - return load32((int32_t*)ptr, 1) != 1 || - load32((int32_t*)ptr, -1) != -1; - } - - static inline bool isReadableRange(const void* start, size_t size) { - assert(size > 0); - // Reject addresses where start + size - 1 would wrap around UINTPTR_MAX, - // which would produce a bogus end_page below start_page. - if (reinterpret_cast(start) > UINTPTR_MAX - (size - 1)) { - return false; - } - void* start_page = (void*)align_down((uintptr_t)start, OS::page_size); - void* end_page = (void*)align_down((uintptr_t)start + size - 1, OS::page_size); - // Check readability page by page. The loop only increments when page != end_page, - // so (uintptr_t)page + OS::page_size never wraps even when end_page is near UINTPTR_MAX. - for (void* page = start_page; page != end_page; page = (void*)((uintptr_t)page + OS::page_size)) { - if (!isReadable(page)) { - return false; - } - } - return isReadable(end_page); - } -}; - -#endif // _SAFEACCESS_H diff --git a/ddprof-lib/src/main/cpp/sframe.cpp b/ddprof-lib/src/main/cpp/sframe.cpp deleted file mode 100644 index ed0de992e..000000000 --- a/ddprof-lib/src/main/cpp/sframe.cpp +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "sframe.h" -#include "log.h" -#include -#include -#include - -SFrameParser::SFrameParser(const char* name, const char* section_base, - size_t section_size, u32 section_offset) - : _name(name), - _section_base(section_base), - _section_size(section_size), - _section_offset(section_offset), - _capacity(128), - _count(0), - _table(static_cast(malloc(128 * sizeof(FrameDesc)))), - _linked_frame_size(-1), - _oom(false) {} - -SFrameParser::~SFrameParser() { - free(_table); // safe: free(nullptr) is a no-op; table() nulls _table on success -} - -FrameDesc* SFrameParser::table() { - FrameDesc* t = _table; - _table = nullptr; - return t; -} - -const FrameDesc& SFrameParser::detectedDefaultFrame() const { - if (_linked_frame_size == LINKED_FRAME_CLANG_SIZE && - LINKED_FRAME_CLANG_SIZE != LINKED_FRAME_SIZE) { - return FrameDesc::default_clang_frame; - } - return FrameDesc::default_frame; -} - -FrameDesc* SFrameParser::addRecord(u32 loc, u32 cfa, int fp_off, int pc_off) { - if (!_table) return nullptr; // constructor malloc failed - if (_count >= _capacity) { - if (_capacity > (INT_MAX / 2)) return nullptr; // overflow guard - FrameDesc* resized = static_cast( - realloc(_table, _capacity * 2 * sizeof(FrameDesc))); - if (!resized) { - _oom = true; - return nullptr; - } - _capacity *= 2; - _table = resized; - } - FrameDesc* fd = &_table[_count++]; - fd->loc = loc; - fd->cfa = cfa; - fd->fp_off = fp_off; - fd->pc_off = pc_off; - return fd; -} - -bool SFrameParser::parseFDE(const SFrameHeader* hdr, const SFrameFDE* fde, - const char* fre_section, const char* fre_end) { - // Determine FRE start address size - int addr_size; - switch (SFRAME_FUNC_FRE_TYPE(fde->info)) { - case SFRAME_FRE_TYPE_ADDR1: addr_size = 1; break; - case SFRAME_FRE_TYPE_ADDR2: addr_size = 2; break; - case SFRAME_FRE_TYPE_ADDR4: addr_size = 4; break; - default: return false; - } - - // Bounds-check fre_off before pointer arithmetic - size_t fre_section_len = static_cast(fre_end - fre_section); - if (fde->fre_off >= fre_section_len) return false; - - const char* fre_ptr = fre_section + fde->fre_off; - - for (uint32_t j = 0; j < fde->fre_num; j++) { - // (a) Entry-level bounds check - if (fre_ptr >= fre_end) return false; - - // (b) Read FRE start address offset (unsigned) - if (fre_ptr + addr_size > fre_end) return false; - uint32_t fre_start = 0; - if (addr_size == 1) { - fre_start = *reinterpret_cast(fre_ptr); - } else if (addr_size == 2) { - uint16_t v; memcpy(&v, fre_ptr, 2); fre_start = v; - } else { - memcpy(&fre_start, fre_ptr, 4); - } - fre_ptr += addr_size; - - // (c) Read FRE info byte - if (fre_ptr + 1 > fre_end) return false; - uint8_t fre_info = *reinterpret_cast(fre_ptr); - fre_ptr += 1; - - // (d) Determine offset encoding size - int off_size; - switch (SFRAME_FRE_OFFSET_SIZE(fre_info)) { - case SFRAME_FRE_OFFSET_1B: off_size = 1; break; - case SFRAME_FRE_OFFSET_2B: off_size = 2; break; - case SFRAME_FRE_OFFSET_4B: off_size = 4; break; - default: return false; - } - - // Decide what to read from the stream (governed by FRE info byte alone) - bool fp_tracked = SFRAME_FRE_FP_TRACKED(fre_info); - bool ra_in_fre = SFRAME_FRE_RA_TRACKED(fre_info); - - // (e) Bounds check all remaining reads for this FRE - int n_offsets = 1 + (fp_tracked ? 1 : 0) + (ra_in_fre ? 1 : 0); - if (fre_ptr + n_offsets * off_size > fre_end) return false; - - // (f) Read CFA offset (signed) - int32_t cfa_offset = 0; - if (off_size == 1) { - cfa_offset = *reinterpret_cast(fre_ptr); - } else if (off_size == 2) { - int16_t v; memcpy(&v, fre_ptr, 2); cfa_offset = v; - } else { - memcpy(&cfa_offset, fre_ptr, 4); - } - fre_ptr += off_size; - - // Guard: CFA offset must fit in the 24-bit field packed into FrameDesc::cfa - if (cfa_offset < -8388608 || cfa_offset > 8388607) return false; - - // (g) Read RA offset if tracked - int32_t ra_offset = 0; - if (ra_in_fre) { - if (off_size == 1) { - ra_offset = *reinterpret_cast(fre_ptr); - } else if (off_size == 2) { - int16_t v; memcpy(&v, fre_ptr, 2); ra_offset = v; - } else { - memcpy(&ra_offset, fre_ptr, 4); - } - fre_ptr += off_size; - } - - // (h) Read FP offset if tracked - int32_t fp_offset = 0; - if (fp_tracked) { - if (off_size == 1) { - fp_offset = *reinterpret_cast(fre_ptr); - } else if (off_size == 2) { - int16_t v; memcpy(&v, fre_ptr, 2); fp_offset = v; - } else { - memcpy(&fp_offset, fre_ptr, 4); - } - fre_ptr += off_size; - } - - // (i) Translate to FrameDesc fields - // Use unsigned arithmetic to avoid implementation-defined signed cast for large offsets - u32 loc = _section_offset + static_cast(fde->start_addr) + fre_start; - - u32 cfa_reg = SFRAME_FRE_BASE_REG(fre_info) ? static_cast(DW_REG_FP) - : static_cast(DW_REG_SP); - u32 cfa = (static_cast(cfa_offset) << 8) | cfa_reg; - - // aarch64 GCC vs Clang detection: first FP-based entry with cfa_offset > 0 - if (_linked_frame_size < 0 && cfa_reg == static_cast(DW_REG_FP) && cfa_offset > 0) { - _linked_frame_size = cfa_offset; - } - - // Determine fp_off: per-FRE value takes priority; fall back to header fixed offset - int fp_off; - if (fp_tracked) { - fp_off = static_cast(fp_offset); - } else if (hdr->cfa_fixed_fp_offset != 0) { - fp_off = static_cast(hdr->cfa_fixed_fp_offset); - } else { - fp_off = DW_SAME_FP; - } - - // Header fixed RA offset takes priority over per-FRE value - int pc_off; - if (hdr->cfa_fixed_ra_offset != 0) { - pc_off = static_cast(hdr->cfa_fixed_ra_offset); - } else if (ra_in_fre) { - pc_off = static_cast(ra_offset); - } else { - pc_off = DW_LINK_REGISTER; - } - - // (j) Append record - if (!addRecord(loc, cfa, fp_off, pc_off)) return false; - } - - return true; -} - -bool SFrameParser::parse() { - // 1. Size check - if (_section_size < sizeof(SFrameHeader)) { - Log::warn("SFrame section too small in %s", _name); - return false; - } - - const SFrameHeader* hdr = reinterpret_cast(_section_base); - - // 2-4. Header field validation - if (hdr->magic != SFRAME_MAGIC) { - Log::warn("SFrame bad magic in %s", _name); - return false; - } - if (hdr->version != SFRAME_VERSION_2) { - Log::warn("SFrame unsupported version %d in %s", (int)hdr->version, _name); - return false; - } - -#if defined(__x86_64__) - if (hdr->abi_arch != SFRAME_ABI_AMD64_ENDIAN_LITTLE) { - Log::warn("SFrame wrong ABI 0x%x in %s", (int)hdr->abi_arch, _name); - return false; - } -#elif defined(__aarch64__) - if (hdr->abi_arch != SFRAME_ABI_AARCH64_ENDIAN_LITTLE) { - Log::warn("SFrame wrong ABI 0x%x in %s", (int)hdr->abi_arch, _name); - return false; - } -#else - return false; -#endif - - // 5. Bounds check auxhdr_len before computing data_start - if (sizeof(SFrameHeader) + hdr->auxhdr_len > _section_size) { - Log::warn("SFrame auxhdr_len overflows section in %s", _name); - return false; - } - - // SFrame V2 PCREL func-start encoding makes start_addr field-relative, which - // our section-relative loc arithmetic (parseFDE) does not implement. Bail so - // the caller falls back to DWARF rather than building a scrambled table. - if (hdr->flags & SFRAME_F_FDE_FUNC_START_PCREL) { - Log::warn("SFrame PCREL func-start encoding unsupported in %s; falling back to DWARF", _name); - return false; - } - - const char* data_start = _section_base + sizeof(SFrameHeader) + hdr->auxhdr_len; - const char* section_end = _section_base + _section_size; - - // Bounds-check fdeoff, freoff, and fre_len before pointer arithmetic - size_t data_len = static_cast(section_end - data_start); - if (hdr->fdeoff > data_len) { - Log::warn("SFrame fdeoff out of bounds in %s", _name); - return false; - } - if (hdr->freoff > data_len) { - Log::warn("SFrame freoff out of bounds in %s", _name); - return false; - } - if (hdr->fre_len > data_len - hdr->freoff) { - Log::warn("SFrame fre_len overflows section in %s", _name); - return false; - } - - const SFrameFDE* fde_array = reinterpret_cast(data_start + hdr->fdeoff); - const char* fre_section = data_start + hdr->freoff; - const char* fre_end = fre_section + hdr->fre_len; - - // 6-7. Bounds checks for FDE array and FRE section - if (reinterpret_cast(fde_array) + - (size_t)hdr->num_fdes * sizeof(SFrameFDE) > section_end) { - Log::warn("SFrame FDE array overflows section in %s", _name); - return false; - } - if (fre_end > section_end) { - Log::warn("SFrame FRE section overflows in %s", _name); - return false; - } - - // 8. FDE array / FRE section overlap check - if (hdr->num_fdes > 0 && hdr->fre_len > 0) { - const char* fde_start_ptr = reinterpret_cast(fde_array); - const char* fde_end_ptr = fde_start_ptr + (size_t)hdr->num_fdes * sizeof(SFrameFDE); - if (fde_end_ptr > fre_section && fde_start_ptr < fre_end) { - Log::warn("SFrame FDE array overlaps FRE section in %s", _name); - return false; - } - } - - // 9. Iterate FDEs - for (uint32_t i = 0; i < hdr->num_fdes; i++) { - const SFrameFDE* fde = &fde_array[i]; - if (SFRAME_FUNC_FDE_TYPE(fde->info) != 0) continue; // skip PCMASK - if (fde->fre_num == 0) continue; // empty FDE - int saved_count = _count; // safe: addRecord capacity guard keeps _count <= INT_MAX/2 - int saved_linked_frame_size = _linked_frame_size; - if (!parseFDE(hdr, fde, fre_section, fre_end)) { - if (_oom) return false; // OOM: partial table is not safe to use - // Intentional divergence from DwarfParser::addRecord, which drops only the - // offending record and keeps going. Here a single bad FRE fails the whole - // FDE: we conservatively discard all FREs added for this function rather - // than ship a partially-decoded function with a coverage gap. Cost: one - // out-of-range FRE forfeits unwinding for the entire function. - _count = saved_count; - _linked_frame_size = saved_linked_frame_size; - } - } - - // 10. Sort, unless the producer asserts sorted order via SFRAME_F_FDE_SORTED. - // findFrameDesc binary-searches by loc, so the flattened table must be - // globally sorted. We trust the flag here (qsort of a real, large table is - // non-trivial). A malformed file that sets SORTED without honoring FDE/FRE - // ordering and non-overlap yields silent lookup misses (not memory-unsafe). - if (_count > 0 && !(hdr->flags & SFRAME_F_FDE_SORTED)) { - qsort(_table, _count, sizeof(FrameDesc), FrameDesc::comparator); - } - - return _count > 0; -} diff --git a/ddprof-lib/src/main/cpp/sframe.h b/ddprof-lib/src/main/cpp/sframe.h deleted file mode 100644 index b2a21d8b6..000000000 --- a/ddprof-lib/src/main/cpp/sframe.h +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _SFRAME_H -#define _SFRAME_H - -#include "dwarf.h" -#include - -#ifndef PT_GNU_SFRAME -#define PT_GNU_SFRAME 0x6474e554 -#endif - -// SFrame V2 magic, version, flags -const uint16_t SFRAME_MAGIC = 0xDEE2; -const uint8_t SFRAME_VERSION_2 = 2; -const uint8_t SFRAME_F_FDE_SORTED = 0x01; -// SFrame V2 erratum: when set, FDE start_addr is relative to the start_addr -// field location, not the section start. Unsupported here -> DWARF fallback. -const uint8_t SFRAME_F_FDE_FUNC_START_PCREL = 0x04; - -// ABI/architecture identifiers -const uint8_t SFRAME_ABI_AARCH64_ENDIAN_LITTLE = 2; -const uint8_t SFRAME_ABI_AMD64_ENDIAN_LITTLE = 3; - -// FDE info byte: bit 0 = FDE type (0=PCINC,1=PCMASK), bits 1-2 = FRE start address size -#define SFRAME_FUNC_FDE_TYPE(info) ((info) & 0x1) -#define SFRAME_FUNC_FRE_TYPE(info) (((info) >> 1) & 0x3) - -// FRE info byte: bit 0 = CFA base (0=SP,1=FP), bits 1-2 = offset size, bit 3 = RA tracked, bit 4 = FP tracked -#define SFRAME_FRE_BASE_REG(info) ((info) & 0x1) -#define SFRAME_FRE_OFFSET_SIZE(info) (((info) >> 1) & 0x3) -#define SFRAME_FRE_RA_TRACKED(info) (((info) >> 3) & 0x1) -#define SFRAME_FRE_FP_TRACKED(info) (((info) >> 4) & 0x1) - -// FRE offset size codes -const int SFRAME_FRE_OFFSET_1B = 0; -const int SFRAME_FRE_OFFSET_2B = 1; -const int SFRAME_FRE_OFFSET_4B = 2; - -// FRE start address size codes (from FDE info bits 1-2) -const int SFRAME_FRE_TYPE_ADDR1 = 0; -const int SFRAME_FRE_TYPE_ADDR2 = 1; -const int SFRAME_FRE_TYPE_ADDR4 = 2; - -struct __attribute__((packed, may_alias)) SFrameHeader { // 28 bytes - uint16_t magic; - uint8_t version; - uint8_t flags; - uint8_t abi_arch; - int8_t cfa_fixed_fp_offset; - int8_t cfa_fixed_ra_offset; // -8 on x86_64; 0 on aarch64 - uint8_t auxhdr_len; - uint32_t num_fdes; - uint32_t num_fres; - uint32_t fre_len; - uint32_t fdeoff; - uint32_t freoff; -}; - -struct __attribute__((packed, may_alias)) SFrameFDE { // 20 bytes - int32_t start_addr; // signed, relative to .sframe section start (V2) - uint32_t func_size; - uint32_t fre_off; // byte offset into FRE sub-section - uint32_t fre_num; // number of FREs - uint8_t info; // FDE type (bit 0) | FRE addr size (bits 1-2) - uint8_t rep_size; - uint16_t padding; -}; - -class SFrameParser { - private: - const char* _name; - const char* _section_base; - size_t _section_size; - u32 _section_offset; - - int _capacity; - int _count; - FrameDesc* _table; - int _linked_frame_size; // for aarch64 GCC vs Clang detection; -1 = undetected - bool _oom; // set by addRecord on realloc failure - - bool parseFDE(const SFrameHeader* hdr, const SFrameFDE* fde, - const char* fre_section, const char* fre_end); - FrameDesc* addRecord(u32 loc, u32 cfa, int fp_off, int pc_off); - - public: - SFrameParser(const char* name, const char* section_base, - size_t section_size, u32 section_offset); - ~SFrameParser(); - - // Returns false when the section is invalid or unsupported (triggers DWARF fallback). - bool parse(); - - // Transfers table ownership to caller. Nulls _table so the destructor does not double-free. - // Call only after a successful parse(). Caller must free() the returned pointer. - FrameDesc* table(); - int count() const { return _count; } - - const FrameDesc& detectedDefaultFrame() const; -}; - -#endif // _SFRAME_H diff --git a/ddprof-lib/src/main/cpp/signalCookie.cpp b/ddprof-lib/src/main/cpp/signalCookie.cpp deleted file mode 100644 index 7c2688248..000000000 --- a/ddprof-lib/src/main/cpp/signalCookie.cpp +++ /dev/null @@ -1,22 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "signalCookie.h" - -namespace SignalCookie { - namespace detail { - // Place tags in a named section on Linux to prevent LTO from merging - // or reordering them across TUs (their addresses must be unique per DSO). -#ifdef __linux__ - [[gnu::section(".data.signal_cookie")]] char cpu_tag; - [[gnu::section(".data.signal_cookie")]] char wallclock_tag; -#else - char cpu_tag; - char wallclock_tag; -#endif - } - void* cpu() { return &detail::cpu_tag; } - void* wallclock() { return &detail::wallclock_tag; } -} diff --git a/ddprof-lib/src/main/cpp/signalCookie.h b/ddprof-lib/src/main/cpp/signalCookie.h deleted file mode 100644 index 8744307b0..000000000 --- a/ddprof-lib/src/main/cpp/signalCookie.h +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef SIGNAL_COOKIE_H -#define SIGNAL_COOKIE_H - -// Sentinels carried in siginfo_t::si_value.sival_ptr by profiler-originated -// signals (timer_create for CPU engines, rt_tgsigqueueinfo for wallclock). -// Handlers use these to distinguish signals raised by this profiler from -// signals of the same number raised by other sources in the process (e.g. -// Go's setitimer(ITIMER_PROF), foreign tgkill, or kill/raise). -// -// CTimer and ITimer are mutually exclusive CPU engines — a profiler lifetime -// uses at most one of them — so they share SignalCookie::cpu(). (ITimer -// carries no payload, so it is not actually gated; the cookie is reserved.) -// -// Cookies are addresses of static tag variables, not hand-rolled magic -// numbers: tag addresses are unique per shared-library image, cannot be -// forged by an unrelated in-process sender without reading our symbols, and -// never collide with legitimate user-space pointers in third-party code. -namespace SignalCookie { - void* cpu(); - void* wallclock(); -} - -#endif // SIGNAL_COOKIE_H diff --git a/ddprof-lib/src/main/cpp/signalSafety.cpp b/ddprof-lib/src/main/cpp/signalSafety.cpp deleted file mode 100644 index 0f8dc3038..000000000 --- a/ddprof-lib/src/main/cpp/signalSafety.cpp +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Header-only module — see signalSafety.h for DEBUG_ASSERT_NOT_IN_SIGNAL(). -// The signal-context depth counter (used by both the assertion and the -// production AS-safe deferred path) lives in guards.{h,cpp}. -#include "signalSafety.h" diff --git a/ddprof-lib/src/main/cpp/signalSafety.h b/ddprof-lib/src/main/cpp/signalSafety.h deleted file mode 100644 index e322fdcbb..000000000 --- a/ddprof-lib/src/main/cpp/signalSafety.h +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _SIGNAL_SAFETY_H -#define _SIGNAL_SAFETY_H - -#include "guards.h" // isInSignalContext, SIGNAL_HANDLER_GUARD, ... -#include "thread.h" // ProfiledThread::currentSignalSafe - -// Detect ASAN using compiler-provided macros so the ASAN_ENABLED guard below -// works in every TU that includes this header, independent of include order. -#ifdef __has_feature -# if __has_feature(address_sanitizer) -# ifndef ASAN_ENABLED -# define ASAN_ENABLED 1 -# endif -# endif -#endif -#ifdef __SANITIZE_ADDRESS__ -# ifndef ASAN_ENABLED -# define ASAN_ENABLED 1 -# endif -#endif - -// Debug-only AS-safety assertion. Aborts with a file:line diagnostic when -// invoked from inside a signal handler in debug / ASAN builds; compiles to a -// no-op in release builds (NDEBUG). -// -// The depth counter itself lives in ProfiledThread::_signal_depth (see -// guards.h for the rationale). The check is skipped when ProfiledThread -// is null — uninstrumented threads (VM Thread, JIT, GC) have no thread -// context, so the assertion has no way to know whether they're really in -// a signal frame. Treating "unknown" as a violation would produce false -// positives every time AS-unsafe code legitimately ran on such a thread. -// -// write(2) is POSIX async-signal-safe. abort() generates a core dump and -// triggers ASAN's stack-trace symbolization, making it far more debuggable -// than _exit(1). The macro is only active in debug/ASAN builds where we -// intentionally trade AS-safety of the abort path for diagnosability. -#define _SIGNAL_SAFETY_STR(x) #x -#define _SIGNAL_SAFETY_TOSTR(x) _SIGNAL_SAFETY_STR(x) - -#if !defined(NDEBUG) || defined(ASAN_ENABLED) -#include // write, STDERR_FILENO, open, close -#include // abort -#include // O_WRONLY, O_CREAT, O_APPEND - -// Path for the diagnostic file — picked up as a CI artifact on failure. -#define _SIGNAL_SAFETY_DIAG_FILE "/tmp/signal-safety-violation.txt" - -#define DEBUG_ASSERT_NOT_IN_SIGNAL() \ - do { \ - ProfiledThread *_pt_for_assert = ProfiledThread::currentSignalSafe(); \ - /* Skip when no thread context — see comment above. */ \ - if (_pt_for_assert != nullptr && _pt_for_assert->signalDepth() != 0) { \ - static const char _msg[] = \ - "[java-profiler] AS-safety violation at " \ - __FILE__ ":" _SIGNAL_SAFETY_TOSTR(__LINE__) \ - ": async-signal-unsafe call made from signal handler context\n"; \ - (void)write(STDERR_FILENO, _msg, sizeof(_msg) - 1); \ - /* Also write to a file so CI can capture it regardless of output routing. */ \ - int _diag_fd = open(_SIGNAL_SAFETY_DIAG_FILE, \ - O_WRONLY | O_CREAT | O_APPEND, 0644); \ - if (_diag_fd >= 0) { \ - (void)write(_diag_fd, _msg, sizeof(_msg) - 1); \ - close(_diag_fd); \ - } \ - abort(); \ - } \ - } while (0) -#else -#define DEBUG_ASSERT_NOT_IN_SIGNAL() ((void)0) -#endif - -#endif // _SIGNAL_SAFETY_H diff --git a/ddprof-lib/src/main/cpp/spinLock.h b/ddprof-lib/src/main/cpp/spinLock.h deleted file mode 100644 index a8ec1bb72..000000000 --- a/ddprof-lib/src/main/cpp/spinLock.h +++ /dev/null @@ -1,166 +0,0 @@ -/* - * Copyright 2017 Andrei Pangin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _SPINLOCK_H -#define _SPINLOCK_H - -#include "arch.h" -#include - -// Cannot use regular mutexes inside signal handler. -// This lock is based on CAS busy loop. GCC atomic builtins imply full barrier. -// Aligned to cache line size (64 bytes) to prevent false sharing between SpinLock instances -class alignas(DEFAULT_CACHE_LINE_SIZE) SpinLock { -private: - // 0 - unlocked - // 1 - exclusive lock - // <0 - shared lock - volatile int _lock; - char _padding[DEFAULT_CACHE_LINE_SIZE - sizeof(_lock)]; -public: - explicit constexpr SpinLock(int initial_state = 0) : _lock(initial_state), _padding() { - static_assert(sizeof(SpinLock) == DEFAULT_CACHE_LINE_SIZE); - } - - void reset() { __atomic_store_n(&_lock, 0, __ATOMIC_RELAXED); } - - bool tryLock() { return __sync_bool_compare_and_swap(&_lock, 0, 1); } - - void lock() { - while (!tryLock()) { - spinPause(); - } - } - - void unlock() { - assert(__atomic_load_n(&_lock, __ATOMIC_RELAXED) == 1); - __sync_fetch_and_sub(&_lock, 1); - } - - // Spin budget for bounded shared acquisition in signal handlers. - // Bounds the number of CAS-retry iterations under reader contention; - // does NOT bound wall-clock latency (CAS stall time is hardware-dependent). - static constexpr int DEFAULT_SHARED_SPIN_BUDGET = 256; - - bool tryLockShared() { - // Spins while no exclusive lock is held and the CAS to acquire a shared - // lock fails (due to concurrent reader contention). Returns false ONLY when - // an exclusive lock is observed (_lock > 0); never returns false spuriously. - int value; - while ((value = __atomic_load_n(&_lock, __ATOMIC_ACQUIRE)) <= 0) { - if (__sync_bool_compare_and_swap(&_lock, value, value - 1)) { - return true; - } - spinPause(); - } - return false; - } - - // Bounded variant for signal-handler paths. Returns false when an exclusive - // lock is observed OR the spin budget is exhausted under reader contention. - bool tryLockShared(int max_spins) { - int value; - int spins = 0; - while ((value = __atomic_load_n(&_lock, __ATOMIC_ACQUIRE)) <= 0) { - if (__sync_bool_compare_and_swap(&_lock, value, value - 1)) { - return true; - } - if (++spins >= max_spins) { - return false; - } - spinPause(); - } - return false; - } - - void lockShared() { - int value; - while ((value = __atomic_load_n(&_lock, __ATOMIC_ACQUIRE)) > 0 || - !__sync_bool_compare_and_swap(&_lock, value, value - 1)) { - spinPause(); - } - } - - void unlockShared() { - assert(__atomic_load_n(&_lock, __ATOMIC_RELAXED) < 0); - __sync_fetch_and_add(&_lock, 1); - } -}; - -// RAII guard classes for automatic lock management -class SharedLockGuard { -private: - SpinLock* _lock; -public: - explicit SharedLockGuard(SpinLock* lock) : _lock(lock) { - _lock->lockShared(); - } - ~SharedLockGuard() { - _lock->unlockShared(); - } - // Non-copyable and non-movable - SharedLockGuard(const SharedLockGuard&) = delete; - SharedLockGuard& operator=(const SharedLockGuard&) = delete; - SharedLockGuard(SharedLockGuard&&) = delete; - SharedLockGuard& operator=(SharedLockGuard&&) = delete; -}; - -// Acquires a shared lock with a bounded CAS-retry budget. Returns without -// acquiring (ownsLock() == false) when an exclusive lock is observed or -// the spin budget is exhausted under reader contention. Safe to use in -// signal-handler paths. -class OptionalSharedLockGuard { - SpinLock* _lock; -public: - explicit OptionalSharedLockGuard( - SpinLock* lock, int max_spins = SpinLock::DEFAULT_SHARED_SPIN_BUDGET) - : _lock(lock) { - if (!_lock->tryLockShared(max_spins)) { - _lock = nullptr; - } - } - ~OptionalSharedLockGuard() { - if (_lock != nullptr) { - _lock->unlockShared(); - } - } - bool ownsLock() const { return _lock != nullptr; } - - // Non-copyable and non-movable - OptionalSharedLockGuard(const OptionalSharedLockGuard&) = delete; - OptionalSharedLockGuard& operator=(const OptionalSharedLockGuard&) = delete; - OptionalSharedLockGuard(OptionalSharedLockGuard&&) = delete; - OptionalSharedLockGuard& operator=(OptionalSharedLockGuard&&) = delete; -}; - -class ExclusiveLockGuard { -private: - SpinLock* _lock; -public: - explicit ExclusiveLockGuard(SpinLock* lock) : _lock(lock) { - _lock->lock(); - } - ~ExclusiveLockGuard() { - _lock->unlock(); - } - // Non-copyable and non-movable - ExclusiveLockGuard(const ExclusiveLockGuard&) = delete; - ExclusiveLockGuard& operator=(const ExclusiveLockGuard&) = delete; - ExclusiveLockGuard(ExclusiveLockGuard&&) = delete; - ExclusiveLockGuard& operator=(ExclusiveLockGuard&&) = delete; -}; - -#endif // _SPINLOCK_H diff --git a/ddprof-lib/src/main/cpp/stackFrame.h b/ddprof-lib/src/main/cpp/stackFrame.h deleted file mode 100644 index 83bd24d66..000000000 --- a/ddprof-lib/src/main/cpp/stackFrame.h +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _STACKFRAME_H -#define _STACKFRAME_H - -#include -#include -#include -#include "arch.h" -class StackFrame { - protected: - ucontext_t* _ucontext; - - static bool withinCurrentStack(uintptr_t address) { - // Check that the address is not too far from the stack pointer of current context - void* real_sp; - return address - (uintptr_t)&real_sp <= 0xffff; - } - - public: - explicit StackFrame(void* ucontext) { - _ucontext = (ucontext_t*)ucontext; - } - - void restore(uintptr_t saved_pc, uintptr_t saved_sp, uintptr_t saved_fp) { - if (_ucontext != nullptr) { - pc() = saved_pc; - sp() = saved_sp; - fp() = saved_fp; - } - } - - uintptr_t stackAt(int slot) { - return ((uintptr_t*)sp())[slot]; - } - - uintptr_t& pc(); - uintptr_t& sp(); - uintptr_t& fp(); - - uintptr_t& retval(); - uintptr_t link() const; - uintptr_t arg0() const; - uintptr_t arg1() const; - uintptr_t arg2() const; - uintptr_t arg3() const; - uintptr_t jarg0() const; - uintptr_t method() const; - uintptr_t senderSP() const; - - void ret(); - - void adjustSP(const void* entry, const void* pc, uintptr_t& sp); - - bool skipFaultInstruction(); - - bool checkInterruptedSyscall(); - - // Check if PC points to a syscall instruction - static bool isSyscall(instruction_t* pc); -}; - -#endif // _STACKFRAME_H diff --git a/ddprof-lib/src/main/cpp/stackFrame_aarch64.cpp b/ddprof-lib/src/main/cpp/stackFrame_aarch64.cpp deleted file mode 100644 index 11174114c..000000000 --- a/ddprof-lib/src/main/cpp/stackFrame_aarch64.cpp +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifdef __aarch64__ - -#include -#include -#include -#include "stackFrame.h" -#include "safeAccess.h" -#include "counters.h" - - -#ifdef __APPLE__ -# define REG(l, m) _ucontext->uc_mcontext->__ss.__##m -#else -# define REG(l, m) _ucontext->uc_mcontext.l -#endif - - -uintptr_t& StackFrame::pc() { - return (uintptr_t&)REG(pc, pc); -} - -uintptr_t& StackFrame::sp() { - return (uintptr_t&)REG(sp, sp); -} - -uintptr_t& StackFrame::fp() { - return (uintptr_t&)REG(regs[29], fp); -} - -uintptr_t& StackFrame::retval() { - return (uintptr_t&)REG(regs[0], x[0]); -} - -uintptr_t StackFrame::link() const { - return (uintptr_t)REG(regs[30], lr); -} - -uintptr_t StackFrame::arg0() const { - return (uintptr_t)REG(regs[0], x[0]); -} - -uintptr_t StackFrame::arg1() const { - return (uintptr_t)REG(regs[1], x[1]); -} - -uintptr_t StackFrame::arg2() const { - return (uintptr_t)REG(regs[2], x[2]); -} - -uintptr_t StackFrame::arg3() const { - return (uintptr_t)REG(regs[3], x[3]); -} - -uintptr_t StackFrame::jarg0() const { - return arg1(); -} - -uintptr_t StackFrame::method() const { - return (uintptr_t)REG(regs[12], x[12]); -} - -uintptr_t StackFrame::senderSP() const { - return (uintptr_t)REG(regs[19], x[19]); -} - -void StackFrame::ret() { - pc() = link(); -} - -NOSANALIGSANITIZE void StackFrame::adjustSP(const void* entry, const void* pc, uintptr_t& sp) { - instruction_t* ip = (instruction_t*)pc; - if (ip > entry && (ip[-1] == 0xa9bf27ff || (ip[-1] == 0xd63f0100 && ip[-2] == 0xa9bf27ff))) { - // When calling a leaf native from Java, JVM puts a dummy frame link onto the stack, - // thus breaking the invariant: sender_sp == current_sp + frame_size. - // Since JDK 21, there are more instructions between `blr` and `add`, - // ignore them now for the sake of simplicity. - // stp xzr, x9, [sp, #-16]! - // blr x8 - // ... - // add sp, sp, #0x10 - sp += 16; - } -} - -bool StackFrame::skipFaultInstruction() { - return false; -} - -NOSANALIGSANITIZE bool StackFrame::checkInterruptedSyscall() { -#ifdef __APPLE__ - // We are not interested in syscalls that do not check error code, e.g. semaphore_wait_trap - if (*(instruction_t*)pc() == 0xd65f03c0) { - return true; - } - // If carry flag is set, the error code is in low byte of x0 - if (REG(pstate, cpsr) & (1 << 29)) { - return (retval() & 0xff) == EINTR || (retval() & 0xff) == ETIMEDOUT; - } else { - return retval() == (uintptr_t)-EINTR; - } -#else - if (retval() == (uintptr_t)-EINTR) { - // Workaround for JDK-8237858: restart the interrupted poll / epoll_wait manually - uintptr_t nr = (uintptr_t)REG(regs[8], x[8]); - if (nr == SYS_ppoll || (nr == SYS_epoll_pwait && (int)arg3() == -1)) { - // Check against unreadable page for the loop below - const uintptr_t max_distance = 24; - if ((pc() & 0xfff) < max_distance && SafeAccess::load32((int32_t*)(pc() - max_distance)) == 0) { - return true; - } - // Try to restore the original value of x0 saved in another register - for (uintptr_t prev_pc = pc() - 4; pc() - prev_pc <= max_distance; prev_pc -= 4) { - instruction_t insn = *(instruction_t*)prev_pc; - unsigned int reg = (insn >> 16) & 31; - if ((insn & 0xffe0ffff) == 0xaa0003e0 && reg >= 6) { - // mov x0, reg - REG(regs[0], x[0]) = REG(regs[reg], x[reg]); - pc() -= sizeof(instruction_t); - break; - } - } - } - return true; - } - return false; -#endif -} - -bool StackFrame::isSyscall(instruction_t* pc) { - // svc #0 or svc #80 - return (*pc & 0xffffefff) == 0xd4000001; -} - -#endif // __aarch64__ diff --git a/ddprof-lib/src/main/cpp/stackFrame_x64.cpp b/ddprof-lib/src/main/cpp/stackFrame_x64.cpp deleted file mode 100644 index 11bd8b0b1..000000000 --- a/ddprof-lib/src/main/cpp/stackFrame_x64.cpp +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifdef __x86_64__ - -#include -#include -#include -#include "stackFrame.h" - -#ifdef __APPLE__ -# define REG(l, m) _ucontext->uc_mcontext->__ss.__##m -#else -# define REG(l, m) _ucontext->uc_mcontext.gregs[REG_##l] -#endif - - -uintptr_t& StackFrame::pc() { - return (uintptr_t&)REG(RIP, rip); -} - -uintptr_t& StackFrame::sp() { - return (uintptr_t&)REG(RSP, rsp); -} - -uintptr_t& StackFrame::fp() { - return (uintptr_t&)REG(RBP, rbp); -} - -uintptr_t& StackFrame::retval() { - return (uintptr_t&)REG(RAX, rax); -} - -uintptr_t StackFrame::link() const { - // No link register on x86 - return 0; -} - -uintptr_t StackFrame::arg0() const { - return (uintptr_t)REG(RDI, rdi); -} - -uintptr_t StackFrame::arg1() const { - return (uintptr_t)REG(RSI, rsi); -} - -uintptr_t StackFrame::arg2() const { - return (uintptr_t)REG(RDX, rdx); -} - -uintptr_t StackFrame::arg3() const { - return (uintptr_t)REG(RCX, rcx); -} - -uintptr_t StackFrame::jarg0() const { - return arg1(); -} - -uintptr_t StackFrame::method() const { - return (uintptr_t)REG(RBX, rbx); -} - -uintptr_t StackFrame::senderSP() const { - return (uintptr_t)REG(R13, r13); -} - -void StackFrame::ret() { - pc() = stackAt(0); - sp() += 8; -} - -void StackFrame::adjustSP(const void* entry, const void* pc, uintptr_t& sp) { - // Not needed -} - -// Skip failed MOV instruction by writing 0 to destination register -bool StackFrame::skipFaultInstruction() { - unsigned int insn = *(unsigned int*)pc(); - if ((insn & 0x80fff8) == 0x008b48) { - // mov r64, [r64 + offs] - unsigned int reg = ((insn << 1) & 8) | ((insn >> 19) & 7); - switch (reg) { - case 0x0: REG(RAX, rax) = 0; break; - case 0x1: REG(RCX, rcx) = 0; break; - case 0x2: REG(RDX, rdx) = 0; break; - case 0x3: REG(RBX, rbx) = 0; break; - case 0x4: return false; // Do not modify RSP - case 0x5: REG(RBP, rbp) = 0; break; - case 0x6: REG(RSI, rsi) = 0; break; - case 0x7: REG(RDI, rdi) = 0; break; - case 0x8: REG(R8 , r8 ) = 0; break; - case 0x9: REG(R9 , r9 ) = 0; break; - case 0xa: REG(R10, r10) = 0; break; - case 0xb: REG(R11, r11) = 0; break; - case 0xc: REG(R12, r12) = 0; break; - case 0xd: REG(R13, r13) = 0; break; - case 0xe: REG(R14, r14) = 0; break; - case 0xf: REG(R15, r15) = 0; break; - } - - unsigned int insn_size = 3; - if ((insn & 0x070000) == 0x040000) insn_size++; - if ((insn & 0x400000) == 0x400000) insn_size++; - pc() += insn_size; - return true; - } - return false; -} - -__attribute__((no_sanitize("address"))) bool StackFrame::checkInterruptedSyscall() { -#ifdef __APPLE__ - // We are not interested in syscalls that do not check error code, e.g. semaphore_wait_trap - if (*(instruction_t*)pc() == 0xc3) { - return true; - } - // If CF is set, the error code is in low byte of eax, - // some other syscalls (ulock_wait) do not set CF when interrupted - if (REG(EFL, rflags) & 1) { - return (retval() & 0xff) == EINTR || (retval() & 0xff) == ETIMEDOUT; - } else { - return retval() == (uintptr_t)-EINTR; - } -#else - if (retval() == (uintptr_t)-EINTR) { - // Workaround for JDK-8237858: restart the interrupted poll() manually. - // Check if the previous instruction is mov eax, SYS_poll with infinite timeout or - // mov eax, SYS_ppoll with any timeout (ppoll adjusts timeout automatically) - uintptr_t pc = this->pc(); - if ((pc & 0xfff) >= 7 && *(instruction_t*)(pc - 7) == 0xb8) { - int nr = ([&] { int val; memcpy(&val, (const void*)(pc - 6), sizeof(val)); return val; }()); - if (nr == SYS_ppoll - || (nr == SYS_poll && (int)REG(RDX, rdx) == -1) - || (nr == SYS_epoll_wait && (int)REG(R10, r10) == -1) - || (nr == SYS_epoll_pwait && (int)REG(R10, r10) == -1)) { - this->pc() = pc - 7; - } - } - return true; - } - return false; -#endif -} - -bool StackFrame::isSyscall(instruction_t* pc) { - return pc[0] == 0x0f && pc[1] == 0x05; -} - -#endif // __x86_64__ diff --git a/ddprof-lib/src/main/cpp/stackWalker.cpp b/ddprof-lib/src/main/cpp/stackWalker.cpp deleted file mode 100644 index 9e619644f..000000000 --- a/ddprof-lib/src/main/cpp/stackWalker.cpp +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "stackWalker.inline.h" -#include "dwarf.h" -#include "profiler.h" -#include "stackFrame.h" -#include "symbols.h" -#include "jvmSupport.inline.h" -#include "jvmThread.h" -#include "thread.h" - -// Use validation helpers from header (shared with tests) -using StackWalkValidation::inDeadZone; -using StackWalkValidation::aligned; -using StackWalkValidation::MAX_FRAME_SIZE; -using StackWalkValidation::sameStack; - - -int StackWalker::walkFP(void* ucontext, const void** callchain, int max_depth, StackContext* java_ctx, bool* truncated) { - const void* pc; - uintptr_t fp; - uintptr_t sp; - uintptr_t bottom = (uintptr_t)&sp + MAX_WALK_SIZE; - - StackFrame frame(ucontext); - if (ucontext == NULL) { - pc = callerPC(); - fp = (uintptr_t)callerFP(); - sp = (uintptr_t)callerSP(); - } else { - pc = (const void*)frame.pc(); - fp = frame.fp(); - sp = frame.sp(); - } - - int depth = 0; - int actual_max_depth = truncated ? max_depth + 1 : max_depth; - - // Walk until the bottom of the stack or until the first Java frame - while (depth < actual_max_depth) { - if (JVMSupport::isJitCode(pc) && !(depth == 0 && JVMSupport::canUnwind(frame, pc)) && - JVMThread::current() != nullptr) { // If it is not a JVM thread, it cannot have Java frame - java_ctx->set(pc, sp, fp); - break; - } - - callchain[depth++] = pc; - - // Check if the next frame is below on the current stack - if (fp < sp || fp >= sp + MAX_FRAME_SIZE || fp >= bottom) { - break; - } - - // Frame pointer must be word aligned - if (!aligned(fp)) { - break; - } - - pc = stripPointer(SafeAccess::load((void**)fp + FRAME_PC_SLOT)); - if (inDeadZone(pc)) { - break; - } - - sp = fp + (FRAME_PC_SLOT + 1) * sizeof(void*); - fp = (uintptr_t)SafeAccess::load((void**)fp); - } - - if (truncated && depth > max_depth) { - *truncated = true; - depth = max_depth; - } - - return depth; -} - -int StackWalker::walkDwarf(void* ucontext, const void** callchain, int max_depth, StackContext* java_ctx, bool* truncated) { - const void* pc; - uintptr_t fp; - uintptr_t sp; - uintptr_t bottom = (uintptr_t)&sp + MAX_WALK_SIZE; - - StackFrame frame(ucontext); - if (ucontext == NULL) { - pc = callerPC(); - fp = (uintptr_t)callerFP(); - sp = (uintptr_t)callerSP(); - } else { - pc = (const void*)frame.pc(); - fp = frame.fp(); - sp = frame.sp(); - } - - int depth = 0; - Profiler* profiler = Profiler::instance(); - int actual_max_depth = truncated ? max_depth + 1 : max_depth; - - // Walk until the bottom of the stack or until the first Java frame - while (depth < actual_max_depth) { - if (JVMSupport::isJitCode(pc) && !(depth == 0 && JVMSupport::canUnwind(frame, pc)) && - JVMThread::current() != nullptr) { // If it is not a JVM thread, it cannot have Java frame - // Don't dereference pc as it may point to unreadable memory - // frame.adjustSP(page_start, pc, sp); - java_ctx->set(pc, sp, fp); - break; - } - - callchain[depth++] = pc; - - uintptr_t prev_sp = sp; - CodeCache* cc = profiler->findLibraryByAddress(pc); - FrameDesc f = cc != NULL ? cc->findFrameDesc(pc) : FrameDesc::fallback_default_frame(); - - u8 cfa_reg = (u8)f.cfa; - int cfa_off = f.cfa >> 8; - if (cfa_reg == DW_REG_SP) { - sp = sp + cfa_off; - } else if (cfa_reg == DW_REG_FP) { - sp = fp + cfa_off; - } else if (cfa_reg == DW_REG_PLT) { - sp += ((uintptr_t)pc & 15) >= 11 ? cfa_off * 2 : cfa_off; - } else { - break; - } - - // Check if the next frame is below on the current stack - if (sp < prev_sp || sp >= prev_sp + MAX_FRAME_SIZE || sp >= bottom) { - break; - } - - // Stack pointer must be word aligned - if (!aligned(sp)) { - break; - } - - const void* prev_pc = pc; - if (f.fp_off & DW_PC_OFFSET) { - pc = (const char*)pc + (f.fp_off >> 1); - } else { - if (f.fp_off != DW_SAME_FP && f.fp_off < MAX_FRAME_SIZE && f.fp_off > -MAX_FRAME_SIZE) { - uintptr_t fp_addr = sp + f.fp_off; - if (!aligned(fp_addr)) { - break; - } - fp = (uintptr_t)SafeAccess::load((void**)fp_addr); - } - - if (EMPTY_FRAME_SIZE > 0 || f.pc_off != DW_LINK_REGISTER) { - uintptr_t pc_addr = sp + f.pc_off; - if (!aligned(pc_addr)) { - break; - } - pc = stripPointer(SafeAccess::load((void**)pc_addr)); - } else if (depth == 1) { - pc = (const void*)frame.link(); - } else { - break; - } - - if (EMPTY_FRAME_SIZE == 0 && cfa_off == 0 && f.fp_off != DW_SAME_FP) { - // AArch64 default_frame - sp = defaultSenderSP(sp, fp); - if (sp < prev_sp || sp >= bottom || !aligned(sp)) { - break; - } - } - } - - if (inDeadZone(pc) || (pc == prev_pc && sp == prev_sp)) { - break; - } - } - - if (truncated && depth > max_depth) { - *truncated = true; - depth = max_depth; - } - - return depth; -} diff --git a/ddprof-lib/src/main/cpp/stackWalker.h b/ddprof-lib/src/main/cpp/stackWalker.h deleted file mode 100644 index 5b9127d6c..000000000 --- a/ddprof-lib/src/main/cpp/stackWalker.h +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _STACKWALKER_H -#define _STACKWALKER_H - -#include -#include -#include "arguments.h" -#include "event.h" -#include "vmEntry.h" - - -class VMJavaFrameAnchor; -class ProfiledThread; - -struct StackContext { - const void* pc; - uintptr_t sp; - uintptr_t fp; - u64 cpu; - - void set(const void* pc, uintptr_t sp, uintptr_t fp) { - this->pc = pc; - this->sp = sp; - this->fp = fp; - } -}; - -// Stack walking validation helpers (used by implementation and tests) -namespace StackWalkValidation { - const intptr_t MAX_INTERPRETER_FRAME_SIZE = 0x1000; - const uintptr_t DEAD_ZONE = 0x1000; - const intptr_t MAX_FRAME_SIZE = 0x40000; - const uintptr_t SAME_STACK_DISTANCE = 8192; - - // Check if pointer is in dead zone (very low or very high address) - static inline bool inDeadZone(const void* ptr) { - return ptr < (const void*)DEAD_ZONE || ptr > (const void*)-DEAD_ZONE; - } - - // Check if pointer is properly aligned - static inline bool aligned(uintptr_t ptr) { - return (ptr & (sizeof(uintptr_t) - 1)) == 0; - } - - // Check if two pointers are on the same stack - static inline bool sameStack(void* hi, void* lo) { - return (uintptr_t)hi - (uintptr_t)lo < SAME_STACK_DISTANCE; - } - - // Check if a frame pointer is plausibly valid (not in dead zone, properly aligned) - static inline bool isValidFP(uintptr_t fp) { - return !inDeadZone((const void*)fp) && aligned(fp); - } - - // Check if a stack pointer is within [lo, hi) and properly aligned - static inline bool isValidSP(uintptr_t sp, uintptr_t lo, uintptr_t hi) { - return sp > lo && sp < hi && aligned(sp); - } - - // Drop unknown leaf frame (method_id == NULL at index 0). - // Returns the new depth after removal. - static inline int dropUnknownLeaf(ASGCT_CallFrame* frames, int depth) { - if (depth > 0 && frames[0].method_id == NULL) { - depth--; - if (depth > 0) { - memmove(frames, frames + 1, depth * sizeof(frames[0])); - } - } - return depth; - } - - static inline bool isPlausibleInterpreterFrame(uintptr_t fp, uintptr_t sp, int bcp_offset){ - return fp != 0 && aligned(fp) && !inDeadZone((const void*)fp) - && sp != 0 && sp > fp - MAX_INTERPRETER_FRAME_SIZE - && sp < fp + bcp_offset * (intptr_t)sizeof(void*); - } -} - -typedef struct { - jint event_type; - u32 lock_index; - void* ucontext; - ASGCT_CallFrame* frames; - int max_depth; - StackContext* java_ctx; - bool* truncated; -} StackWalkRequest; - -class StackWalker { - public: - static int walkFP(void* ucontext, const void** callchain, int max_depth, StackContext* java_ctx, bool* truncated = nullptr); - static int walkDwarf(void* ucontext, const void** callchain, int max_depth, StackContext* java_ctx, bool* truncated = nullptr); -}; - -#endif // _STACKWALKER_H diff --git a/ddprof-lib/src/main/cpp/stackWalker.inline.h b/ddprof-lib/src/main/cpp/stackWalker.inline.h deleted file mode 100644 index d8399242c..000000000 --- a/ddprof-lib/src/main/cpp/stackWalker.inline.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _STACKWALKER_INLINE_H -#define _STACKWALKER_INLINE_H - -#include "stackWalker.h" -#include "safeAccess.h" - -#include - -inline constexpr uintptr_t MAX_WALK_SIZE = 0x100000; -inline constexpr intptr_t MAX_FRAME_SIZE_WORDS = StackWalkValidation::MAX_FRAME_SIZE / sizeof(void*); // 0x8000 = 32768 words - -// AArch64: on Linux, frame link is stored at the top of the frame, -// while on macOS, frame link is at the bottom. -inline uintptr_t defaultSenderSP(uintptr_t sp, uintptr_t fp) { -#ifdef __APPLE__ - return sp + 2 * sizeof(void*); -#else - return fp; -#endif -} - -inline void fillFrame(ASGCT_CallFrame& frame, ASGCT_CallFrameType type, const char* name) { - frame.bci = type; - frame.method_id = (jmethodID)name; -} - -// Overload for RemoteFrameInfo* (passed as void* to support both char* and RemoteFrameInfo*) -inline void fillFrame(ASGCT_CallFrame& frame, int bci, void* method_id_ptr) { - frame.bci = bci; - frame.method_id = (jmethodID)method_id_ptr; -} - -inline void fillFrame(ASGCT_CallFrame& frame, ASGCT_CallFrameType type, u32 class_id) { - frame.bci = type; - frame.method_id = (jmethodID)(uintptr_t)class_id; -} - -inline void fillFrame(ASGCT_CallFrame& frame, FrameTypeId type, int bci, jmethodID method) { - frame.bci = FrameType::encode(type, bci); - frame.method_id = method; -} - -#endif // _STACKWALKER_INLINE_H diff --git a/ddprof-lib/src/main/cpp/stringDictionary.h b/ddprof-lib/src/main/cpp/stringDictionary.h deleted file mode 100644 index 05bb62d3d..000000000 --- a/ddprof-lib/src/main/cpp/stringDictionary.h +++ /dev/null @@ -1,606 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _STRINGDICTIONARY_H -#define _STRINGDICTIONARY_H - -#include "counters.h" -#include "log.h" -#include "refCountGuard.h" -#include "tripleBuffer.h" -#include "arch.h" -#include -#include -#include -#include -#include -#include - -// Reuse the same table geometry as Dictionary for cache-friendly layout. -// The two headers share these macros via this #ifndef guard; the static_asserts -// below catch any divergence at compile time so a future change to dictionary.h -// cannot silently change SBTable geometry depending on include order. -#ifndef ROW_BITS -#define ROW_BITS 7 -#define ROWS (1 << ROW_BITS) -#define CELLS 3 -#endif -static_assert(ROW_BITS == 7, "StringDictionary assumes ROW_BITS == 7"); -static_assert(ROWS == 128, "StringDictionary assumes ROWS == 128 (sized into freeOverflowNodes/collectTable traversal)"); -static_assert(CELLS == 3, "StringDictionary assumes CELLS == 3 (SBRow layout)"); - -// ─── Internal storage types ──────────────────────────────────────────────── - -struct SBTable; - -struct SBRow { - char* keys[CELLS]; // null = empty; CAS-claimed by the inserting thread - u32 ids[CELLS]; // set AFTER winning the key CAS (release); 0 until published - SBTable* next; // overflow chain; CAS-created on overflow -}; - -struct SBTable { - SBRow rows[ROWS]; -}; - -// ─── StringArena ────────────────────────────────────────────────────────── -// -// Auto-growing bump allocator for key strings inside StringDictionaryBuffer. -// -// Memory is organised as a linked list of 512 KB chunks. alloc() is a single -// atomic fetch_add on the current chunk — fully contention-free as long as -// the chunk is not full. When a chunk fills up, grow() serialises creation of -// the next chunk via a CAS spinlock; contention here is extremely rare. -// -// Threads that lose a CAS race in insert_with_id leave their arena allocation -// as waste; space is recovered on the next reset(). -// -// reset() frees all chunks after the first and resets the first chunk's -// position counter. The first chunk is kept to avoid a malloc on the next -// use. reset() must only be called when no concurrent alloc() calls are in -// flight. -class StringArena { - static constexpr size_t CHUNK_SIZE = 512 * 1024; - - // Plain struct allocated via calloc (zero-initialised); pos accessed via - // __atomic builtins, consistent with the rest of the file. - struct Chunk { - Chunk* next; // singly-linked for traversal in reset() / ~StringArena() - size_t pos; // bump pointer within data[] - char data[CHUNK_SIZE]; - }; - - Chunk* _first; // head of chain; kept across resets - std::atomic _active; // current allocation target - std::atomic _growing{false}; // serialises new-chunk creation - int _counter_offset{0};// 0 = no DICTIONARY_BYTES tracking - bool _oom_logged{false};// latched per generation; cleared by reset() - - static Chunk* make_chunk() { - return static_cast(calloc(1, sizeof(Chunk))); - } - - void countChunkAlloc() { - if (_counter_offset != 0) { - Counters::increment(DICTIONARY_BYTES, (long long)sizeof(Chunk), _counter_offset); - } - } - - void countChunkFree(int n) { - if (_counter_offset != 0 && n > 0) { - Counters::decrement(DICTIONARY_BYTES, (long long)(n * sizeof(Chunk)), _counter_offset); - } - } - - void grow(Chunk* full) { - // One thread at a time creates the next chunk. Others spin briefly - // then re-check _active; if it has already advanced they return. - bool expected = false; - while (!_growing.compare_exchange_weak(expected, true, - std::memory_order_acquire, std::memory_order_relaxed)) { - if (_active.load(std::memory_order_relaxed) != full) return; - expected = false; - spinPause(); - } - if (_active.load(std::memory_order_relaxed) != full) { - _growing.store(false, std::memory_order_release); - return; - } - Chunk* fresh = make_chunk(); - // On OOM store nullptr so alloc() returns nullptr instead of spinning. - _active.store(fresh, std::memory_order_release); - if (fresh) { - full->next = fresh; // link into chain for reset() traversal - countChunkAlloc(); - } else { - // Make the failure observable in production logs. Latched per - // arena instance: only the first OOM in the current generation - // logs; reset() clears the latch. - if (!_oom_logged) { - _oom_logged = true; - Log::warn("StringArena: chunk allocation failed; new inserts will " - "be dropped on this buffer until the next clearAll/reset"); - } - } - _growing.store(false, std::memory_order_release); - } - -public: - StringArena() : _first(make_chunk()), _active(_first) {} - - ~StringArena() { - Chunk* c = _first; - while (c) { Chunk* n = c->next; free(c); c = n; } - } - - StringArena(const StringArena&) = delete; - StringArena& operator=(const StringArena&) = delete; - - // Enable DICTIONARY_BYTES tracking for arena chunks beyond the initial one. - // The initial chunk is counted by StringDictionaryBuffer::initCounters(), - // which calls this method after construction. - void initCounters(int offset) { - _counter_offset = offset; - if (offset != 0 && _first != nullptr) { - Counters::increment(DICTIONARY_BYTES, (long long)sizeof(Chunk), offset); - } - } - - char* alloc(size_t n) { - n = (n + alignof(void*) - 1) & ~(alignof(void*) - 1); - for (;;) { - Chunk* c = _active.load(std::memory_order_acquire); - if (!c) return nullptr; // OOM - size_t off = __atomic_fetch_add(&c->pos, n, __ATOMIC_RELAXED); - if (off + n <= CHUNK_SIZE) return c->data + off; - grow(c); - } - } - - // Free all chunks after the first; reset the first. - // O(extra_chunks). Also clears the OOM state: if alloc() had returned - // nullptr after a failed make_chunk(), the next alloc() after reset() - // will succeed again (up to one chunk's worth). - void reset() { - Chunk* c = _first ? _first->next : nullptr; - int freed = 0; - while (c) { Chunk* n = c->next; free(c); c = n; ++freed; } - if (_first) { - _first->next = nullptr; - __atomic_store_n(&_first->pos, (size_t)0, __ATOMIC_RELAXED); - } - _active.store(_first, std::memory_order_release); - countChunkFree(freed); - _oom_logged = false; - } -}; - -// ─── StringDictionaryBuffer ──────────────────────────────────────────────── -// -// Open-addressing concurrent hash table mapping string keys to u32 IDs. -// -// Key strings are owned by the per-buffer StringArena. Overflow SBTable -// nodes are heap-allocated (calloc) and freed by freeOverflowNodes() on -// clear() and destruction. This makes clear() O(number-of-overflow-nodes) -// rather than O(number-of-entries), and eliminates per-key malloc/free. -// -// Concurrency model: -// - Inserts (insert_with_id, copyFrom): CAS on keys[c] to claim a slot. -// id stored AFTER winning CAS (release); readers check ids[c] != 0. -// Losers of the CAS leave their arena allocation as recoverable waste. -// - Reads (lookup): acquire-load keys[c]; miss on null or unpublished id. -// - clear(): called only when no concurrent readers/writers are active. -// -// Not signal-safe for insert_with_id / copyFrom (arena alloc + calloc). -// Signal-safe for lookup (read-only, no allocation). -class StringDictionaryBuffer { -private: - SBTable* _table; - std::atomic _size{0}; - StringArena _arena; - int _counter_offset{0}; // 0 = no page/byte tracking - - static unsigned int hash(const char* key, size_t length) { - unsigned int h = 2166136261U; - for (size_t i = 0; i < length; i++) h = (h ^ (unsigned char)key[i]) * 16777619; - return h; - } - - static bool keyEquals(const char* candidate, const char* key, size_t length) { - return strncmp(candidate, key, length) == 0 && candidate[length] == '\0'; - } - - // Common-case overflow-chain depth; std::vector reserves this many frames - // up front so the typical traversal never reallocates. Deeper chains grow - // the vector — no silent truncation. - static constexpr int RESERVED_TRAVERSAL_DEPTH = 34; - - // Free only overflow SBTable chain nodes (not key strings — arena-owned). - // Returns the number of overflow nodes freed (excludes the root table). - static int freeOverflowNodes(SBTable* table) { - struct Frame { SBTable* t; int row; }; - std::vector stk; - stk.reserve(RESERVED_TRAVERSAL_DEPTH); - int freed = 0; - stk.push_back({table, 0}); - while (!stk.empty()) { - Frame& f = stk.back(); - if (f.row >= ROWS) { - if (f.t != table) { free(f.t); freed++; } - stk.pop_back(); - continue; - } - SBRow* row = &f.t->rows[f.row++]; - if (row->next) stk.push_back({row->next, 0}); - } - return freed; - } - - static void collectTable(const SBTable* table, - std::map& out) { - struct Frame { const SBTable* t; int row; }; - std::vector stk; - stk.reserve(RESERVED_TRAVERSAL_DEPTH); - stk.push_back({table, 0}); - while (!stk.empty()) { - Frame& f = stk.back(); - if (f.row >= ROWS) { stk.pop_back(); continue; } - const SBRow* row = &f.t->rows[f.row++]; - for (int j = 0; j < CELLS; j++) { - const char* k = __atomic_load_n(&row->keys[j], __ATOMIC_ACQUIRE); - if (k) { - u32 eid = __atomic_load_n(&row->ids[j], __ATOMIC_ACQUIRE); - if (eid != 0) out[eid] = k; - } - } - const SBTable* next = __atomic_load_n(&row->next, __ATOMIC_ACQUIRE); - if (next) stk.push_back({next, 0}); - } - } - -public: - StringDictionaryBuffer() { - _table = static_cast(calloc(1, sizeof(SBTable))); - } - - ~StringDictionaryBuffer() { - if (_table != nullptr) { - freeOverflowNodes(_table); - free(_table); - _table = nullptr; - } - } - - StringDictionaryBuffer(const StringDictionaryBuffer&) = delete; - StringDictionaryBuffer& operator=(const StringDictionaryBuffer&) = delete; - StringDictionaryBuffer(StringDictionaryBuffer&&) = delete; - StringDictionaryBuffer& operator=(StringDictionaryBuffer&&) = delete; - - // Enable DICTIONARY_PAGES / DICTIONARY_BYTES tracking for this buffer. - // Called by StringDictionary after construction; counts the root SBTable - // and the initial arena Chunk. Subsequent arena growth and reset() are - // accounted for by StringArena itself. - void initCounters(int offset) { - _counter_offset = offset; - if (_table != nullptr) { - Counters::increment(DICTIONARY_PAGES, 1, offset); - Counters::increment(DICTIONARY_BYTES, (long long)sizeof(SBTable), offset); - } - _arena.initCounters(offset); - } - - // Signal-safe read-only probe. Returns 0 on miss. - u32 lookup(const char* key, size_t len) const { - const SBTable* table = _table; - unsigned int h = hash(key, len); - while (table) { - const SBRow* row = &table->rows[h % ROWS]; - for (int c = 0; c < CELLS; c++) { - const char* k = __atomic_load_n(&row->keys[c], __ATOMIC_ACQUIRE); - if (!k) return 0; - if (keyEquals(k, key, len)) { - u32 id = __atomic_load_n(&row->ids[c], __ATOMIC_ACQUIRE); - return id; - } - } - table = __atomic_load_n(&row->next, __ATOMIC_ACQUIRE); - h = (h >> ROW_BITS) | (h << (32 - ROW_BITS)); - } - return 0; - } - - // Insert with the given id. Returns the id stored for this key. - // NOT signal-safe (arena alloc; calloc for overflow nodes). - u32 insert_with_id(const char* key, size_t len, u32 id) { - SBTable* table = _table; - if (table == nullptr) return 0; // calloc OOM at ctor; match lookup() contract - unsigned int h = hash(key, len); - while (true) { - SBRow* row = &table->rows[h % ROWS]; - for (int c = 0; c < CELLS; c++) { - char* existing = __atomic_load_n(&row->keys[c], __ATOMIC_ACQUIRE); - if (!existing) { - char* new_key = _arena.alloc(len + 1); - if (!new_key) return 0; - memcpy(new_key, key, len); - new_key[len] = '\0'; - if (__sync_bool_compare_and_swap(&row->keys[c], nullptr, new_key)) { - __atomic_store_n(&row->ids[c], id, __ATOMIC_RELEASE); - _size.fetch_add(1, std::memory_order_relaxed); - return id; - } - // CAS lost — new_key is arena waste, recovered on clear(). - // Bump-allocator design does not support per-slot reclaim; - // expose the waste so operators can quantify the cost. - if (_counter_offset != 0) { - size_t wasted = (len + 1 + alignof(void*) - 1) & ~(alignof(void*) - 1); - Counters::increment(DICTIONARY_ARENA_WASTE_BYTES, - (long long)wasted, _counter_offset); - } - existing = __atomic_load_n(&row->keys[c], __ATOMIC_ACQUIRE); - } - if (existing && keyEquals(existing, key, len)) { - u32 stored_id; - while ((stored_id = __atomic_load_n(&row->ids[c], __ATOMIC_ACQUIRE)) == 0) { spinPause(); } - return stored_id; - } - } - // Relaxed is fine here: the optimization hint may be stale; the CAS - // below will handle that, and the ACQUIRE load of row->next below - // provides the necessary happens-before for the newly-created SBTable's contents. - if (!__atomic_load_n(&row->next, __ATOMIC_RELAXED)) { - SBTable* nt = static_cast(calloc(1, sizeof(SBTable))); - if (nt == nullptr) return 0; - if (!__sync_bool_compare_and_swap(&row->next, nullptr, nt)) { - free(nt); - } else if (_counter_offset != 0) { - Counters::increment(DICTIONARY_PAGES, 1, _counter_offset); - Counters::increment(DICTIONARY_BYTES, (long long)sizeof(SBTable), _counter_offset); - } - } - table = __atomic_load_n(&row->next, __ATOMIC_ACQUIRE); - h = (h >> ROW_BITS) | (h << (32 - ROW_BITS)); - } - } - - // Copy all entries from src into this buffer preserving their ids. - // NOT signal-safe. - void copyFrom(const StringDictionaryBuffer& src) { - std::map entries; - src.collect(entries); - for (auto& kv : entries) { - insert_with_id(kv.second, strlen(kv.second), kv.first); - } - } - - // Populate out with {id -> key} for all entries in this buffer. - void collect(std::map& out) const { - collectTable(_table, out); - } - - // Free overflow nodes, zero the root table, reset the arena. - // Call only with no concurrent accessors. - void clear() { - if (_table == nullptr) { _size.store(0, std::memory_order_relaxed); return; } - int freed = freeOverflowNodes(_table); - memset(_table, 0, sizeof(SBTable)); - _arena.reset(); - _size.store(0, std::memory_order_relaxed); - if (_counter_offset != 0 && freed > 0) { - Counters::decrement(DICTIONARY_PAGES, freed, _counter_offset); - Counters::decrement(DICTIONARY_BYTES, (long long)(freed * sizeof(SBTable)), _counter_offset); - } - } - - int size() const { return _size.load(std::memory_order_relaxed); } -}; - -// ─── StringDictionary ───────────────────────────────────────────────────── -// -// Triple-buffered wrapper around StringDictionaryBuffer. -// -// Roles cycle through three buffers: -// active — receives new writes (lookup, insert_with_id) -// dump — stable snapshot for the current JFR chunk (after rotate()) -// scratch — two rotations behind; cleared by clearStandby() -// -// _next_id is a global monotonic counter that never resets until clearAll(). -// rotate() does a two-phase ID-preserving copy so no entry is lost due to -// concurrent inserts in the rotation window: -// phase 1: copy active → clearTarget (before rotate) -// phase 2: copy old_active → new_active (after drain, catch late inserts) -// lookupDuringDump(key): probes dump then active; inserts into both if new. -// -// Concurrency: -// bounded_lookup acquires RefCountGuard on active before reading. -// lookup also acquires RefCountGuard before inserting (not signal-safe due to -// arena alloc, but the guard protects the buffer pointer lifetime). -// lookupDuringDump is NOT signal-safe; call from dump thread only. -// -// _accepting gates new guard creation during clearAll(). A thread that -// passed the outer acquire-load check before clearAll() sets _accepting=false -// may create its guard after waitForAllRefCountsToClear() returns, missing -// the drain. A seq_cst recheck inside the guard scope catches this TOCTOU -// window: the thread sees _accepting=false and returns 0 before touching any -// buffer data (overflow nodes or arena chunks that clearAll() is freeing). -class StringDictionary { - std::atomic _next_id{1}; // starts at 1; id=0 reserved as "no entry" - std::atomic _accepting{true}; // false while clearAll() is resetting buffers - StringDictionaryBuffer _a, _b, _c; - TripleBufferRotator _rot; - int _counter_offset; // offset into DICTIONARY_KEYS / DICTIONARY_KEYS_BYTES counter rows - - u32 nextId() { - // id 0 is the "no entry" sentinel. After ~4 billion lookup() calls - // _next_id wraps; skip the resulting 0 so insert_with_id never stores - // 0 in ids[c] (which would make readers in the spin-wait loop hang). - u32 id; - do { - id = _next_id.fetch_add(1, std::memory_order_relaxed); - } while (__builtin_expect(id == 0, 0)); - return id; - } - - void countInsert(size_t len) { - Counters::increment(DICTIONARY_KEYS, 1, _counter_offset); - Counters::increment(DICTIONARY_KEYS_BYTES, (long long)(len + 1), _counter_offset); - } - -public: - explicit StringDictionary(int counter_offset = 0) - : _rot(&_a, &_b, &_c), _counter_offset(counter_offset) { - if (counter_offset != 0) { - _a.initCounters(counter_offset); - _b.initCounters(counter_offset); - _c.initCounters(counter_offset); - } - } - - // Insert into active buffer; returns globally stable id. NOT signal-safe. - u32 lookup(const char* key, size_t len) { - if (!_accepting.load(std::memory_order_acquire)) return 0; - while (true) { - StringDictionaryBuffer* active = _rot.active(); - RefCountGuard guard(active); - if (!guard.isActive()) return 0; - // Re-check _accepting after guard creation to close the TOCTOU window: - // clearAll() sets _accepting=false then drains; a thread that passed the - // outer check but hadn't yet incremented its guard count would be missed - // by the drain and could access freed overflow nodes or arena chunks. - if (!_accepting.load(std::memory_order_seq_cst)) return 0; - if (_rot.active() != active) continue; - u32 id = active->lookup(key, len); - if (id != 0) return id; - u32 new_id = nextId(); - u32 result = active->insert_with_id(key, len, new_id); - if (result == new_id) countInsert(len); - return result; - } - } - - // Insert into active buffer if size < size_limit; returns 0 when at cap. - // NOT signal-safe. - u32 bounded_lookup(const char* key, size_t len, int size_limit) { - if (!_accepting.load(std::memory_order_acquire)) return 0; - while (true) { - StringDictionaryBuffer* active = _rot.active(); - RefCountGuard guard(active); - if (!guard.isActive()) return 0; - if (!_accepting.load(std::memory_order_seq_cst)) return 0; - if (_rot.active() != active) continue; - u32 id = active->lookup(key, len); - if (id != 0) return id; - if (active->size() >= size_limit) return 0; - u32 new_id = nextId(); - u32 result = active->insert_with_id(key, len, new_id); - if (result == new_id) countInsert(len); - return result; - } - } - - // Signal-safe read-only probe of active. Returns 0 on miss. - u32 bounded_lookup(const char* key, size_t len) { - if (!_accepting.load(std::memory_order_acquire)) return 0; - while (true) { - StringDictionaryBuffer* active = _rot.active(); - RefCountGuard guard(active); - if (!guard.isActive()) return 0; - if (!_accepting.load(std::memory_order_seq_cst)) return 0; - if (_rot.active() != active) continue; - return active->lookup(key, len); - } - } - - // Returns the dump buffer (snapshot of old active after rotate()). - StringDictionaryBuffer* standby() { - return _rot.dumpBuffer(); - } - - // Two-phase ID-preserving rotate. - // StringDictionary makes no assumption about which callers are blocked. - // In the Profiler context, three caller-side invariants reduce the - // concurrency that phase 2 must handle: - // - Signal paths: the caller (rotateDictsAndRun) holds a SignalBlocker - // that masks SIGPROF/SIGVTALRM on the calling thread during rotate(), - // so no profiler signal fires on this thread between Phase 1 and 2. - // - JNI callers (e.g. recordTrace0): they bypass lockAll() and CAN - // still insert into old_active after Phase 1. Phase 2's - // waitForRefCountToClear(old_active) drains those in-flight inserts - // before copying — that is the reason phase 2 exists. - // - lookupDuringDump(): same thread as the rotate() caller — no - // concurrency. - // clearTarget() is the buffer that becomes the new active after rotate(). - // The caller is responsible for ensuring it is empty on entry (Profiler - // achieves this by calling clearStandby() after every cycle and - // serialising JFR operations with _state_lock). - void rotate() { - StringDictionaryBuffer* old_active = _rot.active(); - // Phase 1: pre-populate clearTarget from active (before rotate). - _rot.clearTarget()->copyFrom(*old_active); - _rot.rotate(); - // Drain all in-flight accessors on old_active (now the dump buffer). - RefCountGuard::waitForRefCountToClear(old_active); - // Phase 2: catch any entries inserted into old_active between Phase 1 - // and the drain completing. - _rot.active()->copyFrom(*old_active); - } - - // Resolve a key during the dump phase. Safe to call from the dump thread - // after rotate(); must NOT be called from signal handlers or concurrently - // with another lookupDuringDump call. - u32 lookupDuringDump(const char* key, size_t len) { - StringDictionaryBuffer* dump = _rot.dumpBuffer(); - - u32 id = dump->lookup(key, len); - if (id != 0) return id; - - { - StringDictionaryBuffer* active = _rot.active(); - RefCountGuard guard(active); - if (!guard.isActive()) return 0; - id = active->lookup(key, len); - } - if (id != 0) { - dump->insert_with_id(key, len, id); - return id; - } - - { - StringDictionaryBuffer* active = _rot.active(); - RefCountGuard guard(active); - if (!guard.isActive()) return 0; - u32 new_id = nextId(); - new_id = active->insert_with_id(key, len, new_id); - if (new_id != 0) dump->insert_with_id(key, len, new_id); - return new_id; - } - } - - // Clear the scratch buffer (two rotations behind active; safe to clear). - // Resets per-dump counters to 0 so they track only post-clearStandby inserts. - void clearStandby() { - _rot.clearTarget()->clear(); - Counters::set(DICTIONARY_KEYS, 0, _counter_offset); - Counters::set(DICTIONARY_KEYS_BYTES, 0, _counter_offset); - } - - // Reset all three buffers and restart the ID counter. - // _accepting=false gates new RefCountGuard creation; the subsequent drain - // ensures no concurrent accessor is mid-read when clear() zeroes the root - // table. clear() is O(overflow_nodes + extra_arena_chunks); both are - // typically zero for small-to-medium dictionaries. - void clearAll() { - _accepting.store(false, std::memory_order_seq_cst); - RefCountGuard::waitForAllRefCountsToClear(); - _a.clear(); _b.clear(); _c.clear(); - _rot.reset(); - _next_id.store(1, std::memory_order_relaxed); - Counters::set(DICTIONARY_KEYS, 0, _counter_offset); - Counters::set(DICTIONARY_KEYS_BYTES, 0, _counter_offset); - _accepting.store(true, std::memory_order_release); - } -}; - -#endif // _STRINGDICTIONARY_H diff --git a/ddprof-lib/src/main/cpp/symbols.h b/ddprof-lib/src/main/cpp/symbols.h deleted file mode 100644 index b315d51ef..000000000 --- a/ddprof-lib/src/main/cpp/symbols.h +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _SYMBOLS_H -#define _SYMBOLS_H - -#include "codeCache.h" -#include "mutex.h" - -#include - - -class Symbols { - private: - static Mutex _parse_lock; - static bool _have_kernel_symbols; - static bool _libs_limit_reported; - - public: - static void initLibraryRanges(); - static void parseKernelSymbols(CodeCache* cc); - static void parseLibraries(CodeCacheArray* array, bool kernel_symbols); - - static bool haveKernelSymbols() { - return _have_kernel_symbols; - } - // Clear internal caches - mainly for test purposes - static void clearParsingCaches(); - // Fast range check: does this PC lie in libc or libpthread? - static bool isLibcOrPthreadAddress(uintptr_t pc); -}; - -class UnloadProtection { - private: - void* _lib_handle; - bool _valid; - - public: - UnloadProtection(const CodeCache *cc); - ~UnloadProtection(); - - UnloadProtection& operator=(const UnloadProtection& other) = delete; - - bool isValid() const { return _valid; } -}; - -#endif // _SYMBOLS_H diff --git a/ddprof-lib/src/main/cpp/symbols_linux.cpp b/ddprof-lib/src/main/cpp/symbols_linux.cpp deleted file mode 100644 index b328fcfd5..000000000 --- a/ddprof-lib/src/main/cpp/symbols_linux.cpp +++ /dev/null @@ -1,1476 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifdef __linux__ - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "common.h" -#include "symbols.h" -#include "dwarf.h" -#include "sframe.h" -#include "fdtransferClient.h" -#include "log.h" -#include "os.h" -#include "symbols_linux.h" - -// Simple address range -struct Range { - uintptr_t start; - uintptr_t end; -}; - -static bool range_valid(const Range* r) { - return r->start && r->end && r->end > r->start; -} - -static Range g_libc = {0, 0}; -static Range g_libpthread = {0, 0}; -static bool g_lib_ranges_inited = false; - -// Unified dl_iterate_phdr callback context -struct UnifiedCtx { - void* fbase; // For range_for_fbase functionality - Range* out; // For range_for_fbase functionality - const void** main_phdr; // For getMainPhdr functionality - void* libc_fbase; // For init_lib_ranges_once functionality - void* pthread_fbase; // For init_lib_ranges_once functionality - Range* libc_range; // For init_lib_ranges_once functionality - Range* pthread_range; // For init_lib_ranges_once functionality -}; - -// Unified callback for both range computation and main phdr collection -static int unified_phdr_cb(dl_phdr_info* info, size_t /*unused*/, void* data) { - UnifiedCtx* ctx = (UnifiedCtx*)data; - - // Main executable's program header (first entry) - if (ctx->main_phdr != NULL && *ctx->main_phdr == NULL) { - *ctx->main_phdr = info->dlpi_phdr; - } - - // Range computation for specific fbase (range_for_fbase functionality) - if (ctx->fbase != NULL && (void*)info->dlpi_addr == ctx->fbase) { - uintptr_t minv = (uintptr_t)-1; - uintptr_t maxv = 0; - for (int i = 0; i < info->dlpi_phnum; i++) { - const ElfW(Phdr)* ph = &info->dlpi_phdr[i]; - if (ph->p_type != PT_LOAD) continue; - uintptr_t vaddr = (uintptr_t)info->dlpi_addr + ph->p_vaddr; - uintptr_t vend = vaddr + ph->p_memsz; - if (vaddr < minv) minv = vaddr; - if (vend > maxv) maxv = vend; - } - if (minv != (uintptr_t)-1 && maxv > minv) { - ctx->out->start = minv; - ctx->out->end = maxv; - } - } - - // Library range computation (init_lib_ranges_once functionality) - if (ctx->libc_fbase != NULL && (void*)info->dlpi_addr == ctx->libc_fbase) { - uintptr_t minv = (uintptr_t)-1; - uintptr_t maxv = 0; - for (int i = 0; i < info->dlpi_phnum; i++) { - const ElfW(Phdr)* ph = &info->dlpi_phdr[i]; - if (ph->p_type != PT_LOAD) continue; - uintptr_t vaddr = (uintptr_t)info->dlpi_addr + ph->p_vaddr; - uintptr_t vend = vaddr + ph->p_memsz; - if (vaddr < minv) minv = vaddr; - if (vend > maxv) maxv = vend; - } - if (minv != (uintptr_t)-1 && maxv > minv) { - ctx->libc_range->start = minv; - ctx->libc_range->end = maxv; - } - } - - if (ctx->pthread_fbase != NULL && (void*)info->dlpi_addr == ctx->pthread_fbase) { - uintptr_t minv = (uintptr_t)-1; - uintptr_t maxv = 0; - for (int i = 0; i < info->dlpi_phnum; i++) { - const ElfW(Phdr)* ph = &info->dlpi_phdr[i]; - if (ph->p_type != PT_LOAD) continue; - uintptr_t vaddr = (uintptr_t)info->dlpi_addr + ph->p_vaddr; - uintptr_t vend = vaddr + ph->p_memsz; - if (vaddr < minv) minv = vaddr; - if (vend > maxv) maxv = vend; - } - if (minv != (uintptr_t)-1 && maxv > minv) { - ctx->pthread_range->start = minv; - ctx->pthread_range->end = maxv; - } - } - - return 0; // continue iteration -} - -// Main program header - initialized lazily -static const void* _main_phdr = NULL; -static pthread_once_t _main_phdr_once = PTHREAD_ONCE_INIT; -static const char* _ld_base = (const char*)getauxval(AT_BASE); - -// Initialize main phdr once -static void init_main_phdr_once() { - UnifiedCtx ctx = {NULL, NULL, &_main_phdr, NULL, NULL, NULL, NULL}; - dl_iterate_phdr(&unified_phdr_cb, &ctx); -} - -// Ensure main phdr is initialized -static void ensure_main_phdr_initialized() { - pthread_once(&_main_phdr_once, init_main_phdr_once); -} - -static void init_lib_ranges_once() { - if (g_lib_ranges_inited) return; - g_lib_ranges_inited = true; - - // libc anchor: prefer gnu_get_libc_version if present; fallback to strlen - void* libc_sym = dlsym(RTLD_DEFAULT, "gnu_get_libc_version"); - if (!libc_sym) libc_sym = (void*)&strlen; - - Dl_info di = {0}; - void* libc_fbase = NULL; - if (dladdr(libc_sym, &di) && di.dli_fbase) { - libc_fbase = di.dli_fbase; - } - - // pthread anchor: pthread_create (on glibc >= 2.34 this lives in libc) - Dl_info di2 = {0}; - void* pthread_fbase = NULL; - if (dladdr((void*)&pthread_create, &di2) && di2.dli_fbase) { - pthread_fbase = di2.dli_fbase; - } - - // Use unified dl_iterate_phdr call to get all information at once - UnifiedCtx ctx = {NULL, NULL, &_main_phdr, libc_fbase, pthread_fbase, &g_libc, &g_libpthread}; - dl_iterate_phdr(&unified_phdr_cb, &ctx); - - // If pthread couldn't be resolved separately, treat it as libc - if (!range_valid(&g_libpthread)) g_libpthread = g_libc; -} - -static bool pc_in_range(uintptr_t pc, const Range* r) { - return range_valid(r) && pc >= r->start && pc < r->end; -} - -#ifdef __x86_64__ - -#include -#include "vmEntry.h" - -// Workaround for JDK-8312065 on JDK 8: -// replace poll() implementation with ppoll() which is restartable -static int poll_hook(struct pollfd* fds, nfds_t nfds, int timeout) { - if (timeout >= 0) { - struct timespec ts; - ts.tv_sec = timeout / 1000; - ts.tv_nsec = (timeout % 1000) * 1000000; - return ppoll(fds, nfds, &ts, NULL); - } else { - return ppoll(fds, nfds, NULL, NULL); - } -} - -static void applyPatch(CodeCache* cc) { - static bool patch_libnet = VM::hotspot_version() == 8; - - if (patch_libnet) { - size_t len = strlen(cc->name()); - if (len >= 10 && strcmp(cc->name() + len - 10, "/libnet.so") == 0) { - UnloadProtection handle(cc); - if (handle.isValid()) { - cc->patchImport(im_poll, (void*)poll_hook); - patch_libnet = false; - } - } - } -} - -#else - -static void applyPatch(CodeCache* cc) {} - -#endif - - -static bool isMainExecutable(const char* image_base, const void* map_end) { - ensure_main_phdr_initialized(); - return _main_phdr != NULL && _main_phdr >= image_base && _main_phdr < map_end; -} - -static bool isLoader(const char* image_base) { - return _ld_base == image_base; -} - -class SymbolDesc { - private: - const char* _addr; - const char* _desc; - - public: - SymbolDesc(const char* s) { - _addr = s; - _desc = strchr(_addr, ' '); - } - - const char* addr() { return (const char*)strtoul(_addr, NULL, 16); } - char type() { return _desc != NULL ? _desc[1] : 0; } - const char* name() { return _desc + 3; } -}; - -class MemoryMapDesc { - private: - const char* _addr; - const char* _end; - const char* _perm; - const char* _offs; - const char* _dev; - const char* _inode; - const char* _file; - - public: - MemoryMapDesc(const char* s) { - _addr = s; - _end = strchr(_addr, '-') + 1; - _perm = strchr(_end, ' ') + 1; - _offs = strchr(_perm, ' ') + 1; - _dev = strchr(_offs, ' ') + 1; - _inode = strchr(_dev, ' ') + 1; - _file = strchr(_inode, ' '); - - if (_file != NULL) { - while (*_file == ' ') _file++; - } - } - - const char* file() { return _file; } - bool isReadable() { return _perm[0] == 'r'; } - bool isExecutable() { return _perm[2] == 'x'; } - const char* addr() { return (const char*)strtoul(_addr, NULL, 16); } - const char* end() { return (const char*)strtoul(_end, NULL, 16); } - unsigned long offs() { return strtoul(_offs, NULL, 16); } - unsigned long inode() { return strtoul(_inode, NULL, 10); } - - unsigned long dev() { - char* colon; - unsigned long major = strtoul(_dev, &colon, 16); - unsigned long minor = strtoul(colon + 1, NULL, 16); - return major << 8 | minor; - } -}; - -struct SharedLibrary { - char* file; - const char* map_start; - const char* map_end; - const char* image_base; -}; - - -#ifdef __LP64__ -const unsigned char ELFCLASS_SUPPORTED = ELFCLASS64; -typedef Elf64_Ehdr ElfHeader; -typedef Elf64_Shdr ElfSection; -typedef Elf64_Phdr ElfProgramHeader; -typedef Elf64_Nhdr ElfNote; -typedef Elf64_Sym ElfSymbol; -typedef Elf64_Rel ElfRelocation; -typedef Elf64_Dyn ElfDyn; -#define ELF_R_TYPE ELF64_R_TYPE -#define ELF_R_SYM ELF64_R_SYM -#else -const unsigned char ELFCLASS_SUPPORTED = ELFCLASS32; -typedef Elf32_Ehdr ElfHeader; -typedef Elf32_Shdr ElfSection; -typedef Elf32_Phdr ElfProgramHeader; -typedef Elf32_Nhdr ElfNote; -typedef Elf32_Sym ElfSymbol; -typedef Elf32_Rel ElfRelocation; -typedef Elf32_Dyn ElfDyn; -#define ELF_R_TYPE ELF32_R_TYPE -#define ELF_R_SYM ELF32_R_SYM -#endif // __LP64__ - -#if defined(__x86_64__) -# define R_GLOB_DAT R_X86_64_GLOB_DAT -# define R_ABS64 R_X86_64_64 -#elif defined(__i386__) -# define R_GLOB_DAT R_386_GLOB_DAT -# define R_ABS64 -1 -#elif defined(__arm__) || defined(__thumb__) -# define R_GLOB_DAT R_ARM_GLOB_DAT -# define R_ABS64 -1 -#elif defined(__aarch64__) -# define R_GLOB_DAT R_AARCH64_GLOB_DAT -# define R_ABS64 R_AARCH64_ABS64 -#elif defined(__PPC64__) -# define R_GLOB_DAT R_PPC64_GLOB_DAT -# define R_ABS64 -1 -#elif defined(__riscv) && (__riscv_xlen == 64) -// RISC-V does not have GLOB_DAT relocation, use something neutral, -// like the impossible relocation number. -# define R_GLOB_DAT -1 -# define R_ABS64 -1 -#elif defined(__loongarch_lp64) -// LOONGARCH does not have GLOB_DAT relocation, use something neutral, -// like the impossible relocation number. -# define R_GLOB_DAT -1 -# define R_ABS64 -1 -#else -# error "Compiling on unsupported arch" -#endif - - -static char _debuginfod_cache_buf[PATH_MAX] = {0}; - -class ElfParser { - private: - CodeCache* _cc; - const char* _base; - const char* _file_name; - bool _relocate_dyn; - ElfHeader* _header; - const char* _sections; - const char* _vaddr_diff; - const char* _image_end; // one-past-the-end of the mapped ELF image; bounds file-relative reads - - ElfParser(CodeCache* cc, const char* base, const void* addr, size_t image_size, const char* file_name, bool relocate_dyn) { - _cc = cc; - _base = base; - _file_name = file_name; - _relocate_dyn = relocate_dyn; - _header = (ElfHeader*)addr; - _image_end = (const char*)addr + image_size; - // e_shoff sits at a fixed offset inside the header; only compute the pointer - // when the image is at least header-sized AND e_shoff is within the image, - // so the addition cannot overflow and sectionAt()/inImage() can reject it - // cleanly without UB. - _sections = (image_size >= sizeof(ElfHeader) && _header->e_shoff < image_size) - ? (const char*)addr + _header->e_shoff - : NULL; - } - - bool validHeader() { - // A valid ELF image is at least a full header; this also makes the - // e_ident / e_shstrndx reads below in-bounds for tiny inputs. - if (_image_end < (const char*)_header + sizeof(ElfHeader)) { - return false; - } - unsigned char* ident = _header->e_ident; - return ident[0] == 0x7f && ident[1] == 'E' && ident[2] == 'L' && ident[3] == 'F' - && ident[4] == ELFCLASS_SUPPORTED && ident[5] == ELFDATA2LSB && ident[6] == EV_CURRENT - && _header->e_shstrndx != SHN_UNDEF; - } - - // --- Bounds-checked accessors for the file/section path ----------------- - // These guard parsing of section headers, symbol tables and string tables, - // all of which use file-offset-relative pointers that must lie inside the - // mapped image [_header, _image_end). The dynamic-section path uses - // virtual-address-relative pointers into live memory and is intentionally - // NOT routed through inImage(). - - // True when [ptr, ptr+len) lies entirely within the mapped image. - bool inImage(const void* ptr, size_t len) const { - const char* p = (const char*)ptr; - return p >= (const char*)_header - && p <= _image_end - && len <= (size_t)(_image_end - p); - } - - // Section header at `index`, or NULL when the index or entry is out of bounds. - ElfSection* sectionAt(int index) { - if (_sections == NULL || index < 0 || index >= _header->e_shnum - || _header->e_shentsize < sizeof(ElfSection)) { - return NULL; - } - ElfSection* s = (ElfSection*)(_sections + (size_t)index * _header->e_shentsize); - return inImage(s, sizeof(ElfSection)) ? s : NULL; - } - - // Start of a section's first `want` content bytes, or NULL if not fully mapped. - const char* contentAt(ElfSection* s, size_t want) { - if (s == NULL) { - return NULL; - } - // Validate sh_offset in integer space before forming the pointer so that - // a large attacker-controlled offset cannot cause pointer-overflow UB - // (the project builds with -fsanitize=pointer-overflow -fno-sanitize-recover). - size_t img_size = (size_t)(_image_end - (const char*)_header); - if (s->sh_offset > img_size || want > img_size - s->sh_offset) { - return NULL; - } - return (const char*)_header + s->sh_offset; - } - - // NUL-terminated string at `off` within a [strtab, strtab+size) string table, - // or NULL if the offset is out of range or the string is not terminated in it. - static const char* strAt(const char* strtab, size_t size, uint32_t off) { - if (strtab == NULL || off >= size) { - return NULL; - } - if (memchr(strtab + off, '\0', size - off) == NULL) { - return NULL; - } - return strtab + off; - } - - // Program-header entry at `index`, or NULL when the index or entry is out of bounds. - ElfProgramHeader* phdrAt(int index) { - if (index < 0 || index >= _header->e_phnum - || _header->e_phentsize < sizeof(ElfProgramHeader)) { - return NULL; - } - // Validate entirely in integer space before forming any pointer. - // Both e_phoff and index*e_phentsize are attacker-controlled; either - // can be large enough to wrap a pointer under -fsanitize=pointer-overflow. - size_t img_size = (size_t)(_image_end - (const char*)_header); - size_t phoff = _header->e_phoff; - size_t stride = (size_t)index * _header->e_phentsize; - if (phoff > img_size || stride > img_size - phoff) { - return NULL; - } - ElfProgramHeader* ph = (ElfProgramHeader*)((const char*)_header + phoff + stride); - return inImage(ph, sizeof(ElfProgramHeader)) ? ph : NULL; - } - - const char* at(ElfProgramHeader* pheader) { - if (_header->e_type == ET_EXEC) { - return (const char*)pheader->p_vaddr; - } - return _vaddr_diff == NULL ? (const char*)pheader->p_vaddr : _vaddr_diff + pheader->p_vaddr; - } - - const char* base() { - return _header->e_type == ET_EXEC ? NULL : _vaddr_diff; - } - - char* dyn_ptr(ElfDyn* dyn) { - // GNU dynamic linker relocates pointers in the dynamic section, while musl doesn't. - // Also, [vdso] is not relocated, and its vaddr may differ from the load address. - if (_relocate_dyn || (_base != NULL && (char*)dyn->d_un.d_ptr < _base)) { - return _vaddr_diff == NULL ? (char*)dyn->d_un.d_ptr : (char*)_vaddr_diff + dyn->d_un.d_ptr; - } else { - return (char*)dyn->d_un.d_ptr; - } - } - - ElfSection* findSection(uint32_t type, const char* name); - ElfProgramHeader* findProgramHeader(uint32_t type); - - void calcVirtualLoadAddress(); - void parseDynamicSection(); - void parseDwarfInfo(); - uint32_t getSymbolCount(uint32_t* gnu_hash); - void loadSymbols(bool use_debug); - bool loadSymbolsFromDebug(const char* build_id, const int build_id_len); - bool loadSymbolsFromDebuginfodCache(const char* build_id, const int build_id_len); - bool loadSymbolsUsingBuildId(); - bool loadSymbolsUsingDebugLink(); - void loadSymbolTable(const char* symbols, size_t total_size, size_t ent_size, const char* strings, size_t strings_size); - void addRelocationSymbols(ElfSection* reltab, const char* plt); - const char* getDebuginfodCache(); - - public: - static void parseProgramHeaders(CodeCache* cc, const char* base, const char* end, bool relocate_dyn); - static bool parseFile(CodeCache* cc, const char* base, const char* file_name, bool use_debug); -}; - - -ElfSection* ElfParser::findSection(uint32_t type, const char* name) { - // The section-header string table must be present and fully mapped before - // any section name can be resolved. Untrusted e_shoff/e_shentsize/e_shstrndx - // and sh_offset values are all validated here. - ElfSection* shstr = sectionAt(_header->e_shstrndx); - if (shstr == NULL) { - return NULL; - } - size_t strtab_size = shstr->sh_size; - const char* strtab = contentAt(shstr, strtab_size); - if (strtab == NULL) { - return NULL; - } - - for (int i = 0; i < _header->e_shnum; i++) { - ElfSection* section = sectionAt(i); - if (section == NULL) { - continue; - } - if (section->sh_type == type && section->sh_name != 0) { - const char* sname = strAt(strtab, strtab_size, section->sh_name); - if (sname != NULL && strcmp(sname, name) == 0) { - return section; - } - } - } - - return NULL; -} - -ElfProgramHeader* ElfParser::findProgramHeader(uint32_t type) { - for (int i = 0; i < _header->e_phnum; i++) { - ElfProgramHeader* pheader = phdrAt(i); - if (pheader != NULL && pheader->p_type == type) { - return pheader; - } - } - return NULL; -} - -bool ElfParser::parseFile(CodeCache* cc, const char* base, const char* file_name, bool use_debug) { - int fd = open(file_name, O_RDONLY); - if (fd == -1) { - return false; - } - - size_t length = (size_t)lseek(fd, 0, SEEK_END); - void* addr = mmap(NULL, length, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - - if (addr == MAP_FAILED) { - Log::warn("Could not parse symbols from %s: %s", file_name, strerror(errno)); - } else { - ElfParser elf(cc, base, addr, length, file_name, false); - if (elf.validHeader()) { - elf.calcVirtualLoadAddress(); - elf.loadSymbols(use_debug); - } - munmap(addr, length); - } - return true; -} - -void ElfParser::parseProgramHeaders(CodeCache* cc, const char* base, const char* end, bool relocate_dyn) { - ElfParser elf(cc, base, base, (size_t)(end - base), NULL, relocate_dyn); - if (elf.validHeader() && base + elf._header->e_phoff < end) { - cc->setTextBase(base); - elf.calcVirtualLoadAddress(); - elf.parseDynamicSection(); - elf.parseDwarfInfo(); - } -} - -void ElfParser::calcVirtualLoadAddress() { - // Find a difference between the virtual load address (often zero) and the actual DSO base - if (_base == NULL) { - _vaddr_diff = NULL; - return; - } - for (int i = 0; i < _header->e_phnum; i++) { - ElfProgramHeader* pheader = phdrAt(i); - if (pheader != NULL && pheader->p_type == PT_LOAD) { - _vaddr_diff = _base - pheader->p_vaddr; - return; - } - } - _vaddr_diff = _base; -} - -void ElfParser::parseDynamicSection() { - ElfProgramHeader* dynamic = findProgramHeader(PT_DYNAMIC); - if (dynamic != NULL) { - const char* symtab = NULL; - const char* strtab = NULL; - char* jmprel = NULL; - char* rel = NULL; - size_t pltrelsz = 0; - size_t relsz = 0; - size_t relent = 0; - size_t relcount = 0; - size_t syment = 0; - size_t strsz = 0; - uint32_t nsyms = 0; - - const char* dyn_start = at(dynamic); - const char* dyn_end = dyn_start + dynamic->p_memsz; - for (ElfDyn* dyn = (ElfDyn*)dyn_start; dyn < (ElfDyn*)dyn_end; dyn++) { - switch (dyn->d_tag) { - case DT_SYMTAB: - symtab = dyn_ptr(dyn); - break; - case DT_STRTAB: - strtab = dyn_ptr(dyn); - break; - case DT_SYMENT: - syment = dyn->d_un.d_val; - break; - case DT_STRSZ: - strsz = dyn->d_un.d_val; - break; - case DT_HASH: - nsyms = ((uint32_t*)dyn_ptr(dyn))[1]; - break; - case DT_GNU_HASH: - if (nsyms == 0) { - nsyms = getSymbolCount((uint32_t*)dyn_ptr(dyn)); - } - break; - case DT_JMPREL: - jmprel = dyn_ptr(dyn); - break; - case DT_PLTRELSZ: - pltrelsz = dyn->d_un.d_val; - break; - case DT_RELA: - case DT_REL: - rel = dyn_ptr(dyn); - break; - case DT_RELASZ: - case DT_RELSZ: - relsz = dyn->d_un.d_val; - break; - case DT_RELAENT: - case DT_RELENT: - relent = dyn->d_un.d_val; - break; - case DT_RELACOUNT: - case DT_RELCOUNT: - relcount = dyn->d_un.d_val; - break; - } - } - - if (symtab == NULL || strtab == NULL || syment == 0 || relent == 0) { - return; - } - - // DT_STRSZ is required by the ELF spec whenever DT_STRTAB is present. - // When it is absent (strsz == 0) all string lookups via strAt() would - // be rejected, silently dropping every symbol. Cap to 1 MB: real dynamic - // string tables are well under that, and live linker memory guarantees - // NUL termination, so memchr will always find a terminator before the cap. - if (strsz == 0) { - Log::warn("DT_STRSZ absent from dynamic section in %s; capping string-table scan to 1 MB", - _file_name != NULL ? _file_name : "unknown"); - strsz = 1u << 20; - } - - if (!_cc->hasDebugSymbols() && nsyms > 0) { - loadSymbolTable(symtab, syment * nsyms, syment, strtab, strsz); - } - - const char* base = this->base(); - if (jmprel != NULL && pltrelsz != 0) { - // Parse .rela.plt table - for (size_t offs = 0; offs < pltrelsz; offs += relent) { - ElfRelocation* r = (ElfRelocation*)(jmprel + offs); - ElfSymbol* sym = (ElfSymbol*)(symtab + ELF_R_SYM(r->r_info) * syment); - if (sym->st_name != 0) { - const char* sym_name = strAt(strtab, strsz, sym->st_name); - if (sym_name != NULL) { - _cc->addImport((void**)(base + r->r_offset), sym_name); - } - } - } - } - - if (rel != NULL && relsz != 0) { - // Relocation entries for imports can be found in .rela.dyn, for example - // if a shared library is built without PLT (-fno-plt). However, if both - // entries exist, addImport saves them both. - for (size_t offs = relcount * relent; offs < relsz; offs += relent) { - ElfRelocation* r = (ElfRelocation*)(rel + offs); - if (ELF_R_TYPE(r->r_info) == R_GLOB_DAT || ELF_R_TYPE(r->r_info) == R_ABS64) { - ElfSymbol* sym = (ElfSymbol*)(symtab + ELF_R_SYM(r->r_info) * syment); - if (sym->st_name != 0) { - const char* sym_name = strAt(strtab, strsz, sym->st_name); - if (sym_name != NULL) { - _cc->addImport((void**)(base + r->r_offset), sym_name); - } - } - } - } - } - } -} - -void ElfParser::parseDwarfInfo() { - if (!DWARF_SUPPORTED) return; - - // Try SFrame first (simpler format, faster parsing, no opcode interpretation). - ElfProgramHeader* sframe_phdr = findProgramHeader(PT_GNU_SFRAME); - if (sframe_phdr != NULL && sframe_phdr->p_vaddr != 0) { - const char* section_base = at(sframe_phdr); - uintptr_t section_offset_full = static_cast(section_base - _base); - if (section_offset_full <= static_cast(UINT32_MAX)) { - u32 section_offset = static_cast(section_offset_full); - SFrameParser sframe(_cc->name(), section_base, - static_cast(sframe_phdr->p_filesz), section_offset); - if (sframe.parse()) { - _cc->setDwarfTable(sframe.table(), sframe.count(), - sframe.detectedDefaultFrame()); - return; - } - // SFrame parse failed; fall through to DWARF. - } else { - Log::warn("SFrame section offset too large for u32 in %s; falling back to DWARF", _cc->name()); - } - } - - // DWARF fallback (reached when SFrame is absent, offset too large, or parse failed). - ElfProgramHeader* eh_frame_hdr = findProgramHeader(PT_GNU_EH_FRAME); - if (eh_frame_hdr != NULL) { - if (eh_frame_hdr->p_vaddr != 0) { - // Parse per-PC frame descriptions and detect per-library default frame layout. - // On aarch64 this distinguishes GCC (LINKED_FRAME_SIZE=0) from clang - // (LINKED_FRAME_CLANG_SIZE=16) conventions for each shared library. - // Compute image_end from the highest end address of all LOAD segments so - // the DWARF parser can validate FDE pointers against mapped memory. - const char* image_end = _base; - for (int i = 0; i < _header->e_phnum; i++) { - ElfProgramHeader* ph = phdrAt(i); - if (ph != NULL && ph->p_type == PT_LOAD) { - const char* seg_end = at(ph) + ph->p_memsz; - if (seg_end > image_end) image_end = seg_end; - } - } - DwarfParser dwarf(_cc->name(), _base, at(eh_frame_hdr), eh_frame_hdr->p_memsz, - DwarfParser::EhFrameHdrTag{}, image_end); - _cc->setDwarfTable(dwarf.table(), dwarf.count(), dwarf.detectedDefaultFrame()); - } else if (strcmp(_cc->name(), "[vdso]") == 0) { - FrameDesc* table = (FrameDesc*)malloc(sizeof(FrameDesc)); - *table = FrameDesc::empty_frame; - _cc->setDwarfTable(table, 1); - } - } -} - -uint32_t ElfParser::getSymbolCount(uint32_t* gnu_hash) { - uint32_t nbuckets = gnu_hash[0]; - uint32_t* buckets = &gnu_hash[4] + gnu_hash[2] * (sizeof(size_t) / 4); - - uint32_t nsyms = 0; - for (uint32_t i = 0; i < nbuckets; i++) { - if (buckets[i] > nsyms) nsyms = buckets[i]; - } - - if (nsyms > 0) { - uint32_t* chain = &buckets[nbuckets] - gnu_hash[1]; - while (!(chain[nsyms++] & 1)); - } - return nsyms; -} - -void ElfParser::loadSymbols(bool use_debug) { - ElfSection* symtab = findSection(SHT_SYMTAB, ".symtab"); - if (symtab != NULL) { - // Parse debug symbols from the original .so. The symbol table and its - // linked string table are file-offset-relative, so every range is - // validated against the mapped image before it is read. - ElfSection* strtab = sectionAt(symtab->sh_link); - const char* symbols = contentAt(symtab, symtab->sh_size); - const char* strings = strtab != NULL ? contentAt(strtab, strtab->sh_size) : NULL; - if (symbols != NULL && strings != NULL) { - loadSymbolTable(symbols, symtab->sh_size, symtab->sh_entsize, strings, strtab->sh_size); - _cc->setDebugSymbols(true); - } - } else if (use_debug) { - // Try to load symbols from an external debuginfo library - loadSymbolsUsingBuildId() || loadSymbolsUsingDebugLink(); - } - - if (use_debug) { - // Synthesize names for PLT stubs - ElfSection* plt = findSection(SHT_PROGBITS, ".plt"); - if (plt != NULL) { - _cc->setPlt(plt->sh_addr, plt->sh_size); - ElfSection* reltab = findSection(SHT_RELA, ".rela.plt"); - if (reltab != NULL || (reltab = findSection(SHT_REL, ".rel.plt")) != NULL) { - addRelocationSymbols(reltab, base() + plt->sh_addr + PLT_HEADER_SIZE); - } - } - } -} - -const char* ElfParser::getDebuginfodCache() { - if (_debuginfod_cache_buf[0]) { - return _debuginfod_cache_buf; - } - - const char* env_vars[] = {"DEBUGINFOD_CACHE_PATH", "XDG_CACHE_HOME", "HOME"}; - const char* suffixes[] = {"/", "debuginfod_client/", ".cache/debuginfod_client/"}; - - for (size_t i = 0; i < sizeof(env_vars) / sizeof(env_vars[0]); i++) { - const char* env_val = getenv(env_vars[i]); - if (!env_val || !env_val[0]) { - continue; - } - - if (snprintf(_debuginfod_cache_buf, sizeof(_debuginfod_cache_buf), "%s/%s", env_val, suffixes[i]) < static_cast(sizeof(_debuginfod_cache_buf))) { - return _debuginfod_cache_buf; - } - } - - _debuginfod_cache_buf[0] = '\0'; - return _debuginfod_cache_buf; -} - -bool ElfParser::loadSymbolsFromDebug(const char* build_id, const int build_id_len) { - char path[PATH_MAX]; - char* p = path + snprintf(path, sizeof(path), "/usr/lib/debug/.build-id/%02hhx/", build_id[0]); - for (int i = 1; i < build_id_len; i++) { - p += snprintf(p, 3, "%02hhx", build_id[i]); - } - strcpy(p, ".debug"); - - return parseFile(_cc, _base, path, false); -} - -bool ElfParser::loadSymbolsFromDebuginfodCache(const char* build_id, const int build_id_len) { - const char* debuginfod_cache = getDebuginfodCache(); - if (debuginfod_cache == NULL || !debuginfod_cache[0]) { - return false; - } - - char path[PATH_MAX]; - const int debuginfod_cache_len = strlen(debuginfod_cache); - if (debuginfod_cache_len + build_id_len + strlen("/debuginfo") >= sizeof(path)) { - Log::warn("Path too long, skipping loading symbols: %s", debuginfod_cache); - return false; - } - - char* p = strcpy(path, debuginfod_cache); - p += debuginfod_cache_len; - for (int i = 0; i < build_id_len; i++) { - p += snprintf(p, 3, "%02hhx", build_id[i]); - } - strcpy(p, "/debuginfo"); - - return parseFile(_cc, _base, path, false); -} - -// Load symbols from the first file that exists in the following locations, in order, where abcdef1234 is Build ID. -// /usr/lib/debug/.build-id/ab/cdef1234.debug -// $DEBUGINFOD_CACHE_PATH/abcdef1234/debuginfo -// $XDG_CACHE_HOME/debuginfod_client/abcdef1234/debuginfo -// $HOME/.cache/debuginfod_client/abcdef1234/debuginfo -bool ElfParser::loadSymbolsUsingBuildId() { - ElfSection* section = findSection(SHT_NOTE, ".note.gnu.build-id"); - if (section == NULL || section->sh_size <= 16) { - return false; - } - - // The whole note section must be mapped before reading the note header. - const char* note_base = contentAt(section, section->sh_size); - if (note_base == NULL || section->sh_size < sizeof(ElfNote)) { - return false; - } - ElfNote* note = (ElfNote*)note_base; - if (note->n_namesz != 4 || note->n_descsz < 2 || note->n_descsz > 64) { - return false; - } - - // The descriptor (build-id bytes) follows the header and a 4-byte aligned - // "GNU\0" name; ensure it lies inside the note section. - size_t desc_off = sizeof(ElfNote) + 4; - if (desc_off + note->n_descsz > section->sh_size) { - return false; - } - const char* build_id = note_base + desc_off; - int build_id_len = note->n_descsz; - - return loadSymbolsFromDebug(build_id, build_id_len) - || loadSymbolsFromDebuginfodCache(build_id, build_id_len); -} - -// Look for debuginfo file specified in .gnu_debuglink section -bool ElfParser::loadSymbolsUsingDebugLink() { - ElfSection* section = findSection(SHT_PROGBITS, ".gnu_debuglink"); - if (section == NULL || section->sh_size <= 4) { - return false; - } - - // The debuglink is a NUL-terminated filename at the start of the section; - // validate it is mapped and terminated before it feeds strcmp()/snprintf(). - const char* debuglink = contentAt(section, section->sh_size); - if (debuglink == NULL || memchr(debuglink, '\0', section->sh_size) == NULL) { - return false; - } - - const char* basename = strrchr(_file_name, '/'); - if (basename == NULL) { - return false; - } - - char* dirname = strndup(_file_name, basename - _file_name); - if (dirname == NULL) { - return false; - } - - char path[PATH_MAX]; - bool result = false; - - // 1. /path/to/libjvm.so.debug - if (strcmp(debuglink, basename + 1) != 0 && - snprintf(path, PATH_MAX, "%s/%s", dirname, debuglink) < PATH_MAX) { - result = parseFile(_cc, _base, path, false); - } - - // 2. /path/to/.debug/libjvm.so.debug - if (!result && snprintf(path, PATH_MAX, "%s/.debug/%s", dirname, debuglink) < PATH_MAX) { - result = parseFile(_cc, _base, path, false); - } - - // 3. /usr/lib/debug/path/to/libjvm.so.debug - if (!result && snprintf(path, PATH_MAX, "/usr/lib/debug%s/%s", dirname, debuglink) < PATH_MAX) { - result = parseFile(_cc, _base, path, false); - } - - free(dirname); - return result; -} - -void ElfParser::loadSymbolTable(const char* symbols, size_t total_size, size_t ent_size, const char* strings, size_t strings_size) { - // A stride smaller than one symbol entry would never advance past (or would - // re-read) an entry; reject it to avoid an infinite loop / over-read. - if (ent_size < sizeof(ElfSymbol)) { - return; - } - const char* base = this->base(); - // Iterate by a size_t offset rather than incrementing the pointer: a huge - // attacker-controlled ent_size would otherwise overflow `symbols + ent_size` - // to a small pointer that still compares <= end, walking off the image. The - // `ent_size <= total_size - off` form keeps off <= total_size with no overflow. - for (size_t off = 0; ent_size <= total_size - off; off += ent_size) { - ElfSymbol* sym = (ElfSymbol*)(symbols + off); - if (sym->st_name != 0 && sym->st_value != 0) { - // Resolve the name through the bounded string table; a bad st_name - // offset (or unterminated string) drops the symbol instead of reading - // out of bounds. - const char* sym_name = strAt(strings, strings_size, sym->st_name); - if (sym_name == NULL) { - continue; - } - // Skip special AArch64 mapping symbols: $x and $d - if (sym->st_size != 0 || sym->st_info != 0 || sym_name[0] != '$') { - const char* addr; - if (base != NULL) { - // Check for overflow when adding sym->st_value to base - uintptr_t base_addr = (uintptr_t)base; - uint64_t symbol_value = sym->st_value; - - // Skip this symbol if addition would overflow - // First check if symbol_value exceeds the address space - if (symbol_value > UINTPTR_MAX) { - continue; - } - // Then check if addition would overflow - if (base_addr > UINTPTR_MAX - (uintptr_t)symbol_value) { - continue; - } - - // Perform addition using integer arithmetic to avoid pointer overflow - addr = (const char*)(base_addr + (uintptr_t)symbol_value); - } else { - addr = (const char*)sym->st_value; - } - _cc->add(addr, (int)sym->st_size, sym_name); - } - } - } -} - -void ElfParser::addRelocationSymbols(ElfSection* reltab, const char* plt) { - // Resolve and bounds-check the linked symbol and string tables. Any missing - // or out-of-image section aborts relocation naming rather than reading wild - // pointers built from attacker-controlled sh_link / r_info / sh_entsize. - ElfSection* symtab = sectionAt(reltab->sh_link); - ElfSection* strtab = symtab != NULL ? sectionAt(symtab->sh_link) : NULL; - if (symtab == NULL || strtab == NULL) { - return; - } - size_t sym_region = symtab->sh_size; - size_t strings_size = strtab->sh_size; - size_t sym_ent = symtab->sh_entsize; - size_t rel_ent = reltab->sh_entsize; - const char* symbols = contentAt(symtab, sym_region); - const char* strings = contentAt(strtab, strings_size); - size_t reltab_size = reltab->sh_size; - const char* relocations = contentAt(reltab, reltab_size); - if (symbols == NULL || strings == NULL || relocations == NULL - || rel_ent < sizeof(ElfRelocation) - || sym_ent < sizeof(ElfSymbol) - || sym_region < sizeof(ElfSymbol)) { - return; - } - - // Largest symbol index whose full ElfSymbol still fits in the table. Written - // as a division so the index * sym_ent product can never overflow. - size_t max_sym_index = (sym_region - sizeof(ElfSymbol)) / sym_ent; - - // Offset-based iteration (see loadSymbolTable) so a huge rel_ent cannot - // overflow the relocation pointer past the section end. - for (size_t off = 0; rel_ent <= reltab_size - off; off += rel_ent, plt += PLT_ENTRY_SIZE) { - ElfRelocation* r = (ElfRelocation*)(relocations + off); - if (ELF_R_SYM(r->r_info) > max_sym_index) { - continue; - } - ElfSymbol* sym = (ElfSymbol*)(symbols + (size_t)ELF_R_SYM(r->r_info) * sym_ent); - - char name[256]; - if (sym->st_name == 0) { - strcpy(name, "@plt"); - } else { - const char* sym_name = strAt(strings, strings_size, sym->st_name); - if (sym_name == NULL) { - continue; // plt advances via the for-increment - } - // sym_name is NUL-terminated within the string table, so sym_name[1] - // is safe to read (it is at worst the terminator). - char sep = sym_name[0] == '_' && sym_name[1] == 'Z' ? '.' : '@'; - snprintf(name, sizeof(name), "%s%cplt", sym_name, sep); - name[sizeof(name) - 1] = 0; - } - - _cc->add(plt, PLT_ENTRY_SIZE, name); - } -} - - -Mutex Symbols::_parse_lock; -bool Symbols::_have_kernel_symbols = false; -bool Symbols::_libs_limit_reported = false; -static std::unordered_set _parsed_inodes; -static bool _in_parse_libraries = false; - -void Symbols::parseKernelSymbols(CodeCache* cc) { - int fd; - if (FdTransferClient::hasPeer()) { - fd = FdTransferClient::requestKallsymsFd(); - } else { - fd = open("/proc/kallsyms", O_RDONLY); - } - - if (fd == -1) { - Log::warn("open(\"/proc/kallsyms\"): %s", strerror(errno)); - return; - } - - FILE* f = fdopen(fd, "r"); - if (f == NULL) { - Log::warn("fdopen(): %s", strerror(errno)); - close(fd); - return; - } - - char str[256]; - while (fgets(str, sizeof(str) - 8, f) != NULL) { - size_t len = strlen(str) - 1; // trim the '\n' - strcpy(str + len, "_[k]"); - - SymbolDesc symbol(str); - char type = symbol.type(); - if (type == 'T' || type == 't' || type == 'W' || type == 'w') { - const char* addr = symbol.addr(); - if (addr != NULL) { - if (!_have_kernel_symbols) { - if (strncmp(symbol.name(), "__LOAD_PHYSICAL_ADDR", 20) == 0 || - strncmp(symbol.name(), "phys_startup", 12) == 0) { - continue; - } - _have_kernel_symbols = true; - } - cc->add(addr, 0, symbol.name()); - } - } - } - - fclose(f); -} - -static void collectSharedLibraries(std::unordered_map& libs, int max_count) { - FILE* f = fopen("/proc/self/maps", "r"); - if (f == NULL) { - return; - } - - const char* image_base = NULL; - u64 last_inode = 0; - char* str = NULL; - size_t str_size = 0; - ssize_t len; - - while (max_count > 0 && (len = getline(&str, &str_size, f)) > 0) { - str[len - 1] = 0; - - MemoryMapDesc map(str); - if (!map.isReadable() || map.file() == NULL || map.file()[0] == 0) { - continue; - } - - u64 inode = u64(map.dev()) << 32 | map.inode(); - if (_parsed_inodes.find(inode) != _parsed_inodes.end()) { - continue; // shared object is already parsed - } - if (inode == 0 && strcmp(map.file(), "[vdso]") != 0) { - continue; // all shared libraries have inode, except vDSO - } - - const char* map_start = map.addr(); - const char* map_end = map.end(); - if (inode != last_inode && map.offs() == 0) { - image_base = map_start; - last_inode = inode; - } - - if (map.isExecutable()) { - SharedLibrary& lib = libs[inode]; - if (lib.file == nullptr) { - lib.file = strdup(map.file()); - lib.map_start = map_start; - lib.map_end = map_end; - lib.image_base = inode == last_inode ? image_base : NULL; - max_count--; - } else { - // The same library may have multiple executable segments mapped - lib.map_end = map_end; - } - } - } - - free(str); - fclose(f); -} - -void Symbols::parseLibraries(CodeCacheArray* array, bool kernel_symbols) { - MutexLocker ml(_parse_lock); - - if (_in_parse_libraries || array->count() >= MAX_NATIVE_LIBS) { - return; - } - _in_parse_libraries = true; - - if (kernel_symbols && !haveKernelSymbols()) { - CodeCache* cc = new CodeCache("[kernel]"); - parseKernelSymbols(cc); - - if (haveKernelSymbols()) { - cc->sort(); - if (!array->add(cc)) { - delete cc; - } - } else { - delete cc; - } - } - - std::unordered_map libs; - collectSharedLibraries(libs, MAX_NATIVE_LIBS - array->count()); - - for (auto& it : libs) { - u64 inode = it.first; - _parsed_inodes.insert(inode); - - SharedLibrary& lib = it.second; - CodeCache* cc = new CodeCache(lib.file, array->count(), lib.map_start, lib.map_end, lib.image_base); - - if (strchr(lib.file, ':') != NULL) { - // Do not try to parse pseudofiles like anon_inode:name, /memfd:name - } else if (strcmp(lib.file, "[vdso]") == 0) { - ElfParser::parseProgramHeaders(cc, lib.map_start, lib.map_end, true); - } else if (lib.image_base == NULL) { - // Unlikely case when image base has not been found: not safe to access program headers. - // Be careful: executable file is not always ELF, e.g. classes.jsa - TEST_LOG("parseLibraries: image_base==NULL for %s, skipping program headers", lib.file); - ElfParser::parseFile(cc, lib.map_start, lib.file, true); - } else { - // Parse debug symbols first - ElfParser::parseFile(cc, lib.image_base, lib.file, true); - - UnloadProtection handle(cc); - if (handle.isValid()) { - ElfParser::parseProgramHeaders(cc, lib.image_base, lib.map_end, OS::isMusl()); - } else { - TEST_LOG("parseLibraries: UnloadProtection invalid for %s, skipping program headers", lib.file); - } - } - - free(lib.file); - - cc->sort(); - applyPatch(cc); - if (!array->add(cc)) { - delete cc; - } - } - - if (array->count() >= MAX_NATIVE_LIBS && !_libs_limit_reported) { - Log::warn("Number of parsed libraries reached the limit of %d", MAX_NATIVE_LIBS); - _libs_limit_reported = true; - } - - _in_parse_libraries = false; -} - -// Check that the base address of the shared object has not changed -static bool verifyBaseAddress(const CodeCache* cc, void* lib_handle) { - Dl_info dl_info; - struct link_map* map; - - if (dlinfo(lib_handle, RTLD_DI_LINKMAP, &map) != 0 || dladdr(map->l_ld, &dl_info) == 0) { - return false; - } - - return cc->imageBase() == (const char*)dl_info.dli_fbase; -} - -UnloadProtection::UnloadProtection(const CodeCache *cc) { - if (OS::isMusl() || isMainExecutable(cc->imageBase(), cc->maxAddress()) || isLoader(cc->imageBase())) { - _lib_handle = NULL; - _valid = true; - return; - } - - // dlopen() can reopen previously loaded libraries even if the underlying file has been deleted - const char* stripped_name = cc->name(); - size_t name_len = strlen(stripped_name); - if (name_len > 10 && strcmp(stripped_name + name_len - 10, " (deleted)") == 0) { - char* buf = (char*) alloca(name_len - 9); - *stpncpy(buf, stripped_name, name_len - 10) = 0; - stripped_name = buf; - } - - // Protect library from unloading while parsing in-memory ELF program headers. - // Also, dlopen() ensures the library is fully loaded. - _lib_handle = dlopen(stripped_name, RTLD_LAZY | RTLD_NOLOAD); - _valid = _lib_handle != NULL && verifyBaseAddress(cc, _lib_handle); -} - -UnloadProtection::~UnloadProtection() { - if (_lib_handle != NULL) { - dlclose(_lib_handle); - } -} - -void Symbols::initLibraryRanges() { - init_lib_ranges_once(); -} - -bool Symbols::isLibcOrPthreadAddress(uintptr_t pc) { - // Fast, allocation-free integer checks — no strings involved. - // initLibraryRanges() must have been called during profiler startup. - if (pc_in_range(pc, &g_libc)) return true; - if (pc_in_range(pc, &g_libpthread)) return true; - return false; -} - - -// Implementation of clearParsingCaches for test compatibility -void Symbols::clearParsingCaches() { - _parsed_inodes.clear(); -} - -// GNU build-id extraction implementation -// -// The build-id is a unique identifier embedded in ELF binaries and shared libraries. -// It is stored in a PT_NOTE program header segment as an ELF note with type NT_GNU_BUILD_ID. -// -// References: -// - ELF Specification: https://refspecs.linuxfoundation.org/elf/elf.pdf -// - ELF Note Section: https://refspecs.linuxfoundation.org/LSB_5.0.0/LSB-Core-generic/LSB-Core-generic/noteobject.html -// - GNU build-id: https://fedoraproject.org/wiki/Releases/FeatureBuildId -// - GNU binutils ld --build-id: https://sourceware.org/binutils/docs/ld/Options.html -// - readelf(1) --notes: https://man7.org/linux/man-pages/man1/readelf.1.html -// -// Build-ID format: -// - Located in PT_NOTE program header segments (p_type == PT_NOTE) -// - Stored as ELF note with: -// - n_namesz = 4 (length of "GNU\0") -// - n_descsz = build-id length (typically 20 bytes for SHA1) -// - n_type = NT_GNU_BUILD_ID (3) -// - name = "GNU\0" -// - desc = build-id bytes -// - All fields are 4-byte aligned as per ELF note format - -// GNU build-id note constants -#define NT_GNU_BUILD_ID 3 -#define GNU_BUILD_ID_NAME "GNU" - -char* SymbolsLinux::extractBuildId(const char* file_path, size_t* build_id_len) { - if (!file_path || !build_id_len) { - return nullptr; - } - - int fd = open(file_path, O_RDONLY); - if (fd < 0) { - return nullptr; - } - - struct stat st; - if (fstat(fd, &st) < 0) { - close(fd); - return nullptr; - } - - void* elf_base = mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0); - close(fd); - - if (elf_base == MAP_FAILED) { - return nullptr; - } - - char* result = extractBuildIdFromMemory(elf_base, st.st_size, build_id_len); - - munmap(elf_base, st.st_size); - return result; -} - -char* SymbolsLinux::extractBuildIdFromMemory(const void* elf_base, size_t elf_size, size_t* build_id_len) { - if (!elf_base || !build_id_len || elf_size < sizeof(Elf64_Ehdr)) { - return nullptr; - } - - const Elf64_Ehdr* ehdr = static_cast(elf_base); - - // Verify ELF magic - if (memcmp(ehdr->e_ident, ELFMAG, SELFMAG) != 0) { - return nullptr; - } - - // Only handle 64-bit ELF for now - if (ehdr->e_ident[EI_CLASS] != ELFCLASS64) { - return nullptr; - } - - // Check if we have program headers - if (ehdr->e_phoff == 0 || ehdr->e_phnum == 0) { - return nullptr; - } - - // Verify program header table is within file bounds. Written as subtractions - // so a huge e_phoff cannot wrap the addition and slip past the check, which - // would leave `phdr` pointing outside the mapped image. - if (ehdr->e_phoff > elf_size || - ehdr->e_phnum * sizeof(Elf64_Phdr) > elf_size - ehdr->e_phoff) { - return nullptr; - } - - // Verify program header offset is properly aligned - if (ehdr->e_phoff % alignof(Elf64_Phdr) != 0) { - return nullptr; - } - - const char* base = static_cast(elf_base); - const Elf64_Phdr* phdr = reinterpret_cast(base + ehdr->e_phoff); - - // Search for PT_NOTE segments - for (int i = 0; i < ehdr->e_phnum; i++) { - if (phdr[i].p_type == PT_NOTE && phdr[i].p_filesz > 0) { - // Ensure note segment is within file bounds. Subtraction form avoids - // a u64 overflow in p_offset + p_filesz that would otherwise yield a - // wild note_data pointer passed to findBuildIdInNotes(). - if (phdr[i].p_offset > elf_size || - phdr[i].p_filesz > elf_size - phdr[i].p_offset) { - continue; - } - - const void* note_data = base + phdr[i].p_offset; - const uint8_t* build_id_bytes = findBuildIdInNotes(note_data, phdr[i].p_filesz, build_id_len); - - if (build_id_bytes) { - return buildIdToHex(build_id_bytes, *build_id_len); - } - } - } - - return nullptr; -} - -const uint8_t* SymbolsLinux::findBuildIdInNotes(const void* note_data, size_t note_size, size_t* build_id_len) { - const char* data = static_cast(note_data); - size_t offset = 0; - - // Parse ELF note entries within the PT_NOTE segment - // Each note has the structure: - // typedef struct { - // Elf64_Word n_namesz; // Length of name field (including null terminator) - // Elf64_Word n_descsz; // Length of descriptor (build-id bytes) - // Elf64_Word n_type; // Note type (NT_GNU_BUILD_ID == 3) - // // Followed by name (4-byte aligned) - // // Followed by descriptor (4-byte aligned) - // } Elf64_Nhdr; - // Reference: https://refspecs.linuxfoundation.org/LSB_5.0.0/LSB-Core-generic/LSB-Core-generic/noteobject.html - while (offset < note_size) { - // Ensure there is enough space for the note header itself - if (note_size - offset < sizeof(Elf64_Nhdr)) { - break; - } - - // Copy the note header into an aligned local: note_data is base + - // p_offset and p_offset is attacker-controlled, so dereferencing an - // Elf64_Nhdr* in place could be a misaligned load (UB, and a fault on - // alignment-strict architectures). The size check above guarantees the - // whole header is in bounds. - Elf64_Nhdr nhdr; - memcpy(&nhdr, data + offset, sizeof(nhdr)); - - // Calculate aligned sizes (4-byte alignment as per ELF spec). Promote to - // size_t before the +3 so a near-UINT32_MAX n_namesz/n_descsz cannot wrap - // to a small value and defeat the bounds checks below. - size_t name_size_aligned = (static_cast(nhdr.n_namesz) + 3) & ~static_cast(3); - size_t desc_size_aligned = (static_cast(nhdr.n_descsz) + 3) & ~static_cast(3); - - // Check bounds using subtraction to avoid overflow - size_t remaining = note_size - offset - sizeof(Elf64_Nhdr); - if (name_size_aligned > remaining) { - break; - } - remaining -= name_size_aligned; - if (desc_size_aligned > remaining) { - break; - } - - // Check if this is a GNU build-id note - if (nhdr.n_type == NT_GNU_BUILD_ID && nhdr.n_namesz > 0 && nhdr.n_descsz > 0) { - const char* name = data + offset + sizeof(Elf64_Nhdr); - - // Verify GNU build-id name (including null terminator) - if (nhdr.n_namesz == 4 && strncmp(name, GNU_BUILD_ID_NAME, 3) == 0 && name[3] == '\0') { - const uint8_t* desc = reinterpret_cast(data + offset + sizeof(Elf64_Nhdr) + name_size_aligned); - *build_id_len = nhdr.n_descsz; - return desc; - } - } - - offset += sizeof(Elf64_Nhdr) + name_size_aligned + desc_size_aligned; - } - - return nullptr; -} - -char* SymbolsLinux::buildIdToHex(const uint8_t* build_id_bytes, size_t byte_len) { - if (!build_id_bytes || byte_len == 0) { - return nullptr; - } - - // Allocate string for hex representation (2 chars per byte + null terminator) - char* hex_str = static_cast(malloc(byte_len * 2 + 1)); - if (!hex_str) { - return nullptr; - } - - for (size_t i = 0; i < byte_len; i++) { - snprintf(hex_str + i * 2, 3, "%02x", build_id_bytes[i]); - } - - hex_str[byte_len * 2] = '\0'; - return hex_str; -} - -#endif // __linux__ diff --git a/ddprof-lib/src/main/cpp/symbols_linux.h b/ddprof-lib/src/main/cpp/symbols_linux.h deleted file mode 100644 index 787ffa0da..000000000 --- a/ddprof-lib/src/main/cpp/symbols_linux.h +++ /dev/null @@ -1,54 +0,0 @@ -#ifndef _SYMBOLS_LINUX_H -#define _SYMBOLS_LINUX_H - -#include "symbols.h" - -/** - * Datadog-specific extensions to Linux symbol handling. - * Provides build-id extraction for remote symbolication support. - */ -class SymbolsLinux { -public: - /** - * Extract GNU build-id from ELF file on disk. - * Build-id is stored in .note.gnu.build-id section and provides - * unique identification for libraries/executables for remote symbolication. - * - * @param file_path Path to ELF file - * @param build_id_len Output parameter for build-id length in bytes - * @return Hex-encoded build-id string (caller must free), or NULL on error - */ - static char* extractBuildId(const char* file_path, size_t* build_id_len); - - /** - * Extract GNU build-id from ELF file already mapped in memory. - * - * @param elf_base Base address of mapped ELF file - * @param elf_size Size of mapped ELF file - * @param build_id_len Output parameter for build-id length in bytes - * @return Hex-encoded build-id string (caller must free), or NULL on error - */ - static char* extractBuildIdFromMemory(const void* elf_base, size_t elf_size, size_t* build_id_len); - -private: - /** - * Convert binary build-id to hex string. - * - * @param build_id_bytes Raw build-id bytes - * @param byte_len Length of raw build-id in bytes - * @return Hex-encoded string (caller must free) - */ - static char* buildIdToHex(const uint8_t* build_id_bytes, size_t byte_len); - - /** - * Parse ELF note section to find GNU build-id. - * - * @param note_data Start of note section data - * @param note_size Size of note section - * @param build_id_len Output parameter for build-id length - * @return Raw build-id bytes, or NULL if not found - */ - static const uint8_t* findBuildIdInNotes(const void* note_data, size_t note_size, size_t* build_id_len); -}; - -#endif // _SYMBOLS_LINUX_H diff --git a/ddprof-lib/src/main/cpp/symbols_macos.cpp b/ddprof-lib/src/main/cpp/symbols_macos.cpp deleted file mode 100644 index e09c0f149..000000000 --- a/ddprof-lib/src/main/cpp/symbols_macos.cpp +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifdef __APPLE__ - -#include -#include -#include -#include -#include -#include -#include "dwarf.h" -#include "symbols.h" -#include "log.h" - -UnloadProtection::UnloadProtection(const CodeCache *cc) { - // Protect library from unloading while parsing in-memory ELF program headers. - // Also, dlopen() ensures the library is fully loaded. - _lib_handle = dlopen(cc->name(), RTLD_LAZY | RTLD_NOLOAD); - _valid = _lib_handle != NULL; -} - -UnloadProtection::~UnloadProtection() { - if (_lib_handle != NULL) { - dlclose(_lib_handle); - } -} - -class MachOParser { - private: - CodeCache* _cc; - const mach_header* _image_base; - const char* _vmaddr_slide; - - static const char* add(const void* base, uint64_t offset) { - return (const char*)base + offset; - } - - void findSymbolPtrSection(const segment_command_64* sc, const section_64** section_ptr) { - const section_64* section = (const section_64*)add(sc, sizeof(segment_command_64)); - for (uint32_t i = 0; i < sc->nsects; i++) { - uint32_t section_type = section->flags & SECTION_TYPE; - if (section_type == S_NON_LAZY_SYMBOL_POINTERS) { - section_ptr[0] = section; - } else if (section_type == S_LAZY_SYMBOL_POINTERS) { - section_ptr[1] = section; - } - section++; - } - } - - const section_64* findSection(const segment_command_64* sc, const char* section_name) { - const section_64* section = (const section_64*)add(sc, sizeof(segment_command_64)); - for (uint32_t i = 0; i < sc->nsects; i++) { - if (strcmp(section->sectname, section_name) == 0) { - return section; - } - section++; - } - return NULL; - } - - void loadSymbols(const symtab_command* symtab, const char* link_base) { - const nlist_64* sym = (const nlist_64*)add(link_base, symtab->symoff); - const char* str_table = add(link_base, symtab->stroff); - bool debug_symbols = false; - - for (uint32_t i = 0; i < symtab->nsyms; i++) { - if ((sym->n_type & 0xee) == 0x0e && sym->n_value != 0) { - const char* addr = _vmaddr_slide + sym->n_value; - const char* name = str_table + sym->n_un.n_strx; - if (name[0] == '_') name++; - _cc->add(addr, 0, name); - debug_symbols = true; - } - sym++; - } - - _cc->setDebugSymbols(debug_symbols); - } - - void loadStubSymbols(const symtab_command* symtab, const dysymtab_command* dysymtab, - const section_64* stubs_section, const char* link_base) { - const nlist_64* sym = (const nlist_64*)add(link_base, symtab->symoff); - const char* str_table = add(link_base, symtab->stroff); - - const uint32_t* isym = (const uint32_t*)add(link_base, dysymtab->indirectsymoff) + stubs_section->reserved1; - uint32_t isym_count = stubs_section->size / stubs_section->reserved2; - const char* stubs_start = _vmaddr_slide + stubs_section->addr; - - for (uint32_t i = 0; i < isym_count; i++) { - if ((isym[i] & (INDIRECT_SYMBOL_LOCAL | INDIRECT_SYMBOL_ABS)) == 0) { - const char* name = str_table + sym[isym[i]].n_un.n_strx; - if (name[0] == '_') name++; - - char stub_name[256]; - snprintf(stub_name, sizeof(stub_name), "stub:%s", name); - _cc->add(stubs_start + i * stubs_section->reserved2, stubs_section->reserved2, stub_name); - } - } - - _cc->setPlt(stubs_section->addr, isym_count * stubs_section->reserved2); - } - - void loadImports(const symtab_command* symtab, const dysymtab_command* dysymtab, - const section_64* symbol_ptr_section, const char* link_base) { - const nlist_64* sym = (const nlist_64*)add(link_base, symtab->symoff); - const char* str_table = add(link_base, symtab->stroff); - - const uint32_t* isym = (const uint32_t*)add(link_base, dysymtab->indirectsymoff) + symbol_ptr_section->reserved1; - uint32_t isym_count = symbol_ptr_section->size / sizeof(void*); - void** slot = (void**)(_vmaddr_slide + symbol_ptr_section->addr); - - for (uint32_t i = 0; i < isym_count; i++) { - if ((isym[i] & (INDIRECT_SYMBOL_LOCAL | INDIRECT_SYMBOL_ABS)) == 0) { - const char* name = str_table + sym[isym[i]].n_un.n_strx; - if (name[0] == '_') name++; - _cc->addImport(&slot[i], name); - } - } - } - - public: - MachOParser(CodeCache* cc, const mach_header* image_base, const char* vmaddr_slide) : - _cc(cc), _image_base(image_base), _vmaddr_slide(vmaddr_slide) {} - - bool parse() { - if (_image_base->magic != MH_MAGIC_64) { - return false; - } - - const mach_header_64* header = (const mach_header_64*)_image_base; - const load_command* lc = (const load_command*)(header + 1); - - const char* link_base = NULL; - const section_64* symbol_ptr[2] = {NULL, NULL}; - const symtab_command* symtab = NULL; - const dysymtab_command* dysymtab = NULL; - const section_64* stubs_section = NULL; - const char* eh_frame = NULL; - size_t eh_frame_size = 0; - for (uint32_t i = 0; i < header->ncmds; i++) { - if (lc->cmd == LC_SEGMENT_64) { - const segment_command_64* sc = (const segment_command_64*)lc; - if (strcmp(sc->segname, "__TEXT") == 0) { - _cc->updateBounds(_image_base, add(_image_base, sc->vmsize)); - stubs_section = findSection(sc, "__stubs"); - const section_64* eh_frame_section = findSection(sc, "__eh_frame"); - if (eh_frame_section != NULL) { - eh_frame = _vmaddr_slide + eh_frame_section->addr; - eh_frame_size = eh_frame_section->size; - } - } else if (strcmp(sc->segname, "__LINKEDIT") == 0) { - link_base = _vmaddr_slide + sc->vmaddr - sc->fileoff; - } else if (strcmp(sc->segname, "__DATA") == 0 || strcmp(sc->segname, "__DATA_CONST") == 0) { - findSymbolPtrSection(sc, symbol_ptr); - } - } else if (lc->cmd == LC_SYMTAB) { - symtab = (const symtab_command*)lc; - } else if (lc->cmd == LC_DYSYMTAB) { - dysymtab = (const dysymtab_command*)lc; - } - lc = (const load_command*)add(lc, lc->cmdsize); - } - - if (symtab != NULL && link_base != NULL) { - loadSymbols(symtab, link_base); - - if (dysymtab != NULL) { - if (symbol_ptr[0] != NULL) loadImports(symtab, dysymtab, symbol_ptr[0], link_base); - if (symbol_ptr[1] != NULL) loadImports(symtab, dysymtab, symbol_ptr[1], link_base); - if (stubs_section != NULL) loadStubSymbols(symtab, dysymtab, stubs_section, link_base); - } - } - - if (DWARF_SUPPORTED && eh_frame != NULL && eh_frame_size > 0) { - DwarfParser dwarf(_cc->name(), _vmaddr_slide, eh_frame, eh_frame_size); - _cc->setDwarfTable(dwarf.table(), dwarf.count(), dwarf.detectedDefaultFrame()); - } else { - // No __eh_frame (clang compact-unwind-only libraries): fall back to the - // library-wide default frame. On aarch64, clang uses a different frame - // layout from GCC, so we must pass fallback_default_frame() rather than - // letting CodeCache keep its constructor default of FrameDesc::default_frame. - _cc->setDwarfTable(NULL, 0, FrameDesc::fallback_default_frame()); - } - - return true; - } -}; - - -Mutex Symbols::_parse_lock; -bool Symbols::_have_kernel_symbols = false; -bool Symbols::_libs_limit_reported = false; -static std::unordered_set _parsed_libraries; - -void Symbols::parseKernelSymbols(CodeCache* cc) { -} - -void Symbols::parseLibraries(CodeCacheArray* array, bool kernel_symbols) { - MutexLocker ml(_parse_lock); - uint32_t images = _dyld_image_count(); - - for (uint32_t i = 0; i < images; i++) { - const mach_header* image_base = _dyld_get_image_header(i); - if (image_base == NULL || !_parsed_libraries.insert(image_base).second) { - continue; // the library was already parsed - } - - int count = array->count(); - if (count >= MAX_NATIVE_LIBS) { - if (!_libs_limit_reported) { - Log::warn("Number of parsed libraries reached the limit of %d", MAX_NATIVE_LIBS); - _libs_limit_reported = true; - } - break; - } - - const char* path = _dyld_get_image_name(i); - const char* vmaddr_slide = (const char*)_dyld_get_image_vmaddr_slide(i); - - CodeCache* cc = new CodeCache(path, count); - cc->setTextBase(vmaddr_slide); - - UnloadProtection handle(cc); - if (handle.isValid()) { - MachOParser parser(cc, image_base, vmaddr_slide); - if (!parser.parse()) { - Log::warn("Could not parse symbols from %s", path); - } - cc->sort(); - if (!array->add(cc)) { - delete cc; - } - } else { - delete cc; - } - } -} - -void Symbols::initLibraryRanges() { - // No initialization needed on macOS -} - -bool Symbols::isLibcOrPthreadAddress(uintptr_t pc) { - return false; -} - -void Symbols::clearParsingCaches() { - _parsed_libraries.clear(); -} - -#endif // __APPLE__ diff --git a/ddprof-lib/src/main/cpp/thread.cpp b/ddprof-lib/src/main/cpp/thread.cpp deleted file mode 100644 index 16f482fc0..000000000 --- a/ddprof-lib/src/main/cpp/thread.cpp +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "thread.h" -#include "context_api.h" -#include "guards.h" -#include "otel_context.h" -#include "os.h" -#include -#include - -pthread_key_t ProfiledThread::_tls_key; -bool ProfiledThread::_tls_key_initialized = false; - -void ProfiledThread::initTLSKey() { - static pthread_once_t tls_initialized = PTHREAD_ONCE_INIT; - pthread_once(&tls_initialized, doInitTLSKey); -} - -void ProfiledThread::doInitTLSKey() { - pthread_key_create(&_tls_key, freeKey); - // Must be set AFTER pthread_key_create so signal handlers see a valid key. - // Store-release pairs with the acquire loads in currentSignalSafe() and release() - // to prevent hardware load-load reordering on weakly-ordered architectures (aarch64): - // a plain volatile write is not sufficient there. - __atomic_store_n(&_tls_key_initialized, true, __ATOMIC_RELEASE); -} - -inline void ProfiledThread::freeKey(void *key) { - ProfiledThread *tls_ref = (ProfiledThread *)(key); - if (tls_ref != NULL) { - SignalBlocker blocker; - delete tls_ref; - } -} - -void ProfiledThread::initCurrentThread() { - // JVMTI callback path - does NOT use buffer - // Allocate dedicated ProfiledThread for Java threads (not from buffer) - // This MUST happen here to prevent lazy allocation in signal handler - initTLSKey(); - - if (pthread_getspecific(_tls_key) != NULL) { - return; // Already initialized - } - - int tid = OS::threadId(); - ProfiledThread *tls = ProfiledThread::forTid(tid); - pthread_setspecific(_tls_key, (const void *)tls); -} - -void ProfiledThread::release() { - if (!__atomic_load_n(&_tls_key_initialized, __ATOMIC_ACQUIRE)) { - return; - } - pthread_key_t key = _tls_key; - ProfiledThread *tls = (ProfiledThread *)pthread_getspecific(key); - if (tls != NULL) { - SignalBlocker blocker; - pthread_setspecific(key, NULL); - delete tls; - } -} - -int ProfiledThread::currentTid() { - ProfiledThread *tls = current(); - if (tls != NULL) { - return tls->tid(); - } - return OS::threadId(); -} - -ProfiledThread *ProfiledThread::current() { - initTLSKey(); - - ProfiledThread *tls = (ProfiledThread *)pthread_getspecific(_tls_key); - if (tls == NULL) { - // Lazy allocation - safe since current() is never called from signal handlers - int tid = OS::threadId(); - tls = ProfiledThread::forTid(tid); - pthread_setspecific(_tls_key, (const void *)tls); - } - return tls; -} - -ProfiledThread *ProfiledThread::currentSignalSafe() { - // Signal-safe: never allocate, just return existing TLS or null. - // Use _tls_key_initialized instead of key != 0 because pthread_key_create - // can legitimately return key 0 (common on musl where keys start at 0). - return __atomic_load_n(&_tls_key_initialized, __ATOMIC_ACQUIRE) ? (ProfiledThread *)pthread_getspecific(_tls_key) : nullptr; -} - - -Context ProfiledThread::snapshotContext(size_t numAttrs) { - Context ctx = {}; - u64 span_id = 0, root_span_id = 0; - if (ContextApi::get(span_id, root_span_id)) { - ctx.spanId = span_id; - ctx.rootSpanId = root_span_id; - size_t count = numAttrs < DD_TAGS_CAPACITY ? numAttrs : DD_TAGS_CAPACITY; - for (size_t i = 0; i < count; i++) { - ctx.tags[i].value = _otel_tag_encodings[i]; - } - } - return ctx; -} diff --git a/ddprof-lib/src/main/cpp/thread.h b/ddprof-lib/src/main/cpp/thread.h deleted file mode 100644 index a15cf8fc1..000000000 --- a/ddprof-lib/src/main/cpp/thread.h +++ /dev/null @@ -1,301 +0,0 @@ -/* - * Copyright 2025, 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _THREAD_H -#define _THREAD_H - -#include "context.h" -#include "otel_context.h" -#include "os.h" -#include "threadLocalData.h" -#include "threadState.h" -#include "unwindStats.h" -#include -#include -#include -#include -#include -#include -#include - -class ProfiledThread : public ThreadLocalData { -public: - enum ThreadType : u32 { - TYPE_UNKNOWN = 0, - TYPE_JAVA_THREAD = 0x1, - TYPE_NOT_JAVA_THREAD = 0x2, - TYPE_MASK = TYPE_JAVA_THREAD | TYPE_NOT_JAVA_THREAD - }; - - static constexpr u32 FLAG_PARKED = 0x4u; // next free bit after TYPE_MASK (0x1|0x2) - -private: - // We are allowing several levels of nesting because we can be - // eg. in a crash handler when wallclock signal kicks in, - // catching sigseg while also triggering CPU signal handler - // which would also potentially trigger sigseg we need to handle. - // This means 3 levels but we allow for some wiggling space, just in case. - // Even with 5 levels cap we will need any highly recursing signal handlers - static constexpr u32 CRASH_HANDLER_NESTING_LIMIT = 5; - static pthread_key_t _tls_key; - static bool _tls_key_initialized; - - static void initTLSKey(); - static void doInitTLSKey(); - static inline void freeKey(void *key); - - u64 _pc; - u64 _sp; - u64 _span_id; // Wall-clock collapsing cache: last-seen span ID (not a context store — read from _otel_ctx_record on each signal, cached here to detect "same as last time") - volatile u32 _crash_depth; - int _tid; - u32 _cpu_epoch; - u32 _wall_epoch; - u64 _call_trace_id; - u32 _recording_epoch; - u32 _misc_flags; - u64 _park_block_token; - int _filter_slot_id; // Slot ID for thread filtering - uint8_t _init_window; // Countdown for JVM thread init race window (PROF-13072) - uint8_t _signal_depth; // Nested signal-handler depth (see SignalHandlerScope) - UnwindFailures _unwind_failures; - bool _otel_ctx_initialized; - bool _crash_protection_active; - // alignas(8) + sizeof(OtelThreadContextRecord)==640 (multiple of 8) guarantee - // _otel_tag_encodings sits at +640 with no padding, so the three fields form one - // 688-byte contiguous region exposed as a combined DirectByteBuffer. - alignas(8) OtelThreadContextRecord _otel_ctx_record; - // These two fields MUST be contiguous and 8-byte aligned — the JNI layer - // exposes them as a single DirectByteBuffer (sidecar), and VarHandle long - // views require 8-byte alignment for the buffer base address. - // Read invariant: sidecar readers must gate on record->valid (see ContextApi::get). - // ThreadContext.restore() relies on this to perform a bulk memcpy under valid=0. - alignas(8) u32 _otel_tag_encodings[DD_TAGS_CAPACITY]; - u64 _otel_local_root_span_id; - - ProfiledThread(int tid) - : ThreadLocalData(), _pc(0), _sp(0), _span_id(0), _crash_depth(0), _tid(tid), _cpu_epoch(0), - _wall_epoch(0), _call_trace_id(0), _recording_epoch(0), _misc_flags(0), - _park_block_token(0), _filter_slot_id(-1), _init_window(0), - _signal_depth(0), - _otel_ctx_initialized(false), _crash_protection_active(false), - _otel_ctx_record{}, _otel_tag_encodings{}, _otel_local_root_span_id(0) {}; - - virtual ~ProfiledThread() { } -public: - static ProfiledThread *forTid(int tid) { return new ProfiledThread(tid); } - - static void initCurrentThread(); - static void release(); -#ifdef UNIT_TEST - // Simulates the moment inside release() after pthread_setspecific(NULL) but - // before delete — the race window the clearCurrentThreadTLS fix covers. - // Returns the detached pointer so the caller can delete it after assertions. - static ProfiledThread* clearCurrentThreadTLS() { - if (__atomic_load_n(&_tls_key_initialized, __ATOMIC_ACQUIRE)) { - ProfiledThread *pt = (ProfiledThread *)pthread_getspecific(_tls_key); - pthread_setspecific(_tls_key, nullptr); - return pt; - } - return nullptr; - } - // Deletes a ProfiledThread returned by clearCurrentThreadTLS(). - // Needed because the destructor is private. - static void deleteForTest(ProfiledThread *pt) { delete pt; } -#endif - - static ProfiledThread *current(); - static ProfiledThread *currentSignalSafe(); // Signal-safe version that never allocates - static int currentTid(); - - inline int tid() { return _tid; } - - inline u64 noteCPUSample(u32 recording_epoch) { - _recording_epoch = recording_epoch; - return ++_cpu_epoch; - } - - /** - * Attempts to reuse a cached call trace ID for wallclock sample collapsing. - * Collapsing is allowed only when the execution state (PC, SP) and trace - * context (spanId, rootSpanId) are identical to the previous sample. - * - * @param pc Program counter from ucontext - * @param sp Stack pointer from ucontext - * @param recording_epoch Current profiling session epoch - * @param context_valid True if the OTEP valid flag was set; controls whether _otel_local_root_span_id is updated - * @param span_id Current trace span ID - * @param root_span_id Current trace root span ID - * @return Cached call_trace_id if collapsing is allowed, 0 otherwise - */ - u64 lookupWallclockCallTraceId(u64 pc, u64 sp, u32 recording_epoch, - bool context_valid, u64 span_id, u64 root_span_id) { - if (_pc == pc && _sp == sp && _span_id == span_id && - _otel_local_root_span_id == root_span_id && _recording_epoch == recording_epoch && - _call_trace_id != 0) { - return _call_trace_id; - } - _pc = pc; - _sp = sp; - _span_id = span_id; - // Only update the sidecar when context is valid (valid=1). If the signal fires - // between detach() and attach() in Java, ContextApi::get returns valid=0 with - // root_span_id=0; writing that would clobber the value Java just stored. - if (context_valid) { - // Plain store is safe: naturally-aligned u64 stores/loads are atomic on - // x86-64 and aarch64 (the only supported targets). The Java writer uses - // sidecarBuffer.putLong() which is a single aligned 8-byte store. - _otel_local_root_span_id = root_span_id; - } - _recording_epoch = recording_epoch; - return 0; - } - - inline void recordCallTraceId(u64 call_trace_id) { - _call_trace_id = call_trace_id; - } - - // this is called in the crash handler to avoid recursing - bool enterCrashHandler() { - u32 prev = _crash_depth; - // This is thread local; no need for atomic cmpxchg - if (prev < CRASH_HANDLER_NESTING_LIMIT) { - _crash_depth++; - return true; - } - return false; - } - - // needs to be called when the crash handler exits - void exitCrashHandler() { - // failsafe check - do not attempt to decrement if there are no crash handlers on stack - if (_crash_depth > 0) _crash_depth--; - } - - void resetCrashHandler() { - _crash_depth = 0; - } - - bool isDeepCrashHandler() { - return _crash_depth > CRASH_HANDLER_NESTING_LIMIT; - } - - // Signal-handler depth counter used by SignalHandlerScope (guards.h). All - // access happens on the owning thread (signal handlers are delivered to the - // thread that's interrupted), so plain reads/writes are AS-safe — no locks, - // no malloc, no syscalls. See guards.h for the public API. - inline uint8_t signalDepth() const { return _signal_depth; } - inline void enterSignalScope() { ++_signal_depth; } - inline void exitSignalScope() { if (_signal_depth > 0) --_signal_depth; } - - UnwindFailures* unwindFailures(bool reset = true) { - if (reset) { - _unwind_failures.clear(); - } - return &_unwind_failures; - } - - int filterSlotId() { return _filter_slot_id; } - void setFilterSlotId(int slotId) { _filter_slot_id = slotId; } - - // JVM thread init race window (PROF-13072): skip at most one signal that fires - // between Profiler::registerThread() and the JVM's pd_set_thread() call. - // Pure native threads (e.g. NativeThreadCreator) also see nullptr from - // JVMThread::current(), so the window auto-expires after one skip, allowing - // their subsequent samples through. - inline bool inInitWindow() const { return _init_window > 0; } - inline void startInitWindow() { _init_window = 1; } - inline void tickInitWindow() { if (_init_window > 0) --_init_window; } - - // Signal handler reentrancy protection - bool tryEnterCriticalSection() { - // Uses GCC atomic builtin (no malloc, async-signal-safe) - bool expected = false; - return __atomic_compare_exchange_n(&_in_critical_section, &expected, true, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); - } - void exitCriticalSection() { - // Uses GCC atomic builtin (no malloc, async-signal-safe) - __atomic_store_n(&_in_critical_section, false, __ATOMIC_RELEASE); - } - - // Context TLS (OTEP #4947) - inline void markContextInitialized() { - _otel_ctx_initialized = true; - } - - inline bool isContextInitialized() { - return _otel_ctx_initialized; - } - - inline OtelThreadContextRecord* getOtelContextRecord() { - return &_otel_ctx_record; - } - - // CAS RMW to update only TYPE_MASK bits without clobbering FLAG_PARKED, which - // is managed independently by the Java park hooks on the owning thread. - inline void setJavaThread(bool is_java) { - const u32 type_bits = is_java ? static_cast(TYPE_JAVA_THREAD) : static_cast(TYPE_NOT_JAVA_THREAD); - u32 cur = __atomic_load_n(&_misc_flags, __ATOMIC_RELAXED); - u32 desired; - do { - desired = (cur & ~static_cast(TYPE_MASK)) | type_bits; - } while (!__atomic_compare_exchange_n(&_misc_flags, &cur, desired, - /*weak=*/true, - __ATOMIC_ACQ_REL, __ATOMIC_RELAXED)); - } - - inline enum ThreadType threadType() const { - u32 flags = __atomic_load_n(&_misc_flags, __ATOMIC_ACQUIRE); - return static_cast(flags & TYPE_MASK); - } - - inline bool isCrashProtectionActive() const { return _crash_protection_active; } - inline void setCrashProtectionActive(bool active) { _crash_protection_active = active; } - - // JFR tag encoding sidecar — populated by JNI thread, read by signal handler - // (flightRecorder.cpp writeCurrentContext / wallClock.cpp collapsing). - inline u32* getOtelTagEncodingsPtr() { return _otel_tag_encodings; } - inline u32 getOtelTagEncoding(u32 idx) const { - return idx < DD_TAGS_CAPACITY ? _otel_tag_encodings[idx] : 0; - } - inline u64 getOtelLocalRootSpanId() const { return _otel_local_root_span_id; } - - inline void clearOtelSidecar() { - memset(_otel_tag_encodings, 0, sizeof(_otel_tag_encodings)); - _otel_local_root_span_id = 0; - } - - inline bool parkEnter() { - u32 prev = __atomic_fetch_or(&_misc_flags, FLAG_PARKED, __ATOMIC_RELEASE); - return (prev & FLAG_PARKED) == 0; - } - - inline void setParkBlockToken(u64 token) { - _park_block_token = token; - } - - // Returns false if the thread was not parked (idempotent). - inline bool parkExit(u64 &park_block_token) { - u32 prev = __atomic_fetch_and(&_misc_flags, ~FLAG_PARKED, __ATOMIC_ACQ_REL); - if ((prev & FLAG_PARKED) == 0) { - return false; - } - park_block_token = _park_block_token; - _park_block_token = 0; - return true; - } - - Context snapshotContext(size_t numAttrs); - -private: - // Atomic flag for signal handler reentrancy protection within the same thread - // Must be atomic because a signal handler can interrupt normal execution mid-instruction, - // and both contexts may attempt to enter the critical section. Without atomic exchange(), - // both could see the flag as false and both would think they successfully entered. - // The atomic exchange() is uninterruptible, ensuring only one context succeeds. - bool _in_critical_section{false}; -}; - -#endif // _THREAD_H diff --git a/ddprof-lib/src/main/cpp/threadFilter.cpp b/ddprof-lib/src/main/cpp/threadFilter.cpp deleted file mode 100644 index a189be3dc..000000000 --- a/ddprof-lib/src/main/cpp/threadFilter.cpp +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -// High-performance lock-free thread filter implementation -// -// PERFORMANCE CONTRACT: -// - add(), remove(), accept() are optimized for signal-safe hot paths -// - These methods assume slot_id comes from registerThread() (undefined behavior otherwise) - -#include "threadFilter.h" -#include "arch.h" -#include "os.h" -#include "thread.h" -#include -#include -#include -#include -#include -#include - -ThreadFilter::ShardHead ThreadFilter::_free_heads[ThreadFilter::kShardCount] {}; - -ThreadFilter::ThreadFilter() : _enabled(false) { - // Initialize chunk pointers to null (lazy allocation) - for (int i = 0; i < kMaxChunks; ++i) { - _chunks[i].store(nullptr, std::memory_order_relaxed); - } - _free_list = std::make_unique(kFreeListSize); - - // Initialize the first chunk - initializeChunk(0); - // ordering is fine because we are not enabled yet - initFreeList(); -} - -// This function is not called as we keep the profiler alive -// Memory orders should be adjusted if we want to unload the profiler -// This could have a perf impact on reading chunk variables. -ThreadFilter::~ThreadFilter() { - // Make the filter inert for any concurrent readers - _enabled.store(false, std::memory_order_release); - // Reset free-list heads and nodes first - for (int s = 0; s < kShardCount; ++s) { - _free_heads[s].head.store(-1, std::memory_order_relaxed); - } - for (int i = 0; i < kFreeListSize; ++i) { - _free_list[i].value.store(-1, std::memory_order_relaxed); - _free_list[i].next.store(-1, std::memory_order_relaxed); - } - // Publish 0 chunks to stop range scans (collect) - _num_chunks.store(0, std::memory_order_release); - // Detach and delete chunks - for (int i = 0; i < kMaxChunks; ++i) { - ChunkStorage* chunk = _chunks[i].exchange(nullptr, std::memory_order_acquire); - delete chunk; - } -} - -void ThreadFilter::initializeChunk(int chunk_idx) { - if (chunk_idx >= kMaxChunks) return; - - // Check if chunk already exists - ChunkStorage* existing = _chunks[chunk_idx].load(std::memory_order_acquire); - if (existing != nullptr) return; - - // Allocate and initialize new chunk completely before swapping - ChunkStorage* new_chunk = new ChunkStorage(); - for (auto& slot : new_chunk->slots) { - slot.value.store(-1, std::memory_order_relaxed); - slot.active_block_state.store(OSThreadState::UNKNOWN, std::memory_order_relaxed); - } - - // Try to install it atomically - ChunkStorage* expected = nullptr; - if (_chunks[chunk_idx].compare_exchange_strong(expected, new_chunk, std::memory_order_release)) { - // Successfully installed - } else { - // Another thread beat us to it - clean up our allocation - delete new_chunk; - } -} - -ThreadFilter::SlotID ThreadFilter::registerThread() { - // If disabled, block new registrations - if (!_enabled.load(std::memory_order_acquire)) { - return -1; - } - - // First, try to get a slot from the free list (lock-free stack) - SlotID reused_slot = popFromFreeList(); - if (reused_slot >= 0) { - return reused_slot; - } - - // Allocate a new slot - SlotID index = _next_index.fetch_add(1, std::memory_order_relaxed); - if (index >= kMaxThreads) { - // Revert the increment and return failure - _next_index.fetch_sub(1, std::memory_order_relaxed); - return -1; - } - - const int chunk_idx = index >> kChunkShift; - - // Ensure the chunk is initialized (lock-free) - if (chunk_idx >= _num_chunks.load(std::memory_order_acquire)) { - // Update the chunk count atomically - int expected_chunks = chunk_idx; - int desired_chunks = chunk_idx + 1; - while (!_num_chunks.compare_exchange_weak(expected_chunks, desired_chunks, - std::memory_order_acq_rel)) { - if (expected_chunks > chunk_idx) { - break; // Another thread already updated it - } - desired_chunks = expected_chunks + 1; - } - } - - // Initialize the chunk if needed - initializeChunk(chunk_idx); - - return index; -} - -void ThreadFilter::initFreeList() { - // Initialize the free list storage - for (int i = 0; i < kFreeListSize; ++i) { - _free_list[i].value.store(-1, std::memory_order_relaxed); - _free_list[i].next.store(-1, std::memory_order_relaxed); - } - - // Reset the free heads for each shard - for (int s = 0; s < kShardCount; ++s) { - _free_heads[s].head.store(-1, std::memory_order_relaxed); - } -} - -bool ThreadFilter::accept(SlotID slot_id) const { - // Fast path: if disabled, accept everything (relaxed to avoid fences on hot path) - if (unlikely(!_enabled.load(std::memory_order_relaxed))) { - return true; - } - if (unlikely(slot_id < 0)) return false; - - int chunk_idx = slot_id >> kChunkShift; - int slot_idx = slot_id & kChunkMask; - - // This is not a fast path like the add operation. - ChunkStorage* chunk = _chunks[chunk_idx].load(std::memory_order_acquire); - if (likely(chunk != nullptr)) { - return chunk->slots[slot_idx].value.load(std::memory_order_relaxed) != -1; - } - return false; -} - -void ThreadFilter::add(int tid, SlotID slot_id) { - // PRECONDITION: slot_id must be from registerThread() or negative - // Undefined behavior for invalid positive slot_ids (performance optimization) - if (slot_id < 0) return; - - int chunk_idx = slot_id >> kChunkShift; - int slot_idx = slot_id & kChunkMask; - - // Fast path: assume valid slot_id from registerThread() - ChunkStorage* chunk = _chunks[chunk_idx].load(std::memory_order_acquire); - if (likely(chunk != nullptr)) { - chunk->slots[slot_idx].value.store(tid, std::memory_order_release); - } -} - -void ThreadFilter::remove(SlotID slot_id) { - // PRECONDITION: slot_id must be from registerThread() or negative - // Undefined behavior for invalid positive slot_ids (performance optimization) - if (slot_id < 0) return; - - int chunk_idx = slot_id >> kChunkShift; - int slot_idx = slot_id & kChunkMask; - - if (unlikely(chunk_idx >= kMaxChunks)) { - assert(false && "Invalid slot_id in ThreadFilter::remove - should not happen after wall clock fix"); - return; - } - - ChunkStorage* chunk = _chunks[chunk_idx].load(std::memory_order_acquire); - if (unlikely(chunk == nullptr)) { - return; - } - - chunk->slots[slot_idx].value.store(-1, std::memory_order_release); -} - -void ThreadFilter::unregisterThread(SlotID slot_id) { - if (slot_id < 0) return; - remove(slot_id); - resetSlotRunState(slot_id); - pushToFreeList(slot_id); -} - -bool ThreadFilter::pushToFreeList(SlotID slot_id) { - // Lock-free sharded Treiber stack push - const int shard = shardOfSlot(slot_id); - auto& head = _free_heads[shard].head; // private cache-line - - for (int i = 0; i < kFreeListSize; ++i) { - int expected = -1; - if (_free_list[i].value.compare_exchange_strong( - expected, slot_id, std::memory_order_acq_rel)) { - // Link node into this shard’s Treiber stack - int old_head; - do { - old_head = head.load(std::memory_order_acquire); - _free_list[i].next.store(old_head, std::memory_order_relaxed); - } while (!head.compare_exchange_weak(old_head, i, - std::memory_order_release, std::memory_order_relaxed)); - return true; - } - } - return false; // Free list full, slot is lost but this is rare -} - -ThreadFilter::SlotID ThreadFilter::popFromFreeList() { - // Lock-free sharded Treiber stack pop - int hash = static_cast(std::hash{}(std::this_thread::get_id())); - int start = shardOf(hash); - - for (int pass = 0; pass < kShardCount; ++pass) { - int s = (start + pass) & (kShardCount - 1); - auto& head = _free_heads[s].head; - - while (true) { - int node = head.load(std::memory_order_acquire); - if (node == -1) break; // shard empty → try next - - int next = _free_list[node].next.load(std::memory_order_relaxed); - if (head.compare_exchange_weak(node, next, - std::memory_order_release, - std::memory_order_relaxed)) - { - int id = _free_list[node].value.exchange(-1, - std::memory_order_relaxed); - _free_list[node].next.store(-1, std::memory_order_relaxed); - return id; - } - } - } - return -1; // Empty list -} - -void ThreadFilter::collect(std::vector& tids) const { - tids.clear(); - - // Reserve space for efficiency - // The eventual resize is not the bottleneck, so we reserve a reasonable size - tids.reserve(512); - - // Scan only initialized chunks - int num_chunks = _num_chunks.load(std::memory_order_relaxed); - for (int chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) { - ChunkStorage* chunk = _chunks[chunk_idx].load(std::memory_order_acquire); - if (chunk == nullptr) { - continue; // Skip unallocated chunks - } - - for (const auto& slot : chunk->slots) { - int slot_tid = slot.value.load(std::memory_order_relaxed); - if (slot_tid != -1) { - tids.push_back(slot_tid); - } - } - } - - // Optional: shrink if we over-reserved significantly - if (tids.capacity() > tids.size() * 2) { - tids.shrink_to_fit(); - } -} - -void ThreadFilter::collect(std::vector& entries) const { - entries.clear(); - entries.reserve(512); - - int num_chunks = _num_chunks.load(std::memory_order_relaxed); - for (int chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) { - ChunkStorage* chunk = _chunks[chunk_idx].load(std::memory_order_acquire); - if (chunk == nullptr) { - continue; - } - - for (auto& slot : chunk->slots) { - int slot_tid = slot.value.load(std::memory_order_acquire); - if (slot_tid != -1) { - entries.push_back({slot_tid, &slot}); - } - } - } -} - -void ThreadFilter::clearActive() { - int num_chunks = _num_chunks.load(std::memory_order_acquire); - for (int chunk_idx = 0; chunk_idx < num_chunks; ++chunk_idx) { - ChunkStorage* chunk = _chunks[chunk_idx].load(std::memory_order_acquire); - if (chunk == nullptr) { - continue; - } - - for (auto& slot : chunk->slots) { - slot.value.store(-1, std::memory_order_release); - slot.clearActiveBlockRun(OSThreadState::UNKNOWN); - } - } -} - -void ThreadFilter::resetSlotRunState(SlotID slot_id) { - if (slot_id < 0) return; - int chunk_idx = slot_id >> kChunkShift; - int slot_idx = slot_id & kChunkMask; - ChunkStorage* chunk = _chunks[chunk_idx].load(std::memory_order_acquire); - if (chunk != nullptr) { - // Clear stale suppression state so a new thread in this slot cannot inherit - // its predecessor's active block or once-per-run sampled marker. - chunk->slots[slot_idx].clearActiveBlockRun(OSThreadState::UNKNOWN); - } -} - -u64 ThreadFilter::enterBlockedRun(SlotID slot_id, OSThreadState state, - BlockRunOwner owner) { - Slot* s = slotForId(slot_id); - if (s != nullptr) { - u32 generation = 0; - if (!s->trySetActiveBlockRun(state, owner, &generation)) { - return 0; - } - return encodeBlockRunToken(slot_id, generation); - } - return 0; -} - -void ThreadFilter::exitBlockedRun(SlotID slot_id) { - Slot* s = slotForId(slot_id); - if (s != nullptr) { - s->clearActiveBlockRun(OSThreadState::RUNNABLE); - } -} - -bool ThreadFilter::exitBlockedRun(SlotID slot_id, u32 generation) { - Slot* s = slotForId(slot_id); - if (s == nullptr || generation == 0 || s->blockGeneration() != generation) { - return false; - } - s->clearActiveBlockRun(OSThreadState::RUNNABLE); - return true; -} - -void ThreadFilter::init(const char* filter) { - // Simple logic: any filter value (including "0") enables filtering - // Only explicitly registered threads via addThread() will be sampled - // Previously we had a syntax where we could manually force some thread IDs. - // This is no longer supported. - _enabled.store(filter != nullptr && strlen(filter) > 0, std::memory_order_release); -} - -bool ThreadFilter::enabled() const { - return _enabled.load(std::memory_order_acquire); -} diff --git a/ddprof-lib/src/main/cpp/threadFilter.h b/ddprof-lib/src/main/cpp/threadFilter.h deleted file mode 100644 index 541249e4c..000000000 --- a/ddprof-lib/src/main/cpp/threadFilter.h +++ /dev/null @@ -1,265 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#ifndef _THREADFILTER_H -#define _THREADFILTER_H - -#include -#include -#include -#include -#include - -#include "arch.h" -#include "threadState.h" - -struct ThreadEntry; // defined after ThreadFilter; carries a pointer to a ThreadFilter::Slot - -enum class BlockRunOwner : int { - NONE = 0, - JAVA = 1, - JVMTI = 2, - NATIVE = 3, -}; - -class ThreadFilter { -public: - using SlotID = int; - - // Optimized limits for reasonable memory usage - static constexpr int kChunkSize = 256; - static constexpr int kChunkShift = 8; // log2(256) - static constexpr int kChunkMask = kChunkSize - 1; - static constexpr int kMaxThreads = 2048; - static constexpr int kMaxChunks = (kMaxThreads + kChunkSize - 1) / kChunkSize; // = 8 chunks - // High-performance free list using Treiber stack, 64 shards - static constexpr int kFreeListSize = 1024; // power-of-two for fast modulo - static constexpr int kShardCount = 64; // power-of-two for fast modulo - - // One cache line per slot to avoid false sharing. Slot instances are never freed - // (ChunkStorage is process-lifetime), so a captured Slot* is always dereferenceable. - struct alignas(DEFAULT_CACHE_LINE_SIZE) Slot { - static constexpr u64 kUnownedBlockedFallbackRatio = 10; - - std::atomic unowned_blocked_pending_weight{0}; - std::atomic unowned_blocked_decision_count{0}; - std::atomic unowned_blocked_call_trace_id{0}; - std::atomic unowned_blocked_state{OSThreadState::UNKNOWN}; - std::atomic value{-1}; - std::atomic active_block_owner{static_cast(BlockRunOwner::NONE)}; - std::atomic block_generation{0}; - // Wall-clock once-per-run suppression state. The signal handler records the - // last sampled blocked state; the signal handler and timer thread read it to - // suppress duplicate samples, while lifecycle/block-exit paths reset it. - // Release/acquire on sampled_this_run pairs with relaxed last_sampled_state, - // following the standard flag+payload pattern. - std::atomic last_sampled_state{OSThreadState::UNKNOWN}; // 4 bytes - // Set by explicit block enter/exit hooks. It lets the timer skip sending a signal - // only while instrumentation still owns a suppressible blocking interval. - std::atomic active_block_state{OSThreadState::UNKNOWN}; - std::atomic sampled_this_run{false}; - char padding[2 * DEFAULT_CACHE_LINE_SIZE - - sizeof(std::atomic) - - sizeof(std::atomic) - - sizeof(std::atomic) - - sizeof(std::atomic) - - sizeof(std::atomic) - - sizeof(std::atomic) - - sizeof(std::atomic) - - sizeof(std::atomic) - - sizeof(std::atomic) - - sizeof(std::atomic)]; - - inline bool sampledThisRun() const { - return sampled_this_run.load(std::memory_order_acquire); - } - inline OSThreadState lastSampledState() const { - return last_sampled_state.load(std::memory_order_relaxed); - } - inline void markSampledThisRun(OSThreadState state) { - last_sampled_state.store(state, std::memory_order_relaxed); - sampled_this_run.store(true, std::memory_order_release); - } - inline void resetSampledRun(OSThreadState state) { - resetUnownedBlockedSampling(); - last_sampled_state.store(state, std::memory_order_relaxed); - sampled_this_run.store(false, std::memory_order_release); - } - inline OSThreadState activeBlockState() const { - return active_block_state.load(std::memory_order_acquire); - } - inline void setActiveBlockState(OSThreadState state) { - active_block_state.store(state, std::memory_order_release); - } - inline BlockRunOwner activeBlockOwner() const { - return static_cast(active_block_owner.load(std::memory_order_acquire)); - } - inline u32 blockGeneration() const { - return block_generation.load(std::memory_order_acquire); - } - inline void resetUnownedBlockedSampling() { - unowned_blocked_pending_weight.store(0, std::memory_order_relaxed); - unowned_blocked_decision_count.store(0, std::memory_order_relaxed); - unowned_blocked_state.store(OSThreadState::UNKNOWN, std::memory_order_relaxed); - unowned_blocked_call_trace_id.store(0, std::memory_order_release); - } - inline bool shouldRecordUnownedBlockedSample() { - u64 decision = unowned_blocked_decision_count.fetch_add(1, std::memory_order_relaxed) + 1; - if ((decision % kUnownedBlockedFallbackRatio) == 1) { - return true; - } - unowned_blocked_pending_weight.fetch_add(1, std::memory_order_relaxed); - return false; - } - inline u64 consumeUnownedBlockedWeight() { - return unowned_blocked_pending_weight.exchange(0, std::memory_order_relaxed) + 1; - } - inline void restoreUnownedBlockedWeight(u64 weight) { - if (weight > 1) { - unowned_blocked_pending_weight.fetch_add(weight - 1, std::memory_order_relaxed); - } - } - inline void recordUnownedBlockedSample(u64 call_trace_id, OSThreadState state) { - unowned_blocked_state.store(state, std::memory_order_relaxed); - unowned_blocked_call_trace_id.store(call_trace_id, std::memory_order_release); - } - inline bool flushUnownedBlockedTail(u64& call_trace_id, u64& weight, - OSThreadState& state) { - call_trace_id = unowned_blocked_call_trace_id.exchange(0, std::memory_order_acq_rel); - weight = unowned_blocked_pending_weight.exchange(0, std::memory_order_relaxed); - state = unowned_blocked_state.exchange(OSThreadState::UNKNOWN, std::memory_order_relaxed); - unowned_blocked_decision_count.store(0, std::memory_order_relaxed); - if (call_trace_id == 0 || weight == 0 || state == OSThreadState::UNKNOWN) { - return false; - } - return true; - } - inline bool trySetActiveBlockRun(OSThreadState state, BlockRunOwner owner, - u32* generation_out) { - int expected_owner = static_cast(BlockRunOwner::NONE); - if (!active_block_owner.compare_exchange_strong( - expected_owner, static_cast(owner), std::memory_order_acq_rel, - std::memory_order_acquire)) { - return false; - } - u32 generation = block_generation.fetch_add(1, std::memory_order_acq_rel) + 1; - resetUnownedBlockedSampling(); - last_sampled_state.store(OSThreadState::UNKNOWN, std::memory_order_relaxed); - sampled_this_run.store(false, std::memory_order_relaxed); - active_block_state.store(state, std::memory_order_release); - *generation_out = generation; - return true; - } - inline void clearActiveBlockRun(OSThreadState state) { - active_block_state.store(OSThreadState::UNKNOWN, std::memory_order_release); - resetSampledRun(state); - active_block_owner.store(static_cast(BlockRunOwner::NONE), std::memory_order_release); - } - }; - static_assert(sizeof(Slot) == 2 * DEFAULT_CACHE_LINE_SIZE, "Slot must be exactly two cache lines"); - static_assert(std::atomic::is_always_lock_free, - "Slot OSThreadState fields must be lock-free for signal-handler safety"); - static_assert(std::atomic::is_always_lock_free, - "Slot::sampled_this_run must be lock-free for signal-handler safety"); - - ThreadFilter(); - ~ThreadFilter(); - - void init(const char* filter); - void initFreeList(); - bool enabled() const; - // Hot path methods - slot_id MUST be from registerThread(), undefined behavior otherwise - bool accept(SlotID slot_id) const; - void add(int tid, SlotID slot_id); - void remove(SlotID slot_id); - void collect(std::vector& tids) const; - void collect(std::vector& entries) const; - // Clears per-recording membership and suppression state while keeping - // process-lifetime slot ownership intact. Threads must opt in again with add(). - void clearActive(); - void resetSlotRunState(SlotID slot_id); - u64 enterBlockedRun(SlotID slot_id, OSThreadState state, - BlockRunOwner owner = BlockRunOwner::JAVA); - // Unconditional cleanup for reset/unregister paths only. Normal block - // lifecycles must use the generation-checked overload so they cannot clear - // another owner. - void exitBlockedRun(SlotID slot_id); - bool exitBlockedRun(SlotID slot_id, u32 generation); - - static inline u64 encodeBlockRunToken(SlotID slot_id, u32 generation) { - return (static_cast(generation) << 32) | static_cast(slot_id + 1); - } - static inline SlotID tokenSlotId(u64 token) { - return static_cast(static_cast(token) - 1); - } - static inline u32 tokenGeneration(u64 token) { - return static_cast(token >> 32); - } - - // Returns nullptr if slot_id is invalid or its chunk has not been allocated. - inline Slot* slotForId(SlotID slot_id) const { - if (slot_id < 0) return nullptr; - int chunk_idx = slot_id >> kChunkShift; - int slot_idx = slot_id & kChunkMask; - if (chunk_idx >= kMaxChunks) return nullptr; - ChunkStorage* chunk = _chunks[chunk_idx].load(std::memory_order_acquire); - return chunk != nullptr ? &chunk->slots[slot_idx] : nullptr; - } - - SlotID registerThread(); - void unregisterThread(SlotID slot_id); - -private: - - // Lock-free free list using a stack-like structure - struct FreeListNode { - std::atomic value{-1}; - std::atomic next{-1}; - }; - - // Pre-allocated chunk storage to eliminate mutex contention - struct ChunkStorage { - std::array slots; - }; - - std::atomic _enabled{false}; - - // Lazily allocated storage for chunks - std::atomic _chunks[kMaxChunks]; - std::atomic _num_chunks{1}; - - // Lock-free slot allocation - std::atomic _next_index{0}; - std::unique_ptr _free_list; - - // Cache line aligned to prevent false sharing between shards - struct alignas(DEFAULT_CACHE_LINE_SIZE) ShardHead { std::atomic head{-1}; }; - static ShardHead _free_heads[kShardCount]; // one cache-line each - - static inline int shardOf(int tid) { return tid & (kShardCount - 1); } - static inline int shardOfSlot(int s){ return s & (kShardCount - 1); } - // Helper methods for lock-free operations - void initializeChunk(int chunk_idx); - bool pushToFreeList(SlotID slot_id); - SlotID popFromFreeList(); -}; - -// Snapshot entry produced by ThreadFilter::collect for the wall-clock timer. -struct ThreadEntry { - int tid; - ThreadFilter::Slot* slot; -}; - -#endif // _THREADFILTER_H diff --git a/ddprof-lib/src/main/cpp/threadIdTable.h b/ddprof-lib/src/main/cpp/threadIdTable.h deleted file mode 100644 index 03a278c1f..000000000 --- a/ddprof-lib/src/main/cpp/threadIdTable.h +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2025 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#ifndef _THREADIDTABLE_H -#define _THREADIDTABLE_H - -#include -#include -#include "arch.h" - -// Simple fixed size thread ID table -class ThreadIdTable { -private: - // We have 256 slots per concurrency level (currently 16) - // This should cater for 4096 threads - if it turns out to be too small, we - // can increase it or make it configurable - static const int TABLE_SIZE = 256; // power of 2 - static const int TABLE_MASK = TABLE_SIZE - 1; // For fast bit masking - std::atomic table[TABLE_SIZE]; - - int hash(int tid) const { - // Improved hash function with bit mixing to reduce clustering - unsigned int utid = static_cast(tid); - utid ^= utid >> 16; // Mix high and low bits - return utid & TABLE_MASK; // Fast bit masking instead of modulo - } - -public: - ThreadIdTable() { - clear(); - } - - // Signal-safe insertion using atomic operations only - void insert(int tid) { - if (unlikely(tid == 0)) return; // Invalid thread ID, 0 is reserved for empty slots - - int start_slot = hash(tid); - for (int probe = 0; probe < TABLE_SIZE; probe++) { - int slot = (start_slot + probe) & TABLE_MASK; // Fast bit masking - int expected = 0; - - // Try to claim empty slot - if (table[slot].compare_exchange_strong(expected, tid, std::memory_order_relaxed)) { - return; // Successfully inserted - } - - // Check if already present (common case - threads insert multiple times) - if (likely(table[slot].load(std::memory_order_relaxed) == tid)) { - return; // Already exists - } - } - // Table full - thread ID will be lost, but this is rare and non-critical - // Could increment a counter here for diagnostics if needed - } - - // Clear the table (not signal-safe, called during buffer switch) - void clear() { - for (int i = 0; i < TABLE_SIZE; i++) { - table[i].store(0, std::memory_order_relaxed); - } - } - - // Collect all thread IDs into a set (not signal-safe, called during buffer switch) - void collect(std::unordered_set& result) { - for (int i = 0; i < TABLE_SIZE; i++) { - int tid = table[i].load(std::memory_order_relaxed); - if (tid != 0) { - result.insert(tid); - } - } - } -}; - -#endif // _THREADIDTABLE_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/threadInfo.cpp b/ddprof-lib/src/main/cpp/threadInfo.cpp deleted file mode 100644 index c2b80ca4c..000000000 --- a/ddprof-lib/src/main/cpp/threadInfo.cpp +++ /dev/null @@ -1,97 +0,0 @@ -#include "threadInfo.h" -#include "counters.h" -#include "mutex.h" - -void ThreadInfo::set(int tid, const char *name, u64 java_thread_id) { - MutexLocker ml(_ti_lock); - _thread_names[tid] = std::string(name); - _thread_ids[tid] = java_thread_id; -} - -std::pair, u64> ThreadInfo::get(int threadId) { - MutexLocker ml(_ti_lock); - auto it = _thread_names.find(threadId); - if (it != _thread_names.end()) { - return std::make_pair(std::make_shared(it->second), - _thread_ids[threadId]); - } - return std::make_pair(nullptr, 0); -} - -int ThreadInfo::getThreadId(int threadId) { - MutexLocker ml(_ti_lock); - auto it = _thread_ids.find(threadId); - if (it != _thread_ids.end()) { - return it->second; - } - return -1; -} - -void ThreadInfo::clearAll() { - MutexLocker ml(_ti_lock); - _thread_names.clear(); - _thread_ids.clear(); -} - -void ThreadInfo::clearAll(std::set &live_thread_ids) { - // Reset thread names and IDs - MutexLocker ml(_ti_lock); - if (live_thread_ids.empty()) { - // take the fast path - _thread_names.clear(); - _thread_ids.clear(); - } else { - // we need to honor the thread referenced from the liveness tracker - std::map::iterator name_itr = _thread_names.begin(); - while (name_itr != _thread_names.end()) { - if (live_thread_ids.find(name_itr->first) == live_thread_ids.end()) { - name_itr = _thread_names.erase(name_itr); - } else { - ++name_itr; - } - } - std::map::iterator id_itr = _thread_ids.begin(); - while (id_itr != _thread_ids.end()) { - if (live_thread_ids.find(id_itr->first) == live_thread_ids.end()) { - id_itr = _thread_ids.erase(id_itr); - } else { - ++id_itr; - } - } - } -} - -int ThreadInfo::size() { - MutexLocker ml(_ti_lock); - return _thread_names.size(); -} - -void ThreadInfo::updateThreadName( - int tid, std::function resolver) { - // Fast path: bail out if the name is already known, holding the lock only - // for the lookup. - { - MutexLocker ml(_ti_lock); - if (_thread_names.find(tid) != _thread_names.end()) { - return; - } - } - // Resolve OUTSIDE the lock: the resolver may perform blocking I/O (e.g. - // reading /proc/self/task//comm). Holding _ti_lock across that would - // stall every concurrent set/get/clearAll caller for the duration of the - // syscall, once per unknown tid. - std::string name = resolver(tid); - if (name.empty()) { - return; - } - // emplace is a no-op if a concurrent caller inserted this tid in the - // meantime, so the brief unlocked window is harmless. - MutexLocker ml(_ti_lock); - _thread_names.emplace(tid, std::move(name)); -} - -void ThreadInfo::reportCounters() { - MutexLocker ml(_ti_lock); - Counters::set(THREAD_IDS_COUNT, _thread_ids.size()); - Counters::set(THREAD_NAMES_COUNT, _thread_names.size()); -} \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/threadInfo.h b/ddprof-lib/src/main/cpp/threadInfo.h deleted file mode 100644 index 9fab8d2f5..000000000 --- a/ddprof-lib/src/main/cpp/threadInfo.h +++ /dev/null @@ -1,36 +0,0 @@ -#include "mutex.h" -#include "os.h" -#include -#include -#include -#include -#include - -class ThreadInfo { -private: - Mutex _ti_lock; - std::map _thread_names; - std::map _thread_ids; - -public: - // disallow copy and assign to avoid issues with the mutex - ThreadInfo(const ThreadInfo &) = delete; - ThreadInfo &operator=(const ThreadInfo &) = delete; - - ThreadInfo() {} - - void set(int tid, const char *name, u64 java_thread_id); - std::pair, u64> get(int tid); - - void updateThreadName(int tid, std::function resolver); - - int size(); - - void clearAll(std::set &live_thread_ids); - void clearAll(); - - void reportCounters(); - - // For testing - int getThreadId(int threadId); -}; diff --git a/ddprof-lib/src/main/cpp/threadLocal.h b/ddprof-lib/src/main/cpp/threadLocal.h deleted file mode 100644 index 47c632696..000000000 --- a/ddprof-lib/src/main/cpp/threadLocal.h +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _THREADLOCAL_H -#define _THREADLOCAL_H - -#include -#include -#include -#include "arch.h" - -/** - * This file implements an alternative to C/C++ thread local. - * Due to some restrictions of the language implementations, especially, on musl/aarch64, - * they cannot be safely used in profiler. - * - * How to use? - * A ThreadLocal should be declared as a static variable, e.g. - * - * static void* create_my_object() { - * return new MyObject(); - * } - * - * static void delete_my_object(void* p) { - * MyObject* obj = (MyObject*)p; - * delete obj; - * } - * - * static ThreadLocal var; - * MyObject* value = var.get(); - * ... - * var.clear(); - * ... - * MyObject* new_value = new MyObject(); - * var.set(new_value); - * ... - * var.clear(); - * - */ - -// The function to create value if it does not exist -typedef void* (*CREATE_FUNC)(void); -// Cleanup the value when deleting the key -typedef void (*CLEAN_FUNC)(void*); -template -class ThreadLocal { -protected: - pthread_key_t _key; - bool _key_valid; - -public: - ThreadLocal(const ThreadLocal&) = delete; - ThreadLocal& operator=(const ThreadLocal&) = delete; - - ThreadLocal() { - static_assert(sizeof(T) == sizeof(void*), - "ThreadLocal requires sizeof(T)==sizeof(void*); use a pointer type or add a specialization"); - _key_valid = pthread_key_create(&_key, F) == 0; - // What to do if we can not create a key? - // We probably want to shutdown profiler gracefully, instead of - // aborting user application - We will need this mechanism globally, - // defer to a separate task. - assert(_key_valid); - } - - ~ThreadLocal() { - if(_key_valid) { - pthread_key_delete(_key); - } else { - assert(false && "Invalid pthread key"); - } - } - - /** - * set(nullptr) will result in the value being recreated when get() is called - * when CREATE_FUNC is not nullptr. - * Note: caller is responsible to free old value, which mirrors thread_local - */ - void set(T value) { - assert(_key_valid && "Invalid pthread key"); - int err = pthread_setspecific(_key, reinterpret_cast(value)); - assert(err == 0); - } - - T get() { - assert(_key_valid && "Invalid pthread key"); - void* p = pthread_getspecific(_key); - if (p == nullptr && C != nullptr) { - p = C(); - set((T)p); - } - return reinterpret_cast(p); - } - - // Clear the value - void clear() { - assert(_key_valid && "Invalid pthread key"); - void* p = pthread_getspecific(_key); - if (p == nullptr) return; - int err = pthread_setspecific(_key, nullptr); - // Safety: if reset the value failed, get() can see staled value if - // it is freed. - if (err == 0 && F != nullptr) { - F(p); - } - } -}; - -template <> -class ThreadLocal { -protected: - pthread_key_t _key; - bool _key_valid; - -public: - ThreadLocal(const ThreadLocal&) = delete; - ThreadLocal& operator=(const ThreadLocal&) = delete; - - ThreadLocal() { - // Only support 64-bit platforms, double and void* are the same size - static_assert(sizeof(void*) == 8); - static_assert(sizeof(double) == 8); - _key_valid = pthread_key_create(&_key, nullptr) == 0; - // What to do if we can not create a key? - assert(_key_valid && "Invalid pthread key"); - } - - ~ThreadLocal() { - if(_key_valid) { - pthread_key_delete(_key); - } else { - assert(_key_valid && "Invalid pthread key"); - } - } - - // double <--> u64 cast, preserve bit format - // Can use std::bit_cast after upgrade C++ version to 20 - void set(double value) { - assert(_key_valid && "Invalid pthread key"); - u64 val; - memcpy(&val, &value, sizeof(value)); - int err = pthread_setspecific(_key, reinterpret_cast(val)); - assert(err == 0); - } - - double get() { - assert(_key_valid && "Invalid pthread key"); - void* p = pthread_getspecific(_key); - if (p == nullptr) { - return 0.0; - } - - u64 val = reinterpret_cast(p); - double value; - memcpy(&value, &val, sizeof(val)); - return value; - } - - void clear() { - assert(_key_valid && "Invalid pthread key"); - int err = pthread_setspecific(_key, nullptr); - assert(err == 0); - } -}; - -#endif // _THREADLOCAL_H diff --git a/ddprof-lib/src/main/cpp/threadLocalData.h b/ddprof-lib/src/main/cpp/threadLocalData.h deleted file mode 100644 index 4f348f298..000000000 --- a/ddprof-lib/src/main/cpp/threadLocalData.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifndef THREADLOCALDATA_H -#define THREADLOCALDATA_H - -class ThreadLocalData { -protected: - bool _unwinding_Java; - -public: - ThreadLocalData() : _unwinding_Java(false) {} - virtual bool is_unwinding_Java() final { return _unwinding_Java; } - - virtual void set_unwinding_Java(bool unwinding_Java) final { - _unwinding_Java = unwinding_Java; - } -}; - -#endif // THREADLOCALDATA_H diff --git a/ddprof-lib/src/main/cpp/threadState.h b/ddprof-lib/src/main/cpp/threadState.h deleted file mode 100644 index 786c96fb6..000000000 --- a/ddprof-lib/src/main/cpp/threadState.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _THREADSTATE_H -#define _THREADSTATE_H - -enum class OSThreadState : int { - UNKNOWN = 0, - NEW = 1, // The thread has been initialized but yet started - RUNNABLE = 2, // Has been started and is runnable, but not necessarily running - MONITOR_WAIT = 3, // Waiting on a contended monitor lock - CONDVAR_WAIT = 4, // Waiting on a condition variable - OBJECT_WAIT = 5, // Waiting on an Object.wait() call - BREAKPOINTED = 6, // Suspended at breakpoint - SLEEPING = 7, // Thread.sleep() - TERMINATED = 8, // All done, but not reclaimed yet - SYSCALL = 9 // does not originate in the JVM, used when the current frame is - // known to be a syscall -}; - -enum class ExecutionMode : int { - UNKNOWN = 0, - JAVA = 1, - JVM = 2, - NATIVE = 3, - SAFEPOINT = 4, - SYSCALL = 5 -}; - -inline ExecutionMode getThreadExecutionMode(); -inline OSThreadState getOSThreadState(); - -#endif // _THREADSTATE_H diff --git a/ddprof-lib/src/main/cpp/threadState.inline.h b/ddprof-lib/src/main/cpp/threadState.inline.h deleted file mode 100644 index 5ef2ebd0b..000000000 --- a/ddprof-lib/src/main/cpp/threadState.inline.h +++ /dev/null @@ -1,23 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _THREADSTATE_INLINE_H -#define _THREADSTATE_INLINE_H - -#include "threadState.h" -#include "hotspot/vmStructs.h" - -inline ExecutionMode getThreadExecutionMode() { - return VM::isHotspot() ? VMThread::getExecutionMode() : - ExecutionMode::UNKNOWN; -} - -inline OSThreadState getOSThreadState() { - return VM::isHotspot() ? VMThread::getOSThreadState() : - OSThreadState::UNKNOWN; -} - -#endif // _THREADSTATE_INLINE_H diff --git a/ddprof-lib/src/main/cpp/trap.cpp b/ddprof-lib/src/main/cpp/trap.cpp deleted file mode 100644 index 38e47fc4a..000000000 --- a/ddprof-lib/src/main/cpp/trap.cpp +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "trap.h" -#include "os.h" - - -uintptr_t Trap::_page_start[TRAP_COUNT] = {0}; - - -bool Trap::isFaultInstruction(uintptr_t pc) { - for (int i = 0; i < TRAP_COUNT; i++) { - if (pc - _page_start[i] < OS::page_size) { - return true; - } - } - return false; -} - -void Trap::assign(const void* address, uintptr_t offset) { - _entry = (uintptr_t)address; - if (_entry == 0) { - return; - } - _entry += offset; - -#if defined(__arm__) || defined(__thumb__) - _breakpoint_insn = (_entry & 1) ? BREAKPOINT_THUMB : BREAKPOINT; - _entry &= ~(uintptr_t)1; -#endif - - _saved_insn = *(instruction_t*)_entry; - _page_start[_id] = _entry & -OS::page_size; -} - -// Two allocation traps are always enabled/disabled together. -// If both traps belong to the same page, protect/unprotect it just once. -void Trap::pair(Trap& second) { - if (_page_start[_id] == _page_start[second._id]) { - _protect = false; - second._unprotect = false; - } -} - -// Patch instruction at the entry point -bool Trap::patch(instruction_t insn) { - if (_unprotect) { - int prot = WX_MEMORY ? (PROT_READ | PROT_WRITE) : (PROT_READ | PROT_WRITE | PROT_EXEC); - if (OS::mprotect((void*)(_entry & -OS::page_size), OS::page_size, prot) != 0) { - return false; - } - } - - *(instruction_t*)_entry = insn; - flushCache(_entry); - - if (_protect) { - OS::mprotect((void*)(_entry & -OS::page_size), OS::page_size, PROT_READ | PROT_EXEC); - } - return true; -} diff --git a/ddprof-lib/src/main/cpp/trap.h b/ddprof-lib/src/main/cpp/trap.h deleted file mode 100644 index 97620e44c..000000000 --- a/ddprof-lib/src/main/cpp/trap.h +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _TRAP_H -#define _TRAP_H - -#include -#include "arch.h" - - -const int TRAP_COUNT = 4; - - -class Trap { - private: - int _id; - bool _unprotect; - bool _protect; - uintptr_t _entry; - instruction_t _breakpoint_insn; - instruction_t _saved_insn; - - bool patch(instruction_t insn); - - static uintptr_t _page_start[TRAP_COUNT]; - - public: - Trap(int id) : _id(id), _unprotect(true), _protect(WX_MEMORY), _entry(0), _breakpoint_insn(BREAKPOINT) { - } - - uintptr_t entry() { - return _entry; - } - - bool covers(uintptr_t pc) { - // PC points either to BREAKPOINT instruction or to the next one - return pc - _entry <= sizeof(instruction_t); - } - - void assign(const void* address, uintptr_t offset = BREAKPOINT_OFFSET); - void pair(Trap& second); - - bool install() { - return _entry == 0 || patch(_breakpoint_insn); - } - - bool uninstall() { - return _entry == 0 || patch(_saved_insn); - } - - static bool isFaultInstruction(uintptr_t pc); -}; - -#endif // _TRAP_H diff --git a/ddprof-lib/src/main/cpp/tripleBuffer.h b/ddprof-lib/src/main/cpp/tripleBuffer.h deleted file mode 100644 index 32a2ff1ed..000000000 --- a/ddprof-lib/src/main/cpp/tripleBuffer.h +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _TRIPLE_BUFFER_H -#define _TRIPLE_BUFFER_H - -/** - * Generic triple-buffer rotation manager. - * - * Manages three externally-owned objects that cycle through three roles: - * - * active — receives new writes (signal handlers, fill-path) - * dump — snapshot being read by the current dump (old active after rotate) - * scratch — two rotations behind active; ready to be cleared. At least one - * full dump cycle has elapsed since this buffer was last in the - * "dump" role, which gives any writer that loaded a stale active - * pointer time to complete its lookup before the buffer is freed. - * - * Lifecycle per dump cycle: - * rotate() — advance active index; dump thread reads dumpBuffer() - * ...dump runs, populating dumpBuffer() with fill-path data... - * ...caller clears clearTarget() (the scratch buffer)... - * - * Memory: at most two non-empty buffers at any time (active + dump). - * - * Thread safety: - * _active_index is accessed via __atomic_* with acquire/release ordering. - * Writers may briefly operate on a buffer that has just transitioned to - * "dump" or "scratch" (TOCTOU at rotate); this is safe because Dictionary - * (and similar) operations are lock-free internally and the scratch buffer - * is not cleared until one full dump cycle later. - */ -template -class TripleBufferRotator { - T* const _buf[3]; - volatile int _active_index; // 0/1/2; cycles on rotate() - -public: - // a/b/c must remain valid for the lifetime of this rotator. - TripleBufferRotator(T* a, T* b, T* c) - : _buf{a, b, c}, _active_index(0) {} - - T* active() const { - return _buf[__atomic_load_n(&_active_index, __ATOMIC_ACQUIRE)]; - } - - // Buffer the dump thread reads from: (active_index + 2) % 3 after rotate(). - T* dumpBuffer() const { - return _buf[(__atomic_load_n(&_active_index, __ATOMIC_ACQUIRE) + 2) % 3]; - } - - // Buffer scheduled for the next clear: (active_index + 1) % 3. - // At least one full dump cycle has elapsed since this buffer was the - // "dump" role, so any stale writer pointer to it is no longer in use. - T* clearTarget() const { - return _buf[(__atomic_load_n(&_active_index, __ATOMIC_ACQUIRE) + 1) % 3]; - } - - // Advance _active_index by 1 mod 3. - // After this call the old active is accessible via dumpBuffer(). - // Uses a CAS loop so concurrent callers (e.g. stop() racing dump()) each - // advance by exactly one step without silently aliasing the same index. - void rotate() { - int old = __atomic_load_n(&_active_index, __ATOMIC_ACQUIRE); - int next = (old + 1) % 3; - while (!__atomic_compare_exchange_n(&_active_index, &old, next, - /*weak=*/false, - __ATOMIC_ACQ_REL, - __ATOMIC_ACQUIRE)) { - next = (old + 1) % 3; - } - } - - // Reset to initial state (no concurrent writers/readers). - void reset() { - __atomic_store_n(&_active_index, 0, __ATOMIC_RELEASE); - } -}; - -#endif // _TRIPLE_BUFFER_H diff --git a/ddprof-lib/src/main/cpp/tsc.cpp b/ddprof-lib/src/main/cpp/tsc.cpp deleted file mode 100644 index 8811d4436..000000000 --- a/ddprof-lib/src/main/cpp/tsc.cpp +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "tsc.h" -#include "vmEntry.h" - - -bool TSC::_initialized = false; -bool TSC::_available = false; -bool TSC::_enabled = false; -u64 TSC::_offset = 0; -u64 TSC::_frequency = NANOTIME_FREQ; - -void TSC::enable(Clock clock) { - if (!TSC_SUPPORTED || clock == CLK_MONOTONIC) { - _enabled = false; - return; - } - - if (!_initialized) { - if (VM::loaded()) { - JNIEnv* env = VM::jni(); - - jfieldID jvm; - jmethodID getTicksFrequency, counterTime; - jclass cls = env->FindClass("jdk/jfr/internal/JVM"); - if (cls != NULL - && ((jvm = env->GetStaticFieldID(cls, "jvm", "Ljdk/jfr/internal/JVM;")) != NULL) - && ((getTicksFrequency = env->GetMethodID(cls, "getTicksFrequency", "()J")) != NULL) - && ((counterTime = env->GetStaticMethodID(cls, "counterTime", "()J")) != NULL)) { - u64 frequency = env->CallLongMethod(env->GetStaticObjectField(cls, jvm), getTicksFrequency); - if (frequency > NANOTIME_FREQ) { - // Default 1GHz frequency might mean that rdtsc is not available - u64 jvm_ticks = env->CallStaticLongMethod(cls, counterTime); - _offset = rdtsc() - jvm_ticks; - _frequency = frequency; - _available = true; - } - } - - env->ExceptionClear(); - } else if (cpuHasGoodTimestampCounter()) { - _offset = 0; - _available = true; - } - - _initialized = true; - } - - _enabled = _available; -} diff --git a/ddprof-lib/src/main/cpp/tsc.h b/ddprof-lib/src/main/cpp/tsc.h deleted file mode 100644 index ac8047029..000000000 --- a/ddprof-lib/src/main/cpp/tsc.h +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _TSC_H -#define _TSC_H - -#include "arguments.h" -#include "os.h" - - -const u64 NANOTIME_FREQ = 1000000000; - - -#if defined(__x86_64__) || defined(__i386__) - -#include - -#define TSC_SUPPORTED true - -static inline u64 rdtsc() { -#if defined(__x86_64__) - u32 lo, hi; - asm volatile("rdtsc" : "=a" (lo), "=d" (hi)); - return ((u64)hi << 32) | lo; -#else - u64 result; - asm volatile("rdtsc" : "=A" (result)); - return result; -#endif -} - -// Returns true if this CPU has a good ("invariant") timestamp counter -inline bool cpuHasGoodTimestampCounter() { - unsigned int eax, ebx, ecx, edx; - - // Check if CPUID supports misc feature flags - __cpuid(0x80000000, eax, ebx, ecx, edx); - if (eax < 0x80000007) { - return 0; - } - - // Get misc feature flags - __cpuid(0x80000007, eax, ebx, ecx, edx); - - // Bit 8 of EDX indicates invariant TSC - return (edx & (1 << 8)) != 0; -} - -#elif defined(__aarch64__) - -#define TSC_SUPPORTED true - -static inline u64 rdtsc() { - u64 value; - asm volatile("mrs %0, cntvct_el0" : "=r"(value)); - return value; -} - -inline bool cpuHasGoodTimestampCounter() { - // AARCH64 always has a good timestamp counter. - return true; -} - -#else - -#define TSC_SUPPORTED false -#define rdtsc() 0 - -static bool cpuHasGoodTimestampCounter() { - return false; -} - -#endif - - -class TSC { - private: - static bool _initialized; - static bool _available; - static bool _enabled; - static u64 _offset; - static u64 _frequency; - - public: - static void enable(Clock clock); - - static bool enabled() { - return TSC_SUPPORTED && _enabled; - } - - static u64 ticks() { - return enabled() ? rdtsc() - _offset : OS::nanotime(); - } - - // Ticks per second. - // When using the TSC with no JVM, since there is no calibration, - // this function will return an incorrect value. - static u64 frequency() { - return enabled() ? _frequency : NANOTIME_FREQ; - } - - // Convert ticks to milliseconds - static u64 ticks_to_millis(u64 ticks) { - return TSC_SUPPORTED ? 1000 * ticks / frequency() : ticks / 1000 / 1000; - } -}; - -#endif // _TSC_H diff --git a/ddprof-lib/src/main/cpp/unwindStats.cpp b/ddprof-lib/src/main/cpp/unwindStats.cpp deleted file mode 100644 index 82a38cf17..000000000 --- a/ddprof-lib/src/main/cpp/unwindStats.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "unwindStats.h" - -// initialize static members -SpinLock UnwindStats::_lock; -UnwindFailures UnwindStats::_unwind_failures; diff --git a/ddprof-lib/src/main/cpp/unwindStats.h b/ddprof-lib/src/main/cpp/unwindStats.h deleted file mode 100644 index 1eb4eab29..000000000 --- a/ddprof-lib/src/main/cpp/unwindStats.h +++ /dev/null @@ -1,224 +0,0 @@ -#ifndef STUB_UNWIND_STATS_H -#define STUB_UNWIND_STATS_H - -#include "common.h" -#include "spinLock.h" - -#include -#include - -enum UnwindFailureKind { - UNWIND_FAILURE_STUB = 1, - UNWIND_FAILURE_COMPILED = 2, - UNWIND_FAILURE_ANY = 3, // UNWIND_FAILURE_STUB | UNWIND_FAILURE_COMPILED -}; - -// Maximum number of unique names that can be tracked -#define MAX_UNWIND_FAILURE_NAMES 1024 -// Maximum length of a name string -#define MAX_NAME_LENGTH 256 - -class UnwindFailures { - private: - char (*_names)[MAX_NAME_LENGTH]; - volatile int _nameCount; - volatile u64 (*_counters)[UNWIND_FAILURE_ANY + 1]; - - public: - UnwindFailures() : _nameCount(0) { - _names = new char[MAX_UNWIND_FAILURE_NAMES][MAX_NAME_LENGTH]; - _counters = new u64[MAX_UNWIND_FAILURE_NAMES][UNWIND_FAILURE_ANY + 1]; - memset((void*)_names, 0, MAX_UNWIND_FAILURE_NAMES * MAX_NAME_LENGTH); - memset((void*)_counters, 0, MAX_UNWIND_FAILURE_NAMES * (UNWIND_FAILURE_ANY + 1) * sizeof(u64)); - } - - ~UnwindFailures() { - delete[] _names; - delete[] _counters; - } - - // Disable copy constructor and assignment operator - UnwindFailures(const UnwindFailures&) = delete; - UnwindFailures& operator=(const UnwindFailures&) = delete; - - void record(UnwindFailureKind kind, const char *name) { - if (!name) return; - - // Fast path: try to find existing name first - int index = findName(name); - if (index >= 0) { - _counters[index][kind - 1]++; - return; - } - - // Slow path: create new entry if needed - index = findOrCreateName(name); - if (index >= 0) { - _counters[index][kind - 1]++; - } - } - - int findName(const char *name) const { - for (int i = 0; i < _nameCount; i++) { - if (strcmp(_names[i], name) == 0) { - return i; - } - } - return -1; - } - - int findOrCreateName(const char *name) { - int index = findName(name); - if (index >= 0) { - return index; - } - - int newIndex = _nameCount++; - if (newIndex < MAX_UNWIND_FAILURE_NAMES) { - size_t len = strlen(name); - if (len >= MAX_NAME_LENGTH) { - len = MAX_NAME_LENGTH - 1; - } - memcpy(_names[newIndex], name, len); - _names[newIndex][len] = '\0'; - return newIndex; - } - - return -1; - } - - u64 count(const char *name, UnwindFailureKind kind = UNWIND_FAILURE_ANY) const { - int index = findName(name); - if (index < 0) { - return 0; - } - - if (kind == UNWIND_FAILURE_ANY) { - return _counters[index][UNWIND_FAILURE_STUB - 1] + - _counters[index][UNWIND_FAILURE_COMPILED - 1]; - } - return _counters[index][kind - 1]; - } - - void clear() { - _nameCount = 0; - memset((void*)_names, 0, MAX_UNWIND_FAILURE_NAMES * MAX_NAME_LENGTH); - memset((void*)_counters, 0, MAX_UNWIND_FAILURE_NAMES * (UNWIND_FAILURE_ANY + 1) * sizeof(u64)); - } - - bool empty() const { - return _nameCount == 0; - } - - void merge(const UnwindFailures &other) { - for (int i = 0; i < other._nameCount; i++) { - int index = findOrCreateName(other._names[i]); - if (index >= 0) { - for (int j = 0; j < UNWIND_FAILURE_ANY; j++) { - _counters[index][j] += other._counters[i][j]; - } - } - } - } - - void swap(UnwindFailures &other) { - // Swap pointers - char (*temp_names)[MAX_NAME_LENGTH] = const_cast(_names); - _names = other._names; - other._names = temp_names; - - u64 (*temp_counters)[UNWIND_FAILURE_ANY + 1] = const_cast(_counters); - _counters = other._counters; - other._counters = temp_counters; - - // Swap name count - int temp_count = _nameCount; - _nameCount = other._nameCount; - other._nameCount = temp_count; - } - - template - void forEach(Func fn) const - { - for (int i = 0; i < _nameCount; i++) - { - const char *name = _names[i]; - for (int j = 0; j < 2; j++) - { - UnwindFailureKind kind = static_cast(j + 1); - u64 count = _counters[i][j]; - if (count > 0) - { - fn(kind, name, count); - } - } - } - } -}; - -class ExclusiveLock { - private: - SpinLock &_lock; - public: - ExclusiveLock(SpinLock &lock) : _lock(lock) { - _lock.lock(); - } - ~ExclusiveLock() { - _lock.unlock(); - } - - ExclusiveLock(const ExclusiveLock &) = delete; - ExclusiveLock &operator=(const ExclusiveLock &) = delete; -}; - -class SharedLock { - private: - SpinLock &_lock; - public: - SharedLock(SpinLock &lock) : _lock(lock) { - _lock.lockShared(); - } - ~SharedLock() { - _lock.unlockShared(); - } - - SharedLock(const SharedLock &) = delete; - SharedLock &operator=(const SharedLock &) = delete; -}; - -class UnwindStats -{ - private: - static SpinLock _lock; - static UnwindFailures _unwind_failures; - public: - static void recordFailures(UnwindFailures &failures) { - if (_lock.tryLock()) { - _unwind_failures.merge(failures); - _lock.unlock(); - } - } - - static void recordFailures(UnwindFailures *failures) { - if (failures) { - recordFailures(*failures); - } - } - - static u64 countFailures(const char* name, UnwindFailureKind kind = UNWIND_FAILURE_ANY) { - SharedLock l(_lock); - return _unwind_failures.count(name, kind); - } - - static void collectAndReset(UnwindFailures& result) { - ExclusiveLock l(_lock); - result.swap(_unwind_failures); - } - - static void reset() { - ExclusiveLock l(_lock); - _unwind_failures.clear(); - } -}; - -#endif // STUB_UNWIND_STATS_H diff --git a/ddprof-lib/src/main/cpp/utils.h b/ddprof-lib/src/main/cpp/utils.h deleted file mode 100644 index ecc4d2b00..000000000 --- a/ddprof-lib/src/main/cpp/utils.h +++ /dev/null @@ -1,36 +0,0 @@ -// Unless explicitly stated otherwise all files in this repository are licensed under the Apache License (Version 2.0). -// This product includes software developed at Datadog (https://www.datadoghq.com/) Copyright 2025 Datadog, Inc. - -#ifndef _UTILS_H -#define _UTILS_H - -#include -#include -#include - -inline bool is_power_of_2(size_t size) { - return size > 0 && (size & (size - 1)) == 0; -} - -template -inline bool is_aligned(const T* ptr, size_t alignment) noexcept { - assert(is_power_of_2(alignment)); - // Convert the pointer to an integer type - auto iptr = reinterpret_cast(ptr); - - // Check if the integer value is a multiple of the alignment - return ((iptr & (alignment - 1)) == 0); -} - -inline size_t align_down(size_t size, size_t alignment) noexcept { - assert(is_power_of_2(alignment)); - return size & (~(alignment - 1)); -} - -inline size_t align_up(size_t size, size_t alignment) noexcept { - assert(is_power_of_2(alignment)); - return align_down(size + alignment - 1, alignment); -} - - -#endif // _UTILS_H \ No newline at end of file diff --git a/ddprof-lib/src/main/cpp/vmEntry.cpp b/ddprof-lib/src/main/cpp/vmEntry.cpp deleted file mode 100644 index f93edeea9..000000000 --- a/ddprof-lib/src/main/cpp/vmEntry.cpp +++ /dev/null @@ -1,729 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "common.h" -#include "vmEntry.h" -#include "arguments.h" -#include "context.h" -#include "counters.h" -#include "j9/j9Support.h" -#include "jniHelper.h" -#include "jvmThread.h" -#include "libraries.h" -#include "log.h" -#include "os.h" -#include "profiler.h" -#include "safeAccess.h" -#include "hotspot/vmStructs.h" -#include "hotspot/jitCodeCache.h" -#include -#include -#include -#include "guards.h" -#include -#include - -// JVM TI agent return codes -const int ARGUMENTS_ERROR = 100; -const int COMMAND_ERROR = 200; - -static Arguments _agent_args(true); - -JavaVM *VM::_vm; -jvmtiEnv *VM::_jvmti = NULL; - -int VM::_java_version = 0; -int VM::_java_update_version = 0; -int VM::_hotspot_version = 0; -bool VM::_openj9 = false; -bool VM::_hotspot = false; -bool VM::_zing = false; -bool VM::_can_sample_objects = false; -bool VM::_can_intercept_binding = false; -bool VM::_is_adaptive_gc_boundary_flag_set = false; - -jvmtiExtensionFunction VM::_request_stack_trace = nullptr; -jvmtiExtensionFunction VM::_init_request_stack_trace = nullptr; - -jvmtiError(JNICALL *VM::_orig_RedefineClasses)(jvmtiEnv *, jint, - const jvmtiClassDefinition *); -jvmtiError(JNICALL *VM::_orig_RetransformClasses)(jvmtiEnv *, jint, - const jclass *classes); - -void *VM::_libjvm; -AsyncGetCallTrace VM::_asyncGetCallTrace; -JVM_GetManagement VM::_getManagement; - -static void wakeupHandler(int signo) { - SIGNAL_HANDLER_GUARD(); - // Dummy handler for interrupting syscalls -} - -static bool isVmRuntimeEntry(const char* blob_name) { - return strcmp(blob_name, "_ZNK12MemAllocator8allocateEv") == 0 - || strncmp(blob_name, "_Z22post_allocation_notify", 26) == 0 - || strncmp(blob_name, "_ZN11OptoRuntime", 16) == 0 - || strncmp(blob_name, "_ZN8Runtime1", 12) == 0 - || strncmp(blob_name, "_ZN13SharedRuntime", 18) == 0 - || strncmp(blob_name, "_ZN18InterpreterRuntime", 23) == 0; -} - -static bool isZingRuntimeEntry(const char* blob_name) { - return strncmp(blob_name, "_ZN14DolphinRuntime", 19) == 0 - || strncmp(blob_name, "_ZN37JvmtiSampledObjectAllocEventCollector", 42) == 0; -} - -static bool isZeroInterpreterMethod(const char *blob_name) { - return strncmp(blob_name, "_ZN15ZeroInterpreter", 20) == 0 || - strncmp(blob_name, "_ZN19BytecodeInterpreter3run", 28) == 0; -} - -static bool isOpenJ9InterpreterMethod(const char *blob_name) { - return strncmp(blob_name, "_ZN32VM_BytecodeInterpreter", 27) == 0 || - strncmp(blob_name, "_ZN26VM_BytecodeInterpreter", 27) == 0 || - strncmp(blob_name, "bytecodeLoop", 12) == 0 || - strcmp(blob_name, "cInterpreter") == 0; -} - -static bool isOpenJ9JitStub(const char *blob_name) { - if (strncmp(blob_name, "jit", 3) == 0) { - blob_name += 3; - return strcmp(blob_name, "NewObject") == 0 || - strcmp(blob_name, "NewArray") == 0 || - strcmp(blob_name, "ANewArray") == 0; - } - return false; -} - -static bool isOpenJ9Resolve(const char* blob_name) { - return strncmp(blob_name, "resolve", 7) == 0; -} - -static bool isOpenJ9JitAlloc(const char* blob_name) { - return strncmp(blob_name, "old_", 4) == 0; -} - -static bool isOpenJ9GcAlloc(const char* blob_name) { - return strncmp(blob_name, "J9Allocate", 10) == 0; -} - -static bool isOpenJ9JvmtiAlloc(const char* blob_name) { - return strcmp(blob_name, "jvmtiHookSampledObjectAlloc") == 0; -} - -static bool isCompilerEntry(const char* blob_name) { - return strncmp(blob_name, "_ZN13CompileBroker25invoke_compiler_on_method", 45) == 0; -} - -static bool isThreadEntry(const char* blob_name) { - // Match thread entry point patterns for JVM threads (both HotSpot and OpenJ9) and Rust threads - return strstr(blob_name, "thread_native_entry") != NULL || - strstr(blob_name, "JavaThread::") != NULL || - strstr(blob_name, "_ZN10JavaThread") != NULL || // Mangled JavaThread:: names - strstr(blob_name, "thread_main_inner") != NULL || - strstr(blob_name, "thread_start") != NULL || // Rust thread roots - strstr(blob_name, "start_thread") != NULL; // glibc pthread entry -} - -static void* resolveMethodId(void** mid) { - return mid == NULL || *mid < (void*)4096 ? NULL : *mid; -} - -static void resolveMethodIdEnd() {} - -JavaFullVersion JavaVersionAccess::get_java_version(char* prop_value) { - JavaFullVersion version = {8, 362}; // initial value is 8u362; an arbitrary java version - TEST_LOG("version property: %s", prop_value); - if (strncmp(prop_value, "1.8.0_", 6) == 0) { - version.major = 8; - version.update = atoi(prop_value + 6); - } else if (strncmp(prop_value, "8.0.", 4) == 0) { - version.major = 8; - version.update = atoi(prop_value + 4); - } else if (strncmp(prop_value, "25.", 3) == 0 && prop_value[3] != '0') { - // Java 8 encoded in java.vm.version system property looks like 25.352-b08 - // The upcoming Java 25 version will have a form of '25.0.' instead - version.major = 8; - version.update = atoi(prop_value + 3); - } else if (strncmp(prop_value, "JRE 1.8.0", 9) == 0) { - // Old IBM SDK JDK 8 embeds only a build date (YYYYMMDD) in its version - // string, not the update number. Map known date ranges to update numbers: - // >= 20250328: IBM SDK SR8 FP45 (JDK 8u451, J9 VM build date for OpenJ9 - // 0.51 which fixes the ASGCT livelock; eclipse-openj9#20577) - // >= 20230313: IBM SDK SR6 FP11 (JDK 8u361, first 2023 quarterly release) - // otherwise: IBM SDK SR6 FP10 or earlier (JDK 8u351) - version.major = 8; - char *idx = strstr(prop_value, " 202"); - if (idx != NULL) { - long date = atol(idx + 1); - if (date >= 20250328L) { - version.update = 451; - } else if (date >= 20230313L) { - version.update = 361; - } else { - version.update = 351; - } - } - } else { - version.major = atoi(prop_value); - if (version.major < 9) { - version.major = 9; - } - char* peg_char = strchr(prop_value, '+'); - if (peg_char) { - // terminate before the build specification - *peg_char = '\0'; - } - // format is 11.0.17 - // this shortcut for parsing the update version should hold till Java 99 - version.update = atoi(prop_value + 5); - } - return version; -} - -int JavaVersionAccess::get_hotspot_version(char* prop_value) { - int hs_version = 0; - if (strncmp(prop_value, "25.", 3) == 0 && prop_value[3] > '0') { - hs_version = 8; - } else if (strncmp(prop_value, "24.", 3) == 0 && prop_value[3] > '0') { - hs_version = 7; - } else if (strncmp(prop_value, "20.", 3) == 0 && prop_value[3] > '0') { - hs_version = 6; - } else if ((hs_version = atoi(prop_value)) < 9) { - hs_version = 9; - } - return hs_version; -} - -CodeCache* VM::openJvmLibrary() { - if ((void*)_asyncGetCallTrace == nullptr) { - return nullptr; - } - - Libraries* libraries = Libraries::instance(); - CodeCache *lib = - isOpenJ9() - ? libraries->findJvmLibrary("libj9vm") - : libraries->findLibraryByAddress((const void *)_asyncGetCallTrace); - return lib; -} - -bool VM::initShared(JavaVM* vm) { - if (_jvmti != NULL) - return true; - - _vm = vm; - if (_vm->GetEnv((void **)&_jvmti, JVMTI_VERSION_1_0) != 0) { - return false; - } - -#ifdef __APPLE__ - Dl_info dl_info; - if (dladdr((const void *)wakeupHandler, &dl_info) && - dl_info.dli_fname != NULL) { - // Make sure java-profiler DSO cannot be unloaded, since it contains JVM - // callbacks. On Linux, we use 'nodelete' linker option. - dlopen(dl_info.dli_fname, RTLD_LAZY | RTLD_NODELETE); - } -#endif - - bool is_zero_vm = false; - char *prop; - if (_jvmti->GetSystemProperty("java.vm.name", &prop) == 0) { - _hotspot = strstr(prop, "OpenJDK") != NULL || - strstr(prop, "HotSpot") != NULL || - strstr(prop, "GraalVM") != NULL || - strstr(prop, "Dynamic Code Evolution") != NULL; - is_zero_vm = strstr(prop, "Zero") != NULL; - _zing = !_hotspot && strstr(prop, "Zing") != NULL; - _openj9 = !_hotspot && strstr(prop, "OpenJ9") != NULL; - - _jvmti->Deallocate((unsigned char *)prop); - prop = NULL; - } - - _libjvm = getLibraryHandle("libjvm.so"); - _asyncGetCallTrace = (AsyncGetCallTrace)dlsym(_libjvm, "AsyncGetCallTrace"); - _getManagement = (JVM_GetManagement)dlsym(_libjvm, "JVM_GetManagement"); - - Libraries *libraries = Libraries::instance(); - libraries->updateSymbols(false); - - _openj9 = !_hotspot && J9Support::initialize( - _jvmti, libraries->resolveSymbol("j9thread_self*")); - - if (_openj9) { - if (_jvmti->GetSystemProperty("jdk.extensions.version", &prop) == 0) { - // OpenJ9 Semeru will report the version here - // insert debug output here - } else { - if (prop != NULL) { - _jvmti->Deallocate((unsigned char *)prop); - prop = NULL; - } - if (_jvmti->GetSystemProperty("java.fullversion", &prop) == 0) { - // IBM JDK 8 will report something here - // The reported string contains JRE 1.8.0 and then later year and - // (possibly) hash code Although not very precise, this is best we can - // get :/ insert debug output here - } else { - if (prop != NULL) { - _jvmti->Deallocate((unsigned char *)prop); - prop = NULL; - } - } - } - } - if (prop == NULL) { - if (_jvmti->GetSystemProperty("java.runtime.version", &prop) == 0) { - // insert debug output here - } else { - if (prop != NULL) { - _jvmti->Deallocate((unsigned char *)prop); - prop = NULL; - } - } - } - TEST_LOG("java.runtime.version: %s", prop); - if (prop != NULL) { - JavaFullVersion version = JavaVersionAccess::get_java_version(prop); - _java_version = version.major; - _java_update_version = version.update; - _jvmti->Deallocate((unsigned char *)prop); - prop = NULL; - } - if (_jvmti->GetSystemProperty("java.vm.version", &prop) == 0) { - TEST_LOG("java.vm.version: %s", prop); - if (_java_version == 0) { - JavaFullVersion version = JavaVersionAccess::get_java_version(prop); - _java_version = version.major; - _java_update_version = version.update; - } - _hotspot_version = JavaVersionAccess::get_hotspot_version(prop); - _jvmti->Deallocate((unsigned char *)prop); - prop = NULL; - } - - if (prop != NULL) { - _jvmti->Deallocate((unsigned char *)prop); - } - - if (_java_version == 0 && _hotspot_version > 0) { - // sanity fallback: - // - if we failed to resolve the _java_version but have _hotspot_version, let's use the hotspot version as java version - _java_version = _hotspot_version; - } - TEST_LOG("jvm_version#%d.%d.%d", _java_version, 0, _java_update_version); - - CodeCache *lib = openJvmLibrary(); - if (lib == nullptr) { - return false; - } - - VMStructs::init(lib); - - // Mark thread entry points for all JVMs (critical for correct stack unwinding) - lib->mark(isThreadEntry, MARK_THREAD_ENTRY); - - // Mark OS-level pthread entry points across ALL loaded native libraries. - // On glibc these live in libc.so.6 or libpthread.so.0 (merged in glibc 2.34+); - // on musl in libc.musl-.so.1; on Rust they may be in the app binary itself. - // Scanning all libs avoids fragile name-based lookup (findLibraryByName uses a - // prefix match that can return the wrong library, e.g. libcap instead of libc). - // walkVM stops unwinding when it reaches the top of a pure-native thread stack - // without finding an anchor; marking start_thread/thread_start here gives the - // walker a clean stopping point for any pthread-managed thread. - const CodeCacheArray& all_native_libs = libraries->native_libs(); - for (int i = 0; i < all_native_libs.count(); i++) { - CodeCache *cc = all_native_libs[i]; - if (cc != NULL) { - cc->mark(isThreadEntry, MARK_THREAD_ENTRY); - } - } - - if (isOpenJ9()) { - lib->mark(isOpenJ9InterpreterMethod, MARK_INTERPRETER); - lib->mark(isOpenJ9Resolve, MARK_VM_RUNTIME); - CodeCache* libjit = libraries->findJvmLibrary("libj9jit"); - if (libjit != NULL) { - libjit->mark(isOpenJ9JitStub, MARK_INTERPRETER); - libjit->mark(isOpenJ9JitAlloc, MARK_VM_RUNTIME); - } - CodeCache* libgc = libraries->findJvmLibrary("libj9gc"); - if (libgc != NULL) { - libgc->mark(isOpenJ9GcAlloc, MARK_VM_RUNTIME); - } - CodeCache* libjvmti = libraries->findJvmLibrary("libj9jvmti"); - if (libjvmti != NULL) { - libjvmti->mark(isOpenJ9JvmtiAlloc, MARK_VM_RUNTIME); - } - } else { - lib->mark(isVmRuntimeEntry, MARK_VM_RUNTIME); - if (isZing()) { - lib->mark(isZingRuntimeEntry, MARK_VM_RUNTIME); - } else if (is_zero_vm) { - lib->mark(isZeroInterpreterMethod, MARK_INTERPRETER); - } else { - lib->mark(isCompilerEntry, MARK_COMPILER_ENTRY); - } - } - return true; -} - -bool VM::initLibrary(JavaVM *vm) { - TEST_LOG("VM::initLibrary"); - if (!initShared(vm)) { - return false; - } - ready(jvmti(), jni()); - return true; -} - -void VM::probeJFRRequestStackTrace() { - jint ext_count = 0; - jvmtiExtensionFunctionInfo *ext_functions = nullptr; - if (_jvmti->GetExtensionFunctions(&ext_count, &ext_functions) == 0) { - for (jint i = 0; i < ext_count; i++) { - if (strcmp(ext_functions[i].id, - "com.sun.hotspot.functions.RequestStackTrace") == 0) { - __atomic_store_n(&_request_stack_trace, ext_functions[i].func, __ATOMIC_RELEASE); - } else if (strcmp(ext_functions[i].id, - "com.sun.hotspot.functions.InitializeRequestStackTrace") == 0) { - _init_request_stack_trace = ext_functions[i].func; - } - for (jint j = 0; j < ext_functions[i].param_count; j++) { - _jvmti->Deallocate((unsigned char *)ext_functions[i].params[j].name); - } - _jvmti->Deallocate((unsigned char *)ext_functions[i].params); - _jvmti->Deallocate((unsigned char *)ext_functions[i].errors); - _jvmti->Deallocate((unsigned char *)ext_functions[i].id); - _jvmti->Deallocate((unsigned char *)ext_functions[i].short_description); - } - _jvmti->Deallocate((unsigned char *)ext_functions); - } - - if (_agent_args._jvmtistacks) { - if (_request_stack_trace == nullptr || _init_request_stack_trace == nullptr) { - Log::warn("jvmtistacks requested but HotSpot RequestStackTrace extension " - "is not available on this JVM"); - Counters::increment(JVMTI_STACKS_INIT_FAILED); - } else { - initializeRequestStackTrace(); - } - } -} - -// Must not be called from a signal handler — invokes JVMTI which is not async-signal-safe. -bool VM::initializeRequestStackTrace() { - if (__atomic_load_n(&_request_stack_trace, __ATOMIC_RELAXED) == nullptr || _init_request_stack_trace == nullptr) { - return false; - } - jvmtiError rc = _init_request_stack_trace(_jvmti); - if (rc == JVMTI_ERROR_NONE) { - Counters::increment(JVMTI_STACKS_INIT_OK); - return true; - } - Log::warn("InitializeRequestStackTrace failed: %d", rc); - __atomic_store_n(&_request_stack_trace, (jvmtiExtensionFunction)nullptr, __ATOMIC_RELEASE); - Counters::increment(JVMTI_STACKS_INIT_FAILED); - return false; -} - -bool VM::initProfilerBridge(JavaVM *vm, bool attach) { - TEST_LOG("VM::initProfilerBridge"); - if (!initShared(vm)) { - return false; - } - - CodeCache *lib = openJvmLibrary(); - if (lib == nullptr) { - return false; - } - - if (!attach && hotspot_version() == 8 && OS::isLinux()) { - // Workaround for JDK-8185348 - char *func = (char *)lib->findSymbol( - "_ZN6Method26checked_resolve_jmethod_idEP10_jmethodID"); - if (func != NULL) { - applyPatch(func, (const char *)resolveMethodId, - (const char *)resolveMethodIdEnd); - } - } - - jvmtiCapabilities potential_capabilities = {0}; - _jvmti->GetPotentialCapabilities(&potential_capabilities); - - _can_sample_objects = - potential_capabilities.can_generate_sampled_object_alloc_events && - (!_hotspot || hotspot_version() >= 11); - _can_intercept_binding = - potential_capabilities.can_generate_native_method_bind_events && - HeapUsage::needsNativeBindingInterception(); - - jvmtiCapabilities capabilities = {0}; - capabilities.can_generate_all_class_hook_events = 1; - capabilities.can_retransform_classes = 1; - capabilities.can_retransform_any_class = isOpenJ9() ? 0 : 1; - // capabilities.can_generate_vm_object_alloc_events = isOpenJ9() ? 1 : 0; - capabilities.can_generate_sampled_object_alloc_events = - _can_sample_objects ? 1 : 0; - capabilities.can_generate_native_method_bind_events = - _can_intercept_binding ? 1 : 0; - capabilities.can_generate_garbage_collection_events = 1; - capabilities.can_get_bytecodes = 1; - capabilities.can_get_constant_pool = 1; - capabilities.can_get_source_file_name = 1; - capabilities.can_get_line_numbers = 1; - capabilities.can_generate_compiled_method_load_events = 1; - capabilities.can_generate_monitor_events = 1; - capabilities.can_tag_objects = 1; - - _jvmti->AddCapabilities(&capabilities); - - if (_hotspot) { - probeJFRRequestStackTrace(); - } - - jvmtiEventCallbacks callbacks = {0}; - callbacks.VMInit = VMInit; - callbacks.VMDeath = VMDeath; - callbacks.ClassLoad = ClassLoad; - callbacks.ClassPrepare = ClassPrepare; - callbacks.CompiledMethodLoad = JitCodeCache::CompiledMethodLoad; - callbacks.DynamicCodeGenerated = JitCodeCache::DynamicCodeGenerated; - callbacks.ThreadStart = Profiler::ThreadStart; - callbacks.ThreadEnd = Profiler::ThreadEnd; - callbacks.SampledObjectAlloc = ObjectSampler::SampledObjectAlloc; - callbacks.GarbageCollectionFinish = LivenessTracker::GarbageCollectionFinish; - callbacks.NativeMethodBind = VMStructs::NativeMethodBind; - _jvmti->SetEventCallbacks(&callbacks, sizeof(callbacks)); - - _jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_VM_DEATH, NULL); - _jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_CLASS_LOAD, NULL); - _jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_CLASS_PREPARE, - NULL); - _jvmti->SetEventNotificationMode(JVMTI_ENABLE, - JVMTI_EVENT_DYNAMIC_CODE_GENERATED, NULL); - _jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_NATIVE_METHOD_BIND, - NULL); - - if (hotspot_version() == 0 || !CodeHeap::available()) { - // Workaround for JDK-8173361: avoid CompiledMethodLoad events when possible - _jvmti->SetEventNotificationMode(JVMTI_ENABLE, - JVMTI_EVENT_COMPILED_METHOD_LOAD, NULL); - } else { - // DebugNonSafepoints is automatically enabled with CompiledMethodLoad, - // otherwise we set the flag manually - VMFlag* f = VMFlag::find("DebugNonSafepoints", {VMFlag::Type::Bool}); - if (f != NULL && f->isDefault()) { - f->set(1); - } - } - - // if the user sets -XX:+UseAdaptiveGCBoundary we will just disable the - // profiler to avoid the risk of crashing flag was made obsolete (inert) in 15 - // (see JDK-8228991) and removed in 16 (see JDK-8231560) - if (hotspot_version() < 15) { - VMFlag *f = VMFlag::find("UseAdaptiveGCBoundary", {VMFlag::Type::Bool}); - _is_adaptive_gc_boundary_flag_set = f != NULL && f->get(); - } - - // Make sure we reload method IDs upon class retransformation - JVMTIFunctions *functions = *(JVMTIFunctions **)_jvmti; - _orig_RedefineClasses = functions->RedefineClasses; - _orig_RetransformClasses = functions->RetransformClasses; - functions->RedefineClasses = RedefineClassesHook; - functions->RetransformClasses = RetransformClassesHook; - - if (attach) { - loadAllMethodIDs(_jvmti, jni()); - _jvmti->GenerateEvents(JVMTI_EVENT_DYNAMIC_CODE_GENERATED); - _jvmti->GenerateEvents(JVMTI_EVENT_COMPILED_METHOD_LOAD); - } else { - _jvmti->SetEventNotificationMode(JVMTI_ENABLE, JVMTI_EVENT_VM_INIT, NULL); - } - - OS::installSignalHandler(WAKEUP_SIGNAL, NULL, wakeupHandler); - - return true; -} - -// Run late initialization when JVM is ready -void VM::ready(jvmtiEnv *jvmti, JNIEnv *jni) { - Profiler::check_JDK_8313796_workaround(); - Profiler::setupSignalHandlers(); - JVMThread::initialize(); - if (isHotspot()) { - JitWriteProtection jit(true); - VMStructs::ready(); - } -} - -void VM::applyPatch(char *func, const char *patch, const char *end_patch) { - size_t size = end_patch - patch; - uintptr_t start_page = (uintptr_t)func & ~OS::page_mask; - uintptr_t end_page = - ((uintptr_t)func + size + OS::page_mask) & ~OS::page_mask; - - if (mprotect((void *)start_page, end_page - start_page, - PROT_READ | PROT_WRITE | PROT_EXEC) == 0) { - memcpy(func, patch, size); - __builtin___clear_cache(func, func + size); - mprotect((void *)start_page, end_page - start_page, PROT_READ | PROT_EXEC); - } -} - -void *VM::getLibraryHandle(const char *name) { - if (OS::isLinux()) { - void *handle = dlopen(name, RTLD_LAZY); - if (handle != NULL) { - return handle; - } - Log::warn("Failed to load %s: %s", name, dlerror()); - } - // JVM symbols are globally visible on macOS - return RTLD_DEFAULT; -} - -void VM::loadMethodIDs(jvmtiEnv *jvmti, JNIEnv *jni, jclass klass) { - bool needs_patch = VM::hotspot_version() == 8; - if (needs_patch) { - // Workaround for JVM bug https://bugs.openjdk.org/browse/JDK-8062116 - // Preallocate space for jmethodIDs at the beginning of the list (rather than at the end) - // This is relevant only for JDK 8 - later versions do not have this bug - if (VMStructs::hasClassLoaderData()) { - VMKlass *vmklass = VMKlass::fromJavaClass(jni, klass); - int method_count = vmklass->methodCount(); - if (method_count > 0) { - VMClassLoaderData *cld = vmklass->classLoaderData(); - cld->lock(); - for (int i = 0; i < method_count; i += MethodList::SIZE) { - *cld->methodList() = new MethodList(*cld->methodList()); - } - cld->unlock(); - } - } - } - - // CRITICAL: GetClassMethods must be called to preallocate jmethodIDs for AsyncGetCallTrace. - // AGCT operates in signal handlers where lock acquisition is forbidden, so jmethodIDs must - // exist before profiling encounters them. Without preallocation, AGCT cannot identify methods - // in stack traces, breaking profiling functionality. - // - // JVM-internal allocation: This triggers JVM to allocate jmethodIDs internally, which persist - // until class unload. High class churn causes significant memory growth, but this is inherent - // to AGCT architecture and necessary for signal-safe profiling. - // - // See: https://mostlynerdless.de/blog/2023/07/17/jmethodids-in-profiling-a-tale-of-nightmares/ - jint method_count; - jmethodID *methods; - if (jvmti->GetClassMethods(klass, &method_count, &methods) == 0) { - jvmti->Deallocate((unsigned char *)methods); - } -} - -void VM::loadAllMethodIDs(jvmtiEnv *jvmti, JNIEnv *jni) { - jint class_count; - jclass *classes; - if (jvmti->GetLoadedClasses(&class_count, &classes) == 0) { - for (int i = 0; i < class_count; i++) { - loadMethodIDs(jvmti, jni, classes[i]); - } - jvmti->Deallocate((unsigned char *)classes); - } -} - -void JNICALL VM::ClassPrepare(jvmtiEnv* jvmti, JNIEnv* jni, jthread thread, - jclass klass) { - loadMethodIDs(jvmti, jni, klass); -} - -void JNICALL VM::VMInit(jvmtiEnv* jvmti, JNIEnv* jni, jthread thread) { - ready(jvmti, jni); - loadAllMethodIDs(jvmti, jni); - - // initialize the heap usage tracking only after the VM is ready - HeapUsage::initJMXUsage(VM::jni()); - - // Delayed start of profiler if agent has been loaded at VM bootstrap - Error error = Profiler::instance()->run(_agent_args); - if (error) { - Log::error("%s", error.message()); - } -} - -void JNICALL VM::VMDeath(jvmtiEnv *jvmti, JNIEnv *jni) { - Profiler::instance()->shutdown(_agent_args); -} - -jvmtiError -VM::RedefineClassesHook(jvmtiEnv *jvmti, jint class_count, - const jvmtiClassDefinition *class_definitions) { - jvmtiError result = - _orig_RedefineClasses(jvmti, class_count, class_definitions); - - if (result == 0) { - // jmethodIDs are invalidated after RedefineClasses - JNIEnv *env = jni(); - for (int i = 0; i < class_count; i++) { - if (class_definitions[i].klass != NULL) { - loadMethodIDs(jvmti, env, class_definitions[i].klass); - } - } - } - - return result; -} - -jvmtiError VM::RetransformClassesHook(jvmtiEnv *jvmti, jint class_count, - const jclass *classes) { - jvmtiError result = _orig_RetransformClasses(jvmti, class_count, classes); - - if (result == 0) { - // jmethodIDs are invalidated after RetransformClasses - JNIEnv *env = jni(); - for (int i = 0; i < class_count; i++) { - if (classes[i] != NULL) { - loadMethodIDs(jvmti, env, classes[i]); - } - } - } - - return result; -} - -extern "C" DLLEXPORT jint JNICALL -Agent_OnLoad(JavaVM* vm, char* options, void* reserved) { - Error error = _agent_args.parse(options); - - Log::open(_agent_args); - - if (error) { - Log::error("%s", error.message()); - return ARGUMENTS_ERROR; - } - - if (!VM::initProfilerBridge(vm, false)) { - Log::error("JVM does not support Tool Interface"); - return COMMAND_ERROR; - } - - return 0; -} - -extern "C" DLLEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *reserved) { - if (!VM::initLibrary(vm)) { - return 0; - } - return JNI_VERSION_1_6; -} - -extern "C" DLLEXPORT void JNICALL JNI_OnUnload(JavaVM *vm, void *reserved) { - Profiler *profiler = Profiler::instance(); - if (profiler != NULL) { - profiler->stop(); - } -} diff --git a/ddprof-lib/src/main/cpp/vmEntry.h b/ddprof-lib/src/main/cpp/vmEntry.h deleted file mode 100644 index 4323dad7f..000000000 --- a/ddprof-lib/src/main/cpp/vmEntry.h +++ /dev/null @@ -1,254 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2021, 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _VMENTRY_H -#define _VMENTRY_H - -#include -#include "arch.h" - -#include "arch.h" -#include "codeCache.h" -#include "frame.h" - -#ifdef __clang__ -#define DLLEXPORT __attribute__((visibility("default"))) -#else -#define DLLEXPORT __attribute__((visibility("default"), externally_visible)) -#endif - -// Denotes ASGCT_CallFrame where method_id has special meaning (not jmethodID) -enum ASGCT_CallFrameType { - BCI_CPU = 0, // cpu time - BCI_WALL = -10, // wall time - BCI_NATIVE_FRAME = -11, // native function name (char*) - BCI_ALLOC = -12, // name of the allocated class - BCI_ALLOC_OUTSIDE_TLAB = -13, // name of the class allocated outside TLAB - BCI_LIVENESS = -14, // name of the allocated class - BCI_LOCK = -15, // class name of the locked object - BCI_PARK = -16, // class name of the park() blocker - BCI_THREAD_ID = -17, // method_id designates a thread - BCI_ERROR = -18, // method_id is an error string - BCI_NATIVE_FRAME_REMOTE = -19, // method_id points to RemoteFrameInfo for remote symbolication - BCI_NATIVE_MALLOC = -20, // native malloc/free sample (size stored in counter) - // method_id holds a VMSymbol* (the receiver class's name Symbol), - // NOT a jmethodID. The pointer is captured in the signal handler - // (hotspotSupport.cpp:walkVM) and resolved at dump time via SafeAccess - // in Lookup::resolveVTableReceiver. Same precedent as BCI_NATIVE_FRAME - // (const char* in method_id) and BCI_NATIVE_FRAME_REMOTE (packed - // 64-bit blob). Any reader iterating frames must check bci BEFORE - // dereferencing method_id as a jmethodID. - // - // Limitation: CallTraceHashTable::calcHash mixes the raw bytes of the - // frames array (including method_id) into the trace id. Two samples - // of the same logical class whose Symbol* address differs (class - // unload + reload within a chunk) produce distinct trace ids; this - // is accepted because normalising at sample time would require an - // in-signal-handler Symbol read, which the redesign explicitly - // avoids. The dump-time MethodMap key is class_id-based (see - // MethodMap::makeKey(u32)), so the synthetic - // MethodInfo collapses across distinct Symbol* addresses even though - // the CallTrace itself does not. - BCI_VTABLE_RECEIVER = -21, - BCI_NATIVE_SOCKET = -22, // native socket I/O sample (bytes stored in counter) -}; - -// See hotspot/src/share/vm/prims/forte.cpp -enum ASGCT_Failure { - ticks_no_Java_frame = 0, - ticks_no_class_load = -1, - ticks_GC_active = -2, - ticks_unknown_not_Java = -3, - ticks_not_walkable_not_Java = -4, - ticks_unknown_Java = -5, - ticks_not_walkable_Java = -6, - ticks_unknown_state = -7, - ticks_thread_exit = -8, - ticks_deopt = -9, - ticks_safepoint = -10, - ticks_skipped = -11, - ASGCT_FAILURE_TYPES = 12 -}; - -/** - * Information for native frames requiring remote symbolication. - * Used when bci == BCI_NATIVE_FRAME_REMOTE. - */ -typedef struct RemoteFrameInfo { - const char* build_id; // GNU build-id for library identification (null-terminated hex string) - uintptr_t pc_offset; // PC offset within the library/module - short lib_index; // Index into CodeCache library table for fast lookup - -#ifdef __cplusplus - // Constructor for C++ convenience - RemoteFrameInfo(const char* bid, uintptr_t offset, short lib_idx) - : build_id(bid), pc_offset(offset), lib_index(lib_idx) {} -#endif -} RemoteFrameInfo; - -typedef struct _asgct_callframe { - jint bci; - LP64_ONLY(jint padding;) - union { - jmethodID method_id; - unsigned long packed_remote_frame; // packed RemoteFrameInfo data - const char* native_function_name; - }; -} ASGCT_CallFrame; - -typedef struct { - JNIEnv *env; - jint num_frames; - ASGCT_CallFrame *frames; -} ASGCT_CallTrace; - -typedef void (*AsyncGetCallTrace)(ASGCT_CallTrace *, jint, void *); - -typedef struct { - void *unused[38]; - jstring(JNICALL *ExecuteDiagnosticCommand)(JNIEnv *, jstring); -} VMManagement; - -typedef VMManagement *(*JVM_GetManagement)(jint); - -typedef struct { - void *unused1[86]; - jvmtiError(JNICALL *RedefineClasses)(jvmtiEnv *, jint, - const jvmtiClassDefinition *); - void *unused2[64]; - jvmtiError(JNICALL *RetransformClasses)(jvmtiEnv *, jint, const jclass *); -} JVMTIFunctions; - -typedef struct { - int major; - int update; -} JavaFullVersion; - -class JavaVersionAccess { - public: - static JavaFullVersion get_java_version(char* prop_value); - static int get_hotspot_version(char* prop_value); -}; - -class VM { -private: - static JavaVM *_vm; - static jvmtiEnv *_jvmti; - - static int _java_version; - static int _java_update_version; - static int _hotspot_version; - static bool _openj9; - static bool _hotspot; - static bool _zing; - static bool _can_sample_objects; - static bool _can_intercept_binding; - static bool _is_adaptive_gc_boundary_flag_set; - - // HotSpot JFR async stack-trace extension (optional, JDK 27+). - // _request_stack_trace is atomic (RELEASE/ACQUIRE) because canRequestStackTrace() - // is called from signal handlers; _init_request_stack_trace is plain because it - // is only ever read by initializeRequestStackTrace(), called once from the same - // init thread before any signal handlers are installed. - static jvmtiExtensionFunction _request_stack_trace; - static jvmtiExtensionFunction _init_request_stack_trace; - - static jvmtiError(JNICALL *_orig_RedefineClasses)( - jvmtiEnv *, jint, const jvmtiClassDefinition *); - static jvmtiError(JNICALL *_orig_RetransformClasses)(jvmtiEnv *, jint, - const jclass *classes); - - static void ready(jvmtiEnv *jvmti, JNIEnv *jni); - static void applyPatch(char *func, const char *patch, const char *end_patch); - static void *getLibraryHandle(const char *name); - static void loadMethodIDs(jvmtiEnv *jvmti, JNIEnv *jni, jclass klass); - static void loadAllMethodIDs(jvmtiEnv *jvmti, JNIEnv *jni); - - static bool initShared(JavaVM *vm); - static void probeJFRRequestStackTrace(); - - static CodeCache* openJvmLibrary(); - -public: - static void *_libjvm; - static AsyncGetCallTrace _asyncGetCallTrace; - static JVM_GetManagement _getManagement; - - static bool initLibrary(JavaVM *vm); - static bool initProfilerBridge(JavaVM *vm, bool attach); - - static jvmtiEnv *jvmti() { return _jvmti; } - - static bool loaded() { return _jvmti != nullptr; } - - static JNIEnv *jni() { - JNIEnv *jni; - return _vm->GetEnv((void **)&jni, JNI_VERSION_1_6) == 0 ? jni : NULL; - } - - static JNIEnv *attachThread(const char *name) { - JNIEnv *jni; - JavaVMAttachArgs args = {JNI_VERSION_1_6, (char *)name, NULL}; - return _vm->AttachCurrentThreadAsDaemon((void **)&jni, &args) == 0 ? jni - : NULL; - } - - static void detachThread() { _vm->DetachCurrentThread(); } - - static VMManagement *management() { - return _getManagement != NULL ? _getManagement(0x20030000) : NULL; - } - - static int java_version() { return _java_version; } - - static int hotspot_version() { return isHotspot() ? _hotspot_version : -1; } - - static int java_update_version() { return _java_update_version; } - - static bool isOpenJ9() { return _openj9; } - static bool isHotspot() { return _hotspot; } - - static bool canSampleObjects() { return _can_sample_objects; } - - static bool isZing() { return _zing; } - - static bool isUseAdaptiveGCBoundarySet() { - return _is_adaptive_gc_boundary_flag_set; - } - - static bool canRequestStackTrace() { - return __atomic_load_n(&_request_stack_trace, __ATOMIC_ACQUIRE) != nullptr; - } - - // Must not be called from a signal handler — invokes JVMTI which is not async-signal-safe. - static bool initializeRequestStackTrace(); - - static jvmtiError requestStackTrace(void* ucontext, jlong user_data) { - jvmtiExtensionFunction fn = __atomic_load_n(&_request_stack_trace, __ATOMIC_ACQUIRE); - if (!fn) return JVMTI_ERROR_NOT_AVAILABLE; - return fn(_jvmti, (jthread)nullptr, ucontext, user_data); - } - - static void JNICALL VMInit(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread); - static void JNICALL VMDeath(jvmtiEnv *jvmti, JNIEnv *jni); - - static void JNICALL ClassLoad(jvmtiEnv *jvmti, JNIEnv *jni, jthread thread, - jclass klass) { - // Needed only for AsyncGetCallTrace support - } - - static void JNICALL ClassPrepare(jvmtiEnv* jvmti, JNIEnv* jni, jthread thread, - jclass klass); - - static jvmtiError JNICALL - RedefineClassesHook(jvmtiEnv *jvmti, jint class_count, - const jvmtiClassDefinition *class_definitions); - static jvmtiError JNICALL RetransformClassesHook(jvmtiEnv *jvmti, - jint class_count, - const jclass *classes); -}; - -#endif // _VMENTRY_H diff --git a/ddprof-lib/src/main/cpp/wallClock.cpp b/ddprof-lib/src/main/cpp/wallClock.cpp deleted file mode 100644 index 40af2147e..000000000 --- a/ddprof-lib/src/main/cpp/wallClock.cpp +++ /dev/null @@ -1,532 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "wallClock.h" - -#include "counters.h" -#include "stackFrame.h" -#include "context.h" -#include "context_api.h" -#include "debugSupport.h" -#include "jvmThread.h" -#include "libraries.h" -#include "log.h" -#include "otel_context.h" -#include "profiler.h" -#include "signalCookie.h" -#include "thread.h" -#include "threadState.inline.h" -#include "guards.h" -#include "wallClockCounters.h" -#include -#include -#include -#include -#include // For std::sort and std::binary_search - -std::atomic BaseWallClock::_enabled{false}; - -static inline bool isPrecheckSuppressionState(OSThreadState state) { - return state == OSThreadState::SLEEPING || - state == OSThreadState::CONDVAR_WAIT || - state == OSThreadState::OBJECT_WAIT || - state == OSThreadState::MONITOR_WAIT; -} - -static inline u64 loadSpanId(OtelThreadContextRecord* record) { - u64 span_id = 0; - for (int i = 0; i < 8; i++) { - span_id = (span_id << 8) | - __atomic_load_n(&record->span_id[i], __ATOMIC_RELAXED); - } - return span_id; -} - -static inline bool hasKnownActiveTraceContext(ProfiledThread* thread) { - if (thread == nullptr || !thread->isContextInitialized()) { - return false; - } - - OtelThreadContextRecord* record = thread->getOtelContextRecord(); - // record->valid is not a context-presence bit. ThreadContext leaves cleared - // records invalid indefinitely, so gating on valid=1 disables wallprecheck for - // the common no-context state after a Java ThreadLocal reset. - return loadSpanId(record) != 0; -} - -struct WallPrecheckResult { - bool suppress = false; - ThreadFilter::Slot* slot_to_arm = nullptr; - OSThreadState state_to_arm = OSThreadState::UNKNOWN; - OSThreadState observed_state = OSThreadState::UNKNOWN; - bool observed_state_valid = false; - ThreadFilter::Slot* unowned_weight_slot = nullptr; - u64 unowned_weight = 1; - bool flush_unowned_tail = false; - u64 flush_call_trace_id = 0; - u64 flush_weight = 0; - OSThreadState flush_state = OSThreadState::UNKNOWN; -}; - -static inline void incrementSuppressedSampledRun() { - Counters::increment(WC_SIGNAL_SUPPRESSED_SAMPLED_RUN); - WallClockCounters::incrementSuppressedSampledRun(); -} - -static inline bool suppressAlreadySampledBlock(ThreadFilter::Slot* slot) { - if (slot == nullptr) { - return false; - } - OSThreadState block_state = slot->activeBlockState(); - if (slot->activeBlockOwner() != BlockRunOwner::NONE && - isPrecheckSuppressionState(block_state) && - slot->sampledThisRun() && - block_state == slot->lastSampledState()) { - incrementSuppressedSampledRun(); - return true; - } - return false; -} - -static inline WallPrecheckResult prepareWallPrecheck(ProfiledThread* current, - bool precheck) { - WallPrecheckResult result; - if (current == nullptr || !precheck || hasKnownActiveTraceContext(current)) { - return result; - } - - ThreadFilter::Slot* slot = - Profiler::instance()->threadFilter()->slotForId(current->filterSlotId()); - if (slot == nullptr) { - return result; - } - - OSThreadState active_block_state = slot->activeBlockState(); - BlockRunOwner active_block_owner = slot->activeBlockOwner(); - bool has_owned_block = - active_block_owner != BlockRunOwner::NONE && - isPrecheckSuppressionState(active_block_state); - if (has_owned_block) { - if (slot->sampledThisRun() && - active_block_state == slot->lastSampledState()) { - incrementSuppressedSampledRun(); - result.suppress = true; - return result; - } - // Arm only after the MethodSample has been successfully recorded. If the - // JFR write is skipped due to lock contention, the next signal must retry - // instead of losing the only stack for this blocked run. - result.slot_to_arm = slot; - result.state_to_arm = active_block_state; - return result; - } - - result.observed_state = getOSThreadState(); - result.observed_state_valid = true; - if (isPrecheckSuppressionState(result.observed_state)) { - if (!slot->shouldRecordUnownedBlockedSample()) { - Counters::increment(WC_UNOWNED_BLOCKED_SUPPRESSED); - result.suppress = true; - return result; - } - result.unowned_weight_slot = slot; - result.unowned_weight = slot->consumeUnownedBlockedWeight(); - } else { - result.flush_unowned_tail = slot->flushUnownedBlockedTail( - result.flush_call_trace_id, result.flush_weight, result.flush_state); - } - return result; -} - -static inline void finishWallPrecheck(const WallPrecheckResult& precheck, - bool recorded, - u64 recorded_call_trace_id = 0) { - if (!recorded && precheck.unowned_weight_slot != nullptr) { - precheck.unowned_weight_slot->restoreUnownedBlockedWeight( - precheck.unowned_weight); - } else if (recorded && precheck.unowned_weight_slot != nullptr) { - Counters::increment(WC_UNOWNED_BLOCKED_RECORDED); - if (recorded_call_trace_id != 0) { - precheck.unowned_weight_slot->recordUnownedBlockedSample( - recorded_call_trace_id, precheck.observed_state); - } - } - if (recorded && precheck.slot_to_arm != nullptr) { - precheck.slot_to_arm->markSampledThisRun(precheck.state_to_arm); - } -} - -static inline void recordDeferredWallSample(int tid, u64 call_trace_id, - ExecutionEvent* event) { - Profiler::instance()->recordDeferredSample(tid, call_trace_id, BCI_WALL, event); -} - -static inline void emitUnownedBlockedTailForWallPrecheck( - int tid, const WallPrecheckResult& precheck) { - if (!precheck.flush_unowned_tail || precheck.flush_call_trace_id == 0) { - return; - } - - ExecutionEvent flush_event; - flush_event._thread_state = precheck.flush_state; - flush_event._execution_mode = ExecutionMode::UNKNOWN; - flush_event._weight = precheck.flush_weight; - recordDeferredWallSample(tid, precheck.flush_call_trace_id, &flush_event); -} - -bool BaseWallClock::inSyscall(void *ucontext) { - StackFrame frame(ucontext); - uintptr_t pc = frame.pc(); - - // Consider a thread sleeping, if it has been interrupted in the middle of - // syscall execution, either when PC points to the syscall instruction, or if - // syscall has just returned with EINTR - if (StackFrame::isSyscall((instruction_t *)pc)) { - return true; - } - - // Make sure the previous instruction address is readable - uintptr_t prev_pc = pc - SYSCALL_SIZE; - if ((pc & 0xfff) >= SYSCALL_SIZE || - Libraries::instance()->findLibraryByAddress((instruction_t *)prev_pc) != - NULL) { - if (StackFrame::isSyscall((instruction_t *)prev_pc) && - frame.checkInterruptedSyscall()) { - return true; - } - } - - return false; -} - -void WallClockASGCT::sharedSignalHandler(int signo, siginfo_t *siginfo, - void *ucontext) { - SIGNAL_HANDLER_GUARD(); - // Reject any SIGVTALRM that did not originate from our rt_tgsigqueueinfo - // send. Defends against stray in-process tgkill / external sigqueue that - // would otherwise drive our wallclock sampling path. - if (!OS::shouldProcessSignal(siginfo, SI_QUEUE, SignalCookie::wallclock())) { - Counters::increment(WALLCLOCK_SIGNAL_FOREIGN); - SIGNAL_HANDLER_GUARD_RELEASE(); - OS::forwardForeignSignal(signo, siginfo, ucontext); - return; - } - Counters::increment(WALLCLOCK_SIGNAL_OWN); - - WallClockASGCT *engine = reinterpret_cast(Profiler::instance()->wallEngine()); - if (signo == SIGVTALRM) { - engine->signalHandler(signo, siginfo, ucontext, engine->_interval); - } -} - -void WallClockASGCT::signalHandler(int signo, siginfo_t *siginfo, void *ucontext, - u64 last_sample) { - // Atomically try to enter critical section - prevents all reentrancy races - CriticalSection cs; - if (!cs.entered()) { - return; // Another critical section is active, defer profiling - } - ProfiledThread *current = ProfiledThread::currentSignalSafe(); - // Guard against the race window between Profiler::registerThread() and - // thread_native_entry setting JVM TLS (PROF-13072): skip at most one signal - // per thread. Pure native threads (where JVMThread::current() is always null) - // are allowed through once the one-shot window expires. - if (current != nullptr && JVMThread::isInitialized() && JVMThread::current() == nullptr - && current->inInitWindow()) { - current->tickInitWindow(); - return; - } - // Once-per-run filter (wallprecheck=true): for untraced threads, exact - // suppression is only valid while an explicit lifecycle hook owns the blocked - // interval. Raw OS thread state is only an observation; it cannot distinguish - // one long sleep from several short sleeps separated by runnable gaps between - // signals. Unowned blocked observations therefore use weighted fallback - // sampling instead of arming sampled_this_run. - WallPrecheckResult precheck = prepareWallPrecheck(current, _precheck); - if (precheck.suppress) { - return; - } - int tid = current != NULL ? current->tid() : OS::threadId(); - Shims::instance().setSighandlerTid(tid); - u64 call_trace_id = 0; - if (current != NULL && _collapsing) { - StackFrame frame(ucontext); - u64 spanId = 0, rootSpanId = 0; - // contextValid is not redundant with (spanId==0 && rootSpanId==0): a cleared - // context has spanId=0 and contextValid=true, while an uninitialized/mid-write - // thread has spanId=0 and contextValid=false. lookupWallclockCallTraceId uses - // contextValid to decide whether to update the sidecar _otel_local_root_span_id. - bool contextValid = ContextApi::get(spanId, rootSpanId); - call_trace_id = current->lookupWallclockCallTraceId( - (u64)frame.pc(), (u64)frame.sp(), - Profiler::instance()->recordingEpoch(), - contextValid, spanId, rootSpanId); - if (call_trace_id != 0) { - Counters::increment(SKIPPED_WALLCLOCK_UNWINDS); - } - } - - ExecutionEvent event; - OSThreadState state = - precheck.observed_state_valid ? precheck.observed_state : getOSThreadState(); - ExecutionMode mode = getThreadExecutionMode(); - if (state == OSThreadState::UNKNOWN) { - if (inSyscall(ucontext)) { - state = OSThreadState::SYSCALL; - mode = ExecutionMode::SYSCALL; - } else { - state = OSThreadState::RUNNABLE; - } - } - event._thread_state = state; - event._execution_mode = mode; - event._weight = precheck.unowned_weight; - u64 recorded_call_trace_id = 0; - bool recorded = Profiler::instance()->recordSample(ucontext, last_sample, tid, - BCI_WALL, call_trace_id, - &event, - &recorded_call_trace_id); - finishWallPrecheck(precheck, recorded, recorded_call_trace_id); - emitUnownedBlockedTailForWallPrecheck(tid, precheck); - Shims::instance().setSighandlerTid(-1); -} - -Error BaseWallClock::start(Arguments &args) { - int interval = args._event != NULL ? args._interval : args._wall; - if (interval < 0) { - return Error("interval must be positive"); - } - _interval = interval ? interval : DEFAULT_WALL_INTERVAL; - - _reservoir_size = - args._wall_threads_per_tick ? - args._wall_threads_per_tick : DEFAULT_WALL_THREADS_PER_TICK; - - initialize(args); - - _running = true; - - if (pthread_create(&_thread, NULL, threadEntry, this) != 0) { - return Error("Unable to create timer thread"); - } - - return Error::OK; -} - -void BaseWallClock::stop() { - _running.store(false); - // the thread join ensures we wait for the thread to finish before returning - // (and possibly removing the object) - pthread_kill(_thread, WAKEUP_SIGNAL); - int res = pthread_join(_thread, NULL); - if (res != 0) { - Log::warn("Unable to join WallClock thread on stop %d", res); - } -} - -bool BaseWallClock::isEnabled() const { - return _enabled.load(std::memory_order_acquire); -} - -void WallClockASGCT::initialize(Arguments& args) { - _collapsing = args._wall_collapsing; - _precheck = args._wall_precheck; - // J9WallClock uses JVMTI GetAllStackTracesExtended polling, not SIGVTALRM - // signals — it has no sharedSignalHandler and needs no signal-origin gate. - // Engines are started sequentially; this call is idempotent (no-op after first). - OS::primeSignalOriginCheck(); - OS::installSignalHandler(SIGVTALRM, sharedSignalHandler); -} - -void WallClockASGCT::timerLoop() { - // todo: re-allocating the vector every time is not efficient - auto collectThreads = [&](std::vector& entries) { - // Get thread IDs from the filter if it's enabled - // Otherwise list all threads in the system - if (Profiler::instance()->threadFilter()->enabled()) { - Profiler::instance()->threadFilter()->collect(entries); - } else { - const int refresher_tid = Libraries::instance()->refresherTid(); - ThreadList *thread_list = OS::listThreads(); - while (thread_list->hasNext()) { - int tid = thread_list->next(); - // Don't include the current thread, nor the Libraries refresher - // thread (profiler-internal — masking SIGVTALRM there is not - // enough; we also want to avoid the kill() round-trip and any - // pending-signal accumulation). - if (tid != OS::threadId() && tid != refresher_tid) { - entries.push_back({tid, nullptr}); // no-filter: precheck fast path is skipped (null guards) - } - } - delete thread_list; - } - }; - - auto sampleThreads = [&](ThreadEntry entry, int& num_failures, int& threads_already_exited, - int& permission_denied) { - // Timer-thread fast path (wallprecheck=true): skip the kernel IPI entirely - // only when an explicit lifecycle hook still owns an already-sampled blocked - // run. Raw OS thread state is intentionally not used here because the timer - // thread cannot prove run boundaries for the target thread. - if (_precheck && suppressAlreadySampledBlock(entry.slot)) { - return false; - } - if (!OS::sendSignalWithCookie(entry.tid, SIGVTALRM, SignalCookie::wallclock())) { - num_failures++; - if (errno != 0) { - if (errno == ESRCH) { - threads_already_exited++; - } else if (errno == EPERM) { - permission_denied++; - } else if (errno == EAGAIN) { - // Signal queue limit (RLIMIT_SIGPENDING) reached; not a permission error. - Counters::increment(WC_SIGNAL_QUEUE_FULL); - } else { - Log::debug("unexpected error %s", strerror(errno)); - } - } - return false; - } - return true; - }; - - auto doNothing = []() { - }; - - timerLoopCommon(collectThreads, sampleThreads, doNothing, _reservoir_size, _interval); -} - -// WallClockJvmti: mirrors WallClockASGCT's dispatch, but the signal handler -// delegates the stack walk to HotSpot's JFR RequestStackTrace JVMTI extension -// instead of invoking ASGCT. Used only when VM::canRequestStackTrace() is true -// and the profiler has opted into jvmtistacks. - -void WallClockJvmti::sharedSignalHandler(int signo, siginfo_t *siginfo, - void *ucontext) { - SIGNAL_HANDLER_GUARD(); - // Reject any SIGVTALRM that did not originate from our rt_tgsigqueueinfo - // send (mirrors WallClockASGCT). Defends against stray in-process tgkill or - // external sigqueue driving the JVMTI RequestStackTrace path. - if (!OS::shouldProcessSignal(siginfo, SI_QUEUE, SignalCookie::wallclock())) { - Counters::increment(WALLCLOCK_SIGNAL_FOREIGN); - SIGNAL_HANDLER_GUARD_RELEASE(); - OS::forwardForeignSignal(signo, siginfo, ucontext); - return; - } - Counters::increment(WALLCLOCK_SIGNAL_OWN); - - WallClockJvmti *engine = - reinterpret_cast(Profiler::instance()->wallEngine()); - if (signo == SIGVTALRM) { - engine->signalHandler(signo, siginfo, ucontext, engine->_interval); - } -} - -void WallClockJvmti::signalHandler(int signo, siginfo_t *siginfo, - void *ucontext, u64 last_sample) { - CriticalSection cs; - if (!cs.entered()) { - return; - } - int saved_errno = errno; - ProfiledThread *current = ProfiledThread::currentSignalSafe(); - if (current != nullptr && JVMThread::isInitialized() && JVMThread::current() == nullptr - && current->inInitWindow()) { - current->tickInitWindow(); - errno = saved_errno; - return; - } - WallPrecheckResult precheck = prepareWallPrecheck(current, _precheck); - if (precheck.suppress) { - errno = saved_errno; - return; - } - int tid = current != NULL ? current->tid() : OS::threadId(); - Shims::instance().setSighandlerTid(tid); - - ExecutionEvent event; - OSThreadState state = - precheck.observed_state_valid ? precheck.observed_state : getOSThreadState(); - ExecutionMode mode = getThreadExecutionMode(); - if (state == OSThreadState::UNKNOWN) { - if (inSyscall(ucontext)) { - state = OSThreadState::SYSCALL; - mode = ExecutionMode::SYSCALL; - } else { - state = OSThreadState::RUNNABLE; - } - } - event._thread_state = state; - event._execution_mode = mode; - event._weight = precheck.unowned_weight; - // Pass nullptr ucontext so the JVM uses safepoint-based stack walking. - // Passing the signal-frame PC causes the extension to reject samples where - // the thread is currently inside JVM-internal (non-Java) code. - // JVMTI-delegated samples carry a correlation_id, not a call_trace_id, so - // unowned tail flushing remains limited to the ASGCT wall engine. - bool recorded = Profiler::instance()->recordSampleDelegated( - nullptr, last_sample, tid, BCI_WALL, &event); - finishWallPrecheck(precheck, recorded); - Shims::instance().setSighandlerTid(-1); - errno = saved_errno; -} - -void WallClockJvmti::initialize(Arguments &args) { - // Caller must have verified VM::canRequestStackTrace() before selecting - // this engine; see Profiler::selectWallEngine(). - _precheck = args._wall_precheck; - OS::primeSignalOriginCheck(); - OS::installSignalHandler(SIGVTALRM, sharedSignalHandler); -} - -void WallClockJvmti::timerLoop() { - auto collectThreads = [&](std::vector &entries) { - const int refresher_tid = Libraries::instance()->refresherTid(); - if (Profiler::instance()->threadFilter()->enabled()) { - Profiler::instance()->threadFilter()->collect(entries); - } else { - ThreadList *thread_list = OS::listThreads(); - while (thread_list->hasNext()) { - int tid = thread_list->next(); - // Exclude the wallclock timer thread itself and the Libraries - // refresher (profiler-internal). - if (tid != OS::threadId() && tid != refresher_tid) { - entries.push_back({tid, nullptr}); - } - } - delete thread_list; - } - }; - - auto sampleThreads = [&](ThreadEntry entry, int &num_failures, - int &threads_already_exited, int &permission_denied) { - if (_precheck && suppressAlreadySampledBlock(entry.slot)) { - return false; - } - if (!OS::sendSignalWithCookie(entry.tid, SIGVTALRM, SignalCookie::wallclock())) { - num_failures++; - if (errno != 0) { - if (errno == ESRCH) { - threads_already_exited++; - } else if (errno == EPERM) { - permission_denied++; - } else if (errno == EAGAIN) { - // Signal queue limit (RLIMIT_SIGPENDING) reached — count as missed. - Counters::increment(WC_SIGNAL_QUEUE_FULL); - } else { - Log::debug("unexpected error %s", strerror(errno)); - } - } - return false; - } - return true; - }; - - auto doNothing = []() {}; - - timerLoopCommon(collectThreads, sampleThreads, doNothing, - _reservoir_size, _interval); -} diff --git a/ddprof-lib/src/main/cpp/wallClock.h b/ddprof-lib/src/main/cpp/wallClock.h deleted file mode 100644 index 13bd90ac1..000000000 --- a/ddprof-lib/src/main/cpp/wallClock.h +++ /dev/null @@ -1,184 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2025, 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _WALLCLOCK_H -#define _WALLCLOCK_H - -#include -#include "engine.h" -#include "os.h" -#include "profiler.h" -#include "reservoirSampler.h" -#include "thread.h" -#include "threadFilter.h" -#include "threadState.h" -#include "tsc.h" -#include "wallClockCounters.h" - -class BaseWallClock : public Engine { - private: - static std::atomic _enabled; - std::atomic _running; - protected: - long _interval; - // Maximum number of threads sampled in one iteration. This limit serves as a - // throttle when generating profiling signals. Otherwise applications with too - // many threads may suffer from a big profiling overhead. Also, keeping this - // limit low enough helps to avoid contention on a spin lock inside - // Profiler::recordSample(). - int _reservoir_size; - - pthread_t _thread; - virtual void timerLoop() = 0; - virtual void initialize(Arguments& args) {}; - - static void *threadEntry(void *wall_clock) { - ((BaseWallClock *)wall_clock)->timerLoop(); - return NULL; - } - - bool isEnabled() const; - static bool inSyscall(void* ucontext); - - template - void timerLoopCommon(CollectThreadsFunc collectThreads, SampleThreadsFunc sampleThreads, CleanThreadFunc cleanThreads, int reservoirSize, u64 interval) { - if (!_enabled.load(std::memory_order_acquire)) { - return; - } - - // Dither the sampling interval to introduce some randomness and prevent step-locking - const double stddev = ((double)_interval) / 10.0; // 10% standard deviation - // Set up random engine and normal distribution - std::random_device rd; - std::mt19937 generator(rd()); - std::normal_distribution distribution(interval, stddev); - - std::vector threads; - threads.reserve(reservoirSize); - int self = OS::threadId(); - ThreadFilter* thread_filter = Profiler::instance()->threadFilter(); - - // We don't want to profile ourselves in wall time. - // current may be null if this thread is still initializing its ProfiledThread - // (wall-clock thread startup races with JVMTI ThreadStart). Safe to skip removal. - ProfiledThread* current = ProfiledThread::current(); - if (current != nullptr) { - int slot_id = current->filterSlotId(); - if (slot_id != -1) { - thread_filter->remove(slot_id); - } - } - - u64 startTime = TSC::ticks(); - WallClockEpochEvent epoch(startTime); - - ReservoirSampler reservoir(reservoirSize); - - while (_running.load(std::memory_order_relaxed)) { - collectThreads(threads); - - int num_failures = 0; - int threads_already_exited = 0; - int permission_denied = 0; - u32 num_successful_samples = 0; - std::vector sample = reservoir.sample(threads); - for (ThreadType thread : sample) { - if (sampleThreads(thread, num_failures, threads_already_exited, permission_denied)) { - num_successful_samples++; - } - } - - epoch.updateNumSamplableThreads(threads.size()); - epoch.updateNumFailedSamples(num_failures); - epoch.updateNumSuccessfulSamples(num_successful_samples); - epoch.addNumSuppressedSampledRun(WallClockCounters::drainSuppressedSampledRun()); - epoch.updateNumExitedThreads(threads_already_exited); - epoch.updateNumPermissionDenied(permission_denied); - u64 endTime = TSC::ticks(); - u64 duration = TSC::ticks_to_millis(endTime - startTime); - if (epoch.hasChanged() || duration >= 1000) { - epoch.endEpoch(duration); - Profiler::instance()->recordWallClockEpoch(self, &epoch); - epoch.newEpoch(endTime); - startTime = endTime; - } else { - epoch.clean(); - } - - threads.clear(); - cleanThreads(); - - // Get a random sleep duration - // clamp the random interval to <1,2N-1> - // the probability of clamping is extremely small, close to zero - OS::sleep(std::min(std::max((long int)1, static_cast(distribution(generator))), ((_interval * 2) - 1))); - } - } - -public: - BaseWallClock() : - _running(false), - _interval(LONG_MAX), - _reservoir_size(0), - _thread(0) {} - virtual ~BaseWallClock() = default; - - const char* units() { - return "ns"; - } - - virtual const char* name() = 0; - - long interval() const { return _interval; } - - inline void enableEvents(bool enabled) { - _enabled.store(enabled, std::memory_order_release); - } - - Error start(Arguments& args); - void stop(); -}; - -class WallClockASGCT : public BaseWallClock { - private: - bool _collapsing; - bool _precheck; - - static void sharedSignalHandler(int signo, siginfo_t* siginfo, void* ucontext); - void signalHandler(int signo, siginfo_t* siginfo, void* ucontext, u64 last_sample); - - void initialize(Arguments& args) override; - void timerLoop() override; - - public: - WallClockASGCT() : BaseWallClock(), _collapsing(false), _precheck(false) {} - const char* name() override { - return "WallClock (ASGCT)"; - } -}; - -// Wall-clock engine that uses BaseWallClock's pthread reservoir sampling loop -// to signal target threads, but in its signal handler delegates the stack walk -// to HotSpot's JFR RequestStackTrace JVMTI extension. Requires -// VM::canRequestStackTrace(). -class WallClockJvmti : public BaseWallClock { - private: - bool _precheck; - - static void sharedSignalHandler(int signo, siginfo_t* siginfo, void* ucontext); - void signalHandler(int signo, siginfo_t* siginfo, void* ucontext, u64 last_sample); - - void initialize(Arguments& args) override; - void timerLoop() override; - - public: - WallClockJvmti() : BaseWallClock(), _precheck(false) {} - const char* name() override { - return "WallClock (JVMTI)"; - } -}; - -#endif // _WALLCLOCK_H diff --git a/ddprof-lib/src/main/cpp/wallClockCounters.h b/ddprof-lib/src/main/cpp/wallClockCounters.h deleted file mode 100644 index f295ce87a..000000000 --- a/ddprof-lib/src/main/cpp/wallClockCounters.h +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _WALLCLOCK_COUNTERS_H -#define _WALLCLOCK_COUNTERS_H - -#include "arch.h" -#include - -static_assert(std::atomic::is_always_lock_free, - "WallClockCounters fields must be lock-free for signal-handler safety"); - -// Holds signal-handler-safe atomic counters for wall-clock sampling statistics. -// Increments use relaxed lock-free atomics; drains use atomic exchange, so each -// increment is counted in either the current drain or a later one. -class WallClockCounters { -private: - inline static std::atomic _suppressed_sampled_run{0}; - -public: - static void incrementSuppressedSampledRun() { - _suppressed_sampled_run.fetch_add(1, std::memory_order_relaxed); - } - - static u64 drainSuppressedSampledRun() { - return (u64)_suppressed_sampled_run.exchange(0, std::memory_order_acq_rel); - } - - static void reset() { - _suppressed_sampled_run.store(0, std::memory_order_relaxed); - } -}; - -#endif // _WALLCLOCK_COUNTERS_H diff --git a/ddprof-lib/src/main/cpp/zing/zingSupport.cpp b/ddprof-lib/src/main/cpp/zing/zingSupport.cpp deleted file mode 100644 index bd8459206..000000000 --- a/ddprof-lib/src/main/cpp/zing/zingSupport.cpp +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "zingSupport.h" - -#include "vmEntry.h" -#include - -void* ZingSupport::initialize(jthread thread) { - JNIEnv* env = VM::jni(); - jclass thread_class = env->FindClass("java/lang/Thread"); - if (thread_class == nullptr) { - env->ExceptionClear(); - return nullptr; - } - jfieldID eetop = nullptr; - // Get eetop field - a bridge from Java Thread to JVMThread - if ((eetop = env->GetFieldID(thread_class, "eetop", "J")) == nullptr) { - // No such field - probably not use standard java library - env->ExceptionClear(); - return nullptr; - } - - return (void*)env->GetLongField(thread, eetop); -} - diff --git a/ddprof-lib/src/main/cpp/zing/zingSupport.h b/ddprof-lib/src/main/cpp/zing/zingSupport.h deleted file mode 100644 index 7c207423e..000000000 --- a/ddprof-lib/src/main/cpp/zing/zingSupport.h +++ /dev/null @@ -1,17 +0,0 @@ -/* - * Copyright The async-profiler authors - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifndef _ZING_ZINGSUPPORT_H -#define _ZING_ZINGSUPPORT_H - -#include - -class ZingSupport { -public: - static void* initialize(jthread thread); -}; - -#endif // _ZING_ZINGSUPPORT_H diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/Arch.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/Arch.java deleted file mode 100644 index ac987c338..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/Arch.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.datadoghq.profiler; - -import java.util.Arrays; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Set; - -/** A simple implementation to detect the current architecture */ -enum Arch { - x64("x86_64", "amd64", "k8"), - x86("x86", "i386", "i486", "i586", "i686"), - arm("ARM", "aarch32"), - arm64("arm64", "aarch64"), - unknown(); - - private final Set identifiers; - - Arch(String... identifiers) { - this.identifiers = new HashSet<>(Arrays.asList(identifiers)); - } - - public static Arch of(String identifier) { - for (Arch arch : EnumSet.allOf(Arch.class)) { - if (arch.identifiers.contains(identifier)) { - return arch; - } - } - return unknown; - } - - public static Arch current() { - return Arch.of(System.getProperty("os.arch")); - } - } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java deleted file mode 100644 index d9fb237f5..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter.java +++ /dev/null @@ -1,180 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.nio.ByteBuffer; - -/** - * Version-agnostic wrapper for direct ByteBuffer memory operations with explicit memory ordering guarantees. - * - *

This class provides low-level memory access operations on direct ByteBuffers with precise control - * over memory ordering semantics. It abstracts differences between Java 8 (using sun.misc.Unsafe) and - * Java 9+ (using VarHandle APIs) to provide consistent behavior across JVM versions. - * - *

The class supports two types of memory ordering: - *

    - *
  • Ordered writes (release semantics): Prevents reordering with prior writes but allows - * subsequent operations to be reordered before this write. More efficient than volatile writes.
  • - *
  • Volatile writes (full barrier): Prevents reordering with both prior and subsequent - * operations. Ensures writes are completed before subsequent operations begin.
  • - *
- * - *

Signal Handler Safety: The primary use case is protecting write sequences from being - * observed in inconsistent states by async signal handlers (e.g., SIGPROF). When a signal interrupts - * a write sequence on the same thread, the signal handler must see either the complete write sequence - * or recognize the write is in-progress. This requires careful memory ordering even though both writer - * and reader execute on the same thread. - * - *

This is primarily used by the profiler for writing thread-local context data (span IDs, checksums) - * to direct ByteBuffers that are shared between Java and native code via JNI. - * - *

Thread Safety: This class is thread-safe. Individual write operations provide their own - * memory ordering guarantees as documented. - */ -public final class BufferWriter { - /** - * Service Provider Interface for version-specific buffer write implementations. - * - *

Implementations of this interface provide the actual low-level memory access operations: - *

    - *
  • {@code BufferWriter8} - Java 8 implementation using sun.misc.Unsafe
  • - *
  • {@code BufferWriter9} - Java 9+ implementation using VarHandle
  • - *
- * - *

The implementation is selected automatically based on the runtime Java version. - */ - public interface Impl { - /** - * Writes a long value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. This is more efficient than a - * volatile write when full bidirectional ordering is not required. - * - *

Used for writing data fields in a sequence protected by a volatile sentinel value. - * Ensures that if a signal handler interrupts after this write, prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the long value to write - */ - void writeOrderedLong(ByteBuffer buffer, int offset, long value); - - /** - * Writes an int value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. - * - *

Used for writing data fields in a sequence protected by a volatile sentinel value. - * Ensures that if a signal handler interrupts after this write, prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the int value to write - */ - void writeInt(ByteBuffer buffer, int offset, int value); - - /** - * Executes a store-store memory fence. - * - *

Ensures that stores before the fence are visible before stores after it. - * Cheaper than fullFence on ARM (~5ns vs ~50ns) since it only orders - * stores, not loads. Sufficient for publication protocols where the writer - * needs to ensure data writes are visible before a flag/pointer write. - */ - void storeFence(); - } - - private final Impl impl; - - /** - * Creates a new BufferWriter with the appropriate implementation for the current Java version. - * - *

The implementation is selected based on the runtime Java version: - *

    - *
  • Java 8: Uses {@code BufferWriter8} which leverages sun.misc.Unsafe for memory operations
  • - *
  • Java 9+: Uses {@code BufferWriter9} which leverages VarHandle for memory operations
  • - *
- * - *

The implementation is loaded reflectively to avoid compile-time dependencies on - * version-specific APIs. - * - * @throws RuntimeException if the implementation class cannot be loaded or instantiated - */ - public BufferWriter() { - try { - if (Platform.isJavaVersion(8)) { - impl = (BufferWriter.Impl) Class.forName("com.datadoghq.profiler.BufferWriter8").getConstructor().newInstance(); - } else { - impl = (BufferWriter.Impl) Class.forName("com.datadoghq.profiler.BufferWriter9").getConstructor().newInstance(); - } - } catch (Throwable t) { - throw new RuntimeException(t); - } - } - - /** - * Writes a long value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. This is more efficient than a - * volatile write when full bidirectional ordering is not required. - * - *

This is commonly used for writing context fields (span IDs, root span IDs) in a write - * sequence that is protected by a volatile sentinel value. Ensures that if a signal handler - * interrupts after this write, all prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the long value to write - */ - public void writeOrderedLong(ByteBuffer buffer, int offset, long value) { - impl.writeOrderedLong(buffer, offset, value); - } - - /** - * Writes an int value to the buffer at the specified offset with ordered write semantics - * (release barrier). - * - *

Ordered write guarantees that this write will not be reordered with respect to prior writes, - * but subsequent operations may be reordered before this write. This is more efficient than a - * volatile write when full bidirectional ordering is not required. - * - *

Used for writing data fields in a sequence protected by a volatile sentinel value. - * Ensures that if a signal handler interrupts after this write, prior writes are visible. - * - * @param buffer the direct ByteBuffer to write to - * @param offset the offset in bytes from the buffer's base address - * @param value the int value to write - */ - public void writeOrderedInt(ByteBuffer buffer, int offset, int value) { - impl.writeInt(buffer, offset, value); - } - - /** - * Executes a store-store memory fence. - * - *

Ensures stores before the fence are globally visible before stores after it. - * On ARM this emits DMB ISHST (~2 ns); on x86 it is a compiler-only barrier. - */ - public void storeFence() { - impl.storeFence(); - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java deleted file mode 100644 index 08af58052..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/BufferWriter8.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import sun.misc.Unsafe; -import sun.nio.ch.DirectBuffer; - -import java.lang.reflect.Field; -import java.nio.ByteBuffer; - -public final class BufferWriter8 implements BufferWriter.Impl { - private static final Unsafe UNSAFE; - static { - Unsafe unsafe = null; - // a safety and testing valve to disable unsafe access - if (Platform.isJavaVersion(8)) { - try { - Field f = Unsafe.class.getDeclaredField("theUnsafe"); - f.setAccessible(true); - unsafe = (Unsafe) f.get(null); - } catch (Exception ignore) { - // On Java 8 will never happen - } - } - UNSAFE = unsafe; - } - - @Override - public void writeOrderedLong(ByteBuffer buffer, int offset, long value) { - UNSAFE.putOrderedLong(null, ((DirectBuffer) buffer).address() + offset, value); - } - - @Override - public void writeInt(ByteBuffer buffer, int offset, int value) { - UNSAFE.putOrderedInt(null, ((DirectBuffer) buffer).address() + offset, value); - } - - @Override - public void storeFence() { - UNSAFE.storeFence(); - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java deleted file mode 100644 index 5dac429dd..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ContextSetter.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -public class ContextSetter { - - private final List attributes; - private final JavaProfiler profiler; - - public ContextSetter(JavaProfiler profiler, List attributes) { - this.profiler = profiler; - Set unique = new HashSet<>(attributes); - this.attributes = new ArrayList<>(unique.size()); - for (int i = 0; i < Math.min(attributes.size(), ThreadContext.MAX_CUSTOM_SLOTS); i++) { - String attribute = attributes.get(i); - if (unique.remove(attribute)) { - this.attributes.add(attribute); - } - } - } - - public int[] snapshotTags() { - int[] snapshot = new int[attributes.size()]; - profiler.copyTags(snapshot); - return snapshot; - } - - /** - * Copies current sidecar encodings into {@code snapshot}. The array must have at least - * {@code attributes.size()} elements; arrays shorter than {@code attributes.size()} are - * silently ignored. Indices {@code [attributes.size(), snapshot.length)} are zeroed after - * copying to prevent stale data from leaking to the caller. - * Use the no-arg {@link #snapshotTags()} overload to obtain a correctly sized array. - */ - public void snapshotTags(int[] snapshot) { - if (snapshot.length >= attributes.size()) { - profiler.copyTags(snapshot); - Arrays.fill(snapshot, attributes.size(), snapshot.length, 0); - } - } - - public int offsetOf(String attribute) { - return attributes.indexOf(attribute); - } - - public boolean setContextValue(String attribute, String value) { - return setContextValue(offsetOf(attribute), value); - } - - public boolean setContextValue(int offset, String value) { - if (offset >= 0) { - return profiler.setContextAttribute(offset, value); - } - return false; - } - - /** - * Re-applies attribute values from precomputed constant IDs and UTF-8 bytes, indexed by slot - * (as produced by {@link #snapshotTags(int[])}). Restores both the DD sidecar encoding and the - * OTEP attrs_data value for every slot whose constantId is {@code > 0}, in a single atomic - * publish — no String allocation, hashing, or cache lookup. Intended for re-applying - * application-managed context after a {@code setContext} span activation wipes the slots. - * - *

Partial-write on overflow. A {@code false} return does not mean the record is - * unchanged: slots that were written before an attrs_data overflow remain published. Overflowed - * slots are zeroed in both the sidecar and attrs_data views. Callers must not assume the record - * is unmodified when {@code false} is returned. - */ - public boolean setContextValuesByIdAndBytes(int[] constantIds, byte[][] utf8) { - return profiler.setContextAttributesByIdAndBytes(constantIds, utf8); - } - - public boolean clearContextValue(String attribute) { - return clearContextValue(offsetOf(attribute)); - } - - public boolean clearContextValue(int offset) { - if (offset >= 0) { - profiler.clearContextAttribute(offset); - return true; - } - return false; - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JVMAccess.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JVMAccess.java deleted file mode 100644 index 57db50e37..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JVMAccess.java +++ /dev/null @@ -1,178 +0,0 @@ -package com.datadoghq.profiler; - -import java.util.function.Consumer; - -/** - * An internal JVM access support. - *

- * We are using vmstructs and dynamic symbol lookups to provide access to some JVM internals. - * There will be dragons here. We are touching and possibly mutating JVM internals. Do not use - * unless you know what you are doing. - *

- */ -public final class JVMAccess { - private static final class SingletonHolder { - static final JVMAccess INSTANCE = new JVMAccess(); - } - - /** - * Flags interface to access JVM flags. - * In general, the flags are read-only. However, some flags can be modified at runtime. - * Currently, only string and boolean flags can be modified. Allowing modification of numeric - * flags would require exact specification of the flag type (int, long, float, double) such - * that the correct number of bytes would be written to the flag and not overwrite the surrounding - * memory. - */ - public interface Flags { - Flags NONE = new Flags() { - @Override - public String getStringFlag(String name) { - return null; - } - - @Override - public void setStringFlag(String name, String value) { - } - - @Override - public boolean getBooleanFlag(String name) { - return false; - } - - @Override - public void setBooleanFlag(String name, boolean value) { - } - - @Override - public long getIntFlag(String name) { - return 0; - } - - @Override - public double getFloatFlag(String name) { - return 0; - } - }; - - String getStringFlag(String name); - void setStringFlag(String name, String value); - boolean getBooleanFlag(String name); - void setBooleanFlag(String name, boolean value); - long getIntFlag(String name); - double getFloatFlag(String name); - } - - private class FlagsImpl implements Flags { - public String getStringFlag(String name) { - return findStringJVMFlag0(name); - } - - public void setStringFlag(String name, String value) { - setStringJVMFlag0(name, value); - } - - public boolean getBooleanFlag(String name) { - return findBooleanJVMFlag0(name); - } - - public void setBooleanFlag(String name, boolean value) { - setBooleanJVMFlag0(name, value); - } - - public long getIntFlag(String name) { - return findIntJVMFlag0(name); - } - - public double getFloatFlag(String name) { - return findFloatJVMFlag0(name); - } - } - - /** - * Get the JVM access instance. - * - * @return the JVM access instance - */ - public static JVMAccess getInstance() { - return SingletonHolder.INSTANCE; - } - - private final LibraryLoader.Result libraryLoadResult; - private final Flags flags; - - private JVMAccess() { - LibraryLoader.Result result = LibraryLoader.builder().load();; - if (result.succeeded) { - // library loaded successfully, check if we can access JVM - try { - healthCheck0(); - } catch (Throwable t) { - // failed to access JVM; update the result - result = new LibraryLoader.Result(false, t); - } - - } - if (!result.succeeded && result.error != null) { - System.out.println("[WARNING] Failed to obtain JVM access.\n" + result.error); - } - flags = result.succeeded ? new FlagsImpl() : Flags.NONE; - libraryLoadResult = result; - } - - /** - * Create a JVM access instance. - * @param libLocation the library location or {@literal null} - * @param scratchDir the scratch directory or {@literal null} - * @param errorHandler the error handler or {@literal null} - */ - public JVMAccess(String libLocation, String scratchDir, Consumer errorHandler) { - LibraryLoader.Result result = LibraryLoader.builder().withLibraryLocation(libLocation).withScratchDir(scratchDir).load(); - if (result.succeeded) { - // library loaded successfully, check if we can access JVM - try { - healthCheck0(); - } catch (Throwable t) { - // failed to access JVM; update the result - result = new LibraryLoader.Result(false, t); - } - - } - if (!result.succeeded && result.error != null) { - if (errorHandler != null) { - errorHandler.accept(result.error); - } else { - System.out.println("[WARNING] Failed to obtain JVM access.\n" + result.error); - } - } - flags = result.succeeded ? new FlagsImpl() : Flags.NONE; - libraryLoadResult = result; - } - - /** - * Get the JVM flags. - * - * @return the JVM flags - */ - public Flags flags() { - return flags; - } - - /** - * Check if the JVM access is active. - * - * @return {@literal true} if the JVM access is active, {@literal false} otherwise - */ - public boolean isActive() { - return libraryLoadResult.succeeded; - } - - // a dummy method to check if the library has loaded properly - private native boolean healthCheck0(); - - private native String findStringJVMFlag0(String name); - private native void setStringJVMFlag0(String name, String value); - private native boolean findBooleanJVMFlag0(String name); - private native void setBooleanJVMFlag0(String name, boolean value); - private native long findIntJVMFlag0(String name); - private native double findFloatJVMFlag0(String name); -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java deleted file mode 100644 index 505e54fbe..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/JavaProfiler.java +++ /dev/null @@ -1,469 +0,0 @@ -/* - * Copyright 2018 Andrei Pangin - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.datadoghq.profiler; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.file.Path; -import java.util.HashMap; -import java.util.Map; - -/** - * Java API for in-process profiling. Serves as a wrapper around - * java-profiler native library. This class is a singleton. - * The first call to {@link #getInstance()} initiates loading of - * libjavaProfiler.so. - */ -public final class JavaProfiler { - static final class TSCFrequencyHolder { - /** - * TSC frequency required to convert ticks into seconds - */ - static final long FREQUENCY = tscFrequency0(); - } - private static JavaProfiler instance; - - // Thread-local storage for profiling context - private final ThreadLocal tlsContextStorage = ThreadLocal.withInitial(JavaProfiler::initializeThreadContext); - - private JavaProfiler() { - } - - /** - * Get a {@linkplain JavaProfiler} instance backed by the bundled native library and using - * the default temp directory as the scratch where the bundled library will be exploded - * before linking. - */ - public static JavaProfiler getInstance() throws IOException { - return getInstance(null, null); - } - - /** - * Get a {@linkplain JavaProfiler} instance backed by the bundled native library and using - * the given directory as the scratch where the bundled library will be exploded - * before linking. - * @param scratchDir directory where the bundled library will be exploded before linking - */ - public static JavaProfiler getInstance(String scratchDir) throws IOException { - return getInstance(null, scratchDir); - } - - /** - * Get a {@linkplain JavaProfiler} instance backed by the given native library and using - * the given directory as the scratch where the bundled library will be exploded - * before linking. - * @param libLocation the path to the native library to be used instead of the bundled one - * @param scratchDir directory where the bundled library will be exploded before linking; ignored when 'libLocation' is {@literal null} - */ - public static synchronized JavaProfiler getInstance(String libLocation, String scratchDir) throws IOException { - if (instance != null) { - return instance; - } - - JavaProfiler profiler = new JavaProfiler(); - LibraryLoader.Result result = LibraryLoader.builder().withLibraryLocation(libLocation).withScratchDir(scratchDir).load(); - if (!result.succeeded) { - throw new IOException("Failed to load Datadog Java profiler library", result.error); - } - init0(); - - instance = profiler; - - String maxArenaValue = System.getProperty("ddprof.debug.malloc_arena_max"); - if (maxArenaValue != null) { - try { - mallocArenaMax0(Integer.parseInt(maxArenaValue)); - } catch (NumberFormatException e) { - System.out.println("[WARN] Invalid value for ddprof.debug.malloc_arena_max: " + maxArenaValue + ". Expecting an integer."); - } - } - - return profiler; - } - - /** - * Stop profiling (without dumping results) - * - * @throws IllegalStateException If profiler is not running - */ - public void stop() throws IllegalStateException { - stop0(); - } - - /** - * Get the number of samples collected during the profiling session - * - * @return Number of samples - */ - public static native long getSamples(); - - /** - * Get profiler agent version, e.g. "1.0" - * - * @return Version string - */ - public String getVersion() { - try { - return execute0("version"); - } catch (IOException e) { - throw new IllegalStateException(e); - } - } - - public String getStatus() { - return getStatus0(); - } - - /** - * Execute an agent-compatible profiling command - - * the comma-separated list of arguments described in arguments.cpp - * - * @param command Profiling command - * @return The command result - * @throws IllegalArgumentException If failed to parse the command - * @throws IOException If failed to create output file - */ - public String execute(String command) throws IllegalArgumentException, IllegalStateException, IOException { - if (command == null) { - throw new NullPointerException(); - } - return execute0(command); - } - - /** - * Records the completion of the trace root - */ - public boolean recordTraceRoot(long rootSpanId, String endpoint, String operation, int sizeLimit) { - return recordTrace0(rootSpanId, endpoint, operation, sizeLimit); - } - - /** - * Records the completion of the trace root - */ - @Deprecated - public boolean recordTraceRoot(long rootSpanId, String endpoint, int sizeLimit) { - return recordTrace0(rootSpanId, endpoint, null, sizeLimit); - } - - /** - * Add the given thread to the set of profiled threads. - * 'filter' option must be enabled to use this method. - */ - public void addThread() { - filterThreadAdd0(); - } - - /** - * Remove the given thread to the set of profiled threads. - * 'filter' option must be enabled to use this method. - */ - public void removeThread() { - filterThreadRemove0(); - } - - /** - * Passing context identifier to a profiler. This ID is thread-local and is dumped in - * the JFR output only. 0 is a reserved value for "no-context". - * - *

Note: {@code rootSpanId} maps to {@code localRootSpanId} internally. A synthetic - * trace_id of {@code [0, spanId]} is written to the OTEP record. For correct W3C - * trace ID interop use {@link #setContext(long, long, long, long)}. - * - * @param spanId Span identifier that should be stored for current thread - * @param rootSpanId Local root span identifier (used for endpoint correlation) - * @deprecated Use {@link #setContext(long, long, long, long)} for full OTEP interop. - */ - @Deprecated - public void setContext(long spanId, long rootSpanId) { - tlsContextStorage.get().put(spanId, rootSpanId); - } - - /** - * Sets trace context with full 128-bit W3C trace ID, span ID, and local root span ID. - * - * @param localRootSpanId Local root span ID (for endpoint correlation) - * @param spanId Span identifier - * @param traceIdHigh Upper 64 bits of the 128-bit trace ID - * @param traceIdLow Lower 64 bits of the 128-bit trace ID - */ - public void setContext(long localRootSpanId, long spanId, long traceIdHigh, long traceIdLow) { - tlsContextStorage.get().put(localRootSpanId, spanId, traceIdHigh, traceIdLow); - } - - /** - * Resets the current thread's context to zero (traceId=0, spanId=0, localRootSpanId=0). - * Custom context attributes are also cleared. - */ - public void clearContext() { - tlsContextStorage.get().put(0, 0, 0, 0); - } - - /** - * Sets a custom context attribute at the given slot offset for the current thread. - * - * @param offset slot index (0-based, in [0, 9]); out-of-range values return {@code false} - * @param value the string value to record; {@code null} returns {@code false} without - * writing; an empty string is written as a zero-length entry (not a clear — - * use {@link #clearContextAttribute(int)} to remove a value) - * @return true if the value was recorded; false if {@code offset} is out of range, - * {@code value} is null, the Dictionary is full, or {@code attrs_data} overflows - * for this slot - */ - public boolean setContextAttribute(int offset, String value) { - return tlsContextStorage.get().setContextAttribute(offset, value); - } - - /** - * Clears the custom context attribute at the given slot offset for the current thread. - * Zeros the sidecar encoding and removes it from OTEP {@code attrs_data}. - * - * @param offset slot index (0-based, in [0, 9]); out-of-range values are silently ignored - */ - public void clearContextAttribute(int offset) { - tlsContextStorage.get().clearContextAttribute(offset); - } - - /** - * Re-applies multiple custom attributes from precomputed constant IDs and UTF-8 bytes for - * the current thread in a single detach/attach window. - * - *

    - *
  • Slots with {@code constantIds[i] <= 0} are skipped.
  • - *
  • Returns {@code false} without writing if the thread's record is not currently valid - * (span-less), to avoid resurrecting a cleared record.
  • - *
  • On {@code attrs_data} overflow, the overflowed slot's sidecar is zeroed and - * {@code false} is returned; slots written before the overflow are retained.
  • - *
- * - * @param constantIds per-slot Dictionary constant IDs; entries {@code <= 0} are skipped - * @param utf8 per-slot UTF-8 value bytes; must be non-null and at most 255 bytes - * (the OTEP attrs_data entry length field is one byte) for every slot - * whose {@code constantId > 0} - * @return true if every slot with {@code constantId > 0} was written; false on a cleared - * (span-less) record, or {@code attrs_data} overflow for any slot - * @throws NullPointerException if {@code constantIds}, {@code utf8}, or any active - * {@code utf8[i]} is null - * @throws IllegalArgumentException if the arrays have different lengths, exceed the slot limit, - * or any active {@code utf8[i]} exceeds 255 bytes - */ - public boolean setContextAttributesByIdAndBytes(int[] constantIds, byte[][] utf8) { - return tlsContextStorage.get().setContextAttributesByIdAndBytes(constantIds, utf8); - } - - void copyTags(int[] snapshot) { - tlsContextStorage.get().copyCustoms(snapshot); - } - - /** - * Dumps the JFR recording at the provided path - * @param recording the path to the recording - * @throws NullPointerException if recording is null - */ - public void dump(Path recording) { - dump0(recording.toAbsolutePath().toString()); - } - - /** - * Records a datadog.ProfilerSetting event with no unit - * @param name the name - * @param value the value - */ - public void recordSetting(String name, String value) { - recordSetting(name, value, ""); - } - - /** - * Records a datadog.ProfilerSetting event - * @param name the name - * @param value the value - * @param unit the unit - */ - public void recordSetting(String name, String value, String unit) { - recordSettingEvent0(name, value, unit); - } - - - /** - * Scales the ticks to milliseconds and applies a threshold - */ - public boolean isThresholdExceeded(long thresholdMillis, long startTicks, long endTicks) { - return endTicks - startTicks > thresholdMillis * TSCFrequencyHolder.FREQUENCY / 1000; - } - - /** - * Records when queueing ended - * @param task the name of the enqueue task - * @param scheduler the name of the thread-pool or executor scheduling the task - * @param origin the thread the task was submitted on - */ - public void recordQueueTime(long startTicks, - long endTicks, - Class task, - Class scheduler, - Class queueType, - int queueLength, - Thread origin) { - recordQueueEnd0(startTicks, endTicks, task.getName(), scheduler.getName(), origin, queueType.getName(), queueLength); - } - - /** - * Internal hook called before {@code LockSupport.park}. This remains package-scoped - * until PR2 wires production TaskBlock instrumentation. - */ - void parkEnter() { - parkEnter0(); - } - - /** - * Internal hook called after {@code LockSupport.park}. Clears the parked flag. - * {@code blocker} and {@code unblockingSpanId} are reserved for PR2 TaskBlock use. - */ - void parkExit(long blocker, long unblockingSpanId) { - parkExit0(blocker, unblockingSpanId); - } - - /** - * Internal hook marking the current platform thread as entering an explicitly instrumented - * blocked interval. This is not public API in this PR; production TaskBlock wiring lands in PR2. - * - * @param state native {@code OSThreadState} value for the blocked interval; - * currently only {@code SLEEPING} is armed - * @return an opaque token to pass to {@link #blockExit(long)}, or 0 if no state was armed - */ - long blockEnter(int state) { - return blockEnter0(state); - } - - /** - * Clears a blocked interval previously armed by {@link #blockEnter(int)}. - */ - void blockExit(long token) { - blockExit0(token); - } - - /** - * Get the ticks for the current thread. - * @return ticks - */ - public long getCurrentTicks() { - return currentTicks0(); - } - - /** - * If the profiler is built in debug mode, returns counters recorded during profile execution. - * These are for whitebox testing and not intended for production use. - * @return a map of counters - */ - public Map getDebugCounters() { - Map counters = new HashMap<>(); - ByteBuffer buffer = getDebugCounters0().order(ByteOrder.LITTLE_ENDIAN); - if (buffer.hasRemaining()) { - String[] names = describeDebugCounters0(); - for (int i = 0; i < names.length && i * 128 < buffer.capacity(); i++) { - counters.put(names[i], buffer.getLong(i * 128)); - } - } - return counters; - } - - private static ThreadContext initializeThreadContext() { - long[] metadata = new long[6]; - ByteBuffer buffer = initializeContextTLS0(metadata); - if (buffer == null) { - throw new IllegalStateException("Failed to initialize OTEL TLS — ProfiledThread not available"); - } - return new ThreadContext(buffer, metadata); - } - - private static native boolean init0(); - private native void stop0() throws IllegalStateException; - private native String execute0(String command) throws IllegalArgumentException, IllegalStateException, IOException; - - private static native void filterThreadAdd0(); - private static native void filterThreadRemove0(); - - private static native int getTid0(); - - private static native boolean recordTrace0(long rootSpanId, String endpoint, String operation, int sizeLimit); - - private static native void dump0(String recordingFilePath); - - private static native ByteBuffer getDebugCounters0(); - - private static native String[] describeDebugCounters0(); - - private static native void recordSettingEvent0(String name, String value, String unit); - - private static native void recordQueueEnd0(long startTicks, long endTicks, String task, String scheduler, Thread origin, String queueType, int queueLength); - - private static native void parkEnter0(); - - private static native void parkExit0(long blocker, long unblockingSpanId); - - private static native long blockEnter0(int state); - - private static native void blockExit0(long token); - - private static native long currentTicks0(); - - private static native long tscFrequency0(); - - private static native void mallocArenaMax0(int max); - - private static native String getStatus0(); - - /** - * Initializes context TLS for the current thread and returns a single DirectByteBuffer - * spanning the OTEP record + tag-encoding sidecar + LRS (688 bytes, contiguous in - * ProfiledThread). Sets otel_thread_ctx_v1 permanently to the thread's - * OtelThreadContextRecord. - * - * @param metadata output array filled with absolute offsets into the returned buffer: - * [0] VALID_OFFSET — offset of 'valid' field - * [1] TRACE_ID_OFFSET — offset of 'trace_id' field - * [2] SPAN_ID_OFFSET — offset of 'span_id' field - * [3] ATTRS_DATA_SIZE_OFFSET — offset of 'attrs_data_size' field - * [4] ATTRS_DATA_OFFSET — offset of 'attrs_data' field - * [5] LRS_OFFSET — offset of local_root_span_id - */ - private static native ByteBuffer initializeContextTLS0(long[] metadata); - - public ThreadContext getThreadContext() { - return tlsContextStorage.get(); - } - -// --- test and debug utility methods - - /** - * Write the profiler TEST_LOG - the message will be in sequence with other profiler logs - * @param msg the log message - */ - public static native void testlog(String msg); - - public static native void dumpContext(); - - /** - * Resets the cached ThreadContext for the current thread. - * The next call to {@link #getThreadContext()} or any {@code setContext} overload - * will re-create it with fresh OTEL TLS buffers. - */ - public void resetThreadContext() { - tlsContextStorage.remove(); - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/LibraryLoader.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/LibraryLoader.java deleted file mode 100644 index f7643e8fb..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/LibraryLoader.java +++ /dev/null @@ -1,152 +0,0 @@ -package com.datadoghq.profiler; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.LockSupport; - -/** - * Encapsulates dynamic library loading logic. - * It is used to load the native library from the classpath or from a custom location. - * When loading from the classpath, the library is extracted to a temporary file and loaded from there. - * - */ -public final class LibraryLoader { - enum LoadingState { - NOT_LOADED, - LOADING, - LOADED, - UNAVAILABLE - } - - /** - * Represents the result of a library loading operation. - */ - public static final class Result { - public static final Result SUCCESS = new Result(true, null); - public static final Result UNAVAILABLE = new Result(false, null); - - public final boolean succeeded; - public final Throwable error; - - public Result(boolean succeeded, Throwable error) { - this.succeeded = succeeded; - this.error = error; - } - } - - /** - * Builder for {@link LibraryLoader}. It allows to specify the library location and the scratch directory. - */ - public static final class Builder { - private String libraryLocation; - private String scratchDir; - - private Builder() {} - - /** - * Sets the library location. - * @param libraryLocation the library location - * @return this builder - */ - public Builder withLibraryLocation(String libraryLocation) { - this.libraryLocation = libraryLocation; - return this; - } - - /** - * Sets the scratch directory where the temp library file can be created. - * @param scratchDir the scratch directory - * @return this builder - */ - public Builder withScratchDir(String scratchDir) { - this.scratchDir = scratchDir; - return this; - } - - /** - * Loads the library. - * @return the result of the library loading operation - */ - public Result load() { - return loadLibrary(libraryLocation, scratchDir); - } - } - - private static final String NATIVE_LIBS = "/META-INF/native-libs"; - private static final String JAVA_PROFILER_LIBRARY_NAME_BASE = "libjavaProfiler"; - private static final String JAVA_PROFILER_LIBRARY_NAME = JAVA_PROFILER_LIBRARY_NAME_BASE + "." + (OperatingSystem.current() == OperatingSystem.macos ? "dylib" : "so"); - - private static final Map> loadingStateMap = new ConcurrentHashMap<>(); - - public static Builder builder() { - return new Builder(); - } - - private static Result loadLibrary(final String libraryLocation, String scratchDir) { - String key = libraryLocation == null ? JAVA_PROFILER_LIBRARY_NAME : libraryLocation; - AtomicReference state = loadingStateMap.computeIfAbsent(key, (k) -> new AtomicReference<>(LoadingState.NOT_LOADED)); - - try { - // first thread to arrive will set the flag to 'loading' and will load the library - if (!state.compareAndSet(LoadingState.NOT_LOADED, LoadingState.LOADING)) { - // if there is already a different thread loading the library we will wait for it to finish - while (state.get() == LoadingState.LOADING) { - LockSupport.parkNanos(5_000_000L); // 5ms - } - // the library has been loaded by another thread, we can return - return state.get() == LoadingState.LOADED ? Result.SUCCESS : Result.UNAVAILABLE; - } - // if the attempt to load the library failed do not try again - if (state.get() == LoadingState.UNAVAILABLE) { - return Result.UNAVAILABLE; - } - Path libraryPath = libraryLocation != null ? Paths.get(libraryLocation) : null; - if (libraryPath == null) { - OperatingSystem os = OperatingSystem.current(); - String qualifier = (os == OperatingSystem.linux && os.isMusl()) ? "musl" : null; - - libraryPath = libraryFromClasspath(os, Arch.current(), qualifier, Paths.get(scratchDir != null ? scratchDir : System.getProperty("java.io.tmpdir"))); - } - System.load(libraryPath.toAbsolutePath().toString()); - return Result.SUCCESS; - } catch (Throwable t) { - state.set(LoadingState.UNAVAILABLE); - return new Result(false, t); - } finally { - state.compareAndSet(LoadingState.LOADING, LoadingState.LOADED); - } - } - - /** - * Locates a library on class-path (eg. in a JAR) and creates a publicly accessible temporary copy - * of the library which can then be used by the application by its absolute path. - * - * @param os The operating system - * @param arch The architecture - * @param qualifier An optional qualifier (eg. musl) - * @param tempDir The working scratch dir where to store the temp library file - * @return The library absolute path. The caller should properly dispose of the file once it is - * not needed. The file is marked for 'delete-on-exit' so it won't survive a JVM restart. - * @throws IOException, IllegalStateException if the resource is not found on the classpath - */ - private static Path libraryFromClasspath(OperatingSystem os, Arch arch, String qualifier, Path tempDir) throws IOException { - String resourcePath = NATIVE_LIBS + "/" + os.name().toLowerCase() + "-" + arch.name().toLowerCase() + ((qualifier != null && !qualifier.isEmpty()) ? "-" + qualifier : "") + "/" + JAVA_PROFILER_LIBRARY_NAME; - - InputStream libraryData = JavaProfiler.class.getResourceAsStream(resourcePath); - - if (libraryData != null) { - Path libFile = Files.createTempFile(tempDir, JAVA_PROFILER_LIBRARY_NAME_BASE + "-dd-tmp", ".so"); - Files.copy(libraryData, libFile, StandardCopyOption.REPLACE_EXISTING); - libFile.toFile().deleteOnExit(); - return libFile; - } - throw new IllegalStateException(resourcePath + " not found on classpath"); - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/Main.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/Main.java deleted file mode 100644 index f09b490a9..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/Main.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.datadoghq.profiler; - -import java.io.IOException; -import java.lang.instrument.Instrumentation; - -public class Main { - public static void main(String... args) throws IOException { - String command = args.length > 0 ? args[0] : "status"; - JavaProfiler profiler = JavaProfiler.getInstance(); - profiler.execute(command); - if (command.contains("start")) { - profiler.stop(); - } - } - - public static void premain(String agentArgs, Instrumentation inst) { - try { - JavaProfiler profiler = JavaProfiler.getInstance(); - profiler.execute(agentArgs); - } catch (IOException e) { - e.printStackTrace(); - } - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java deleted file mode 100644 index 2110ac65c..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/OTelContext.java +++ /dev/null @@ -1,266 +0,0 @@ -package com.datadoghq.profiler; - -import java.util.Objects; -import java.util.concurrent.locks.ReentrantReadWriteLock; -import java.util.function.Consumer; - -/** - * OpenTelemetry Process Context API for sharing process-level context information. - * - *

This class provides functionality to publish OpenTelemetry semantic conventions - * compliant process context information that can be discovered and read by external - * monitoring tools and profilers. The context is shared via platform-specific - * mechanisms (currently Linux-only) and includes service identification metadata. - * - *

Platform Support: - *

    - *
  • Linux: Full support using anonymous memory mappings with prctl naming
  • - *
  • Others: Limited support - API calls are no-ops
  • - *
- * - *

Thread Safety: This class is thread-safe. All public methods can be - * called concurrently from multiple threads. - * - *

Usage Example: - *

{@code
- * // Get the singleton instance
- * OTelContext context = OTelContext.getInstance();
- * 
- * // Set process context for external discovery
- * context.initializeAllContext(...);
- * }
- * - *

External Discovery: Once published, the process context can be - * discovered by external tools by scanning /proc/*/maps for mappings named - * [anon:OTEL_CTX] on Linux systems. - * - * @since 1.30.0 - */ -public final class OTelContext { - private static final class SingletonHolder { - static final OTelContext INSTANCE = new OTelContext(); - } - - /** - * Represents the OpenTelemetry process context data. - */ - public static final class ProcessContext { - public final String deploymentEnvironmentName; - public final String hostName; - public final String serviceInstanceId; - public final String serviceName; - public final String serviceVersion; - public final String telemetrySdkLanguage; - public final String telemetrySdkVersion; - public final String telemetrySdkName; - /** - * The threadlocal.attribute_key_map published in the process context's - * thread_ctx_config, or null if no thread context configuration was - * published. The first entry is always the reserved - * {@code datadog.local_root_span_id} slot; user-registered keys follow - * in registration order. - */ - public final String[] attributeKeyMap; - - public ProcessContext(String deploymentEnvironmentName, String hostName, String serviceInstanceId, String serviceName, String serviceVersion, String telemetrySdkLanguage, String telemetrySdkVersion, String telemetrySdkName, String[] attributeKeyMap) { - this.deploymentEnvironmentName = deploymentEnvironmentName; - this.hostName = hostName; - this.serviceInstanceId = serviceInstanceId; - this.serviceName = serviceName; - this.serviceVersion = serviceVersion; - this.telemetrySdkLanguage = telemetrySdkLanguage; - this.telemetrySdkVersion = telemetrySdkVersion; - this.telemetrySdkName = telemetrySdkName; - this.attributeKeyMap = attributeKeyMap; - } - - @Override - public String toString() { - return String.format("ProcessContext{deploymentEnvironmentName='%s', hostName='%s', serviceInstanceId='%s', serviceName='%s', serviceVersion='%s', telemetrySdkLanguage='%s', telemetrySdkVersion='%s', telemetrySdkName='%s', attributeKeyMap=%s}", - deploymentEnvironmentName, hostName, serviceInstanceId, serviceName, serviceVersion, telemetrySdkLanguage, telemetrySdkVersion, telemetrySdkName, java.util.Arrays.toString(attributeKeyMap)); - } - } - - /** - * Returns the singleton instance of the OpenTelemetry process context. - * - *

This method provides access to the globally shared OTelContext instance - * using the lazy initialization pattern. The instance is created on first access - * and reused for all subsequent calls. - * - *

Note: If library loading fails during initialization, a warning - * will be printed to System.out, but a valid (though non-functional) instance - * will still be returned. - * - * @return the singleton OTelContext instance, never null - */ - public static OTelContext getInstance() { - return SingletonHolder.INSTANCE; - } - - private final LibraryLoader.Result libraryLoadResult; - private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); - - /** - * Private constructor for singleton instance. - * - *

Initializes the native library and handles any loading failures gracefully - * by printing warnings to System.out. - */ - private OTelContext() { - LibraryLoader.Result result = LibraryLoader.builder().load(); - if (!result.succeeded ) { - System.out.println("[WARNING] Failed to obtain OTel context.\n" + result.error); - } - libraryLoadResult = result; - } - - /** - * Creates a custom OTelContext instance with specific library loading configuration. - * - *

This constructor allows for advanced configuration of the native library loading - * process, including custom library locations and error handling. Most users should - * use {@link #getInstance()} instead. - * - *

Warning: Creating multiple instances may lead to undefined behavior - * as the underlying native library maintains global state. - * - * @param libLocation the custom library location, or null to use default discovery - * @param scratchDir the scratch directory for temporary files, or null for system default - * @param errorHandler custom error handler for library loading failures, or null - * to print warnings to System.out - */ - public OTelContext(String libLocation, String scratchDir, Consumer errorHandler) { - LibraryLoader.Result result = LibraryLoader.builder().withLibraryLocation(libLocation).withScratchDir(scratchDir).load(); - if (!result.succeeded && result.error != null) { - if (errorHandler != null) { - errorHandler.accept(result.error); - } else { - System.out.println("[WARNING] Failed to obtain OTelContext access.\n" + result.error); - } - } - libraryLoadResult = result; - } - - /** - * Reads the currently published OpenTelemetry process context, if any. - * - *

This method attempts to read back the process context that was previously - * published via {@link #initializeAllContext(String, String, String, String, String, String, String[])}. This is - * primarily useful for debugging and testing purposes. - * - *

Platform Support: Currently only supported on Linux. On other - * platforms, this method will return null. - * - * @return a ProcessContext object containing the current context data if - * successfully read, or null if no context is published or reading failed - * @since 1.30.0 - */ - public ProcessContext readProcessContext() { - if (!libraryLoadResult.succeeded) { - return null; - } - try { - lock.readLock().lock(); - return readProcessCtx0(); - } finally { - lock.readLock().unlock(); - } - } - - /** - * Initializes the OpenTelemetry context shared with external profilers: it publishes the - * process-level context and, as part of the same call, sets up the custom thread-context - * attribute names ({@code attributeKeys}) as the {@code attribute_key_map}. Callers must - * invoke this method for those custom attribute names to be published. - * - *

Important: if this method is mistakenly not called, the omission is easy to miss - * because nothing visibly breaks; java-profiler keeps working and in-process profiling and - * per-thread context capture are unaffected. The only effect is silent and external: readers - * implementing the OpenTelemetry context sharing specification will be unable to read - * this information (the process context and the thread-context {@code attribute_key_map}). - * - *

This method publishes process-level context information following OpenTelemetry - * semantic conventions. The context is made available to external monitoring tools - * and profilers through platform-specific mechanisms. - * - *

On Linux: Creates a named anonymous memory mapping that can be - * discovered by external tools scanning /proc/*/maps for [anon:OTEL_CTX] - * entries. - * - *

On other platforms: This method is a no-op as process context - * sharing is not currently supported. - * - *

Context Lifecycle: The published context remains active until - * the process exits. Calling this method multiple times will replace the previous - * context with the new values. - * - *

Usage Example: - *

{@code
-     * OTelContext.getInstance().initializeAllContext(
-     *     "staging",           // env
-     *     "my-hostname",       // hostname
-     *     "instance-12345",    // runtime-id
-     *     "my-service",        // service
-     *     "1.0.0",             // version
-     *     "3.5.0",             // tracer-version
-     *     new String[] {"http.route", "db.system"}  // thread-context attribute keys
-     * );
-     * }
- * - * @param env the deployment environment name as defined by OpenTelemetry - * semantic conventions (deployment.environment.name). Must not be null. - * Examples: "production", "staging", "development", "test" - * @param hostname the hostname of the service, recorded under the OpenTelemetry - * semantic convention key host.name as a resource attribute. Must not be null. - * Examples: "my-hostname", "my-hostname.example.com" - * @param runtimeId the unique identifier for this specific instance of the service - * as defined by OpenTelemetry semantic conventions (service.instance.id). - * Must not be null. - * @param service the logical name of the service as defined by OpenTelemetry - * semantic conventions (service.name). Must not be null. - * Examples: "order-service", "user-management", "payment-processor" - * @param version the version of the service as defined by OpenTelemetry - * semantic conventions (service.version). Must not be null. - * Examples: "1.0.0", "2.3.4" - * @param tracerVersion the version of the tracer as defined by OpenTelemetry - * semantic conventions (telemetry.sdk.version). Must not be null. - * Examples: "3.5.0", "4.2.0" - * @param attributeKeys the thread-context attribute key names whose per-thread - * values are recorded in the OTEP thread-local record (e.g. - * "http.route", "db.system"). Published in the process context's - * thread_ctx_config as the attribute_key_map, preceded by the - * reserved datadog.local_root_span_id slot. Must not be null - * (may be empty); keys beyond capacity are clipped. If any - * element is null, the entire process context publish is - * skipped - no context is published (a warning is logged) and - * no exception is thrown. Order must match the indices used - * with {@link ThreadContext#setContextAttribute(int, String)}. - * - * @throws NullPointerException if {@code attributeKeys} is null - * - * @see OpenTelemetry Service Attributes - * @see OpenTelemetry Deployment Attributes - */ - /** @deprecated Use {@link #initializeAllContext(String, String, String, String, String, String, String[])} instead. */ - @Deprecated - public void setProcessContext(String env, String hostname, String runtimeId, String service, String version, String tracerVersion) { - initializeAllContext(env, hostname, runtimeId, service, version, tracerVersion, new String[0]); - } - - public void initializeAllContext(String env, String hostname, String runtimeId, String service, String version, String tracerVersion, String[] attributeKeys) { - Objects.requireNonNull(attributeKeys, "attributeKeys"); - if (!libraryLoadResult.succeeded) { - return; - } - try { - lock.writeLock().lock(); - setProcessCtx0(env, hostname, runtimeId, service, version, tracerVersion, attributeKeys); - } finally { - lock.writeLock().unlock(); - } - } - - private static native void setProcessCtx0(String env, String hostname, String runtimeId, String service, String version, String tracerVersion, String[] attributeKeys); - private static native ProcessContext readProcessCtx0(); -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/OperatingSystem.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/OperatingSystem.java deleted file mode 100644 index 2289914e0..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/OperatingSystem.java +++ /dev/null @@ -1,122 +0,0 @@ -package com.datadoghq.profiler; - -import java.io.BufferedReader; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.EnumSet; -import java.util.HashSet; -import java.util.Set; - -/** A simple way to detect the current operating system */ -enum OperatingSystem { - linux("Linux", "linux"), - macos("Mac OS X", "macOS", "mac"), - unknown(); - - private final Set identifiers; - - OperatingSystem(String... identifiers) { - this.identifiers = new HashSet<>(Arrays.asList(identifiers)); - } - - public static OperatingSystem of(String identifier) { - for (OperatingSystem os : EnumSet.allOf(OperatingSystem.class)) { - if (os.identifiers.contains(identifier)) { - return os; - } - } - return unknown; - } - - public static OperatingSystem current() { - return OperatingSystem.of(System.getProperty("os.name")); - } - - public boolean isMusl() throws IOException { - // check the Java exe then fall back to proc/self maps - try { - return isMuslJavaExecutable(); - } catch (IOException e) { - try { - return isMuslProcSelfMaps(); - } catch (IOException ignore) { - // not finding the Java exe is more interesting than failing to parse /proc/self/maps - throw e; - } - } - } - - // package-private access for testing only - boolean isMuslProcSelfMaps() throws IOException { - try (BufferedReader reader = new BufferedReader(new FileReader("/proc/self/maps"))) { - String line; - while ((line = reader.readLine()) != null) { - if (line.contains("-musl-")) { - return true; - } - if (line.contains("/libc.")) { - return false; - } - } - } - return false; - } - - /** - * There is information about the linking in the ELF file. Since properly parsing ELF is not - * trivial this code will attempt a brute-force approach and will scan the first 4096 bytes - * of the 'java' program image for anything prefixed with `/ld-` - in practice this will contain - * `/ld-musl` for musl systems and probably something else for non-musl systems (eg. `/ld-linux-...`). - * However, if such string is missing should indicate that the system is not a musl one. - */ - // package-private access for testing only - boolean isMuslJavaExecutable() throws IOException { - - byte[] magic = new byte[]{(byte)0x7f, (byte)'E', (byte)'L', (byte)'F'}; - byte[] prefix = new byte[]{(byte)'/', (byte)'l', (byte)'d', (byte)'-'}; // '/ld-*' - byte[] musl = new byte[]{(byte)'m', (byte)'u', (byte)'s', (byte)'l'}; // 'musl' - - Path binary = Paths.get(System.getProperty("java.home"), "bin", "java"); - byte[] buffer = new byte[4096]; - - try (InputStream is = Files.newInputStream(binary)) { - int read = is.read(buffer, 0, 4); - if (read != 4 || !containsArray(buffer, 0, magic)) { - throw new IOException(Arrays.toString(buffer)); - } - read = is.read(buffer); - if (read <= 0) { - throw new IOException(); - } - int prefixPos = 0; - for (int i = 0; i < read; i++) { - if (buffer[i] == prefix[prefixPos]) { - if (++prefixPos == prefix.length) { - return containsArray(buffer, i + 1, musl); - } - } else { - prefixPos = 0; - } - } - } - return false; - } - - private static boolean containsArray(byte[] container, int offset, byte[] contained) { - for (int i = 0; i < contained.length; i++) { - int leftPos = offset + i; - if (leftPos >= container.length) { - return false; - } - if (container[leftPos] != contained[i]) { - return false; - } - } - return true; - } - } diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/Platform.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/Platform.java deleted file mode 100644 index 51d7df3fe..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/Platform.java +++ /dev/null @@ -1,345 +0,0 @@ -package com.datadoghq.profiler; - -import java.io.IOException; -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; - -import java.util.ArrayList; -import java.util.List; - -public final class Platform { - - public enum GC { - SERIAL("marksweep"), - PARALLEL("ps"), - CMS("concurrentmarksweep"), - G1("g1"), - SHENANDOAH("shenandoah"), - Z("z"), - UNKNOWN(""); - - private final String identifierPrefix; - - GC(String identifierPrefix) { - this.identifierPrefix = identifierPrefix; - } - - static GC current() { - for (GarbageCollectorMXBean mxBean : ManagementFactory.getGarbageCollectorMXBeans()) { - if (mxBean.isValid()) { - String name = mxBean.getName().toLowerCase(); - for (GC gc : GC.values()) { - if (gc != UNKNOWN && name.startsWith(gc.identifierPrefix)) { - return gc; - } - } - } - } - return UNKNOWN; - } - } - - private static final Version JAVA_VERSION = parseJavaVersion(System.getProperty("java.version")); - private static final JvmRuntime RUNTIME = new JvmRuntime(); - - private static final GC GARBAGE_COLLECTOR = GC.current(); - - private static final boolean HAS_JFR = checkForJfr(); - private static final boolean IS_NATIVE_IMAGE_BUILDER = checkForNativeImageBuilder(); - - public static GC activeGarbageCollector() { - return GARBAGE_COLLECTOR; - } - - public static boolean hasJfr() { - return HAS_JFR; - } - - public static boolean isNativeImageBuilder() { - return IS_NATIVE_IMAGE_BUILDER; - } - - private static boolean checkForJfr() { - try { - /* Check only for the open-sources JFR implementation. - * If it is ever needed to support also the closed sourced JDK 8 version the check should be - * enhanced. - * Need this custom check because ClassLoaderMatchers.hasClassNamed() does not support bootstrap class loader yet. - * Note: the downside of this is that we load some JFR classes at startup. - */ - return ClassLoader.getSystemClassLoader().getResource("jdk/jfr/Event.class") != null; - } catch (Throwable e) { - return false; - } - } - - private static boolean checkForNativeImageBuilder() { - try { - return "org.graalvm.nativeimage.builder".equals(System.getProperty("jdk.module.main")); - } catch (Throwable e) { - return false; - } - } - - /* The method splits java version string by digits. Delimiters are: dot, underscore and plus */ - private static List splitDigits(String str) { - List results = new ArrayList<>(); - - int len = str.length(); - - int value = 0; - for (int i = 0; i < len; i++) { - char ch = str.charAt(i); - if (ch >= '0' && ch <= '9') { - value = value * 10 + (ch - '0'); - } else if (ch == '.' || ch == '_' || ch == '+') { - results.add(value); - value = 0; - } else { - throw new NumberFormatException(); - } - } - results.add(value); - return results; - } - - static Version parseJavaVersion(String javaVersion) { - // Remove pre-release part, usually -ea - final int indexOfDash = javaVersion.indexOf('-'); - if (indexOfDash >= 0) { - javaVersion = javaVersion.substring(0, indexOfDash); - } - - int major = 0; - int minor = 0; - int update = 0; - - try { - List nums = splitDigits(javaVersion); - major = nums.get(0); - - // for java 1.6/1.7/1.8 - if (major == 1) { - major = nums.get(1); - minor = nums.get(2); - update = nums.get(3); - } else { - minor = nums.get(1); - update = nums.get(2); - } - } catch (NumberFormatException | IndexOutOfBoundsException e) { - // unable to parse version string - do nothing - } - return new Version(major, minor, update); - } - - static final class Version { - public final int major, minor, update; - - public Version(int major, int minor, int update) { - this.major = major; - this.minor = minor; - this.update = update; - } - - public boolean is(int major) { - return this.major == major; - } - - public boolean is(int major, int minor) { - return this.major == major && this.minor == minor; - } - - public boolean is(int major, int minor, int update) { - return this.major == major && this.minor == minor && this.update == update; - } - - public boolean isAtLeast(int major, int minor, int update) { - return isAtLeast(this.major, this.minor, this.update, major, minor, update); - } - - public boolean isBetween( - int fromMajor, int fromMinor, int fromUpdate, int toMajor, int toMinor, int toUpdate) { - return isAtLeast(toMajor, toMinor, toUpdate, fromMajor, fromMinor, fromUpdate) - && isAtLeast(fromMajor, fromMinor, fromUpdate) - && !isAtLeast(toMajor, toMinor, toUpdate); - } - - private static boolean isAtLeast( - int major, int minor, int update, int atLeastMajor, int atLeastMinor, int atLeastUpdate) { - return (major > atLeastMajor) - || (major == atLeastMajor && minor > atLeastMinor) - || (major == atLeastMajor && minor == atLeastMinor && update >= atLeastUpdate); - } - } - - static final class JvmRuntime { - /* - * Example: - * jvm -> "AdoptOpenJDK 1.8.0_265-b01" - * - * name -> "OpenJDK" - * vendor -> "AdoptOpenJDK" - * version -> "1.8.0_265" - * patches -> "b01" - */ - public final String name; - - public final String vendor; - public final String version; - public final String patches; - - public JvmRuntime() { - this( - System.getProperty("java.version"), - System.getProperty("java.runtime.version"), - System.getProperty("java.runtime.name"), - System.getProperty("java.vm.vendor")); - } - - // Only visible for testing - JvmRuntime(String javaVer, String rtVer, String name, String vendor) { - this.name = name == null ? "" : name; - this.vendor = vendor == null ? "" : vendor; - javaVer = javaVer == null ? "" : javaVer; - this.version = javaVer; - rtVer = javaVer.isEmpty() || rtVer == null ? javaVer : rtVer; - int patchStart = javaVer.length() + 1; - this.patches = (patchStart >= rtVer.length()) ? "" : rtVer.substring(javaVer.length() + 1); - } - } - - public static boolean isJavaVersion(int major) { - return JAVA_VERSION.is(major); - } - - public static boolean isJavaVersion(int major, int minor) { - return JAVA_VERSION.is(major, minor); - } - - public static boolean isJavaVersion(int major, int minor, int update) { - return JAVA_VERSION.is(major, minor, update); - } - - public static boolean isJavaVersionAtLeast(int major) { - return isJavaVersionAtLeast(major, 0, 0); - } - - public static boolean isJavaVersionAtLeast(int major, int minor) { - return isJavaVersionAtLeast(major, minor, 0); - } - - public static boolean isJavaVersionAtLeast(int major, int minor, int update) { - return JAVA_VERSION.isAtLeast(major, minor, update); - } - - /** - * Check if the Java version is between {@code fromMajor} (inclusive) and {@code toMajor} - * (exclusive). - * - * @param fromMajor major from version (inclusive) - * @param toMajor major to version (exclusive) - * @return if the current java version is between the from version (inclusive) and the to version - * exclusive - */ - public static boolean isJavaVersionBetween(int fromMajor, int toMajor) { - return isJavaVersionBetween(fromMajor, 0, toMajor, 0); - } - - /** - * Check if the Java version is between {@code fromMajor.fromMinor} (inclusive) and {@code - * toMajor.toMinor} (exclusive). - * - * @param fromMajor major from version (inclusive) - * @param fromMinor minor from version (inclusive) - * @param toMajor major to version (exclusive) - * @param toMinor minor to version (exclusive) - * @return if the current java version is between the from version (inclusive) and the to version - * exclusive - */ - public static boolean isJavaVersionBetween( - int fromMajor, int fromMinor, int toMajor, int toMinor) { - return isJavaVersionBetween(fromMajor, fromMinor, 0, toMajor, toMinor, 0); - } - - /** - * Check if the Java version is between {@code fromMajor.fromMinor.fromUpdate} (inclusive) and - * {@code toMajor.toMinor.toUpdate} (exclusive). - * - * @param fromMajor major from version (inclusive) - * @param fromMinor minor from version (inclusive) - * @param fromUpdate update from version (inclusive) - * @param toMajor major to version (exclusive) - * @param toMinor minor to version (exclusive) - * @param toUpdate update to version (exclusive) - * @return if the current java version is between the from version (inclusive) and the to version - * exclusive - */ - public static boolean isJavaVersionBetween( - int fromMajor, int fromMinor, int fromUpdate, int toMajor, int toMinor, int toUpdate) { - return JAVA_VERSION.isBetween(fromMajor, fromMinor, fromUpdate, toMajor, toMinor, toUpdate); - } - - public static boolean isLinux() { - return System.getProperty("os.name").toLowerCase().contains("linux"); - } - - public static boolean isWindows() { - // https://mkyong.com/java/how-to-detect-os-in-java-systemgetpropertyosname/ - final String os = System.getProperty("os.name").toLowerCase(); - return os.contains("win"); - } - - public static boolean isMac() { - final String os = System.getProperty("os.name").toLowerCase(); - return os.contains("mac"); - } - - public static boolean isOracleJDK8() { - return isJavaVersion(8) - && RUNTIME.vendor.contains("Oracle") - && !RUNTIME.name.contains("OpenJDK"); - } - - public static boolean isJ9() { - return System.getProperty("java.vm.name").contains("J9"); - } - - public static boolean isZing() { - return System.getProperty("java.vm.name").contains("Zing"); - } - - public static boolean isGraal() { - String vendor = System.getProperty("java.vendor.version"); - return vendor != null && vendor.contains(" GraalVM "); - } - - public static String getLangVersion() { - return String.valueOf(JAVA_VERSION.major); - } - - public static String getRuntimeVendor() { - return RUNTIME.vendor; - } - - public static String getRuntimeVersion() { - return RUNTIME.version; - } - - public static String getRuntimePatches() { - return RUNTIME.patches; - } - - public static boolean isAarch64() { - return System.getProperty("os.arch").toLowerCase().contains("aarch64"); - } - - public static boolean isMusl() { - try { - OperatingSystem current = OperatingSystem.current(); - return current == OperatingSystem.linux && current.isMusl(); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java deleted file mode 100644 index 65d6cb332..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ScopeStack.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler; - -import java.util.Arrays; - -/** - * Per-thread stack of {@link ThreadContext} snapshots for nested scopes. - * - *

Provides bulk save/restore of the full OTEP record + sidecar state via one memcpy per - * transition. Not thread-safe: a single stack instance must be accessed only from its - * owning thread. - * - *

Storage is tiered to keep shallow nesting allocation-free: - *

    - *
  • Depths 0 .. {@value #FAST_DEPTH}-1: one contiguous byte[] allocated eagerly.
  • - *
  • Depths {@value #FAST_DEPTH} and beyond: lazily allocated {@value #CHUNK_DEPTH}-slot - * chunks, each a single byte[]. Chunks are allocated once per depth band and reused.
  • - *
- */ -public final class ScopeStack { - private static final int FAST_DEPTH = 6; - private static final int CHUNK_DEPTH = 12; - private static final int SLOT_SIZE = ThreadContext.SNAPSHOT_SIZE; - - private final byte[] fast = new byte[FAST_DEPTH * SLOT_SIZE]; - // chunks[i] covers depths [FAST_DEPTH + i*CHUNK_DEPTH .. FAST_DEPTH + (i+1)*CHUNK_DEPTH). - private byte[][] chunks; - private int depth; - - public void enter(ThreadContext ctx) { - int d = depth; - ctx.snapshot(bufferFor(d), offsetFor(d)); - depth = d + 1; - } - - public void exit(ThreadContext ctx) { - int d = depth - 1; - if (d < 0) { - throw new IllegalStateException("ScopeStack underflow"); - } - ctx.restore(bufferFor(d), offsetFor(d)); - depth = d; - } - - /** Current nesting depth (number of outstanding {@link #enter} calls). */ - public int depth() { - return depth; - } - - private byte[] bufferFor(int d) { - if (d < FAST_DEPTH) { - return fast; - } - // chunkFor is idempotent: if this depth was previously populated (via a matching enter), - // it returns the existing chunk without allocating. - return chunkFor((d - FAST_DEPTH) / CHUNK_DEPTH); - } - - private static int offsetFor(int d) { - int slot = d < FAST_DEPTH ? d : (d - FAST_DEPTH) % CHUNK_DEPTH; - return slot * SLOT_SIZE; - } - - private byte[] chunkFor(int idx) { - byte[][] cs = chunks; - if (cs == null) { - cs = new byte[4][]; - chunks = cs; - } else if (idx >= cs.length) { - int newLen = cs.length; - while (newLen <= idx) { - newLen <<= 1; - } - cs = Arrays.copyOf(cs, newLen); - chunks = cs; - } - byte[] c = cs[idx]; - if (c == null) { - c = new byte[CHUNK_DEPTH * SLOT_SIZE]; - cs[idx] = c; - } - return c; - } -} diff --git a/ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java b/ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java deleted file mode 100644 index d5f9c0e2a..000000000 --- a/ddprof-lib/src/main/java/com/datadoghq/profiler/ThreadContext.java +++ /dev/null @@ -1,638 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.nio.Buffer; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.nio.charset.StandardCharsets; -import java.util.Objects; - -/** - * Thread-local context for trace/span identification. - * - *

Uses OTEP #4947 TLS record for all context storage. - * Context is written directly to the OTEP record via DirectByteBuffer - * for minimal overhead. Only little-endian platforms are supported. - */ -public final class ThreadContext { - static final int MAX_CUSTOM_SLOTS = 10; - // Max UTF-8 byte length for a custom attribute value. Matches the 1-byte length - // field in the OTEP attrs_data entry header. Enforced up front in setContextAttribute - // so replaceOtepAttribute can assume the input always fits. - private static final int MAX_VALUE_BYTES = 255; - private static final int OTEL_MAX_RECORD_SIZE = 640; - private static final int SIDECAR_SIZE = MAX_CUSTOM_SLOTS * Integer.BYTES + Long.BYTES; // 48 - // Package-private so ScopeStack can size its byte[] scratch. - static final int SNAPSHOT_SIZE = OTEL_MAX_RECORD_SIZE + SIDECAR_SIZE; // 688 - private static final int LRS_OTEP_KEY_INDEX = 0; - // LRS is always a fixed 16-hex-char value in attrs_data (zero-padded u64). - // The entry header is 2 bytes (key_index + length), giving 18 bytes total. - private static final int LRS_FIXED_VALUE_LEN = 16; - private static final int LRS_ENTRY_SIZE = 2 + LRS_FIXED_VALUE_LEN; - private static final byte[] HEX_DIGITS = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'}; - - private static final BufferWriter BUFFER_WRITER = new BufferWriter(); - - // ---- Per-thread bounded direct-mapped cache for attribute values ---- - // Stores {int encoding, byte[] utf8Bytes} per entry. - // - encoding: Dictionary constant pool ID for DD JFR sidecar - // - utf8Bytes: UTF-8 value for OTEP attrs_data (external profilers) - // - // Instance (not static) so that only the owning thread ever reads or writes - // the cache arrays — no cross-thread races, no memory barriers needed. - // Collision evicts the old entry; a miss triggers one JNI registerConstant0() call. - // The Dictionary (contextValueMap) is never cleared, so encodings remain valid - // for the JVM lifetime. On profiler restart the ThreadContext instance is recreated, - // clearing the cache; the first miss per value pays one JNI call to re-populate. - private static final int CACHE_SIZE = 256; - private static final int CACHE_MASK = CACHE_SIZE - 1; - - // Attribute value cache: String value → {int encoding, byte[] utf8} - // Keyed by value string (not by keyIndex) — same string always maps to - // same encoding regardless of key slot. - private final String[] attrCacheKeys = new String[CACHE_SIZE]; - private final int[] attrCacheEncodings = new int[CACHE_SIZE]; - private final byte[][] attrCacheBytes = new byte[CACHE_SIZE][]; - - // OTEP record field offsets (from packed struct) - private final int validOffset; - private final int traceIdOffset; - private final int spanIdOffset; - private final int attrsDataSizeOffset; - private final int attrsDataOffset; - private final int maxAttrsDataSize; - private final int lrsOffset; // localRootSpanId offset in the unified buffer - // Base offset of the tag-encoding sidecar within the unified buffer. Every tag slot i - // lives at ctxBuffer[tagEncodingsOffset + i * Integer.BYTES]. Equal to OTEL_MAX_RECORD_SIZE. - private static final int TAG_ENCODINGS_OFFSET = OTEL_MAX_RECORD_SIZE; - - // Single buffer spanning [OTEP record | tag_encodings | LRS] — 688 bytes contiguous. - // Used for per-field access AND for bulk snapshot/restore memcpy. Position state is - // thread-confined to snapshot/restore, which reset it before each bulk op. - private final ByteBuffer ctxBuffer; - - /** - * Creates a ThreadContext from the single DirectByteBuffer returned by native initializeContextTLS0. - * - * @param ctxBuffer 688-byte unified buffer spanning record + tag_encodings + LRS - * @param metadata array with absolute offsets [VALID, TRACE_ID, SPAN_ID, - * ATTRS_DATA_SIZE, ATTRS_DATA, LRS] - */ - public ThreadContext(ByteBuffer ctxBuffer, long[] metadata) { - // Uses native order for uint16_t attrs_data_size (read by C as native uint16_t). - // trace_id/span_id are uint8_t[] arrays requiring big-endian — handled via Long.reverseBytes() - // in setContextDirect(). Only little-endian platforms are supported. - this.ctxBuffer = ctxBuffer.order(ByteOrder.nativeOrder()); - this.validOffset = (int) metadata[0]; - this.traceIdOffset = (int) metadata[1]; - this.spanIdOffset = (int) metadata[2]; - this.attrsDataSizeOffset = (int) metadata[3]; - this.attrsDataOffset = (int) metadata[4]; - this.maxAttrsDataSize = OTEL_MAX_RECORD_SIZE - this.attrsDataOffset; - this.lrsOffset = (int) metadata[5]; - if (ByteOrder.nativeOrder() != ByteOrder.LITTLE_ENDIAN) { - throw new UnsupportedOperationException( - "ByteBuffer context path requires little-endian platform"); - } - // Zero sidecar + record to prevent stale encodings from a previous profiler session. - // The native ProfiledThread survives across sessions, so the buffer may hold - // old tag encodings and the record may hold old attrs_data. - for (int i = 0; i < MAX_CUSTOM_SLOTS; i++) { - this.ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES, 0); - } - this.ctxBuffer.putLong(this.lrsOffset, 0); - this.ctxBuffer.put(this.validOffset, (byte) 0); - // Pre-initialize the fixed-size LRS entry at attrs_data[0..LRS_ENTRY_SIZE-1]: - // key_index=0, length=16, value=16 zero hex bytes. - // The entry is always present; updates overwrite only the 16 value bytes. - this.ctxBuffer.put(this.attrsDataOffset, (byte) LRS_OTEP_KEY_INDEX); - this.ctxBuffer.put(this.attrsDataOffset + 1, (byte) LRS_FIXED_VALUE_LEN); - for (int i = 0; i < LRS_FIXED_VALUE_LEN; i++) { - this.ctxBuffer.put(this.attrsDataOffset + 2 + i, (byte) '0'); - } - this.ctxBuffer.putShort(this.attrsDataSizeOffset, (short) LRS_ENTRY_SIZE); - } - - /** - * Returns the current span ID. - * Reads directly from the OTEP record buffer (big-endian bytes → native long). - */ - public long getSpanId() { - return Long.reverseBytes(ctxBuffer.getLong(spanIdOffset)); - } - - /** - * Returns the current local root span ID. - * Reads directly from the LRS region of ctxBuffer (native long). - */ - public long getRootSpanId() { - return ctxBuffer.getLong(lrsOffset); - } - - /** - * Sets trace context with 2-arg legacy API. - * Maps rootSpanId to localRootSpanId. Uses spanId as traceIdLow so that - * the OTEL clear check (all-zero) only fires when spanId is actually 0. - * Note: this produces a synthetic trace_id of [0, spanId] in the OTEP record. - * External OTEP readers will see this as a real W3C trace ID. Callers needing - * correct OTEP interop must use the 4-arg {@link #put(long, long, long, long)}. - * - *

Parameter mapping to the 4-arg API: {@code spanId} → {@code spanId}, - * {@code rootSpanId} → {@code localRootSpanId} (first argument). - * - * @deprecated Use {@link #put(long, long, long, long)} instead. - */ - @Deprecated - public long put(long spanId, long rootSpanId) { - put(rootSpanId, spanId, 0, spanId); - return 0; // Return type kept for ABI compatibility; value carries no meaning. - } - - /** - * Sets trace context with full 128-bit W3C trace ID and local root span ID. - * - * @param localRootSpanId Local root span ID (for endpoint correlation) - * @param spanId The span ID - * @param traceIdHigh Upper 64 bits of the 128-bit trace ID - * @param traceIdLow Lower 64 bits of the 128-bit trace ID - */ - public void put(long localRootSpanId, long spanId, long traceIdHigh, long traceIdLow) { - setContextDirect(localRootSpanId, spanId, traceIdHigh, traceIdLow); - } - - /** - * Clears a custom attribute: zeros the sidecar encoding and removes it from OTEP attrs_data. - */ - public void clearContextAttribute(int keyIndex) { - if (keyIndex < 0 || keyIndex >= MAX_CUSTOM_SLOTS) { - return; - } - int otepKeyIndex = keyIndex + 1; - detach(); - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * Integer.BYTES, 0); - removeOtepAttribute(otepKeyIndex); - attach(); - } - - public void copyCustoms(int[] value) { - int len = Math.min(value.length, MAX_CUSTOM_SLOTS); - for (int i = 0; i < len; i++) { - value[i] = ctxBuffer.getInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES); - } - } - - /** - * Captures the full record + sidecar state into {@code scratch[offset..offset+SNAPSHOT_SIZE)}. - * Pair with {@link #restore} for nested-scope propagation. - * - *

The detach/memcpy/re-publish pair hides the bulk read from any signal handler going - * through {@code ContextApi::get} — while {@code valid=0}, sidecar reads are gated off. The - * pre-snapshot {@code valid} state is preserved in {@code scratch[offset + validOffset]} so - * {@link #restore} can replay it. If the record was already invalid (e.g. the all-zero clear - * path in {@link #setContextDirect} leaves {@code valid=0} with a stale {@code attrs_data_size} - * / {@code attrs_data}), the live buffer is left invalid after snapshot — re-publishing would - * expose a cleared-but-stale record. - */ - public void snapshot(byte[] scratch, int offset) { - byte priorValid = ctxBuffer.get(validOffset); - detach(); - // Cast to Buffer: ByteBuffer.position(int) only returns ByteBuffer since JDK 9 (covariant - // return). This source is compiled for Java 8 runtimes where the method lives on Buffer. - ((Buffer) ctxBuffer).position(0); - ctxBuffer.get(scratch, offset, SNAPSHOT_SIZE); - // Overwrite the valid byte in scratch (memcpy captured the post-detach 0) with the - // pre-snapshot value. restore() consults this to decide whether to re-attach. - scratch[offset + validOffset] = priorValid; - if (priorValid != 0) { - attach(); - } - } - - /** - * Restores a previously captured state. The detach/memcpy/conditional-attach pair hides the - * memcpy from readers going through {@link #ctxBuffer}'s valid flag ({@code ContextApi::get} - * in native code), which is the sole gate for sidecar reads (see {@code thread.h}). - * - *

The valid byte inside scratch is cleared to 0 for the duration of the memcpy so that - * even if the captured state had {@code valid=1}, the live buffer cannot transiently observe - * {@code valid=1} alongside partially-written fields. The captured value is restored into - * scratch after the memcpy so subsequent snapshot/restore cycles keep working, and - * {@link #attach} re-publishes only when the saved state was itself valid — matching the - * semantics of {@link #snapshot}. - */ - public void restore(byte[] scratch, int offset) { - int validIdx = offset + validOffset; - byte wasValid = scratch[validIdx]; - scratch[validIdx] = 0; - detach(); - ((Buffer) ctxBuffer).position(0); - ctxBuffer.put(scratch, offset, SNAPSHOT_SIZE); - if (wasValid != 0) { - scratch[validIdx] = wasValid; - attach(); - } - } - - /** - * Sets a custom attribute on the current thread's context by string value. - * Uses a per-thread encoding cache to avoid JNI for repeated values - * (the common case). On cache miss, a single JNI call registers the value - * in the native Dictionary; subsequent calls with the same value are - * zero-JNI ByteBuffer writes. - * - *

High-cardinality values are not supported. Each unique value - * permanently occupies one slot in the native Dictionary, which is bounded - * at 65536 entries across all threads for the JVM lifetime. Once exhausted, - * this method returns {@code false} and clears the attribute. Use only - * low-cardinality values (e.g. endpoint names, DB system names). UUIDs, - * request IDs, and other per-request-unique strings will exhaust the - * Dictionary and cause attributes to be silently dropped. - * - *

Value size limit. The UTF-8 encoding of {@code value} must fit in - * {@value #MAX_VALUE_BYTES} bytes (the OTEP attrs_data entry length field is one byte). - * Oversized values are rejected up front — they never reach the Dictionary or attrs_data. - * - * @param keyIndex Index into the registered attribute key map (0-based) - * @param value The string value for this attribute - * @return true if the attribute was set successfully, false if the value is too long, - * the Dictionary is full, attrs_data overflows, or keyIndex is out of range - */ - public boolean setContextAttribute(int keyIndex, String value) { - if (keyIndex < 0 || keyIndex >= MAX_CUSTOM_SLOTS || value == null) { - return false; - } - return setContextAttributeDirect(keyIndex, value); - } - - /** - * Writes both the sidecar encoding (DD signal handler) and OTEP attrs_data - * UTF-8 value (external profilers) via ByteBuffer. - */ - private boolean setContextAttributeDirect(int keyIndex, String value) { - - // Resolve encoding + UTF-8 bytes from per-thread cache - int slot = value.hashCode() & CACHE_MASK; - int encoding; - byte[] utf8; - if (value.equals(attrCacheKeys[slot])) { - // Cache hit — the value was previously validated and cached; no re-check needed. - encoding = attrCacheEncodings[slot]; - utf8 = attrCacheBytes[slot]; - } else { - // Cache miss: encode UTF-8 and validate size BEFORE touching the Dictionary. - // Rejecting here avoids an orphan Dictionary entry (the native Dictionary is - // write-only for the JVM lifetime and cannot be undone). - utf8 = value.getBytes(StandardCharsets.UTF_8); - if (utf8.length > MAX_VALUE_BYTES) { - return false; - } - encoding = registerConstant0(value); - if (encoding < 0) { - // Dictionary full: clear sidecar AND remove the OTEP attrs_data entry - // so both views stay consistent (both report no value for this key). - clearContextAttribute(keyIndex); - return false; - } - attrCacheEncodings[slot] = encoding; - attrCacheBytes[slot] = utf8; - attrCacheKeys[slot] = value; - } - - // Write both sidecar and OTEP attrs_data inside the detach/attach window - // so a signal handler never sees a new sidecar encoding alongside old attrs_data. - detach(); - boolean written = writeSlot(keyIndex, encoding, utf8); - attach(); - return written; - } - - /** - * Writes one slot's sidecar encoding and OTEP attrs_data value. The caller must already hold - * the detach/attach window. On attrs_data overflow the old entry was compacted out and the new - * one couldn't fit, so the sidecar is zeroed to keep both views agreeing there is no value. - * - * @return true if the value was written; false on attrs_data overflow (sidecar left zeroed) - */ - private boolean writeSlot(int keyIndex, int encoding, byte[] utf8) { - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * Integer.BYTES, encoding); - if (!replaceOtepAttribute(keyIndex + 1, utf8)) { - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * Integer.BYTES, 0); - return false; - } - return true; - } - - /** - * Re-applies multiple custom attributes from precomputed constant IDs and UTF-8 bytes in a - * single detach/attach window, without Dictionary lookups or per-thread cache access. - * - *

Both arrays are indexed by key slot, matching the layout produced by - * {@link #copyCustoms(int[])}. For each slot {@code i} with {@code constantIds[i] > 0}, the - * sidecar encoding (DD signal handler) and the OTEP attrs_data value (external profilers) are - * written together; slots with {@code constantIds[i] <= 0} are left untouched. Performing all - * writes inside one detach/attach window means a signal handler never observes a partially - * re-applied set. - * - *

Intended for the reapply-app-context hot path: the caller already holds the constant IDs - * (from {@link #copyCustoms(int[])}) and the UTF-8 bytes (from the original Strings), so this - * does no String allocation, hashing, or cache lookup. The record is re-published only if it - * was valid before the call, so a cleared (span-less) record is not resurrected. - * - *

Caller contract. {@code constantIds[i]} must be an ID previously returned for the - * same value via {@link #copyCustoms(int[])} within the current profiler session, and - * {@code utf8[i]} must be that value's UTF-8 bytes. The (id, bytes) pairing is not verifiable - * here; a mismatch silently diverges the sidecar and attrs_data views. - * - * @param constantIds per-slot Dictionary constant IDs; entries {@code <= 0} are skipped - * @param utf8 per-slot UTF-8 value bytes; must be non-null and at most - * {@value #MAX_VALUE_BYTES} bytes for every slot whose constantId {@code > 0} - * @return true if every slot with {@code constantId > 0} was written successfully; false if - * the record was not valid before the call (nothing is published), or if any slot - * overflowed {@code attrs_data} (that slot's sidecar is zeroed). Note: a {@code false} - * return due to {@code attrs_data} overflow does not mean the record is - * unmodified — slots processed before the overflowed one are durably written. - * @throws NullPointerException if {@code constantIds}, {@code utf8}, or any active - * {@code utf8[i]} (where {@code constantIds[i] > 0}) is null - * @throws IllegalArgumentException if the arrays have different lengths, - * {@code constantIds.length > MAX_CUSTOM_SLOTS}, or any - * active {@code utf8[i].length > MAX_VALUE_BYTES} - */ - public boolean setContextAttributesByIdAndBytes(int[] constantIds, byte[][] utf8) { - Objects.requireNonNull(constantIds, "constantIds"); - Objects.requireNonNull(utf8, "utf8"); - if (constantIds.length != utf8.length) { - throw new IllegalArgumentException("constantIds and utf8 must have the same length"); - } - if (constantIds.length > MAX_CUSTOM_SLOTS) { - throw new IllegalArgumentException("constantIds.length exceeds MAX_CUSTOM_SLOTS"); - } - int len = constantIds.length; - // Validate active slots before touching the buffer so a bad input never leaves - // the record detached (valid=0) after an exception unwinds past attach(). - for (int i = 0; i < len; i++) { - if (constantIds[i] > 0) { - if (utf8[i] == null) { - throw new NullPointerException("utf8[" + i + "]"); - } - if (utf8[i].length > MAX_VALUE_BYTES) { - throw new IllegalArgumentException("utf8[" + i + "].length exceeds MAX_VALUE_BYTES"); - } - } - } - // Never resurrect a cleared (span-less) record: valid=0 means no reader can observe - // what we write, and re-publishing would expose a record with no trace/span context. - if (ctxBuffer.get(validOffset) == 0) { - return false; - } - detach(); - boolean allWritten = true; - for (int i = 0; i < len; i++) { - int constantId = constantIds[i]; - if (constantId <= 0) { - continue; - } - if (!writeSlot(i, constantId, utf8[i])) { - allWritten = false; - } - } - attach(); - return allWritten; - } - - /** - * Write context directly to the OTEP record via ByteBuffer. - * trace_id and span_id are OTEP big-endian byte arrays — Long.reverseBytes() - * converts from native LE to big-endian. - */ - private void setContextDirect(long localRootSpanId, long spanId, long trHi, long trLo) { - detach(); - - if (trHi == 0 && trLo == 0 && spanId == 0) { - clearContextDirect(); - return; - } - - // Write trace_id (big-endian) + span_id (big-endian) - ctxBuffer.putLong(traceIdOffset, Long.reverseBytes(trHi)); - ctxBuffer.putLong(traceIdOffset + 8, Long.reverseBytes(trLo)); - ctxBuffer.putLong(spanIdOffset, Long.reverseBytes(spanId)); - - // Reset custom attribute state so the previous span's values don't leak - // into this span. Callers set attributes again via setContextAttribute(). - for (int i = 0; i < MAX_CUSTOM_SLOTS; i++) { - // offset into ctxBuffer for tag-encoding slot i - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES, 0); - } - // Reset attrs_data_size to contain only the fixed LRS entry, discarding - // any custom attribute entries written during the previous span. - ctxBuffer.putShort(attrsDataSizeOffset, (short) LRS_ENTRY_SIZE); - - // Update LRS sidecar and OTEP attrs_data inside the detach/attach window so a - // signal handler never sees the new LRS with old trace/span IDs. - ctxBuffer.putLong(lrsOffset, localRootSpanId); - writeLrsHex(localRootSpanId); - - attach(); - } - - // ---- LRS helpers ---- - - /** - * Zeros trace/span IDs, sidecar encodings, and LRS. Called between detach() and the - * return in the all-zero path; valid stays 0 (no attach) so no reader can see attrs_data. - * attrs_data_size is not reset here; the next non-zero setContext call will reset it - * before attach(). This is safe because valid remains 0 after clear, so no reader will - * observe the stale attrs_data_size. - * - *

External OTEP readers see valid=0 and skip the record — they cannot distinguish - * "cleared" from "being mutated". This is intentional: without also resetting - * attrs_data_size here, publishing valid=1 with a stale attrs_data_size would expose - * a partially-valid record. The cleared state is effectively invisible to external - * readers until the next non-zero setContext call publishes it. - */ - private void clearContextDirect() { - ctxBuffer.putLong(traceIdOffset, 0); - ctxBuffer.putLong(traceIdOffset + 8, 0); - ctxBuffer.putLong(spanIdOffset, 0); - writeLrsHex(0); - for (int i = 0; i < MAX_CUSTOM_SLOTS; i++) { - ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + i * Integer.BYTES, 0); - } - ctxBuffer.putLong(lrsOffset, 0); - } - - /** - * Overwrite the 16 hex value bytes of the fixed LRS entry in-place. - * The entry header (key_index=0, length=16) is pre-initialized and never touched. - * Called between detach() and attach(); no allocation. - */ - private void writeLrsHex(long val) { - int base = attrsDataOffset + 2; // skip key_index byte + length byte - for (int i = 15; i >= 0; i--) { - ctxBuffer.put(base + i, HEX_DIGITS[(int)(val & 0xF)]); - val >>>= 4; - } - } - - // ---- OTEP record helpers (called between detach/attach) ---- - - /** - * Invalidates the record (sets valid=0). Typically followed by attach(), but the - * clear path intentionally leaves the record invalid without calling attach(). - */ - private void detach() { - ctxBuffer.put(validOffset, (byte) 0); - BUFFER_WRITER.storeFence(); - } - - /** Validate record. */ - private void attach() { - // storeFence ensures all record writes are visible before valid=1. - // The TLS pointer (otel_thread_ctx_v1) is permanent and never - // written here; external profilers rely solely on the valid flag. - BUFFER_WRITER.storeFence(); - // Plain put is sufficient: signal handlers run on the same hardware thread, - // so they observe stores in program order — no volatile needed for same-thread - // visibility. The preceding storeFence() provides the release barrier. - ctxBuffer.put(validOffset, (byte) 1); - } - - /** - * Replace or insert an attribute in attrs_data. Record must be detached. - * Writes the pre-encoded UTF-8 bytes into the record. - * - *

Caller contract: {@code utf8.length <= MAX_VALUE_BYTES}, enforced at the public - * entry point in {@link #setContextAttributeDirect}. - */ - private boolean replaceOtepAttribute(int otepKeyIndex, byte[] utf8) { - int currentSize = compactOtepAttribute(otepKeyIndex); - int valueLen = utf8.length; - int entrySize = 2 + valueLen; - if (currentSize + entrySize <= maxAttrsDataSize) { - int base = attrsDataOffset + currentSize; - ctxBuffer.put(base, (byte) otepKeyIndex); - ctxBuffer.put(base + 1, (byte) valueLen); - for (int i = 0; i < valueLen; i++) { - ctxBuffer.put(base + 2 + i, utf8[i]); - } - currentSize += entrySize; - ctxBuffer.putShort(attrsDataSizeOffset, (short) currentSize); - return true; - } - ctxBuffer.putShort(attrsDataSizeOffset, (short) currentSize); - return false; - } - - /** Remove an attribute from attrs_data by compacting. Record must be detached. */ - private void removeOtepAttribute(int otepKeyIndex) { - int currentSize = compactOtepAttribute(otepKeyIndex); - ctxBuffer.putShort(attrsDataSizeOffset, (short) currentSize); - } - - /** - * Scan attrs_data and compact out the entry with the given key_index. - * Returns the new attrs_data size after compaction. - * - *

{@code otepKeyIndex} is always {@code keyIndex + 1} for user attributes, - * so it is never 0. Index 0 is reserved for the fixed LRS entry. - */ - private int compactOtepAttribute(int otepKeyIndex) { - int currentSize = ctxBuffer.getShort(attrsDataSizeOffset) & 0xFFFF; - int readPos = 0; - int writePos = 0; - boolean found = false; - while (readPos + 2 <= currentSize) { - int k = ctxBuffer.get(attrsDataOffset + readPos) & 0xFF; - int len = ctxBuffer.get(attrsDataOffset + readPos + 1) & 0xFF; - if (readPos + 2 + len > currentSize) { currentSize = writePos; break; } - if (k == otepKeyIndex) { - found = true; - readPos += 2 + len; - } else { - if (found && writePos < readPos) { - for (int i = 0; i < 2 + len; i++) { - ctxBuffer.put(attrsDataOffset + writePos + i, - ctxBuffer.get(attrsDataOffset + readPos + i)); - } - } - writePos += 2 + len; - readPos += 2 + len; - } - } - return found ? writePos : currentSize; - } - - /** - * Reads a custom attribute value by key index by scanning {@code attrs_data}. - * - *

Test-only. The only caller is {@code TagContextTest}, which uses it via - * {@link JavaProfiler#getThreadContext()} to verify that writes to the OTEP record are - * observable after set / clear / span-reset cycles. No production path — neither the DD - * signal handler nor the OTEL eBPF reader — ever calls this method: the DD handler reads - * sidecar encoding IDs and the OTEL reader parses {@code attrs_data} directly from native - * memory. The per-call {@code byte[]} / {@code String} allocation is therefore acceptable; - * do not introduce a readback cache unless a real production consumer appears. - * - * @param keyIndex 0-based user key index (same as passed to setContextAttribute) - * @return the attribute value string, or null if not set - */ - public String readContextAttribute(int keyIndex) { - if (keyIndex < 0 || keyIndex >= MAX_CUSTOM_SLOTS) { - return null; - } - // valid=0 → record was detached or never published. No attrs_data to trust. - if (ctxBuffer.get(validOffset) == 0) { - return null; - } - int otepKeyIndex = keyIndex + 1; - int size = ctxBuffer.getShort(attrsDataSizeOffset) & 0xFFFF; - int pos = 0; - while (pos + 2 <= size) { - int k = ctxBuffer.get(attrsDataOffset + pos) & 0xFF; - int len = ctxBuffer.get(attrsDataOffset + pos + 1) & 0xFF; - if (pos + 2 + len > size) { - break; - } - if (k == otepKeyIndex) { - byte[] bytes = new byte[len]; - for (int i = 0; i < len; i++) { - bytes[i] = ctxBuffer.get(attrsDataOffset + pos + 2 + i); - } - return new String(bytes, StandardCharsets.UTF_8); - } - pos += 2 + len; - } - return null; - } - - /** - * Reads the trace ID from the OTEP record as a 32-char lowercase hex string. - * The trace ID is stored big-endian; this method returns it as-is. - * Intended for tests only. - */ - public String readTraceId() { - StringBuilder sb = new StringBuilder(32); - for (int i = 0; i < 16; i++) { - int b = ctxBuffer.get(traceIdOffset + i) & 0xFF; - sb.append((char) HEX_DIGITS[b >> 4]); - sb.append((char) HEX_DIGITS[b & 0xF]); - } - return sb.toString(); - } - - private static native int registerConstant0(String value); -} diff --git a/ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java b/ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java deleted file mode 100644 index 9fcf77c4e..000000000 --- a/ddprof-lib/src/main/java9/com/datadoghq/profiler/BufferWriter9.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import java.lang.invoke.MethodHandles; -import java.lang.invoke.VarHandle; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -public final class BufferWriter9 implements BufferWriter.Impl { - private static final VarHandle LONG_VIEW_VH; - private static final VarHandle INT_VIEW_VH; - - static { - try { - // Create a view into ByteBuffer as if it were a long array - // The VarHandle coordinates are (ByteBuffer, int index) where index is in bytes - LONG_VIEW_VH = MethodHandles.byteBufferViewVarHandle( - long[].class, ByteOrder.nativeOrder()); - INT_VIEW_VH = MethodHandles.byteBufferViewVarHandle( - int[].class, ByteOrder.nativeOrder()); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void writeOrderedLong(ByteBuffer buffer, int offset, long value) { - // setRelease provides ordered write semantics (matches Unsafe.putOrderedLong) - LONG_VIEW_VH.setRelease(buffer, offset, value); - } - - @Override - public void writeInt(ByteBuffer buffer, int offset, int value) { - // setRelease provides ordered write semantics (matches Unsafe.putOrderedInt) - INT_VIEW_VH.setRelease(buffer, offset, value); - } - - @Override - public void storeFence() { - VarHandle.storeStoreFence(); - } -} diff --git a/ddprof-lib/src/test/cpp/ddprof_ut.cpp b/ddprof-lib/src/test/cpp/ddprof_ut.cpp deleted file mode 100644 index afdb990fe..000000000 --- a/ddprof-lib/src/test/cpp/ddprof_ut.cpp +++ /dev/null @@ -1,421 +0,0 @@ - #include - - #include "asyncSampleMutex.h" - #include "buffers.h" - #include "context.h" - #include "counters.h" - #include "guards.h" - #include "mutex.h" - #include "os.h" - #include "thread.h" - #include "unwindStats.h" - #include "threadFilter.h" - #include "threadInfo.h" - #include "threadLocalData.h" - #include "vmEntry.h" - #include "../../main/cpp/gtest_crash_handler.h" - #include - #include - #include - #include // For std::sort - #include - #include - -// Test name for crash handler -static constexpr char DDPROF_TEST_NAME[] = "DdprofTest"; - -// Global crash handler installation (since this file uses bare TEST() macros) -class DdprofGlobalSetup { -public: - DdprofGlobalSetup() { - installGtestCrashHandler(); - } - ~DdprofGlobalSetup() { - restoreDefaultSignalHandlers(); - } -}; - -// Install global crash handler for all tests in this file -static DdprofGlobalSetup ddprof_global_setup; - - ssize_t callback(char* ptr, int len) { - return len; - } - - TEST(Buffer, var64_encoding) { - RecordingBuffer buf; - - u64 val = 2097150; - - buf.putVar64(val); - } - - TEST(Buffer, skip_flush) { - RecordingBuffer buf; - - // no need to flush empty buffer - EXPECT_FALSE(buf.flushIfNeeded(callback)); - - buf.skip(10); - // no need to flush when well below the limit - EXPECT_FALSE(buf.flushIfNeeded(callback)); - - buf.skip(RECORDING_BUFFER_LIMIT - buf.offset() + 1); - // flush is needed here - EXPECT_TRUE(buf.flushIfNeeded(callback)); - - // buff was flushed this should not trigger the assert - buf.skip(RECORDING_BUFFER_LIMIT); - - // writing over limit must trigger assert - EXPECT_DEATH({ - buf.skip(RECORDING_BUFFER_LIMIT); - buf.skip(RECORDING_BUFFER_LIMIT); - }, "Assertion .*"); - } - - TEST(Buffer, writeString) { - int clen = 4191; - char* str = (char*)malloc(clen + 1); - memset(str, 'a', clen); - str[clen] = 0; - - RecordingBuffer buf; - - buf.skip(buf.limit() - clen * 2); - buf.putUtf8(str); - - EXPECT_FALSE(buf.flushIfNeeded(callback)); - - EXPECT_DEATH({ - buf.putUtf8(str); - }, "Assertion .*"); - free(str); - } - - TEST(Buffer, writeStringWithLength) { - int clen = 1 << 16; - char* str = (char*)malloc(clen + 1); - memset(str, 'a', clen); - str[clen] = 0; - - RecordingBuffer buf[2]; - - buf[0].putUtf8(str, clen); - // should be able to write to the adjacent buffer unaffected - buf[1].put8(1); - EXPECT_EQ(1, buf[1].data()[0]); - // long string should have been truncated to 8191 characters - int prefix = 1 + (31 - __builtin_clz(8191)) / 7 + 1; - EXPECT_EQ(0, buf[0].data()[prefix + 8191]); - free(str); - } - - TEST(OS, threadId_sanity) { - EXPECT_FALSE(OS::getMaxThreadId() < 0); - } - - TEST(ThreadInfoTest, testThreadInfoCleanupAllDead) { - ThreadInfo info; - info.set(1, "main", 1); - info.set(2, "ephemeral", 2); - ASSERT_EQ(2, info.size()); - - std::set live_thread_ids; - live_thread_ids.insert(1); - - // make sure only the non-live threads are removed - info.clearAll(live_thread_ids); - ASSERT_EQ(1, info.size()); - ASSERT_EQ(-1, info.getThreadId(2)); - - // sanity check that all threads are removed when no live threads are provided - std::set empty_set; - info.set(2, "ephemeral-1", 2); - info.clearAll(empty_set); - ASSERT_EQ(0, info.size()); - } - - TEST(ThreadInfoTest, testThreadInfoCleanupAll) { - ThreadInfo info; - info.set(1, "main", 1); - info.set(2, "ephemeral", 2); - ASSERT_EQ(2, info.size()); - - info.clearAll(); - ASSERT_EQ(0, info.size()); - } - - TEST(AsyncSampleMutex, testAsyncSampleMutexInterleaving) { - ThreadLocalData data; - EXPECT_FALSE(data.is_unwinding_Java()); - { - AsyncSampleMutex first(&data); - EXPECT_TRUE(first.acquired()); - EXPECT_TRUE(data.is_unwinding_Java()); - { - AsyncSampleMutex second(&data); - EXPECT_FALSE(second.acquired()); - EXPECT_TRUE(first.acquired()); - EXPECT_TRUE(data.is_unwinding_Java()); - } - EXPECT_TRUE(first.acquired()); - } - EXPECT_FALSE(data.is_unwinding_Java()); - } - - TEST(JavaVersionAccess, testJavaVersionAccess_hs_8) { - char runtime_prop_value_1[] = "1.8.0_292"; - char vm_prop_value_1[] = "25.292-b10"; - char runtime_prop_value_2[] = "8.0.292"; - char vm_prop_value_2[] = "25.292-b10"; - - JavaFullVersion java_version1 = JavaVersionAccess::get_java_version(runtime_prop_value_1); - int hs_version1 = JavaVersionAccess::get_hotspot_version(vm_prop_value_1); - EXPECT_EQ(8, java_version1.major); - EXPECT_EQ(292, java_version1.update); - EXPECT_EQ(8, hs_version1); - - JavaFullVersion java_version2 = JavaVersionAccess::get_java_version(runtime_prop_value_2); - int hs_version2 = JavaVersionAccess::get_hotspot_version(vm_prop_value_2); - EXPECT_EQ(8, java_version2.major); - EXPECT_EQ(292, java_version2.update); - EXPECT_EQ(8, hs_version2); - } - - TEST(JavaVersionAccess, testJavaVersionAccess_hs_11) { - char runtime_prop_value_1[] = "11.0.25"; - char vm_prop_value_1[] = "11.0.25+10"; - - JavaFullVersion java_version1 = JavaVersionAccess::get_java_version(runtime_prop_value_1); - int hs_version1 = JavaVersionAccess::get_hotspot_version(vm_prop_value_1); - EXPECT_EQ(11, java_version1.major); - EXPECT_EQ(25, java_version1.update); - EXPECT_EQ(11, hs_version1); - } - - TEST(JavaVersionAccess, testJavaVersionAccess_hs_default) { - char runtime_prop_value_1[] = "3.11.25x_10"; - char vm_prop_value_1[] = "3.11.25x_10"; - - JavaFullVersion java_version1 = JavaVersionAccess::get_java_version(runtime_prop_value_1); - int hs_version1 = JavaVersionAccess::get_hotspot_version(vm_prop_value_1); - EXPECT_EQ(9, java_version1.major); - EXPECT_EQ(25, java_version1.update); - EXPECT_EQ(9, hs_version1); - } - - TEST(JavaVersionAccess, testJavaVersionAccess_hs_24) { - char runtime_prop_value_1[] = "24+36-FR"; - char vm_prop_value_1[] = "24+36-FR"; - - JavaFullVersion java_version1 = JavaVersionAccess::get_java_version(runtime_prop_value_1); - int hs_version1 = JavaVersionAccess::get_hotspot_version(vm_prop_value_1); - EXPECT_EQ(24, java_version1.major); - EXPECT_EQ(0, java_version1.update); - EXPECT_EQ(24, hs_version1); - } - - TEST(UnwindFailures, BasicFunctionality) { - UnwindFailures failures; - - // Test recording failures - EXPECT_EQ(0, failures.count("test_stub1")); - failures.record(UNWIND_FAILURE_STUB, "test_stub1"); - EXPECT_EQ(1, failures.count("test_stub1")); - - failures.record(UNWIND_FAILURE_COMPILED, "test_stub1"); - EXPECT_EQ(1, failures.count("test_stub1", UNWIND_FAILURE_STUB)); - EXPECT_EQ(1, failures.count("test_stub1", UNWIND_FAILURE_COMPILED)); - EXPECT_EQ(2, failures.count("test_stub1")); - - // Test different stubs - EXPECT_EQ(0, failures.count("test_stub2")); - failures.record(UNWIND_FAILURE_STUB, "test_stub2"); - EXPECT_EQ(2, failures.count("test_stub1")); - EXPECT_EQ(1, failures.count("test_stub2")); - - // Test reset - failures.clear(); - EXPECT_TRUE(failures.empty()); - EXPECT_EQ(0, failures.count("test_stub1")); - EXPECT_EQ(0, failures.count("test_stub2")); - } - - TEST(UnwindFailures, HashCollisions) { - UnwindFailures failures; - - // Test multiple entries that might collide - for (int i = 0; i < 100; i++) { - char name[32]; - snprintf(name, sizeof(name), "stub_%d", i); - EXPECT_EQ(0, failures.count(name)); - failures.record(UNWIND_FAILURE_STUB, name); - EXPECT_EQ(1, failures.count(name)); - } - - // Verify counts are correct - for (int i = 0; i < 100; i++) { - char name[32]; - snprintf(name, sizeof(name), "stub_%d", i); - EXPECT_EQ(1, failures.count(name)); - } - } - - TEST(UnwindFailures, SwapEmptyInstances) { - UnwindFailures failures1; - UnwindFailures failures2; - - // Verify both instances are empty - EXPECT_TRUE(failures1.empty()); - EXPECT_TRUE(failures2.empty()); - - // Swap the empty instances - failures1.swap(failures2); - - // Verify both instances are still empty after swap - EXPECT_TRUE(failures1.empty()); - EXPECT_TRUE(failures2.empty()); - - // Add some data to failures1 - failures1.record(UNWIND_FAILURE_STUB, "test_stub1"); - EXPECT_FALSE(failures1.empty()); - EXPECT_TRUE(failures2.empty()); - - // Swap again - failures1.swap(failures2); - - // Verify data moved correctly - EXPECT_TRUE(failures1.empty()); - EXPECT_FALSE(failures2.empty()); - EXPECT_EQ(1, failures2.count("test_stub1")); - } - - TEST(UnwindStats, CollectAndReset) { - // Record some failures - UnwindFailures failures; - failures.record(UNWIND_FAILURE_STUB, "test_stub1"); - failures.record(UNWIND_FAILURE_STUB, "test_stub2"); - UnwindStats::recordFailures(failures); - - // Collect and reset - UnwindFailures result; - UnwindStats::collectAndReset(result); - - // Verify the result contains the recorded failures - EXPECT_EQ(1, result.count("test_stub1")); - EXPECT_EQ(1, result.count("test_stub2")); - - // Verify the stats are reset - UnwindFailures empty_result; - UnwindStats::collectAndReset(empty_result); - EXPECT_TRUE(empty_result.empty()); - } - - TEST(UnwindStatsTest, ThreadSafety) { - UnwindStats::reset(); - - const int numThreads = 4; - const int iterations = 10000; - std::thread threads[numThreads]; - - fprintf(stderr, "Starting %d threads with %d iterations each\n", numThreads, iterations); - // Each thread records failures for different stubs - for (int i = 0; i < numThreads; i++) { - threads[i] = std::thread([i, iterations]() { - UnwindFailures failures; // per-thread instance - char name[32]; - snprintf(name, sizeof(name), "thread_%d_stub", i); - for (int j = 0; j < iterations; j++) { - failures.record(UNWIND_FAILURE_STUB, name); - } - // record the thread-stats - UnwindStats::recordFailures(failures); - }); - } - - fprintf(stderr, "Waiting for threads to finish\n"); - // Wait for all threads - for (int i = 0; i < numThreads; i++) { - threads[i].join(); - } - - fprintf(stderr, "All threads finished\n"); - fprintf(stderr, "Verifying counts\n"); - - UnwindFailures result; - UnwindStats::collectAndReset(result); - // Verify counts - u64 globalCount = 0; - for (int i = 0; i < numThreads; i++) { - char name[32]; - snprintf(name, sizeof(name), "thread_%d_stub", i); - // due to expected concurrency issues some failures may not be counted - // failure recording prefers dropping the failure over blocking on the lock - u64 count = result.count(name); - EXPECT_TRUE(count <= iterations); - globalCount += count; - } - EXPECT_TRUE(globalCount > 0); - } - - // Deterministic regression for the CriticalSection::_thread_ptr capture fix. - // - // Bug: the old destructor re-fetched currentSignalSafe() at destruction time. - // If TLS was cleared between the ctor and dtor (e.g. release() called inside - // the CS scope as it was in the old onThreadEnd), the re-fetch returned nullptr - // and exitCriticalSection() was silently skipped, leaving _in_critical_section - // stuck true so no subsequent CS could enter on that ProfiledThread. - // - // Fix: the ctor captures _thread_ptr once; the dtor uses that pointer regardless - // of TLS state at destruction time. - // - // This test exercises the exact race window by calling clearCurrentThreadTLS() - // inside a live CriticalSection scope, then verifying the flag is cleared. - // Without the fix tryEnterCriticalSection() returns false (exit 5). - // fork() is unsupported under TSan: the child inherits shadow memory in an - // inconsistent state and crashes before any TSan report can be written. - #if !defined(TSAN_ENABLED) - TEST(ProfiledThreadTeardown, CriticalSectionExitsEvenAfterTLSCleared) { - pid_t pid = fork(); - ASSERT_NE(-1, pid); - - if (pid == 0) { - // ---- child process (fork isolates TLS from other tests) ---- - ProfiledThread::initCurrentThread(); - ProfiledThread* pt = ProfiledThread::currentSignalSafe(); - if (pt == nullptr) _exit(2); - - // Baseline: entering critical section works. - if (!pt->tryEnterCriticalSection()) _exit(3); - pt->exitCriticalSection(); - - // Simulate the race: CriticalSection is constructed while TLS is valid - // (so _thread_ptr is captured), then TLS is cleared before the dtor runs. - { - CriticalSection cs; - if (!cs.entered()) _exit(4); - // Mimics the moment inside release() after pthread_setspecific(NULL). - ProfiledThread::clearCurrentThreadTLS(); - } // dtor: old code → re-fetch nullptr → skip exit → _in_critical_section stuck - // new code → _thread_ptr captured at ctor → exitCriticalSection called - - // _in_critical_section must be false; if the bug is present this fails. - if (!pt->tryEnterCriticalSection()) _exit(5); - pt->exitCriticalSection(); - - _exit(0); // destructor is private; OS reclaims memory on exit. - } - - // ---- parent: reap child and check exit code ---- - int status = 0; - ASSERT_NE(-1, waitpid(pid, &status, 0)); - ASSERT_TRUE(WIFEXITED(status)) << "child crashed (signal " << WTERMSIG(status) << ")"; - ASSERT_EQ(0, WEXITSTATUS(status)) << "child exited with code " << WEXITSTATUS(status); - } - #endif // !TSAN_ENABLED - - int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); - } diff --git a/ddprof-lib/src/test/cpp/demangle_ut.cpp b/ddprof-lib/src/test/cpp/demangle_ut.cpp deleted file mode 100644 index 3f347b335..000000000 --- a/ddprof-lib/src/test/cpp/demangle_ut.cpp +++ /dev/null @@ -1,204 +0,0 @@ -#include - -#include "rustDemangler.h" -#include "../../main/cpp/gtest_crash_handler.h" - -#include - -// Test name for crash handler -static constexpr char DEMANGLE_TEST_NAME[] = "DemangleTest"; - -// Global crash handler installation (since this file uses bare TEST() macros) -class DemangleGlobalSetup { -public: - DemangleGlobalSetup() { - installGtestCrashHandler(); - } - ~DemangleGlobalSetup() { - restoreDefaultSignalHandlers(); - } -}; - -// Install global crash handler for all tests in this file -static DemangleGlobalSetup demangle_global_setup; - -#ifndef __APPLE__ - -struct DemangleTestContent { - std::string test; - std::string answer; -}; - -// Partly borrowed from the LLVM unit tests -std::vector s_demangle_cases = { - {"_", "_"}, - {"_Z3fooi", "foo(int)"}, - {"_RNvC3foo3bar", "foo::bar"}, - {"_ZN4llvm4yaml7yamlizeISt6vectorINSt7__cxx1112basic_stringIcSt11char_" - "traitsIcESaIcEEESaIS8_EENS0_12EmptyContextEEENSt9enable_ifIXsr18has_" - "SequenceTraitsIT_EE5valueEvE4typeERNS0_2IOERSD_bRT0_", - "std::enable_if, std::allocator >, " - "std::allocator, " - "std::allocator > > > >::value, void>::type " - "llvm::yaml::yamlize, std::allocator >, " - "std::allocator, " - "std::allocator > > >, llvm::yaml::EmptyContext>(llvm::yaml::IO&, " - "std::vector, " - "std::allocator >, std::allocator, std::allocator > > >&, bool, " - "llvm::yaml::EmptyContext&)"}, - {"_ZWowThisIsWrong", "_ZWowThisIsWrong"}, - - // Following cases were taken by the libiberty project and are used here - // with recognition - {"_ZN4main4main17he714a2e23ed7db23E", "main::main"}, - {"_ZN4main4main17he714a2e23ed7db23E", "main::main"}, - {"_ZN4main4main18h1e714a2e23ed7db23E", "main::main::h1e714a2e23ed7db23"}, - {"_ZN4main4main16h714a2e23ed7db23E", "main::main::h714a2e23ed7db23"}, - {"_ZN4main4main17he714a2e23ed7db2gE", "main::main::he714a2e23ed7db2g"}, - {"_ZN4main4$99$17he714a2e23ed7db23E", "main::$99$"}, - {"_ZN71_$LT$Test$u20$$u2b$$u20$$u27$static$u20$as$u20$foo..Bar$LT$Test$GT$$" - "GT$3bar17h930b740aa94f1d3aE", - ">::bar"}, - {"_ZN54_$LT$I$u20$as$u20$core..iter..traits..IntoIterator$GT$9into_" - "iter17h8581507801fb8615E", - "::into_iter"}, - {"_ZN10parse_tsan4main17hdbbfdf1c6a7e27d9E", "parse_tsan::main"}, - {"_ZN65_$LT$std..env..Args$u20$as$u20$core..iter..iterator..Iterator$GT$" - "4next17h420a7c8d0c7eef40E", - "::next"}, - {"_ZN4core3str9from_utf817hdcea28871313776dE", "core::str::from_utf8"}, - {"_ZN4core3mem7size_of17h18bde9bb8c22e2cfE", "core::mem::size_of"}, - {"_ZN5alloc4heap8allocate17hd55c03e6cb81d924E", "alloc::heap::allocate"}, - {"_ZN4core3ptr8null_mut17h736cce09ca0ac11aE", "core::ptr::null_mut"}, - {"_ZN40_$LT$alloc..raw_vec..RawVec$LT$T$GT$$GT$6double17h4166e2b47539e1ffE", - ">::double"}, - {"_ZN39_$LT$collections..vec..Vec$LT$T$GT$$GT$4push17hd4b6b23c1b88141aE", - ">::push"}, - {"_ZN70_$LT$collections..vec..Vec$LT$T$GT$$u20$as$u20$core..ops..DerefMut$" - "GT$9deref_mut17hf299b860dc5a831cE", - " as core::ops::DerefMut>::deref_mut"}, - {"_ZN63_$LT$core..ptr..Unique$LT$T$GT$$u20$as$u20$core..ops..Deref$GT$" - "5deref17hc784b4a166cb5e5cE", - " as core::ops::Deref>::deref"}, - {"_ZN40_$LT$alloc..raw_vec..RawVec$LT$T$GT$$GT$3ptr17h7570b6e9070b693bE", - ">::ptr"}, - {"_ZN53_$LT$$u5b$T$u5d$$u20$as$u20$core..slice..SliceExt$GT$10as_mut_" - "ptr17h153241df1c7d1666E", - "<[T] as core::slice::SliceExt>::as_mut_ptr"}, - {"_ZN4core3ptr5write17h651fe53ec860e780E", "core::ptr::write"}, - {"_ZN65_$LT$std..env..Args$u20$as$u20$core..iter..iterator..Iterator$GT$" - "4next17h420a7c8d0c7eef40E", - "::next"}, - {"_ZN54_$LT$I$u20$as$u20$core..iter..traits..IntoIterator$GT$9into_" - "iter17he06cb713aae5b465E", - "::into_iter"}, - {"_ZN71_$LT$collections..vec..IntoIter$LT$T$GT$$u20$as$u20$core..ops..Drop$" - "GT$4drop17hf7f23304ebe62eedE", - " as core::ops::Drop>::drop"}, - {"_ZN86_$LT$collections..vec..IntoIter$LT$T$GT$$u20$as$u20$core..iter.." - "iterator..Iterator$GT$4next17h04b3fbf148c39713E", - " as core::iter::iterator::Iterator>::next"}, - {"_ZN75_$LT$$RF$$u27$a$u20$mut$u20$I$u20$as$u20$core..iter..iterator.." - "Iterator$GT$4next17ha050492063e0fd20E", - "<&'a mut I as core::iter::iterator::Iterator>::next"}, - {"_ZN13drop_contents17hfe3c0a68c8ad1c74E", "drop_contents"}, - {"_ZN13drop_contents17h48cb59bef15bb555E", "drop_contents"}, - {"_ZN4core3mem7size_of17h900b33157bf58f26E", "core::mem::size_of"}, - {"_ZN67_$LT$alloc..raw_vec..RawVec$LT$T$GT$$u20$as$u20$core..ops..Drop$GT$" - "4drop17h96a5cf6e94807905E", - " as core::ops::Drop>::drop"}, - {"_ZN68_$LT$core..nonzero..NonZero$LT$T$GT$$u20$as$u20$core..ops..Deref$GT$" - "5deref17hc49056f882aa46dbE", - " as core::ops::Deref>::deref"}, - {"_ZN63_$LT$core..ptr..Unique$LT$T$GT$$u20$as$u20$core..ops..Deref$GT$" - "5deref17h19f2ad4920655e85E", - " as core::ops::Deref>::deref"}, - {"_ZN11issue_609253foo37Foo$LT$issue_60925..llv$u6d$..Foo$GT$" - "3foo17h059a991a004536adE", - "issue_60925::foo::Foo::foo"}, - {"_RNvC6_123foo3bar", "123foo::bar"}, - {"_RNqCs4fqI2P2rA04_11utf8_identsu30____7hkackfecea1cbdathfdh9hlq6y", - "utf8_idents::საჭმელად_გემრიელი_სადილი"}, - {"_RNCNCNgCs6DXkGYLi8lr_2cc5spawn00B5_", - "cc::spawn::{closure#0}::{closure#0}"}, - {"_RNCINkXs25_NgCsbmNqQUJIY6D_4core5sliceINyB9_4IterhENuNgNoBb_" - "4iter8iterator8Iterator9rpositionNCNgNpB9_6memchr7memrchrs_0E0Bb_", - " as " - "core::iter::iterator::Iterator>::rposition::::{closure#0}"}, - {"_RINbNbCskIICzLVDPPb_5alloc5alloc8box_freeDINbNiB4_" - "5boxed5FnBoxuEp6OutputuEL_ECs1iopQbuBiw2_3std", - "alloc::alloc::box_free::>"}, - {"_RNvMC0INtC8arrayvec8ArrayVechKj7b_E3new", - ">::new"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_8UnsignedKhb_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_6SignedKs98_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_6SignedKanb_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4BoolKb0_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4BoolKb1_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKc76_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKca_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKc2202_E", - ">"}, - {"_RNvNvMCs4fqI2P2rA04_13const_genericINtB4_3FooKpE3foo3FOO", - ">::foo::FOO"}, - {"_RNvC6_123foo3bar", "123foo::bar"}, - {"_RNqCs4fqI2P2rA04_11utf8_identsu30____7hkackfecea1cbdathfdh9hlq6y", - "utf8_idents::საჭმელად_გემრიელი_სადილი"}, - {"_RNCNCNgCs6DXkGYLi8lr_2cc5spawn00B5_", - "cc::spawn::{closure#0}::{closure#0}"}, - {"_RNCINkXs25_NgCsbmNqQUJIY6D_4core5sliceINyB9_4IterhENuNgNoBb_" - "4iter8iterator8Iterator9rpositionNCNgNpB9_6memchr7memrchrs_0E0Bb_", - " as " - "core::iter::iterator::Iterator>::rposition::::{closure#0}"}, - {"_RINbNbCskIICzLVDPPb_5alloc5alloc8box_freeDINbNiB4_" - "5boxed5FnBoxuEp6OutputuEL_ECs1iopQbuBiw2_3std", - "alloc::alloc::box_free::>"}, - {"_RNvMC0INtC8arrayvec8ArrayVechKj7b_E3new", - ">::new"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_8UnsignedKhb_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_6SignedKs98_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_6SignedKanb_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4BoolKb0_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4BoolKb1_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKc76_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKca_E", - ">"}, - {"_RMCs4fqI2P2rA04_13const_genericINtB0_4CharKc2202_E", - ">"}, - {"_RNvNvMCs4fqI2P2rA04_13const_genericINtB4_3FooKpE3foo3FOO", - ">::foo::FOO"}, -}; - -#define BUF_LEN 1024 -TEST(DemangleTest, Positive) { - for (auto const &tcase : s_demangle_cases) { - char *demangled = abi::__cxa_demangle(tcase.test.c_str(), nullptr, nullptr, nullptr); - if (demangled) { - std::string demangled_str{demangled}; - if (!demangled_str.empty()) - if (RustDemangler::is_probably_rust_legacy(demangled_str)) - demangled_str = RustDemangler::demangle(demangled_str); - EXPECT_EQ(demangled_str, tcase.answer); - } - free(demangled); - } -} -#endif diff --git a/ddprof-lib/src/test/cpp/dictionary_ut.cpp b/ddprof-lib/src/test/cpp/dictionary_ut.cpp deleted file mode 100644 index 9d6cf343e..000000000 --- a/ddprof-lib/src/test/cpp/dictionary_ut.cpp +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2025 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "dictionary.h" -#include -#include - -// ── Dictionary ───────────────────────────────────────────────────────────── - -TEST(DictionaryTest, LookupReturnsSameIdForSameKey) { - Dictionary d(0); - unsigned int id1 = d.lookup("hello", 5); - EXPECT_GT(id1, 0U); - EXPECT_EQ(id1, d.lookup("hello", 5)); -} - -TEST(DictionaryTest, LookupReturnsDifferentIdsForDifferentKeys) { - Dictionary d(0); - unsigned int a = d.lookup("alpha", 5); - unsigned int b = d.lookup("beta", 4); - EXPECT_NE(a, b); -} - -TEST(DictionaryTest, BoundedLookupSkipsInsertWhenAtLimit) { - Dictionary d(0); - d.lookup("key1", 4); - // size is 1, limit is 1 → insert not allowed - unsigned int r = d.bounded_lookup("key2", 4, 1); - EXPECT_EQ(r, static_cast(INT_MAX)); -} - -TEST(DictionaryTest, BoundedLookupReturnsExistingIdWhenAtLimit) { - Dictionary d(0); - unsigned int existing = d.lookup("key1", 4); - // size is 1, limit is 1 → existing key still found - EXPECT_EQ(existing, d.bounded_lookup("key1", 4, 1)); -} - -TEST(DictionaryTest, CollectReturnsAllInsertedEntries) { - Dictionary d(0); - d.lookup("a", 1); - d.lookup("b", 1); - d.lookup("c", 1); - std::map m; - d.collect(m); - EXPECT_EQ(m.size(), 3U); -} - -TEST(DictionaryTest, ClearResetsToEmpty) { - Dictionary d(0); - d.lookup("x", 1); - d.clear(); - std::map m; - d.collect(m); - EXPECT_EQ(m.size(), 0U); -} diff --git a/ddprof-lib/src/test/cpp/dwarf_ut.cpp b/ddprof-lib/src/test/cpp/dwarf_ut.cpp deleted file mode 100644 index e45983696..000000000 --- a/ddprof-lib/src/test/cpp/dwarf_ut.cpp +++ /dev/null @@ -1,256 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include "dwarf.h" -#include "../../main/cpp/gtest_crash_handler.h" - -#include -#include -#include - -// Test name for crash handler -static constexpr char DWARF_TEST_NAME[] = "DwarfTest"; - -class DwarfGlobalSetup { - public: - DwarfGlobalSetup() { - installGtestCrashHandler(); - } - ~DwarfGlobalSetup() { - restoreDefaultSignalHandlers(); - } -}; -static DwarfGlobalSetup dwarf_global_setup; - -#if DWARF_SUPPORTED - -// Helpers to write little-endian integers into a byte buffer. -static void put32(std::vector& buf, uint32_t v) { - buf.push_back(static_cast(v)); - buf.push_back(static_cast(v >> 8)); - buf.push_back(static_cast(v >> 16)); - buf.push_back(static_cast(v >> 24)); -} - -static void put8(std::vector& buf, uint8_t v) { - buf.push_back(v); -} - -// Append a minimal CIE with "z" augmentation to buf. -// Layout: [4-len=11][4-cie_id=0][1-ver=1][2-aug="z\0"][1-code_align=4][1-data_align=-8][1-ra=30][1-aug_data_len=0] -// Total: 15 bytes. -static void appendCie(std::vector& buf) { - // body size = cie_id(4) + version(1) + "z\0"(2) + code_align(1) + data_align(1) + ra_col(1) + aug_data_len(1) = 11 - put32(buf, 11); // length - put32(buf, 0); // cie_id = 0 - put8(buf, 1); // version - put8(buf, 'z'); // augmentation "z" - put8(buf, 0); // null terminator - put8(buf, 4); // code_align = 4 (LEB128) - put8(buf, 0x78); // data_align = -8 (SLEB128: 0x78) - put8(buf, 30); // return address column = 30 (lr) - put8(buf, 0); // augmentation data length = 0 (LEB128) -} - -// Append an FDE referencing the CIE at cie_start_offset from the buf start. -// cie_offset in the FDE = offset from FDE's cie_id field to the CIE start. -// range_start is encoded as a 4-byte PC-relative signed integer (pcrel). -// With pcrel=0 and image_base=&buf[0]: range_start = offset_of_pcrel_field_within_buf. -// Layout: [4-len=13][4-cie_offset][4-pcrel=0][4-range_len][1-aug_data_len=0] -// Total: 17 bytes. -static void appendFde(std::vector& buf, uint32_t cie_start_offset, uint32_t range_len) { - // The FDE's cie_id field will be at buf.size() + 4 (after length field). - uint32_t cie_id_field_offset = static_cast(buf.size()) + 4; - uint32_t cie_offset = cie_id_field_offset - cie_start_offset; - - // body = cie_offset(4) + range_start(4) + range_len(4) + aug_data_len(1) = 13 - put32(buf, 13); // length - put32(buf, cie_offset); // cie_offset from this field back to CIE start - put32(buf, 0); // range_start pcrel = 0 (absolute value = field_address - image_base) - put32(buf, range_len); // range_len - put8(buf, 0); // aug data length = 0 (LEB128, for "z" augmentation) - // no DWARF call frame instructions -} - -static void appendTerminator(std::vector& buf) { - put32(buf, 0); -} - -// Parse a raw __eh_frame section using the linear DwarfParser constructor. -// image_base is set to buf.data() so that pcrel=0 yields range_start = field_offset_in_buf. -static DwarfParser* parseBuf(const std::vector& buf) { - const char* base = reinterpret_cast(buf.data()); - return new DwarfParser("test", base, base, buf.size()); -} - -TEST(DwarfEhFrame, EmptySection) { - std::vector buf; - DwarfParser* dwarf = parseBuf(buf); - EXPECT_EQ(dwarf->count(), 0); - free(dwarf->table()); - delete dwarf; -} - -TEST(DwarfEhFrame, TerminatorOnly) { - std::vector buf; - appendTerminator(buf); - DwarfParser* dwarf = parseBuf(buf); - EXPECT_EQ(dwarf->count(), 0); - free(dwarf->table()); - delete dwarf; -} - -TEST(DwarfEhFrame, CieOnly) { - std::vector buf; - appendCie(buf); - appendTerminator(buf); - DwarfParser* dwarf = parseBuf(buf); - // CIE alone generates no frame records. - EXPECT_EQ(dwarf->count(), 0); - free(dwarf->table()); - delete dwarf; -} - -TEST(DwarfEhFrame, CieAndFde) { - // CIE starts at offset 0. - std::vector buf; - appendCie(buf); // 15 bytes - appendFde(buf, 0, 256); // 17 bytes (cie_offset = 19) - appendTerminator(buf); // 4 bytes - ASSERT_EQ(buf.size(), static_cast(36)); - - DwarfParser* dwarf = parseBuf(buf); - // The FDE with no instructions generates two records: - // one from parseInstructions (initial state at range_start) and one sentinel (at range_start + range_len). - EXPECT_EQ(dwarf->count(), 2); - - // Table must be in ascending loc order (sorted). - const FrameDesc* table = dwarf->table(); - ASSERT_NE(table, nullptr); - EXPECT_LT(table[0].loc, table[1].loc); - - // Sentinel record covers the end of the FDE's range. - // range_start = offset of pcrel field in buf = 15+4+4 = 23; range_end = 23+256 = 279. - EXPECT_EQ(table[1].loc, static_cast(279)); - - free(dwarf->table()); - delete dwarf; -} - -// --- Bounds-guard tests --- - -TEST(DwarfEhFrame, TruncatedRecord) { - // Build a valid CIE then truncate the buffer so record_end > section_end. - // The length-overflow guard should fire and produce no records. - std::vector buf; - appendCie(buf); // 15 bytes: length=11, so record_end = 15 - buf.resize(10); // section_end = 10 < record_end → overflow guard triggers - DwarfParser* dwarf = parseBuf(buf); - EXPECT_EQ(dwarf->count(), 0); - free(dwarf->table()); - delete dwarf; -} - -TEST(DwarfEhFrame, ShortCieBody) { - // CIE with length=4: body is exactly cie_id (4 bytes), nothing else. - // After reading cie_id, _ptr == record_end; the version/augmentation guard triggers. - std::vector buf; - put32(buf, 4); // length = 4 - put32(buf, 0); // cie_id = 0 → CIE - appendTerminator(buf); - DwarfParser* dwarf = parseBuf(buf); - EXPECT_EQ(dwarf->count(), 0); - free(dwarf->table()); - delete dwarf; -} - -TEST(DwarfEhFrame, FdeAugDataOverrun) { - // CIE with 'z' augmentation followed by an FDE whose aug-data-length encodes - // a value (100) larger than remaining bytes in the record (0). - // The FDE should be skipped without a crash. - std::vector buf; - appendCie(buf); // 15 bytes; _has_z_augmentation = true - - // FDE body: cie_offset(4) + range_start(4) + range_len(4) + aug_data_len(1) = 13 - // aug_data_len = 100 but no aug data bytes follow → _ptr += 100 > record_end → break - uint32_t cie_id_field_offset = static_cast(buf.size()) + 4; - put32(buf, 13); // length - put32(buf, cie_id_field_offset - 0); // cie_offset back to CIE at offset 0 - put32(buf, 0); // range_start pcrel - put32(buf, 128); // range_len - put8(buf, 100); // aug_data_len = 100 but 0 bytes of aug data follow - appendTerminator(buf); - DwarfParser* dwarf = parseBuf(buf); - EXPECT_EQ(dwarf->count(), 0); - free(dwarf->table()); - delete dwarf; -} - -// CIE + FDE whose body ends at exactly the last byte of the section (no -// terminator appended). Verifies that _image_end-bounded reads are not -// spuriously rejected when the FDE occupies the full section. -TEST(DwarfEhFrame, FdeAtExactImageBoundary) { - std::vector buf; - appendCie(buf); // 15 bytes - appendFde(buf, 0, 256); // 17 bytes; FDE ends at offset 32 == image_end - ASSERT_EQ(buf.size(), static_cast(32)); - DwarfParser* dwarf = parseBuf(buf); - EXPECT_EQ(dwarf->count(), 2); // normal result; boundary must not be spuriously rejected - free(dwarf->table()); - delete dwarf; -} - -// An FDE where fde_len makes fde_end > _image_end. -// parseFde()'s `fde_end > _image_end` guard must reject it without reading past -// the buffer. Uses the .eh_frame_hdr constructor path (parse → parseFde). -// -// Buffer layout (24 bytes): -// [0-3] .eh_frame_hdr header (version + 3 encoding bytes) -// [4-7] eh_frame_ptr = 0 -// [8-11] fde_count = 1 -// [12-15] table[0].initial_loc = 0 -// [16-19] table[0].fde_ptr = 20 (offset from hdr start to fde_len field below) -// [20-23] fde_len = 100 (fde_end = hdr+24+100 = hdr+124 > image_end=hdr+24) -TEST(DwarfEhFrameHdr, FdeExceedsImageEnd) { - std::vector hdr(24, 0); - hdr[0] = 1; // version - hdr[1] = 0x03; // eh_frame_ptr_enc = DW_EH_PE_udata4 - hdr[2] = 0x03; // fde_count_enc = DW_EH_PE_udata4 - hdr[3] = 0x33; // table_enc = DW_EH_PE_datarel | DW_EH_PE_udata4 - hdr[8] = 1; // fde_count = 1 - hdr[16] = 20; // table[0].fde_ptr: points to the fde_len field below - hdr[20] = 100; // fde_len = 100 → fde_end = hdr+124 > image_end = hdr+24 - - const char* base = reinterpret_cast(hdr.data()); - DwarfParser dwarf("test", base, base, hdr.size(), DwarfParser::EhFrameHdrTag{}, base + hdr.size()); - EXPECT_EQ(dwarf.count(), 0); // rejected: fde_end > image_end, no crash - free(dwarf.table()); -} - -// Regression test for the .eh_frame_hdr hardening (found by fuzz_dwarf). -// A hostile .eh_frame_hdr can claim a large fde_count while providing no -// binary-search table; pre-hardening, parse() walked `table[i*2]` off the end -// of the section. The bounded parser rejects a fde_count that cannot fit in the -// section. The header is a heap buffer sized to exactly 16 bytes (header only, -// no table entries), so ASan's redzone catches any over-read deterministically. -TEST(DwarfEhFrameHdr, FdeCountOverrun) { - std::vector hdr(16, 0); // 16-byte header; the table would start at 16 - hdr[0] = 1; // version - hdr[1] = 0x03; // eh_frame_ptr_enc = DW_EH_PE_udata4 - hdr[2] = 0x03; // fde_count_enc = DW_EH_PE_udata4 - hdr[3] = 0x33; // table_enc = DW_EH_PE_datarel | DW_EH_PE_udata4 - // fde_count at offset 8 (little-endian): claim 1024 entries that aren't there. - hdr[8] = 0x00; - hdr[9] = 0x04; - - const char* base = reinterpret_cast(hdr.data()); - DwarfParser dwarf("test", base, base, hdr.size(), DwarfParser::EhFrameHdrTag{}, base + hdr.size()); - EXPECT_EQ(dwarf.count(), 0); // rejected: no records, no crash - free(dwarf.table()); -} - -#endif // DWARF_SUPPORTED diff --git a/ddprof-lib/src/test/cpp/elfparser_ut.cpp b/ddprof-lib/src/test/cpp/elfparser_ut.cpp deleted file mode 100644 index c0e7cf584..000000000 --- a/ddprof-lib/src/test/cpp/elfparser_ut.cpp +++ /dev/null @@ -1,576 +0,0 @@ -#ifdef __linux__ - -#include -#include - -#include "codeCache.h" -#include "libraries.h" -#include "symbols.h" -#include "symbols_linux.h" -#include "log.h" -#include "../../main/cpp/gtest_crash_handler.h" - -#include -#include // For PATH_MAX - -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -// Forward declaration for ElfParser functionality from symbols_linux.cpp -// The actual implementation will be available through the patched upstream file -class ElfParser { -public: - static bool parseFile(CodeCache* cc, const char* base, const char* file_name, bool use_debug); -}; - -// Test name for crash handler -static constexpr char ELF_TEST_NAME[] = "ElfParserTest"; - -// Global crash handler installation (since this file uses bare TEST() macros) -class ElfParserGlobalSetup { -public: - ElfParserGlobalSetup() { - installGtestCrashHandler(); - } - ~ElfParserGlobalSetup() { - restoreDefaultSignalHandlers(); - } -}; - -// Install global crash handler for all tests in this file -static ElfParserGlobalSetup global_setup; - -TEST(Elf, readSymTable) { - char cwd[PATH_MAX - 64]; - if (getcwd(cwd, sizeof(cwd)) == nullptr) { - exit(1); - } - char path[PATH_MAX]; - snprintf(path, sizeof(path) - 1, "%s/../build/test/resources/native-libs/unresolved-functions/main", cwd); - if (access(path, R_OK) != 0) { - fprintf(stdout, "Missing test resource %s. Skipping the test\n", path); - exit(0); - } - CodeCache cc("test"); - ElfParser::parseFile(&cc, nullptr, path, false); -} - -class ElfReladyn : public ::testing::Test { - protected: - Libraries* _libs = nullptr; - CodeCache* _libreladyn = nullptr; - - // This method is called before each test. - void SetUp() override { - char cwd[PATH_MAX - 64]; - if (getcwd(cwd, sizeof(cwd)) == nullptr) { - exit(1); - } - char path[PATH_MAX]; - snprintf(path, sizeof(path) - 1, "%s/../build/test/resources/native-libs/reladyn-lib/libreladyn.so", cwd); - if (access(path, R_OK) != 0) { - fprintf(stdout, "Missing test resource %s. Skipping the test\n", path); - exit(0); - } - void* handle = dlopen(path, RTLD_NOW); - ASSERT_THAT(handle, ::testing::NotNull()); - - _libs = Libraries::instance(); - _libs->updateSymbols(false); - _libreladyn = _libs->findLibraryByName("libreladyn"); - ASSERT_THAT(_libreladyn, ::testing::NotNull()); - } - - // This method is called after each test. - void TearDown() override { - // Clean up resources. - } - - CodeCache* libreladyn() { - return _libreladyn; - } -}; - -TEST_F(ElfReladyn, resolveFromRela_plt) { - void* sym = libreladyn()->findImport(im_pthread_create); - ASSERT_THAT(sym, ::testing::NotNull()); -} - -TEST_F(ElfReladyn, resolveFromRela_dyn_R_GLOB_DAT) { - void* sym = libreladyn()->findImport(im_pthread_setspecific); - ASSERT_THAT(sym, ::testing::NotNull()); -} - -TEST_F(ElfReladyn, resolveFromRela_dyn_R_ABS64) { - void* sym = libreladyn()->findImport(im_pthread_exit); - ASSERT_THAT(sym, ::testing::NotNull()); -} - -class ElfTest : public ::testing::Test { -protected: - void SetUp() override { - // Reset global or static state - Symbols::clearParsingCaches(); - } - - void TearDown() override { - // probably some free of the array cache to be done - } -}; - - -// Define an invalid ELF header -unsigned char invalidElfHeader[64] = { - 0x7f, 'E', 'L', 'F', // Correct magic number - 0x01, // Invalid class (32-bit instead of 64-bit) - 0x01, // Invalid data encoding (little-endian) - 0x01, // Invalid version (original version of ELF) - 0, // OS/ABI - 0, // ABI version - 0, 0, 0, 0, 0, 0, 0 // Padding - // Rest of the header can be zeroed out -}; - - -TEST_F(ElfTest, invalidElf) { - // Create an invalid ELF mapping - const size_t headerSize = sizeof(invalidElfHeader); - int fd = open("/tmp/invalid_elf", O_RDWR | O_CREAT | O_TRUNC, 0700); // Make the file executable - ASSERT_NE(fd, -1) << "Failed to open temporary file"; - - // Write the invalid ELF header to the file - ssize_t written = write(fd, invalidElfHeader, headerSize); - ASSERT_EQ(written, headerSize) << "Failed to write invalid ELF header"; - - // Extend the file to a reasonable size - int res = ftruncate(fd, 4096); - ASSERT_EQ(res, 0) << "Failed to extend the file"; - - // Memory map the file with PROT_EXEC - void* addr = mmap(NULL, 4096, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); - ASSERT_NE(addr, MAP_FAILED) << "Failed to memory map the file"; - - close(fd); - - // Set up the CodeCacheArray and other required structures - CodeCacheArray cc_array; - - // Call the parsing function with the invalid ELF mapping - Symbols::parseLibraries(&cc_array, false); - - munmap(addr, 4096); - unlink("/tmp/invalid_elf"); -} - -// Additional test cases for other invalid ELF scenarios -TEST_F(ElfTest, nonElfFile) { - // Create a non-ELF file mapping - const char* nonElfContent = "This is not an ELF file"; - const size_t contentSize = strlen(nonElfContent) + 1; - int fd = open("/tmp/non_elf", O_RDWR | O_CREAT | O_TRUNC, 0700); // Make the file executable - ASSERT_NE(fd, -1) << "Failed to open temporary file"; - - // Write the non-ELF content to the file - ssize_t written = write(fd, nonElfContent, contentSize); - ASSERT_EQ(written, contentSize) << "Failed to write non-ELF content"; - - // Memory map the file with PROT_EXEC - void* addr = mmap(NULL, contentSize, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); - ASSERT_NE(addr, MAP_FAILED) << "Failed to memory map the file"; - - close(fd); - - // Set up the CodeCacheArray and other required structures - CodeCacheArray cc_array; - - // Call the parsing function with the non-ELF mapping - Symbols::parseLibraries(&cc_array, false); - - // we could add checks here, though I am mainly relying on asan to crash - // if something is wrong - munmap(addr, contentSize); - unlink("/tmp/non_elf"); -} - -TEST_F(ElfTest, invalidElfSmallMapping) { - // Create an invalid ELF mapping - const size_t headerSize = sizeof(invalidElfHeader); - int fd = open("/tmp/invalid_elf_small", O_RDWR | O_CREAT | O_TRUNC, 0700); // Make the file executable - ASSERT_NE(fd, -1) << "Failed to open temporary file"; - - // Write the invalid ELF header to the file - ssize_t written = write(fd, invalidElfHeader, headerSize); - ASSERT_EQ(written, headerSize) << "Failed to write invalid ELF header"; - - // Memory map the file with a size smaller than the ELF header - void* addr = mmap(NULL, 16, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); // Map only 16 bytes - ASSERT_NE(addr, MAP_FAILED) << "Failed to memory map the file"; - - close(fd); - - // Set up the CodeCacheArray and other required structures - CodeCacheArray cc_array; - - // Call the parsing function with the small invalid ELF mapping - Symbols::parseLibraries(&cc_array, false); - - munmap(addr, 16); - unlink("/tmp/invalid_elf_small"); -} - -TEST_F(ElfTest, nonElfFileSmallMapping) { - // Create a non-ELF file mapping - const char* nonElfContent = "Not ELF"; - const size_t contentSize = strlen(nonElfContent); - int fd = open("/tmp/non_elf_small", O_RDWR | O_CREAT | O_TRUNC, 0700); // Make the file executable - ASSERT_NE(fd, -1) << "Failed to open temporary file"; - - // Write the non-ELF content to the file - ssize_t written = write(fd, nonElfContent, contentSize); - ASSERT_EQ(written, contentSize) << "Failed to write non-ELF content"; - - // Memory map the file with a size smaller than expected - void* addr = mmap(NULL, contentSize, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); // Map only the content size - ASSERT_NE(addr, MAP_FAILED) << "Failed to memory map the file"; - - close(fd); - - // Set up the CodeCacheArray and other required structures - CodeCacheArray cc_array; - - // Call the parsing function with the small non-ELF mapping - Symbols::parseLibraries(&cc_array, false); - munmap(addr, contentSize); - unlink("/tmp/non_elf_small"); -} - -class ElfTestParam : public ::testing::TestWithParam { -protected: - void SetUp() override { - // Reset global or static state - Symbols::clearParsingCaches(); - } - - void TearDown() override { - // probably some free of the array cache to be done - } -}; - - -#ifdef UNMAP_DOES_NOT_CRASH // for now we only have a lock on dl_close. unmapping will still crash -// This test does not repro 100% of the time. -// However over a few runs, I get it to reproduce the race condition. -TEST_P(ElfTestParam, invalidElfSmallMappingAfterUnmap) { - // This does not work as expected. There is a follow up to improve logging. - Log::open("stderr", "WARN"); - // Create an invalid ELF mapping (it could be valid for this case, it is not relevant) - const size_t headerSize = sizeof(invalidElfHeader); - int fd = open("/tmp/invalid_elf_small_unmap", O_RDWR | O_CREAT | O_TRUNC, 0700); // Make the file executable - ASSERT_NE(fd, -1) << "Failed to open temporary file"; - ssize_t written = write(fd, invalidElfHeader, headerSize); - ASSERT_EQ(written, headerSize) << "Failed to write invalid ELF header"; - // Memory map the file - void* addr = mmap(NULL, 16, PROT_READ | PROT_EXEC, MAP_PRIVATE, fd, 0); // Map only 16 bytes - ASSERT_NE(addr, MAP_FAILED) << "Failed to memory map the file"; - - close(fd); - - const char* base = static_cast(addr); - const char* end = base + 16; - - // Set up the CodeCacheArray and other required structures - CodeCacheArray cc_array; - int delay = GetParam(); - fprintf(stderr, "-- Test Delay = %d ms\n", delay); - // Create a thread that will unmap the memory after X milliseconds - // We need a timing that allows us to read the - // mapping, but not loop over them in the parsing function - // I was able to reproduce with ~15 ms in asan mode. - std::thread unmapper([addr, delay]() { - std::this_thread::sleep_for(std::chrono::milliseconds(delay)); - munmap(addr, 16); - unlink("/tmp/invalid_elf_small_unmap"); - }); - - // Call the parsing function in the main thread - Symbols::parseLibraries(&cc_array, false); - - // Join the unmapper thread to ensure it has finished - unmapper.join(); -} - -INSTANTIATE_TEST_SUITE_P( - DelayedUnmapTest, - ElfTestParam, - ::testing::Range(3, 21) // This will test delays from 5 to 20 milliseconds inclusive -); - -#else -TEST_P(ElfTestParam, invalidElfSmallMappingAfterUnmap) { - char cwd[PATH_MAX - 64]; - if (getcwd(cwd, sizeof(cwd)) == nullptr) { - exit(1); - } - - // Configure logging (assuming Log is defined elsewhere) - Log::open("stderr", "WARN"); - - // Construct the path to the test resource - char path[PATH_MAX]; - snprintf(path, sizeof(path) - 1, "%s/../build/test/resources/native-libs/small-lib/libsmall-lib.so", cwd); - if (access(path, R_OK) != 0) { - fprintf(stdout, "Missing test resource %s. Skipping the test\n", path); - exit(1); - } - void* handle = dlopen(path, RTLD_NOW); - if (!handle) { - fprintf(stderr, "dlopen failed: %s\n", dlerror()); - exit(1); - } - - CodeCacheArray cc_array; - int delay = GetParam(); - fprintf(stderr, "-- Test Delay = %d ms\n", delay); - - // Create a thread that will unmap (close) the shared library after a delay - std::thread unmapper([handle, delay]() { - std::this_thread::sleep_for(std::chrono::milliseconds(delay)); - // Unload the shared library using dlclose - dlclose(handle); - }); - - // Call the parsing function in the main thread, this is where we can crash - Symbols::parseLibraries(&cc_array, false); - unmapper.join(); -} - -INSTANTIATE_TEST_SUITE_P( - DelayedUnmapTest, - ElfTestParam, - ::testing::Range(3, 21) // This will test delays from 5 to 20 milliseconds inclusive -); -#endif - -// ===================================================================== -// Regression tests for the ELF parser hardening (found by the fuzz_elf -// harness). Each builds a minimal ELF whose single malformed field made the -// pre-hardening parser read out of bounds. On the hardened parser they must -// return cleanly: the global crash handler installed above turns any wild or -// out-of-bounds access back into a gtest failure, so a regression fails CI. -// -// The exact byte layouts were confirmed to crash the pre-fix parser -// (ASan SEGV / heap-buffer-overflow) and to pass after the fix. -// ===================================================================== - -namespace { - -// Minimal valid ELF64 header; callers set the malformed field afterwards. -Elf64_Ehdr validEhdr() { - Elf64_Ehdr e; - memset(&e, 0, sizeof(e)); - e.e_ident[EI_MAG0] = ELFMAG0; - e.e_ident[EI_MAG1] = ELFMAG1; - e.e_ident[EI_MAG2] = ELFMAG2; - e.e_ident[EI_MAG3] = ELFMAG3; - e.e_ident[EI_CLASS] = ELFCLASS64; - e.e_ident[EI_DATA] = ELFDATA2LSB; - e.e_ident[EI_VERSION] = EV_CURRENT; - e.e_type = ET_DYN; - e.e_machine = EM_X86_64; - e.e_version = EV_CURRENT; - e.e_ehsize = sizeof(Elf64_Ehdr); - e.e_shstrndx = 1; // non-zero so validHeader() accepts the image - return e; -} - -// Write the bytes to a unique temp file and run ElfParser::parseFile over it, -// mirroring how Symbols::parseLibraries() parses an on-disk library. -void parseElfBytes(const std::vector& bytes) { - char path[] = "/tmp/elf_regress_XXXXXX"; - int fd = mkstemp(path); - ASSERT_NE(fd, -1); - ssize_t written = write(fd, bytes.data(), bytes.size()); - close(fd); - if (written != (ssize_t)bytes.size()) { - unlink(path); - FAIL() << "short write to " << path; - return; - } - CodeCache cc("regress"); - ElfParser::parseFile(&cc, nullptr, path, /*use_debug=*/false); - unlink(path); -} - -} // namespace - -// Regression test for the build-id parser hardening (found by fuzz_elf). -// extractBuildIdFromMemory()'s `p_offset + p_filesz` bounds check could -// overflow, letting findBuildIdInNotes() walk a PT_NOTE past the buffer. The -// buffer is heap-allocated and sized exactly so ASan's redzone catches the -// over-read deterministically. The hardened parser must return cleanly. -TEST(ElfBuildId, noteOffsetOverflow) { - Elf64_Ehdr e = validEhdr(); - e.e_phoff = sizeof(Elf64_Ehdr); - e.e_phentsize = sizeof(Elf64_Phdr); - e.e_phnum = 1; - - Elf64_Phdr p; - memset(&p, 0, sizeof(p)); - p.p_type = PT_NOTE; - p.p_offset = 0x70; // inside the buffer - p.p_filesz = static_cast(8) - p.p_offset; // sum wraps to 8 (< size) - - const size_t size = sizeof(e) + sizeof(p); // 120 bytes - char* buf = new char[size]; // exact size -> redzone right after - memcpy(buf, &e, sizeof(e)); - memcpy(buf + sizeof(e), &p, sizeof(p)); - - size_t build_id_len = 0; - char* id = SymbolsLinux::extractBuildIdFromMemory(buf, size, &build_id_len); - free(id); // hardened parser returns nullptr; the point is that it must not crash - delete[] buf; -} - -// e_shoff pointing far outside the image made findSection() dereference a wild -// section-header pointer (ElfParser::at). 16 TB is reliably unmapped. -TEST_F(ElfTest, sectionHeaderOffsetOutOfBounds) { - Elf64_Ehdr e = validEhdr(); - e.e_shoff = 0x100000000000ULL; // 16 TB past a 64-byte file - e.e_shentsize = sizeof(Elf64_Shdr); - e.e_shnum = 3; - e.e_shstrndx = 1; - std::vector bytes(reinterpret_cast(&e), - reinterpret_cast(&e) + sizeof(e)); - parseElfBytes(bytes); // must not crash -} - -// A .symtab whose sh_size claims 256 MB in a tiny file made loadSymbolTable() -// walk the symbol table off the end of the mapping. -TEST_F(ElfTest, symbolTableSizeOutOfBounds) { - const uint16_t NSEC = 4; - const uint64_t shoff = sizeof(Elf64_Ehdr); - const uint64_t shstr_off = shoff + NSEC * sizeof(Elf64_Shdr); - // Section-header string table: names at offsets 1, 9, 17. - const char shstrtab[] = "\0.symtab\0.strtab\0.shstrtab"; - const uint64_t sym_off = shstr_off + sizeof(shstrtab); - Elf64_Sym sym; - memset(&sym, 0, sizeof(sym)); - sym.st_name = 1; - sym.st_value = 0x1000; - const uint64_t str_off = sym_off + sizeof(sym); - const char strtab[] = "\0main"; - - Elf64_Ehdr e = validEhdr(); - e.e_shoff = shoff; - e.e_shentsize = sizeof(Elf64_Shdr); - e.e_shnum = NSEC; - e.e_shstrndx = 3; - - Elf64_Shdr sh[4]; - memset(sh, 0, sizeof(sh)); - sh[1].sh_name = 1; // ".symtab" - sh[1].sh_type = SHT_SYMTAB; - sh[1].sh_offset = sym_off; - sh[1].sh_size = 0x10000000; // 256 MB: far past the file - sh[1].sh_link = 2; - sh[1].sh_entsize = sizeof(Elf64_Sym); - sh[2].sh_name = 9; // ".strtab" - sh[2].sh_type = SHT_STRTAB; - sh[2].sh_offset = str_off; - sh[2].sh_size = sizeof(strtab); - sh[3].sh_name = 17; // ".shstrtab" - sh[3].sh_type = SHT_STRTAB; - sh[3].sh_offset = shstr_off; - sh[3].sh_size = sizeof(shstrtab); - - std::vector b; - auto app = [&](const void* p, size_t n) { - const char* c = static_cast(p); - b.insert(b.end(), c, c + n); - }; - app(&e, sizeof(e)); - app(sh, sizeof(sh)); - app(shstrtab, sizeof(shstrtab)); - app(&sym, sizeof(sym)); - app(strtab, sizeof(strtab)); - parseElfBytes(b); // must not crash -} - -// A large e_phoff causes phdrAt() to try forming a pointer past the image. -// The bounds check must reject it before any dereference. -TEST_F(ElfTest, programHeaderOffsetOutOfBounds) { - Elf64_Ehdr e = validEhdr(); - e.e_phoff = 0x100000000000ULL; // 16 TB: reliably unmapped - e.e_phentsize = sizeof(Elf64_Phdr); - e.e_phnum = 1; - std::vector bytes(reinterpret_cast(&e), - reinterpret_cast(&e) + sizeof(e)); - parseElfBytes(bytes); // must not crash -} - -// strAt() bounds check: a symbol whose st_name equals strtab_size (one past -// the end) must be skipped without reading out of bounds. -TEST_F(ElfTest, symbolNameOffsetOutOfBounds) { - const uint16_t NSEC = 4; - const uint64_t shoff = sizeof(Elf64_Ehdr); - const uint64_t shstr_off = shoff + NSEC * sizeof(Elf64_Shdr); - const char shstrtab[] = "\0.symtab\0.strtab\0.shstrtab"; - const uint64_t sym_off = shstr_off + sizeof(shstrtab); - Elf64_Sym sym; - memset(&sym, 0, sizeof(sym)); - sym.st_name = 6; // == sizeof(strtab) below: one past the end - sym.st_value = 0x1000; - sym.st_size = 4; - const uint64_t str_off = sym_off + sizeof(sym); - const char strtab[] = "\0main\0"; // 6 bytes; index 6 is out of bounds - - Elf64_Ehdr e = validEhdr(); - e.e_shoff = shoff; - e.e_shentsize = sizeof(Elf64_Shdr); - e.e_shnum = NSEC; - e.e_shstrndx = 3; - - Elf64_Shdr sh[4]; - memset(sh, 0, sizeof(sh)); - sh[1].sh_name = 1; // ".symtab" - sh[1].sh_type = SHT_SYMTAB; - sh[1].sh_offset = sym_off; - sh[1].sh_size = sizeof(sym); // exactly one entry, within image - sh[1].sh_link = 2; - sh[1].sh_entsize = sizeof(Elf64_Sym); - sh[2].sh_name = 9; // ".strtab" - sh[2].sh_type = SHT_STRTAB; - sh[2].sh_offset = str_off; - sh[2].sh_size = sizeof(strtab); - sh[3].sh_name = 17; // ".shstrtab" - sh[3].sh_type = SHT_STRTAB; - sh[3].sh_offset = shstr_off; - sh[3].sh_size = sizeof(shstrtab); - - std::vector b; - auto app = [&](const void* p, size_t n) { - const char* c = static_cast(p); - b.insert(b.end(), c, c + n); - }; - app(&e, sizeof(e)); - app(sh, sizeof(sh)); - app(shstrtab, sizeof(shstrtab)); - app(&sym, sizeof(sym)); - app(strtab, sizeof(strtab)); - // strtab ends at image_size: also exercises inImage() equality case. - parseElfBytes(b); // must not crash: strAt() rejects st_name == strtab_size -} - -#endif //__linux__ diff --git a/ddprof-lib/src/test/cpp/flightRecorder_result_ut.cpp b/ddprof-lib/src/test/cpp/flightRecorder_result_ut.cpp deleted file mode 100644 index eba3995ed..000000000 --- a/ddprof-lib/src/test/cpp/flightRecorder_result_ut.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "engine.h" -#include "event.h" -#include "flightRecorder.h" - -#include - -TEST(FlightRecorderResultTest, RecordEventReturnsFalseWithoutActiveRecording) { - FlightRecorder recorder; - ExecutionEvent event; - - EXPECT_FALSE(recorder.recordEvent(/*lock_index=*/0, /*tid=*/1, - /*call_trace_id=*/42, BCI_WALL, &event)); -} - -TEST(FlightRecorderResultTest, RecordEventDelegatedReturnsFalseWithoutActiveRecording) { - FlightRecorder recorder; - ExecutionEvent event; - - EXPECT_FALSE(recorder.recordEventDelegated(/*lock_index=*/0, /*tid=*/1, - /*correlation_id=*/42, BCI_WALL, - &event)); -} - -TEST(FlightRecorderResultTest, RecordEventDelegatedReturnsFalseForUnsupportedEventType) { - FlightRecorder recorder; - AllocEvent event; - - EXPECT_FALSE(recorder.recordEventDelegated(/*lock_index=*/0, /*tid=*/1, - /*correlation_id=*/42, BCI_ALLOC, - &event)); -} diff --git a/ddprof-lib/src/test/cpp/forced_unwind_ut.cpp b/ddprof-lib/src/test/cpp/forced_unwind_ut.cpp deleted file mode 100644 index d33cb7611..000000000 --- a/ddprof-lib/src/test/cpp/forced_unwind_ut.cpp +++ /dev/null @@ -1,591 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * Regression test for the static-libgcc / forced-unwind incompatibility in - * start_routine_wrapper (libraryPatcher_linux.cpp). - * - * Root cause: when libjavaProfiler.so is built with -static-libgcc, its - * embedded __gxx_personality_v0 is called by the dynamic libgcc_s.so.1's - * _Unwind_ForcedUnwind (triggered by pthread_exit / pthread_cancel). The two - * libgcc versions have incompatible _Unwind_Context layouts; calling - * _Unwind_SetGR with a cross-version context triggers _Unwind_SetGR.cold, - * which calls abort(). The same crash affects any gtest binary that is also - * built with -static-libgcc. - * - * C++ RAII destructors (struct Cleanup { ~Cleanup() {...} }) add cleanup - * entries to the LSDA, which cause __gxx_personality_v0 to call _Unwind_SetGR - * → crash. The pthread_cleanup_push C++ macro (__pthread_cleanup_class RAII) - * has the same problem. - * - * Cleanup strategy (matches libraryPatcher_linux.cpp): - * - glibc: Use __pthread_register_cancel / __pthread_unregister_cancel - * directly (the same mechanism the C form of pthread_cleanup_push - * uses). This registers cleanup via a setjmp buffer in a runtime - * linked-list, NOT in the LSDA. _Unwind_ForcedUnwind's stop - * function (__pthread_unwind_stop) handles the cleanup without - * ever calling __gxx_personality_v0 for the registered frame, so - * _Unwind_SetGR is never called and no crash occurs. - * - musl: pthread_cleanup_push already uses the C/setjmp form (no RAII). - * pthread_exit on musl does not use _Unwind_ForcedUnwind at all, - * so there is no incompatibility issue. - * - * These tests verify the per-platform cleanup path: - * glibc – __pthread_register_cancel callback fires on pthread_cancel, - * pthread_exit, and normal routine return. - * musl – run_with_cleanup (pthread_cleanup_push/pop) callback fires on - * pthread_cancel, pthread_exit, and normal routine return. - * A shared test verifies ProfiledThread::release() is safe to call from the - * chosen cleanup path. - */ - -#include - -#ifdef __linux__ - -#include "thread.h" - -#include -#include -#include -#include - -// Production function under test — defined in libraryPatcher_linux.cpp. -// Declared here (outside any platform guard) because run_with_cleanup is built -// on both glibc and musl; each platform's tests call it directly. -extern "C++" void run_with_cleanup(void* (*)(void*), void*, void(*)(void*), void*); - -// =========================================================================== -// glibc path: __pthread_register_cancel / __pthread_unregister_cancel tests -// =========================================================================== -#ifdef __GLIBC__ - -// --------------------------------------------------------------------------- - -static std::atomic g_bare_cleanup_called{false}; - -static void bare_cleanup(void*) { - g_bare_cleanup_called.store(true, std::memory_order_relaxed); -} - -static void* bare_spin(void*) { - while (true) { - pthread_testcancel(); - usleep(100); - } -} - -static void* bare_cancel_thread(void*) { - g_bare_cleanup_called.store(false, std::memory_order_relaxed); - run_with_cleanup(bare_spin, nullptr, bare_cleanup, nullptr); - return nullptr; -} - -// Regression (glibc): __pthread_register_cancel callback fires on pthread_cancel. -TEST(ForcedUnwindTest, CleanupCallbackRunsOnPthreadCancel) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, bare_cancel_thread, nullptr)); - - usleep(5000); - pthread_cancel(t); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_TRUE(g_bare_cleanup_called.load()) - << "__pthread_register_cancel callback must run during pthread_cancel"; - EXPECT_EQ(PTHREAD_CANCELED, retval); -} - -// --------------------------------------------------------------------------- - -static std::atomic g_pt_cleanup_called{false}; -static std::atomic g_pt_release_called{false}; - -static void profiled_cleanup(void*) { - g_pt_cleanup_called.store(true, std::memory_order_relaxed); - ProfiledThread::release(); - g_pt_release_called.store(true, std::memory_order_relaxed); -} - -static void* profiled_spin(void*) { - while (true) { - pthread_testcancel(); - usleep(100); - } -} - -static void* profiled_cancel_thread(void*) { - g_pt_cleanup_called.store(false, std::memory_order_relaxed); - g_pt_release_called.store(false, std::memory_order_relaxed); - ProfiledThread::initCurrentThread(); - run_with_cleanup(profiled_spin, nullptr, profiled_cleanup, nullptr); - return nullptr; -} - -// Regression (glibc): ProfiledThread lifecycle survives pthread_cancel without -// abort() from _Unwind_SetGR.cold (the static-libgcc / libgcc_s.so.1 clash). -TEST(ForcedUnwindTest, ProfiledThreadReleasedOnForcedUnwind) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, profiled_cancel_thread, nullptr)); - - usleep(5000); - pthread_cancel(t); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_TRUE(g_pt_cleanup_called.load()) - << "Cleanup callback must run when thread is cancelled"; - EXPECT_TRUE(g_pt_release_called.load()) - << "ProfiledThread::release() must complete inside the cleanup callback"; - EXPECT_EQ(PTHREAD_CANCELED, retval); -} - -// --------------------------------------------------------------------------- - -static std::atomic g_exit_cleanup_called{false}; - -static void exit_cleanup(void*) { - g_exit_cleanup_called.store(true, std::memory_order_relaxed); -} - -static void* exit_fn(void*) { - pthread_exit(reinterpret_cast(42)); -} - -static void* exit_thread(void*) { - g_exit_cleanup_called.store(false, std::memory_order_relaxed); - run_with_cleanup(exit_fn, nullptr, exit_cleanup, nullptr); - return nullptr; -} - -// Regression (glibc): __pthread_register_cancel callback fires on pthread_exit. -TEST(ForcedUnwindTest, CleanupCallbackRunsOnPthreadExit) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, exit_thread, nullptr)); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_TRUE(g_exit_cleanup_called.load()) - << "__pthread_register_cancel callback must also run during pthread_exit"; - EXPECT_EQ(reinterpret_cast(42), retval); -} - -// --------------------------------------------------------------------------- - -static std::atomic g_nr_cleanup_count{0}; -static std::atomic g_nr_past_run{false}; - -static void* nr_routine(void*) { return nullptr; } - -static void nr_cleanup(void*) { - g_nr_cleanup_count.fetch_add(1, std::memory_order_relaxed); -} - -static void* nr_thread(void*) { - g_nr_cleanup_count.store(0, std::memory_order_relaxed); - g_nr_past_run.store(false, std::memory_order_relaxed); - run_with_cleanup(nr_routine, nullptr, nr_cleanup, nullptr); - g_nr_past_run.store(true, std::memory_order_relaxed); - // Spin with a cancellation point so the main thread can probe whether - // __pthread_unregister_cancel took effect after run_with_cleanup returned. - while (true) { - pthread_testcancel(); - usleep(50); - } - __builtin_unreachable(); -} - -// Normal-return path (matches start_routine_wrapper on non-aarch64 glibc): -// cleanup_fn is called once AND __pthread_unregister_cancel removes the buf. -// - no cleanup_fn → g_nr_cleanup_count stays 0 (deterministically detected -// by the count==1 assertion below). -// - no __pthread_unregister_cancel → post-return cancel longjmps into the -// freed cancel_buf. This is best-effort detection only: the resulting UB -// may crash, re-fire cleanup (count==2), or — depending on stack contents, -// and especially under ASAN/MSAN — do neither and pass. Do NOT rely on -// this as a deterministic mutation detector for the unregister call; the -// guaranteed coverage here is the positive count==1 assertion. -TEST(ForcedUnwindTest, CleanupCalledExactlyOnceOnNormalReturn) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, nr_thread, nullptr)); - - while (!g_nr_past_run.load(std::memory_order_relaxed)) { - usleep(100); - } - EXPECT_EQ(1, g_nr_cleanup_count.load()) - << "cleanup_fn must be called once on the normal-return path of run_with_cleanup"; - - // Cancel the spinning thread. If __pthread_unregister_cancel was not - // called, glibc still holds a pointer to run_with_cleanup's freed - // cancel_buf; the cancel would longjmp into garbage and either crash or - // fire nr_cleanup a second time (count == 2). - pthread_cancel(t); - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_EQ(1, g_nr_cleanup_count.load()) - << "__pthread_unregister_cancel must remove the buf so the " - "post-return cancel does not re-fire cleanup_fn"; - EXPECT_EQ(PTHREAD_CANCELED, retval); -} - -// --------------------------------------------------------------------------- - -static std::atomic g_pt_nr_called{false}; -static std::atomic g_pt_nr_release_called{false}; - -static void* pt_nr_routine(void*) { return nullptr; } - -static void pt_nr_cleanup(void*) { - g_pt_nr_called.store(true, std::memory_order_relaxed); - ProfiledThread::release(); - g_pt_nr_release_called.store(true, std::memory_order_relaxed); -} - -static void* pt_nr_thread(void*) { - g_pt_nr_called.store(false, std::memory_order_relaxed); - g_pt_nr_release_called.store(false, std::memory_order_relaxed); - ProfiledThread::initCurrentThread(); - run_with_cleanup(pt_nr_routine, nullptr, pt_nr_cleanup, nullptr); - return reinterpret_cast(77); -} - -// Regression (glibc): cleanup_fn fires and ProfiledThread is released when -// routine returns normally. This is the start_routine_wrapper path for -// threads that finish naturally (no cancel, no pthread_exit). -// Removing cleanup_fn(cleanup_arg) after __pthread_unregister_cancel leaves -// g_pt_nr_called false and leaks the ProfiledThread entry. -TEST(ForcedUnwindTest, ProfiledThreadReleasedOnNormalReturn) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, pt_nr_thread, nullptr)); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_TRUE(g_pt_nr_called.load()) - << "cleanup_fn must be called on the normal-return path"; - EXPECT_TRUE(g_pt_nr_release_called.load()) - << "ProfiledThread::release() must complete on the normal-return path"; - EXPECT_EQ(reinterpret_cast(77), retval); -} - -#endif // __GLIBC__ - -// =========================================================================== -// musl (non-glibc) path: run_with_cleanup (pthread_cleanup_push/pop) tests -// =========================================================================== -#ifndef __GLIBC__ - -// --------------------------------------------------------------------------- - -static std::atomic g_m_cancel_called{false}; - -static void m_cancel_cleanup(void*) { - g_m_cancel_called.store(true, std::memory_order_relaxed); -} - -static void* m_cancel_spin(void*) { - while (true) { - pthread_testcancel(); - usleep(100); - } -} - -static void* m_cancel_thread(void*) { - g_m_cancel_called.store(false, std::memory_order_relaxed); - run_with_cleanup(m_cancel_spin, nullptr, m_cancel_cleanup, nullptr); - return nullptr; -} - -// musl: run_with_cleanup cleanup fires on pthread_cancel. -TEST(ForcedUnwindTest, CleanupCallbackRunsOnPthreadCancel) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, m_cancel_thread, nullptr)); - - usleep(5000); - pthread_cancel(t); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_TRUE(g_m_cancel_called.load()) - << "run_with_cleanup cleanup must fire during pthread_cancel on musl"; - EXPECT_EQ(PTHREAD_CANCELED, retval); -} - -// --------------------------------------------------------------------------- - -static std::atomic g_m_exit_called{false}; - -static void m_exit_cleanup(void*) { - g_m_exit_called.store(true, std::memory_order_relaxed); -} - -static void* m_exit_fn(void*) { - pthread_exit(reinterpret_cast(42)); -} - -static void* m_exit_thread(void*) { - g_m_exit_called.store(false, std::memory_order_relaxed); - run_with_cleanup(m_exit_fn, nullptr, m_exit_cleanup, nullptr); - return nullptr; -} - -// musl: run_with_cleanup cleanup fires on pthread_exit. -TEST(ForcedUnwindTest, CleanupCallbackRunsOnPthreadExit) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, m_exit_thread, nullptr)); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_TRUE(g_m_exit_called.load()) - << "run_with_cleanup cleanup must fire during pthread_exit on musl"; - EXPECT_EQ(reinterpret_cast(42), retval); -} - -// --------------------------------------------------------------------------- - -static std::atomic g_m_pt_called{false}; -static std::atomic g_m_pt_release_called{false}; - -static void m_profiled_cleanup(void*) { - g_m_pt_called.store(true, std::memory_order_relaxed); - ProfiledThread::release(); - g_m_pt_release_called.store(true, std::memory_order_relaxed); -} - -static void* m_profiled_spin(void*) { - while (true) { - pthread_testcancel(); - usleep(100); - } -} - -static void* m_profiled_cancel_thread(void*) { - g_m_pt_called.store(false, std::memory_order_relaxed); - g_m_pt_release_called.store(false, std::memory_order_relaxed); - ProfiledThread::initCurrentThread(); - run_with_cleanup(m_profiled_spin, nullptr, m_profiled_cleanup, nullptr); - return nullptr; -} - -// musl: ProfiledThread lifecycle survives pthread_cancel via run_with_cleanup. -TEST(ForcedUnwindTest, ProfiledThreadReleasedOnForcedUnwind) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, m_profiled_cancel_thread, nullptr)); - - usleep(5000); - pthread_cancel(t); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_TRUE(g_m_pt_called.load()) - << "Cleanup callback must run when thread is cancelled"; - EXPECT_TRUE(g_m_pt_release_called.load()) - << "ProfiledThread::release() must complete inside the cleanup callback"; - EXPECT_EQ(PTHREAD_CANCELED, retval); -} - -// --------------------------------------------------------------------------- - -static std::atomic g_m_nr_count{0}; - -static void* m_nr_routine(void*) { return nullptr; } - -static void m_nr_cleanup(void*) { - g_m_nr_count.fetch_add(1, std::memory_order_relaxed); -} - -static void* m_nr_thread(void*) { - g_m_nr_count.store(0, std::memory_order_relaxed); - run_with_cleanup(m_nr_routine, nullptr, m_nr_cleanup, nullptr); - return reinterpret_cast(77); -} - -// musl: normal-return path calls cleanup exactly once via pthread_cleanup_pop(1). -// Removing the pop or changing pop(1) to pop(0) leaves count at 0. -TEST(ForcedUnwindTest, CleanupCalledExactlyOnceOnNormalReturn) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, m_nr_thread, nullptr)); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_EQ(1, g_m_nr_count.load()) - << "cleanup_fn must be called once via pthread_cleanup_pop(1) on the normal-return path"; - EXPECT_EQ(reinterpret_cast(77), retval); -} - -#endif // !__GLIBC__ - -// =========================================================================== -// Integration tests: start_routine_for_test → run_with_cleanup chain -// -// These tests exercise the full path that start_routine_wrapper takes in -// production (init TLS → run_with_cleanup → cleanup on cancel/exit), without -// the Profiler::registerThread/unregisterThread calls that require a live -// profiler instance. Profiler registration is tested separately; these tests -// cover the wiring between the wrapper and run_with_cleanup. -// -// Removing run_with_cleanup from start_routine_wrapper, or wiring it to a -// no-op cleanup, would leave the unit tests passing but fail these tests. -// =========================================================================== - -// Test entry points declared in libraryPatcher_linux.cpp under #ifdef UNIT_TEST. -extern "C++" int pthread_create_wrapped_for_test( - pthread_t*, void* (*)(void*), void*, void (*)(void*), void*); -extern "C++" int pthread_create_with_cleanup_unregister_for_test( - pthread_t*, void* (*)(void*), void*); - -// --------------------------------------------------------------------------- - -static std::atomic g_int_cancel_called{false}; -static std::atomic g_int_cancel_released{false}; - -static void int_cancel_cleanup(void*) { - g_int_cancel_called.store(true, std::memory_order_relaxed); - ProfiledThread::release(); - g_int_cancel_released.store(true, std::memory_order_relaxed); -} - -static void* int_cancel_spin(void*) { - while (true) { - pthread_testcancel(); - usleep(100); - } -} - -// Integration: start_routine_for_test → run_with_cleanup → cleanup fires and -// ProfiledThread is released on pthread_cancel. -TEST(ForcedUnwindTest, WrapperReleasesProfiledThreadOnCancel) { - g_int_cancel_called.store(false, std::memory_order_relaxed); - g_int_cancel_released.store(false, std::memory_order_relaxed); - - pthread_t t; - ASSERT_EQ(0, pthread_create_wrapped_for_test( - &t, int_cancel_spin, nullptr, int_cancel_cleanup, nullptr)); - - usleep(5000); - pthread_cancel(t); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_TRUE(g_int_cancel_called.load()) - << "run_with_cleanup cleanup must fire when wrapped thread is cancelled"; - EXPECT_TRUE(g_int_cancel_released.load()) - << "ProfiledThread::release() must complete inside the wrapper cleanup"; - EXPECT_EQ(PTHREAD_CANCELED, retval); -} - -// --------------------------------------------------------------------------- - -static std::atomic g_int_exit_called{false}; -static std::atomic g_int_exit_released{false}; - -static void int_exit_cleanup(void*) { - g_int_exit_called.store(true, std::memory_order_relaxed); - ProfiledThread::release(); - g_int_exit_released.store(true, std::memory_order_relaxed); -} - -static void* int_exit_fn(void*) { - pthread_exit(reinterpret_cast(99)); -} - -// Integration: start_routine_for_test → run_with_cleanup → cleanup fires and -// ProfiledThread is released on pthread_exit. -TEST(ForcedUnwindTest, WrapperReleasesProfiledThreadOnPthreadExit) { - g_int_exit_called.store(false, std::memory_order_relaxed); - g_int_exit_released.store(false, std::memory_order_relaxed); - - pthread_t t; - ASSERT_EQ(0, pthread_create_wrapped_for_test( - &t, int_exit_fn, nullptr, int_exit_cleanup, nullptr)); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_TRUE(g_int_exit_called.load()) - << "run_with_cleanup cleanup must fire when wrapped thread calls pthread_exit"; - EXPECT_TRUE(g_int_exit_released.load()) - << "ProfiledThread::release() must complete inside the wrapper cleanup"; - EXPECT_EQ(reinterpret_cast(99), retval); -} - -// --------------------------------------------------------------------------- -// Production-cleanup integration: cleanup_unregister calls -// Profiler::unregisterThread with the wrapped thread's TID. -// Replacing unregister_and_release with a bare ProfiledThread::release() would -// leave WrapperReleasesProfiledThreadOnCancel/PthreadExit passing but fail here. - -#include "profiler.h" - -static std::atomic g_int_prod_cancel_tid{-1}; - -static void* int_prod_cancel_spin(void*) { - g_int_prod_cancel_tid.store(ProfiledThread::currentTid(), - std::memory_order_relaxed); - while (true) { - pthread_testcancel(); - usleep(100); - } -} - -// Integration: cleanup_unregister calls Profiler::unregisterThread(tid) on cancel. -TEST(ForcedUnwindTest, WrapperCallsProfilerUnregisterOnCancel) { - g_int_prod_cancel_tid.store(-1, std::memory_order_relaxed); - Profiler::resetUnregisterObservableForTest(); - - pthread_t t; - ASSERT_EQ(0, pthread_create_with_cleanup_unregister_for_test( - &t, int_prod_cancel_spin, nullptr)); - - while (g_int_prod_cancel_tid.load(std::memory_order_relaxed) == -1) { - usleep(100); - } - const int expected_tid = g_int_prod_cancel_tid.load(std::memory_order_relaxed); - - pthread_cancel(t); - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_EQ(expected_tid, Profiler::lastUnregisteredTidForTest()) - << "cleanup_unregister must call Profiler::unregisterThread(tid) on cancel"; - EXPECT_EQ(PTHREAD_CANCELED, retval); -} - -// --------------------------------------------------------------------------- - -static std::atomic g_int_prod_exit_tid{-1}; - -static void* int_prod_exit_fn(void*) { - g_int_prod_exit_tid.store(ProfiledThread::currentTid(), - std::memory_order_relaxed); - pthread_exit(reinterpret_cast(55)); -} - -// Integration: cleanup_unregister calls Profiler::unregisterThread(tid) on pthread_exit. -TEST(ForcedUnwindTest, WrapperCallsProfilerUnregisterOnPthreadExit) { - g_int_prod_exit_tid.store(-1, std::memory_order_relaxed); - Profiler::resetUnregisterObservableForTest(); - - pthread_t t; - ASSERT_EQ(0, pthread_create_with_cleanup_unregister_for_test( - &t, int_prod_exit_fn, nullptr)); - - void* retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - - EXPECT_EQ(g_int_prod_exit_tid.load(std::memory_order_relaxed), - Profiler::lastUnregisteredTidForTest()) - << "cleanup_unregister must call Profiler::unregisterThread(tid) on pthread_exit"; - EXPECT_EQ(reinterpret_cast(55), retval); -} - -#endif // __linux__ diff --git a/ddprof-lib/src/test/cpp/libraries_ut.cpp b/ddprof-lib/src/test/cpp/libraries_ut.cpp deleted file mode 100644 index 5e8d6bd63..000000000 --- a/ddprof-lib/src/test/cpp/libraries_ut.cpp +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "../../main/cpp/codeCache.h" -#include "../../main/cpp/findLibraryImpl.h" -#include "../../main/cpp/gtest_crash_handler.h" - -static constexpr char LIBRARIES_TEST_NAME[] = "LibrariesTest"; - -// Three static buffers to serve as fake "text segments". -// Using static storage ensures the addresses are stable and non-overlapping. -static char g_lib0_text[1024]; -static char g_lib1_text[1024]; -static char g_lib2_text[1024]; - -class LibrariesTest : public ::testing::Test { -protected: - void SetUp() override { - installGtestCrashHandler(); - - _lib0 = new CodeCache("libfake0.so", 0, g_lib0_text, g_lib0_text + sizeof(g_lib0_text)); - _lib1 = new CodeCache("libfake1.so", 1, g_lib1_text, g_lib1_text + sizeof(g_lib1_text)); - _lib2 = new CodeCache("libfake2.so", 2, g_lib2_text, g_lib2_text + sizeof(g_lib2_text)); - - _libs.add(_lib0); - _libs.add(_lib1); - _libs.add(_lib2); - } - - void TearDown() override { - restoreDefaultSignalHandlers(); - delete _lib0; - delete _lib1; - delete _lib2; - } - - CodeCacheArray _libs; - CodeCache* _lib0 = nullptr; - CodeCache* _lib1 = nullptr; - CodeCache* _lib2 = nullptr; -}; - -TEST_F(LibrariesTest, KnownAddress) { - // Address inside lib1's range should return lib1. - const void* addr = g_lib1_text + 512; - CodeCache* result = findLibraryByAddressImpl(_libs, addr); - EXPECT_EQ(result, _lib1); -} - -TEST_F(LibrariesTest, NullAddress) { - // NULL should return nullptr without crashing. - CodeCache* result = findLibraryByAddressImpl(_libs, nullptr); - EXPECT_EQ(result, nullptr); -} - -TEST_F(LibrariesTest, InvalidAddress) { - // An address outside all known ranges should return nullptr. - const void* addr = reinterpret_cast(0x1); - CodeCache* result = findLibraryByAddressImpl(_libs, addr); - EXPECT_EQ(result, nullptr); -} - -TEST_F(LibrariesTest, FirstLibrary) { - const void* addr = g_lib0_text + 100; - CodeCache* result = findLibraryByAddressImpl(_libs, addr); - EXPECT_EQ(result, _lib0); -} - -TEST_F(LibrariesTest, LastLibrary) { - const void* addr = g_lib2_text + 900; - CodeCache* result = findLibraryByAddressImpl(_libs, addr); - EXPECT_EQ(result, _lib2); -} - -TEST_F(LibrariesTest, CacheHitOnRepeatedLookup) { - // First lookup primes the cache for lib1. - const void* addr = g_lib1_text + 256; - CodeCache* first = findLibraryByAddressImpl(_libs, addr); - EXPECT_EQ(first, _lib1); - - // Second lookup with the same range should still return lib1 (cache hit path). - CodeCache* second = findLibraryByAddressImpl(_libs, g_lib1_text + 700); - EXPECT_EQ(second, _lib1); -} - -TEST_F(LibrariesTest, CacheMissFallsBackToLinearScan) { - // Prime cache for lib0. - findLibraryByAddressImpl(_libs, g_lib0_text + 10); - - // Now look up an address in lib2 — cache miss, linear scan must find it. - const void* addr = g_lib2_text + 500; - CodeCache* result = findLibraryByAddressImpl(_libs, addr); - EXPECT_EQ(result, _lib2); -} diff --git a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp b/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp deleted file mode 100644 index 814e9b4ca..000000000 --- a/ddprof-lib/src/test/cpp/livenessTracker_ut.cpp +++ /dev/null @@ -1,243 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "../../main/cpp/gtest_crash_handler.h" -#include -#include - -// Test name for crash handler -static constexpr char LIVENESS_TRACKER_TEST_NAME[] = "LivenessTrackerTest"; - -/** - * Mock structure to test buffer capacity management similar to LivenessTracker's - * tracking table resize logic. This tests the fix for the buffer overrun bug - * where _table_cap was being updated even when realloc failed. - */ -struct TrackingTableMock { - void* table; - int table_cap; - int table_max_cap; - - TrackingTableMock(int initial_cap, int max_cap) - : table(nullptr), table_cap(initial_cap), table_max_cap(max_cap) { - table = malloc(sizeof(int) * initial_cap); - } - - ~TrackingTableMock() { - if (table != nullptr) { - free(table); - } - } - - /** - * This is the CORRECT implementation (after the fix). - * Only update table_cap if realloc succeeds. - */ - bool resizeTableCorrect(int newcap) { - void* tmp = realloc(table, sizeof(int) * newcap); - if (tmp != nullptr) { - table = tmp; - table_cap = newcap; // Only update capacity after successful realloc - return true; - } - return false; - } - - /** - * This is the BUGGY implementation (before the fix). - * Updates table_cap even when realloc fails, causing buffer overrun. - */ - bool resizeTableBuggy(int newcap) { - void* tmp = realloc(table, sizeof(int) * (table_cap = newcap)); // BUG: updates table_cap in the call - if (tmp != nullptr) { - table = tmp; - return true; - } - // BUG: table_cap was already updated even though realloc failed! - return false; - } - - int getCapacity() const { - return table_cap; - } -}; - -class LivenessTrackerTest : public ::testing::Test { -protected: - void SetUp() override { - installGtestCrashHandler(); - } - - void TearDown() override { - restoreDefaultSignalHandlers(); - } -}; - -/** - * Test that verifies the correct behavior: capacity should only be updated - * when realloc succeeds. - */ -TEST_F(LivenessTrackerTest, CapacityOnlyUpdatedOnSuccessfulRealloc) { - TrackingTableMock mock(10, 100); - - int initial_cap = mock.getCapacity(); - EXPECT_EQ(initial_cap, 10); - - // Successful resize should update capacity - bool success = mock.resizeTableCorrect(20); - EXPECT_TRUE(success); - EXPECT_EQ(mock.getCapacity(), 20); - - // Another successful resize - success = mock.resizeTableCorrect(40); - EXPECT_TRUE(success); - EXPECT_EQ(mock.getCapacity(), 40); -} - -/** - * Test that demonstrates the bug: with the buggy implementation, - * capacity gets updated even when realloc would fail. - * - * This test documents the bug that was fixed. The buggy implementation - * would update table_cap inside the realloc call itself, meaning that - * if realloc failed, the capacity would still be updated, leading to - * a mismatch between actual allocated size and recorded capacity. - */ -TEST_F(LivenessTrackerTest, BuggyImplementationUpdateCapacityOnFailure) { - TrackingTableMock mock(10, 100); - - int initial_cap = mock.getCapacity(); - EXPECT_EQ(initial_cap, 10); - - // Successful resize updates capacity (both implementations work here) - bool success = mock.resizeTableBuggy(20); - EXPECT_TRUE(success); - EXPECT_EQ(mock.getCapacity(), 20); - - // Now let's demonstrate the bug with a simulated failure scenario - // In the buggy implementation, even if we pass the capacity update inline, - // it would get updated before realloc returns - // - // The buggy code was: - // TrackingEntry *tmp = (TrackingEntry *)realloc( - // _table, sizeof(TrackingEntry) * (_table_cap = newcap)); - // - // This means _table_cap = newcap happens BEFORE checking if tmp != nullptr - // If realloc fails (returns nullptr), _table_cap is already set to newcap, - // but _table still points to the old, smaller buffer. - // - // Result: buffer overrun when code tries to access _table[i] for i >= old_cap - - // To verify this would happen, we'd need to force realloc to fail. - // In practice, realloc fails when: - // 1. System is out of memory - // 2. Requested size is too large - // 3. Memory corruption - - // We can't easily force a failure in a unit test without complex mocking, - // but we've documented the issue and the fix ensures capacity is only - // updated after verifying tmp != nullptr -} - -/** - * Test that verifies the fixed code follows the correct pattern: - * 1. Call realloc and store result in temporary pointer - * 2. Check if temporary pointer is not null - * 3. Only then update the table pointer and capacity - */ -TEST_F(LivenessTrackerTest, CorrectResizePatternVerification) { - TrackingTableMock mock(10, 100); - - // The correct pattern is: - // 1. void* tmp = realloc(table, new_size); - // 2. if (tmp != nullptr) { - // 3. table = tmp; - // 4. table_cap = new_cap; - // 5. } - - int old_cap = mock.getCapacity(); - EXPECT_EQ(old_cap, 10); - - // Simulate the resize logic - int newcap = old_cap * 2; - bool success = mock.resizeTableCorrect(newcap); - - if (success) { - // Capacity should be updated - EXPECT_EQ(mock.getCapacity(), newcap); - } else { - // If resize failed, capacity should remain unchanged - EXPECT_EQ(mock.getCapacity(), old_cap); - } -} - -/** - * Integration-style test that verifies multiple resize operations - * maintain correct capacity tracking. - */ -TEST_F(LivenessTrackerTest, MultipleResizeOperationsMaintainCorrectCapacity) { - TrackingTableMock mock(4, 128); - - std::vector expected_capacities = {4, 8, 16, 32, 64, 128}; - size_t resize_count = 0; - - EXPECT_EQ(mock.getCapacity(), expected_capacities[resize_count]); - - // Perform multiple resize operations (doubling each time) - for (size_t i = 1; i < expected_capacities.size(); i++) { - int newcap = expected_capacities[i]; - bool success = mock.resizeTableCorrect(newcap); - EXPECT_TRUE(success) << "Resize to " << newcap << " failed"; - EXPECT_EQ(mock.getCapacity(), newcap) - << "Capacity mismatch after resize to " << newcap; - } - - // Verify final capacity - EXPECT_EQ(mock.getCapacity(), 128); -} - -/** - * Test that verifies capacity never exceeds max_cap during resize operations. - */ -TEST_F(LivenessTrackerTest, CapacityDoesNotExceedMaxCap) { - TrackingTableMock mock(10, 50); - - // Try to resize beyond max_cap - int newcap = std::min(mock.table_cap * 2, mock.table_max_cap); - EXPECT_LE(newcap, 50); - - // First resize: 10 -> 20 - mock.resizeTableCorrect(newcap); - EXPECT_EQ(mock.getCapacity(), 20); - - // Second resize: 20 -> 40 - newcap = std::min(mock.table_cap * 2, mock.table_max_cap); - mock.resizeTableCorrect(newcap); - EXPECT_EQ(mock.getCapacity(), 40); - - // Third resize: 40 -> 50 (capped at max_cap) - newcap = std::min(mock.table_cap * 2, mock.table_max_cap); - EXPECT_EQ(newcap, 50); // Should be capped at 50, not 80 - mock.resizeTableCorrect(newcap); - EXPECT_EQ(mock.getCapacity(), 50); - - // Fourth resize attempt: should remain at 50 - newcap = std::min(mock.table_cap * 2, mock.table_max_cap); - EXPECT_EQ(newcap, 50); // Already at max, newcap == table_cap - // In the actual code, this would trigger: if (_table_cap != newcap) { ... } - // which would be false, so no resize would be attempted -} diff --git a/ddprof-lib/src/test/cpp/methodInfo_hash_ut.cpp b/ddprof-lib/src/test/cpp/methodInfo_hash_ut.cpp deleted file mode 100644 index 4d37af4d2..000000000 --- a/ddprof-lib/src/test/cpp/methodInfo_hash_ut.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include -#include -#include "../../main/cpp/flightRecorder.h" - - -class MethodInfoHashTest : public ::testing::Test { -}; - -TEST_F(MethodInfoHashTest, makeHashTest) { - unsigned long dummy = 0xABCD1234; - // jmethodID key - unsigned long key1 = MethodMap::makeKey((jmethodID)dummy); - // method name address key - unsigned long key2 = MethodMap::makeKey((const char*)dummy); - // packed remote frame key - unsigned long key3 = MethodMap::makeKey((unsigned long)dummy); - - // The same value of different types should produce different keys - EXPECT_TRUE(key1 != key2); - EXPECT_TRUE(key1 != key3); - EXPECT_TRUE(key2 != key3); -} diff --git a/ddprof-lib/src/test/cpp/methodMapId_ut.cpp b/ddprof-lib/src/test/cpp/methodMapId_ut.cpp deleted file mode 100644 index e2d246aaa..000000000 --- a/ddprof-lib/src/test/cpp/methodMapId_ut.cpp +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * Regression test for PROF-15130: duplicate jdk.types.Method constant-pool - * ids. resolveMethod() used `mi->_key = _method_map->size() + 1`, but - * cleanupUnreferencedMethods() erases unreferenced entries and shrinks the - * map, so size()+1 later reissues an id still owned by a surviving method. - * Two live methods then share an id in one chunk → first-wins strict parsers - * (JMC/jafar/Datadog backend) render a phantom method. - * - * The fix replaced size()+1 with MethodMap::allocId()/freeId(): a free-list - * id allocator that recycles an id only after the owning method is erased. - * These tests drive the allocator directly and assert the live-id-uniqueness - * invariant, including the exact scenario that broke size()+1. - */ - -#include -#include "../../main/cpp/flightRecorder.h" -#include -#include - -// Reference model of the BUGGY scheme that the fix replaced. -// size()+1 derives the id from the current number of live entries. -static u32 buggyNextId(size_t live_count) { - return static_cast(live_count) + 1; // avoid zero key -} - -// 1) Basic monotonic allocation with no frees: ids are unique and start at 1. -TEST(MethodMapIdTest, AllocWithoutFreeIsUniqueAndStartsAtOne) { - MethodMap m; - std::set seen; - u32 prev = 0; - for (int i = 0; i < 1000; i++) { - u32 id = m.allocId(); - EXPECT_NE(id, 0U) << "id 0 is reserved as the no-entry sentinel"; - EXPECT_TRUE(seen.insert(id).second) << "duplicate id handed out: " << id; - EXPECT_GT(id, prev) << "ids must be monotonic before any free"; - prev = id; - } -} - -// 2) freeId(0) is a no-op: the sentinel must never enter the free list. -TEST(MethodMapIdTest, FreeIdZeroIsNoOp) { - MethodMap m; - m.freeId(0); - EXPECT_NE(m.allocId(), 0U); -} - -// 3) A freed id is recycled (LEB128 stays compact) and is only re-handed out -// after it was freed. -TEST(MethodMapIdTest, FreedIdIsRecycled) { - MethodMap m; - u32 a = m.allocId(); // 1 - u32 b = m.allocId(); // 2 - u32 c = m.allocId(); // 3 - EXPECT_EQ(a, 1U); - EXPECT_EQ(b, 2U); - EXPECT_EQ(c, 3U); - - m.freeId(b); // free the middle id - u32 d = m.allocId(); - EXPECT_EQ(d, b) << "the freed id should be recycled before the high water grows"; -} - -// 4) THE INVARIANT (this is what size()+1 violated). -// Simulate the resolve/erase cycle: alloc a batch, free a middle subset -// (the "erased" methods), alloc more, and assert at every step that the set -// of ids currently held by live entries has no duplicates and that a newly -// allocated id never collides with a still-live id. -TEST(MethodMapIdTest, LiveIdsNeverCollideAcrossEraseReuseCycle) { - MethodMap m; - std::set live; // ids currently owned by live methods - - auto alloc = [&]() { - u32 id = m.allocId(); - EXPECT_NE(id, 0U); - // The core invariant: a newly handed-out id must not already be live. - EXPECT_EQ(live.count(id), 0U) - << "allocId() returned id " << id << " that is still owned by a live method"; - live.insert(id); - return id; - }; - auto freeOne = [&](u32 id) { - live.erase(id); - m.freeId(id); - }; - - // Phase 1: allocate ids 1..N for an initial population of live methods. - const int N = 64; - std::vector ids; - for (int i = 0; i < N; i++) ids.push_back(alloc()); - - // Phase 2: free a middle subset (simulating cleanupUnreferencedMethods - // erasing methods that aged out) while a disjoint set survives. - // Free indices 20..39 (ids 21..40); keep 1..20 and 41..64 live. - for (int i = 20; i < 40; i++) freeOne(ids[i]); - - // Phase 3: allocate a new batch. With size()+1 these would collide with the - // surviving high ids; with the free-list they reuse the freed ids. - // The alloc lambda asserts no collision with any live id. - for (int i = 0; i < 40; i++) alloc(); - - // Cross-check: enumerate every live id and confirm uniqueness holds. - std::set uniq(live.begin(), live.end()); - EXPECT_EQ(uniq.size(), live.size()) << "duplicate id among live methods"; -} - -// 5) Demonstrate that the OLD size()+1 scheme DOES produce a duplicate live id -// in exactly the erase+reuse scenario above. This pins the bug so the test -// is not vacuous: if someone reintroduces size()+1, the asserted property -// (no duplicate live ids) is false. -TEST(MethodMapIdTest, BuggySizePlusOneSchemeProducesDuplicateLiveId) { - // Model live methods as a set of ids. size()+1 derives the id from the - // current live count, mirroring `mi->_key = _method_map->size() + 1`. - std::set live; - - // Phase 1: resolve N methods → ids 1..N (count grows each insert). - const int N = 64; - std::vector ids; - for (int i = 0; i < N; i++) { - u32 id = buggyNextId(live.size()); - ids.push_back(id); - live.insert(id); - } - // After phase 1, live = {1..64}. - - // Phase 2: erase a middle subset (ids 21..40), shrinking the map. NOTE the - // buggy scheme has nothing analogous to freeId — it just shrinks. - for (int i = 20; i < 40; i++) live.erase(ids[i]); - // live now = {1..20, 41..64}, size() == 44. - - // Phase 3: resolve ONE more method. size()+1 == 44+1 == 45, but 45 is still - // owned by a surviving phase-1 method → DUPLICATE id in the chunk. - u32 next = buggyNextId(live.size()); - EXPECT_TRUE(live.count(next) > 0) - << "expected size()+1 to collide with a live id, got fresh id " << next; - EXPECT_EQ(next, 45U); - - // Contrast: the fixed allocator does NOT collide for the same sequence. - MethodMap m; - std::set fixedLive; - std::vector fixedIds; - for (int i = 0; i < N; i++) { - u32 id = m.allocId(); - fixedIds.push_back(id); - fixedLive.insert(id); - } - for (int i = 20; i < 40; i++) { - fixedLive.erase(fixedIds[i]); - m.freeId(fixedIds[i]); - } - u32 fixedNext = m.allocId(); - EXPECT_EQ(fixedLive.count(fixedNext), 0U) - << "fixed allocator handed out a still-live id " << fixedNext; -} diff --git a/ddprof-lib/src/test/cpp/nativeSocketSampler_ut.cpp b/ddprof-lib/src/test/cpp/nativeSocketSampler_ut.cpp deleted file mode 100644 index ee81af7e5..000000000 --- a/ddprof-lib/src/test/cpp/nativeSocketSampler_ut.cpp +++ /dev/null @@ -1,438 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include - -#if defined(__linux__) - -#include "nativeSocketSampler.h" -#include "libraryPatcher.h" - -#include -#include - -// --------------------------------------------------------------------------- -// Stub tracking -// --------------------------------------------------------------------------- - -static std::atomic g_send_calls{0}; -static std::atomic g_recv_calls{0}; -static std::atomic g_send_ret{-1}; -static std::atomic g_recv_ret{-1}; -static std::atomic g_write_calls{0}; -static std::atomic g_read_calls{0}; -static std::atomic g_write_ret{-1}; -static std::atomic g_read_ret{-1}; - -static ssize_t stub_send(int /*fd*/, const void* /*buf*/, size_t /*len*/, int /*flags*/) { - g_send_calls++; - return g_send_ret.load(); -} - -static ssize_t stub_recv(int /*fd*/, void* /*buf*/, size_t /*len*/, int /*flags*/) { - g_recv_calls++; - return g_recv_ret.load(); -} - -static ssize_t stub_write(int /*fd*/, const void* /*buf*/, size_t /*len*/) { - g_write_calls++; - return g_write_ret.load(); -} - -static ssize_t stub_read(int /*fd*/, void* /*buf*/, size_t /*len*/) { - g_read_calls++; - return g_read_ret.load(); -} - -// --------------------------------------------------------------------------- -// Test fixture — installs stubs as the "original" function pointers so the -// hooks invoke them without needing GOT patching or a running JVM. -// --------------------------------------------------------------------------- - -class NativeSocketSamplerHookTest : public ::testing::Test { -protected: - NativeSocketSampler::send_fn _saved_send; - NativeSocketSampler::recv_fn _saved_recv; - NativeSocketSampler::write_fn _saved_write; - NativeSocketSampler::read_fn _saved_read; - - void SetUp() override { - NativeSocketSampler::getOriginalFunctions(_saved_send, _saved_recv, _saved_write, _saved_read); - NativeSocketSampler::setOriginalFunctions(stub_send, stub_recv, stub_write, stub_read); - g_send_calls = 0; - g_recv_calls = 0; - g_write_calls = 0; - g_read_calls = 0; - } - - void TearDown() override { - NativeSocketSampler::setOriginalFunctions(_saved_send, _saved_recv, _saved_write, _saved_read); - NativeSocketSampler::send_fn cur_send; NativeSocketSampler::recv_fn cur_recv; - NativeSocketSampler::write_fn cur_write; NativeSocketSampler::read_fn cur_read; - NativeSocketSampler::getOriginalFunctions(cur_send, cur_recv, cur_write, cur_read); - ASSERT_EQ(cur_send, _saved_send) << "Function pointers must be restored in TearDown"; - } -}; - -// --------------------------------------------------------------------------- -// Tests -// --------------------------------------------------------------------------- - -/** - * Verifies that send_hook forwards the call to _orig_send and returns its - * return value when the original function indicates failure (ret <= 0 so the - * sampling path — which needs a JVM — is not entered). - */ -TEST_F(NativeSocketSamplerHookTest, SendHookCallsOrigSendAndReturnsValue) { - g_send_ret = -1; // error path avoids the JVM-dependent sampling code - char buf[16] = {}; - - ssize_t ret = NativeSocketSampler::send_hook(0, buf, sizeof(buf), 0); - - EXPECT_EQ(g_send_calls.load(), 1) << "send_hook must call _orig_send exactly once"; - EXPECT_EQ(ret, -1) << "send_hook must propagate the return value from _orig_send"; -} - -/** - * Verifies that recv_hook forwards the call to _orig_recv and returns its - * return value (same guard: ret <= 0 skips the JVM path). - */ -TEST_F(NativeSocketSamplerHookTest, RecvHookCallsOrigRecvAndReturnsValue) { - g_recv_ret = 0; - char buf[16] = {}; - - ssize_t ret = NativeSocketSampler::recv_hook(0, buf, sizeof(buf), 0); - - EXPECT_EQ(g_recv_calls.load(), 1) << "recv_hook must call _orig_recv exactly once"; - EXPECT_EQ(ret, 0) << "recv_hook must propagate the return value from _orig_recv"; -} - -/** - * Verifies that write_hook forwards the call to _orig_write and returns its - * return value when the original function indicates failure (ret <= 0 skips - * the sampling path that requires a running JVM). - * - * fd=0 (stdin) is not a socket descriptor, so getsockopt fails and isSocket() - * returns false; recordEvent() is never reached — the non-socket branch is - * exercised here. Note: AF_UNIX SOCK_STREAM would return true from isSocket(). - */ -TEST_F(NativeSocketSamplerHookTest, WriteHookCallsOrigWriteAndReturnsValue) { - g_write_ret = -1; // error path avoids the JVM-dependent sampling code - char buf[16] = {}; - - ssize_t ret = NativeSocketSampler::write_hook(0, buf, sizeof(buf)); - - EXPECT_EQ(g_write_calls.load(), 1) << "write_hook must call _orig_write exactly once"; - EXPECT_EQ(ret, -1) << "write_hook must propagate the return value from _orig_write"; -} - -/** - * Verifies that read_hook forwards the call to _orig_read and returns its - * return value (same guard: ret <= 0 skips the JVM path). - * - * fd=0 (stdin) is not a socket descriptor, so getsockopt fails and isSocket() - * returns false; the non-socket branch is exercised. Note: AF_UNIX SOCK_STREAM - * would return true from isSocket(). - */ -TEST_F(NativeSocketSamplerHookTest, ReadHookCallsOrigReadAndReturnsValue) { - g_read_ret = 0; - char buf[16] = {}; - - ssize_t ret = NativeSocketSampler::read_hook(0, buf, sizeof(buf)); - - EXPECT_EQ(g_read_calls.load(), 1) << "read_hook must call _orig_read exactly once"; - EXPECT_EQ(ret, 0) << "read_hook must propagate the return value from _orig_read"; -} - -#endif // __linux__ - -// --------------------------------------------------------------------------- -// PoissonSampler unit tests — no Linux/glibc guard needed. -// --------------------------------------------------------------------------- - -#include "poissonSampler.h" - -/** - * Verifies that sample() returns false when value is 0 or interval is 0. - */ -TEST(PoissonSamplerTest, ReturnsFalseForZeroValueOrInterval) { - PoissonSampler s; - float w = 0.0f; - - // value == 0: must not sample regardless of interval - EXPECT_FALSE(s.sample(0, 1000, /*epoch=*/1, w)) - << "sample() must return false when value is 0"; - - // interval == 0: must not sample regardless of value - EXPECT_FALSE(s.sample(1000, 0, /*epoch=*/1, w)) - << "sample() must return false when interval is 0"; -} - -/** - * Verifies that a change in epoch causes the sampler to reset so subsequent - * calls with large value cross the freshly-drawn threshold. - */ -TEST(PoissonSamplerTest, EpochChangeResetsState) { - PoissonSampler s; - float w = 0.0f; - const u64 interval = 100; - - // Prime with epoch=1 until it fires at least once. - bool fired = false; - for (int i = 0; i < 10000 && !fired; i++) { - fired = s.sample(interval, interval, /*epoch=*/1, w); - } - ASSERT_TRUE(fired) << "Sampler should have fired during priming"; - - // Bump epoch: large value should fire quickly against the fresh threshold. - bool fired_after_reset = false; - for (int i = 0; i < 10000 && !fired_after_reset; i++) { - fired_after_reset = s.sample(interval * 1000, interval, /*epoch=*/2, w); - } - EXPECT_TRUE(fired_after_reset) - << "Sampler should fire quickly after epoch reset with large value"; -} - -/** - * Verifies that when value >> interval the computed weight approaches 1.0. - * For value = 1000 * interval, exp(-1000) ≈ 0 so P ≈ 1 and weight ≈ 1. - */ -TEST(PoissonSamplerTest, HighVolumeWeightApproachesOne) { - PoissonSampler s; - float w = 0.0f; - const u64 interval = 100; - const u64 big_value = interval * 1000; - - // Use successive epoch bumps to guarantee a fresh threshold each iteration. - bool fired = false; - for (int i = 0; i < 100 && !fired; i++) { - fired = s.sample(big_value, interval, /*epoch=*/(u64)(i + 1), w); - } - ASSERT_TRUE(fired) << "Sampler must fire with value >> interval"; - EXPECT_NEAR(w, 1.0f, 1e-3f) - << "Weight must be ~1.0 when value >> interval"; -} - -// --------------------------------------------------------------------------- -// Success-path and isSocket() tests — Linux guard required for socket APIs. -// --------------------------------------------------------------------------- - -#if defined(__linux__) - -#include -#include - -// Global counter incremented by stub_send when it returns a positive value. -// Used to verify that recordEvent is reached on the success path. -static std::atomic g_send_success_calls{0}; - -static ssize_t stub_send_success(int /*fd*/, const void* /*buf*/, size_t len, int /*flags*/) { - g_send_success_calls++; - return (ssize_t)len; // report all bytes sent -} - -/** - * Verifies that send_hook calls _orig_send and propagates a successful return - * value WHEN HOOKS ARE INACTIVE (the most common in-process state during tests). - * Because _socket_active is false the hook short-circuits to _orig_send and - * never reaches recordEvent / Profiler::recordSample — exercising recordEvent - * requires a running profiler with a recorder bound, which is not feasible in - * this gtest unit harness. - * - * To exercise the active-path short-circuit (i.e., the hook's outer guard - * branch), see SendHookActivePathReachesRecorderGuard below. - */ -TEST_F(NativeSocketSamplerHookTest, SendHookSuccessPathReturnsBytes) { - g_send_success_calls = 0; - NativeSocketSampler::setOriginalFunctions(stub_send_success, stub_recv, stub_write, stub_read); - char buf[16] = {}; - - ssize_t ret = NativeSocketSampler::send_hook(0, buf, sizeof(buf), 0); - - EXPECT_EQ(g_send_success_calls.load(), 1) - << "send_hook must call _orig_send exactly once on the inactive path"; - EXPECT_EQ(ret, (ssize_t)sizeof(buf)) - << "send_hook must propagate the byte count from _orig_send"; -} - -/** - * Verifies that with _socket_active flipped to true the hook actually takes - * the active branch (calls TSC::ticks twice and routes through record_if_positive). - * No JVM/recorder is running so recordEvent's downstream Profiler::recordSample - * is benign (returns without recording); we verify the orig fn is still called - * exactly once and the return value is propagated. - */ -TEST_F(NativeSocketSamplerHookTest, SendHookActivePathReachesRecorderGuard) { - g_send_success_calls = 0; - NativeSocketSampler::setOriginalFunctions(stub_send_success, stub_recv, stub_write, stub_read); - - // Manually flip the active flag so the hook traverses the active branch. - // Restore in a guard; tearing down the fixture must observe it cleared. - bool prev = LibraryPatcher::_socket_active.exchange(true, std::memory_order_release); - - char buf[16] = {}; - ssize_t ret = NativeSocketSampler::send_hook(0, buf, sizeof(buf), 0); - - LibraryPatcher::_socket_active.store(prev, std::memory_order_release); - - EXPECT_EQ(g_send_success_calls.load(), 1) - << "send_hook must call _orig_send exactly once on the active path"; - EXPECT_EQ(ret, (ssize_t)sizeof(buf)) - << "send_hook must propagate the byte count from _orig_send on the active path"; -} - -/** - * Verifies write_hook pass-through: when hooks are inactive (_socket_active=false), - * write_hook forwards immediately to _orig_write without calling isSocket(). - * Uses a real AF_UNIX SOCK_STREAM socketpair; the pass-through path must handle - * any fd type correctly. After closing both ends the stub is still called (the - * inactive guard fires before any fd inspection). - */ -TEST(NativeSocketSamplerIsSocketTest, UnixSocketPairReturnsFalseAfterClose) { - int fds[2]; - ASSERT_EQ(socketpair(AF_UNIX, SOCK_STREAM, 0, fds), 0) - << "socketpair() failed: " << strerror(errno); - - NativeSocketSampler* inst = NativeSocketSampler::instance(); - ASSERT_NE(inst, nullptr); - - // _socket_active is false — write_hook forwards to _orig_write without calling isSocket(). - // (write_hook is exercised with a stub that returns the full length, then verify the - // return value is correct.) - NativeSocketSampler::send_fn saved_send; NativeSocketSampler::recv_fn saved_recv; - NativeSocketSampler::write_fn saved_write; NativeSocketSampler::read_fn saved_read; - NativeSocketSampler::getOriginalFunctions(saved_send, saved_recv, saved_write, saved_read); - NativeSocketSampler::setOriginalFunctions(saved_send, saved_recv, - [](int /*fd*/, const void* /*buf*/, size_t len) -> ssize_t { return (ssize_t)len; }, - saved_read); - - char buf[16] = {}; - ssize_t ret = NativeSocketSampler::write_hook(fds[0], buf, sizeof(buf)); - EXPECT_EQ(ret, (ssize_t)sizeof(buf)) - << "write_hook must propagate the byte count even for AF_UNIX fds"; - - NativeSocketSampler::setOriginalFunctions(saved_send, saved_recv, saved_write, saved_read); - close(fds[0]); - close(fds[1]); -} - -// --------------------------------------------------------------------------- -// LRU fd→addr cache tests. -// --------------------------------------------------------------------------- - -TEST(NativeSocketSamplerLruTest, ClearResetsCache) { - NativeSocketSampler* inst = NativeSocketSampler::instance(); - inst->fdAddrCacheInsertForTest(1, "1.2.3.4:100"); - inst->fdAddrCacheInsertForTest(2, "1.2.3.4:200"); - ASSERT_GE(inst->fdAddrCacheSizeForTest(), 2); - - inst->clearFdCache(); - - EXPECT_EQ(inst->fdAddrCacheSizeForTest(), 0) - << "clearFdCache() must empty both the map and the LRU list"; -} - -TEST(NativeSocketSamplerLruTest, InsertAndLookupPreservesEntries) { - NativeSocketSampler* inst = NativeSocketSampler::instance(); - inst->clearFdCache(); - - for (int fd = 0; fd < 16; fd++) { - inst->fdAddrCacheInsertForTest(fd, "10.0.0.1:" + std::to_string(fd)); - } - - EXPECT_EQ(inst->fdAddrCacheSizeForTest(), 16) - << "All 16 distinct-fd inserts must be stored"; - inst->clearFdCache(); -} - -TEST(NativeSocketSamplerLruTest, UpdateExistingEntryDoesNotGrowCache) { - NativeSocketSampler* inst = NativeSocketSampler::instance(); - inst->clearFdCache(); - - inst->fdAddrCacheInsertForTest(42, "1.2.3.4:9000"); - inst->fdAddrCacheInsertForTest(42, "5.6.7.8:9001"); // update same fd - - EXPECT_EQ(inst->fdAddrCacheSizeForTest(), 1) - << "Re-inserting the same fd must update in-place, not add a second entry"; - inst->clearFdCache(); -} - -TEST(NativeSocketSamplerLruTest, EvictsLruEntryAtCapacity) { - NativeSocketSampler* inst = NativeSocketSampler::instance(); - inst->clearFdCache(); - - const int CAP = NativeSocketSampler::MAX_FD_CACHE; - for (int fd = 0; fd < CAP; fd++) { - inst->fdAddrCacheInsertForTest(fd, "x"); - } - ASSERT_EQ(inst->fdAddrCacheSizeForTest(), CAP); - - // Insert one more entry beyond capacity. - inst->fdAddrCacheInsertForTest(CAP, "overflow"); - - EXPECT_EQ(inst->fdAddrCacheSizeForTest(), CAP) - << "Cache size must stay at MAX_FD_CACHE after overflow insert"; - inst->clearFdCache(); -} - -#endif // __linux__ (success-path / isSocket tests) - -// --------------------------------------------------------------------------- -// Arguments parsing tests — no Linux/glibc guard needed. -// --------------------------------------------------------------------------- - -#include "arguments.h" - -/** - * Verifies that a negative natsock interval is rejected by Arguments::parse. - */ -TEST(ArgumentsNatsock, NegativeIntervalRejected) { - Arguments args; - Error e = args.parse("natsock=-1us"); - ASSERT_TRUE(static_cast(e)) << "Expected error for negative natsock interval"; - ASSERT_NE(std::string(e.message()).find("must be >= 0"), std::string::npos) - << "Error message should mention 'must be >= 0'"; -} - -/** - * Verifies that bare natsock (no =value) enables socket profiling with the - * default interval (same as natsock=0). - */ -TEST(ArgumentsNatsock, BareNatsockEnables) { - Arguments args; - Error e = args.parse("natsock"); - ASSERT_FALSE(static_cast(e)) << "Bare 'natsock' should be accepted"; - ASSERT_TRUE(args._nativesocket) << "Bare 'natsock' should enable socket profiling"; - ASSERT_EQ(args._nativesocket_interval, 0L) << "Bare 'natsock' should use default interval"; -} - -/** - * Verifies that natsock=0us is accepted (zero interval is valid). - */ -TEST(ArgumentsNatsock, ZeroIntervalAccepted) { - Arguments args; - Error e = args.parse("natsock=0us"); - ASSERT_FALSE(static_cast(e)) << "natsock=0us should be accepted"; -} - -/** - * Verifies that natsock with overflow value is rejected. - */ -TEST(ArgumentsNatsock, OverflowRejected) { - Arguments args; - Error e = args.parse("natsock=99999999999999999s"); - ASSERT_TRUE(static_cast(e)) << "Expected error for natsock with overflow value"; -} diff --git a/ddprof-lib/src/test/cpp/nativeThreadNames_ut.cpp b/ddprof-lib/src/test/cpp/nativeThreadNames_ut.cpp deleted file mode 100644 index 43e679b64..000000000 --- a/ddprof-lib/src/test/cpp/nativeThreadNames_ut.cpp +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Integration tests for Profiler::updateNativeThreadNames(bool), PROF-15139. -// -// The periodic refresher calls updateNativeThreadNames(true), which must skip -// a thread whose /proc comm still equals the process's own (inherited) name: -// such a thread is likely mid-initialization and recording it would latch a -// provisional name permanently (ThreadInfo is first-writer-wins). The dump-time -// pass calls updateNativeThreadNames(false), which records the name regardless. -// -// These tests spawn real parked threads and observe the recorded names via the -// UNIT_TEST-only Profiler::threadNameForTest hook, so they exercise the actual -// /proc resolution and defer predicate rather than re-stating the ThreadInfo -// contract (covered by threadInfo_ut.cpp). Linux-only: the deferral depends on -// /proc comm and pthread name inheritance. - -#ifdef __linux__ - -#ifndef _GNU_SOURCE -#define _GNU_SOURCE -#endif - -#include -#include -#include -#include -#include -#include "../../main/cpp/profiler.h" -#include "../../main/cpp/os.h" - -namespace { - -// A thread that optionally sets its own pthread name, publishes its tid, then -// parks until released. Parking keeps it visible in OS::listThreads() across -// the updateNativeThreadNames() scans under test. -struct ParkedThread { - std::string set_name; // empty => leave the inherited comm - std::atomic tid{-1}; - std::atomic ready{false}; - std::atomic stop{false}; - pthread_t handle{}; - - static void* run(void* arg) { - ParkedThread* self = static_cast(arg); - if (!self->set_name.empty()) { - // Linux caps the pthread name at 16 bytes incl. NUL; callers keep - // set_name short enough to survive without truncation. - pthread_setname_np(pthread_self(), self->set_name.c_str()); - } - self->tid.store(OS::threadId(), std::memory_order_release); - self->ready.store(true, std::memory_order_release); - while (!self->stop.load(std::memory_order_acquire)) { - struct timespec ts{0, 1000000}; // 1ms - nanosleep(&ts, nullptr); - } - return nullptr; - } - - void start() { - ASSERT_EQ(0, pthread_create(&handle, nullptr, &ParkedThread::run, this)); - // Bounded spin: the thread publishes ready almost immediately. - for (int i = 0; i < 5000 && !ready.load(std::memory_order_acquire); i++) { - struct timespec ts{0, 1000000}; // 1ms - nanosleep(&ts, nullptr); - } - ASSERT_TRUE(ready.load(std::memory_order_acquire)); - ASSERT_GT(tid.load(std::memory_order_acquire), 0); - } - - void join() { - stop.store(true, std::memory_order_release); - pthread_join(handle, nullptr); - } -}; - -std::string processComm() { - char buf[64]; - EXPECT_TRUE(OS::threadName(OS::processId(), buf, sizeof(buf))); - return std::string(buf); -} - -} // namespace - -// A thread that has set its own (non-inherited) name is recorded by the -// deferring periodic scan: it is not mid-initialization. -TEST(NativeThreadNamesTest, deferringScanRecordsRealName) { - const std::string real_name = "ut-real-name"; // 12 chars, fits the 16B cap - ASSERT_NE(real_name, processComm()); - - ParkedThread t; - t.set_name = real_name; - t.start(); - int tid = t.tid.load(); - - Profiler::instance()->updateNativeThreadNames(/*defer_initializing=*/true); - - EXPECT_EQ(real_name, Profiler::instance()->threadNameForTest(tid)); - - t.join(); -} - -// A thread still showing the inherited process name is skipped by the deferring -// scan (so the provisional name is not latched), but recorded by the dump-time -// scan, which does not defer. -TEST(NativeThreadNamesTest, deferringScanSkipsInheritedNameDumpScanRecordsIt) { - const std::string proc_comm = processComm(); - - ParkedThread t; // no set_name => inherits the parent's (process) comm - t.start(); - int tid = t.tid.load(); - // Sanity: the child really is showing the inherited process name. - char buf[64]; - ASSERT_TRUE(OS::threadName(tid, buf, sizeof(buf))); - ASSERT_EQ(proc_comm, std::string(buf)); - - // Deferring scan: must skip it, leaving the tid unrecorded. - Profiler::instance()->updateNativeThreadNames(/*defer_initializing=*/true); - EXPECT_EQ("", Profiler::instance()->threadNameForTest(tid)); - - // Dump-time scan: records the name even though it equals the process comm. - Profiler::instance()->updateNativeThreadNames(/*defer_initializing=*/false); - EXPECT_EQ(proc_comm, Profiler::instance()->threadNameForTest(tid)); - - t.join(); -} - -#endif // __linux__ diff --git a/ddprof-lib/src/test/cpp/nativefunc_ut.cpp b/ddprof-lib/src/test/cpp/nativefunc_ut.cpp deleted file mode 100644 index 7aa72fa19..000000000 --- a/ddprof-lib/src/test/cpp/nativefunc_ut.cpp +++ /dev/null @@ -1,236 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - */ - -#include -#include -#include "../../main/cpp/codeCache.h" -#include "../../main/cpp/utils.h" -#include "../../main/cpp/gtest_crash_handler.h" - -static constexpr char NATIVEFUNC_TEST_NAME[] = "NativeFuncTest"; - -class NativeFuncTest : public ::testing::Test { -protected: - void SetUp() override { - installGtestCrashHandler(); - } - - void TearDown() override { - // Clean up any allocated NativeFunc instances - for (char* name : created_names) { - NativeFunc::destroy(name); - } - created_names.clear(); - restoreDefaultSignalHandlers(); - } - - std::vector created_names; - - char* createAndTrack(const char* name, short lib_index) { - char* result = NativeFunc::create(name, lib_index); - created_names.push_back(result); - return result; - } -}; - -TEST_F(NativeFuncTest, CreateReturnsNonNull) { - char* name = createAndTrack("test_function", 42); - ASSERT_NE(name, nullptr) << "NativeFunc::create() should not return nullptr"; -} - -TEST_F(NativeFuncTest, CreateReturnsAlignedPointer) { - char* name = createAndTrack("test_function", 42); - ASSERT_NE(name, nullptr); - - // The NativeFunc struct pointer (backing up from name) - // should be aligned to sizeof(NativeFunc*) - NativeFunc* func_ptr = (NativeFunc*)(name - sizeof(NativeFunc)); - EXPECT_TRUE(is_aligned(func_ptr, sizeof(NativeFunc*))) - << "NativeFunc structure must be aligned to sizeof(NativeFunc*) = " << sizeof(NativeFunc*) - << " for alignment checks to work correctly"; -} - -TEST_F(NativeFuncTest, CreatePreservesName) { - const char* input_name = "my_test_function"; - char* name = createAndTrack(input_name, 1); - ASSERT_NE(name, nullptr); - - EXPECT_STREQ(name, input_name) - << "NativeFunc::create() should preserve the function name"; -} - -TEST_F(NativeFuncTest, LibIndexAccessible) { - char* name = createAndTrack("my_function", 123); - ASSERT_NE(name, nullptr); - - short lib_index = NativeFunc::libIndex(name); - EXPECT_EQ(lib_index, 123) - << "NativeFunc::libIndex() should return the lib_index set during create()"; -} - -TEST_F(NativeFuncTest, InitialMarkIsZero) { - char* name = createAndTrack("unmarked_function", 1); - ASSERT_NE(name, nullptr); - - EXPECT_FALSE(NativeFunc::is_marked(name)) - << "Newly created NativeFunc should not be marked (is_marked returns false)"; - EXPECT_EQ(NativeFunc::read_mark(name), 0) - << "Newly created NativeFunc should have mark value of 0"; -} - -TEST_F(NativeFuncTest, SetAndReadMark) { - char* name = createAndTrack("marked_function", 1); - ASSERT_NE(name, nullptr); - - // Set mark to MARK_THREAD_ENTRY (5) - NativeFunc::set_mark(name, 5); - - // Verify mark was set - EXPECT_TRUE(NativeFunc::is_marked(name)) - << "After set_mark(5), is_marked() should return true"; - EXPECT_EQ(NativeFunc::read_mark(name), 5) - << "After set_mark(5), read_mark() should return 5"; -} - -TEST_F(NativeFuncTest, SetMultipleDifferentMarks) { - char* name1 = createAndTrack("func1", 1); - char* name2 = createAndTrack("func2", 2); - char* name3 = createAndTrack("func3", 3); - - NativeFunc::set_mark(name1, MARK_VM_RUNTIME); - NativeFunc::set_mark(name2, MARK_INTERPRETER); - NativeFunc::set_mark(name3, MARK_THREAD_ENTRY); - - EXPECT_EQ(NativeFunc::read_mark(name1), MARK_VM_RUNTIME); - EXPECT_EQ(NativeFunc::read_mark(name2), MARK_INTERPRETER); - EXPECT_EQ(NativeFunc::read_mark(name3), MARK_THREAD_ENTRY); -} - -TEST_F(NativeFuncTest, OverwriteMark) { - char* name = createAndTrack("func", 1); - ASSERT_NE(name, nullptr); - - // Set initial mark - NativeFunc::set_mark(name, MARK_INTERPRETER); - EXPECT_EQ(NativeFunc::read_mark(name), MARK_INTERPRETER); - - // Overwrite with different mark - NativeFunc::set_mark(name, MARK_THREAD_ENTRY); - EXPECT_EQ(NativeFunc::read_mark(name), MARK_THREAD_ENTRY) - << "Marks should be overwritable"; -} - -TEST_F(NativeFuncTest, MarkOnNonNativeFuncPointerReturnsSafe) { - // Use a deliberately misaligned pointer rather than a string literal. - // String literals live in .rodata whose base address is linker-determined; - // on some platforms/compilers the literal ends up at an address where - // (ptr - sizeof(NativeFunc)) is also naturally aligned, causing the - // is_aligned() guard to pass and the methods to read garbage. - // A 64-byte-aligned buffer with a +1 offset is guaranteed to be misaligned - // with respect to sizeof(NativeFunc*) (4 or 8 bytes) on every platform, - // so the guard reliably rejects it and returns safe defaults. - alignas(64) char buf[64] = "not_a_nativefunc"; - const char* unaligned = buf + 1; - - EXPECT_FALSE(NativeFunc::is_marked(unaligned)) - << "is_marked() on non-NativeFunc string should return false (safe default)"; - EXPECT_EQ(NativeFunc::read_mark(unaligned), 0) - << "read_mark() on non-NativeFunc string should return 0 (safe default)"; - EXPECT_EQ(NativeFunc::libIndex(unaligned), -1) - << "libIndex() on non-NativeFunc string should return -1 (safe default)"; -} - -TEST_F(NativeFuncTest, NullPointerReturnsSafe) { - // Test that NULL pointer is handled safely - // The from() method computes (NativeFunc*)(nullptr - sizeof(NativeFunc)) - // which is a large negative address that should fail alignment check - - EXPECT_FALSE(NativeFunc::is_marked(nullptr)) - << "is_marked(nullptr) should return false (safe default)"; - EXPECT_EQ(NativeFunc::read_mark(nullptr), 0) - << "read_mark(nullptr) should return 0 (safe default)"; - EXPECT_EQ(NativeFunc::libIndex(nullptr), -1) - << "libIndex(nullptr) should return -1 (safe default)"; - - // set_mark should be a no-op on nullptr (doesn't crash) - NativeFunc::set_mark(nullptr, MARK_THREAD_ENTRY); - // If we got here without crashing, the test passes -} - -TEST_F(NativeFuncTest, MarkingDisabledDetection) { - // *** CRITICAL TEST *** - // This test would have caught the is_aligned() bug where marking was silently disabled. - // If is_aligned() is broken, marks won't be readable even though they're set. - - char* name = createAndTrack("critical_test", 99); - ASSERT_NE(name, nullptr); - - // Set a mark - NativeFunc::set_mark(name, MARK_THREAD_ENTRY); - - // Read it back - if alignment check is broken, this returns 0 - char mark = NativeFunc::read_mark(name); - - ASSERT_EQ(mark, MARK_THREAD_ENTRY) - << "CRITICAL FAILURE: Marking appears to be disabled!\n" - << "Set mark to " << (int)MARK_THREAD_ENTRY << " but read back " << (int)mark << "\n" - << "\n" - << "This likely means:\n" - << " 1. is_aligned() has a bug (inverted logic? wrong mask?)\n" - << " 2. NativeFunc::read_mark() alignment check is failing\n" - << " 3. NativeFunc::set_mark() alignment check is failing\n" - << "\n" - << "Impact: ALL stack unwinding optimizations are broken:\n" - << " - THREAD_ENTRY detection won't work\n" - << " - COMPILER_ENTRY injection won't work\n" - << " - VM_RUNTIME filtering won't work\n" - << "\n" - << "This is a critical performance regression."; -} - -TEST_F(NativeFuncTest, AllMarkTypesWork) { - // Verify all mark enum values can be set and read - const char mark_values[] = { - MARK_VM_RUNTIME, - MARK_INTERPRETER, - MARK_COMPILER_ENTRY, - MARK_ASYNC_PROFILER, - MARK_THREAD_ENTRY - }; - - for (size_t i = 0; i < sizeof(mark_values); i++) { - char mark_value = mark_values[i]; - char* name = createAndTrack("test", i); - ASSERT_NE(name, nullptr); - - NativeFunc::set_mark(name, mark_value); - EXPECT_EQ(NativeFunc::read_mark(name), mark_value) - << "Mark value " << (int)mark_value << " should be readable after setting"; - } -} - -TEST_F(NativeFuncTest, MultipleAllocationsIndependent) { - // Create multiple NativeFunc instances and verify they're independent - char* name1 = createAndTrack("func1", 10); - char* name2 = createAndTrack("func2", 20); - char* name3 = createAndTrack("func3", 30); - - ASSERT_NE(name1, nullptr); - ASSERT_NE(name2, nullptr); - ASSERT_NE(name3, nullptr); - - // Set different marks - NativeFunc::set_mark(name1, 1); - NativeFunc::set_mark(name2, 2); - NativeFunc::set_mark(name3, 3); - - // Verify independence - EXPECT_EQ(NativeFunc::read_mark(name1), 1); - EXPECT_EQ(NativeFunc::read_mark(name2), 2); - EXPECT_EQ(NativeFunc::read_mark(name3), 3); - - EXPECT_EQ(NativeFunc::libIndex(name1), 10); - EXPECT_EQ(NativeFunc::libIndex(name2), 20); - EXPECT_EQ(NativeFunc::libIndex(name3), 30); -} diff --git a/ddprof-lib/src/test/cpp/objectSampler_ut.cpp b/ddprof-lib/src/test/cpp/objectSampler_ut.cpp deleted file mode 100644 index 0def85da5..000000000 --- a/ddprof-lib/src/test/cpp/objectSampler_ut.cpp +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - */ - -#include -#include -#include -#include "../../main/cpp/objectSampler.h" -#include "../../main/cpp/gtest_crash_handler.h" -#include "vmEntry.h" - -static constexpr char OBJECT_SAMPLER_TEST_NAME[] = "ObjectSamplerTest"; -class ObjectSamplerGlobalSetup { -public: - ObjectSamplerGlobalSetup() { installGtestCrashHandler(); } - ~ObjectSamplerGlobalSetup() { restoreDefaultSignalHandlers(); } -}; -static ObjectSamplerGlobalSetup object_sampler_global_setup; - -// --------------------------------------------------------------------------- -// ObjectSamplerTestAccessor — friend of ObjectSampler, exposes internals -// needed by the regression tests. -// --------------------------------------------------------------------------- -class ObjectSamplerTestAccessor { -public: - static void setActive(ObjectSampler *s, bool v) { - __atomic_store_n(&s->_active, v, __ATOMIC_RELEASE); - } - - static void callRecordAllocation(ObjectSampler *s, jvmtiEnv *jvmti, - JNIEnv *jni, jthread thread, - int event_type, jobject object, - jclass klass, jlong size) { - s->recordAllocation(jvmti, jni, thread, event_type, object, klass, size); - } -}; - -// --------------------------------------------------------------------------- -// Mock-JVMTI infrastructure for Deallocate regression tests -// --------------------------------------------------------------------------- - -// Read-only buffer the success mocks hand back as a class signature; never -// modified, so file-scope storage is safe. -static char g_mock_class_name[] = "Ljava/lang/String;"; - -// Test fixture owning the per-test JVMTI function table and Deallocate -// counter. Each TEST_F gets a fresh instance, which (a) prevents shared -// static state from leaking between tests, (b) lets the fixture restore -// the process-global ObjectSampler _active flag in TearDown, and (c) -// removes the use-after-return hazard of returning a struct whose -// _jvmtiEnv::functions points into a per-call static. -class ObjectSamplerDeallocateTest : public ::testing::Test { -protected: - jvmtiInterface_1_ tbl{}; - _jvmtiEnv mock_env{}; - int deallocate_calls = 0; - - static ObjectSamplerDeallocateTest *active_fixture; - - void SetUp() override { - deallocate_calls = 0; - tbl = jvmtiInterface_1_{}; - tbl.Deallocate = &mock_Deallocate; - mock_env.functions = &tbl; - active_fixture = this; - } - - void TearDown() override { - // Restore the process-global singleton flag so subsequent tests start - // from a known state. - ObjectSamplerTestAccessor::setActive(ObjectSampler::instance(), false); - active_fixture = nullptr; - } - - void setMockGetClassSignature( - jvmtiError(JNICALL *fn)(jvmtiEnv *, jclass, char **, char **)) { - tbl.GetClassSignature = fn; - } - - void runAndExpect( - jvmtiError(JNICALL *mock)(jvmtiEnv *, jclass, char **, char **), - bool active, int expected_calls) { - setMockGetClassSignature(mock); - ObjectSampler *s = ObjectSampler::instance(); - ObjectSamplerTestAccessor::setActive(s, active); - ObjectSamplerTestAccessor::callRecordAllocation( - s, &mock_env, nullptr, nullptr, BCI_ALLOC, nullptr, nullptr, 1024); - EXPECT_EQ(deallocate_calls, expected_calls); - } - - static jvmtiError JNICALL mock_Deallocate(jvmtiEnv * /*env*/, - unsigned char * /*mem*/) { - ++active_fixture->deallocate_calls; - return JVMTI_ERROR_NONE; - } -}; - -ObjectSamplerDeallocateTest * - ObjectSamplerDeallocateTest::active_fixture = nullptr; - -static jvmtiError JNICALL mock_GetClassSignature_success( - jvmtiEnv * /*env*/, jclass /*klass*/, - char **signature_ptr, char ** /*generic_ptr*/) { - if (signature_ptr) { - *signature_ptr = g_mock_class_name; - } - return JVMTI_ERROR_NONE; -} - -// Returns error but writes a non-NULL sentinel into *signature_ptr — the UAF -// scenario the fix guards against (Deallocate must not be called on error). -static jvmtiError JNICALL mock_GetClassSignature_error_with_sentinel( - jvmtiEnv * /*env*/, jclass /*klass*/, - char **signature_ptr, char ** /*generic_ptr*/) { - if (signature_ptr) { - *signature_ptr = g_mock_class_name; - } - return JVMTI_ERROR_INVALID_CLASS; -} - -static jvmtiError JNICALL mock_GetClassSignature_error_null( - jvmtiEnv * /*env*/, jclass /*klass*/, - char **signature_ptr, char ** /*generic_ptr*/) { - (void)signature_ptr; - return JVMTI_ERROR_INVALID_CLASS; -} - -static jvmtiError JNICALL mock_GetClassSignature_success_null_name( - jvmtiEnv * /*env*/, jclass /*klass*/, - char **signature_ptr, char ** /*generic_ptr*/) { - if (signature_ptr) { - *signature_ptr = NULL; - } - return JVMTI_ERROR_NONE; -} - -// Regression tests for ObjectSampler::normalizeClassSignature, the -// guard that recordAllocation uses against null, empty, or malformed -// class signatures coming back from JVMTI. -TEST(ObjectSamplerTest, NormalizeRejectsNull) { - const char *out_name = (const char *)0xdeadbeef; - size_t out_len = 99; - EXPECT_FALSE(ObjectSampler::normalizeClassSignature(NULL, &out_name, &out_len)); - // Outputs must be untouched when the call returns false. - EXPECT_EQ(out_name, (const char *)0xdeadbeef); - EXPECT_EQ(out_len, 99u); -} - -TEST(ObjectSamplerTest, NormalizeRejectsEmpty) { - const char *out_name = nullptr; - size_t out_len = 0; - EXPECT_FALSE(ObjectSampler::normalizeClassSignature("", &out_name, &out_len)); -} - -TEST(ObjectSamplerTest, NormalizeRejectsLoneL) { - // Without the underflow guard, len - 2 would wrap to SIZE_MAX and - // poison the lookupClass call. - const char *out_name = nullptr; - size_t out_len = 0; - EXPECT_FALSE(ObjectSampler::normalizeClassSignature("L", &out_name, &out_len)); -} - -TEST(ObjectSamplerTest, NormalizeStripsLname) { - const char *signature = "Ljava/lang/String;"; - const char *out_name = nullptr; - size_t out_len = 0; - EXPECT_TRUE(ObjectSampler::normalizeClassSignature(signature, &out_name, &out_len)); - EXPECT_EQ(out_name, signature + 1); - EXPECT_EQ(out_len, strlen("java/lang/String")); -} - -TEST(ObjectSamplerTest, NormalizeRejectsLnameWithEmptyBody) { - const char *out_name = (const char *)0xdeadbeef; - size_t out_len = 99; - EXPECT_FALSE(ObjectSampler::normalizeClassSignature("L;", &out_name, &out_len)); - EXPECT_EQ(out_name, (const char *)0xdeadbeef); - EXPECT_EQ(out_len, 99u); -} - -TEST(ObjectSamplerTest, NormalizeRejectsLnameMissingTrailingSemicolon) { - const char *out_name = (const char *)0xdeadbeef; - size_t out_len = 99; - EXPECT_FALSE(ObjectSampler::normalizeClassSignature("Ljava/lang/String", &out_name, &out_len)); - EXPECT_EQ(out_name, (const char *)0xdeadbeef); - EXPECT_EQ(out_len, 99u); -} - -TEST(ObjectSamplerTest, NormalizeRejectsNullOutName) { - size_t out_len = 0; - EXPECT_FALSE(ObjectSampler::normalizeClassSignature("Ljava/lang/String;", NULL, &out_len)); -} - -TEST(ObjectSamplerTest, NormalizeRejectsNullOutLen) { - const char *out_name = nullptr; - EXPECT_FALSE(ObjectSampler::normalizeClassSignature("Ljava/lang/String;", &out_name, NULL)); -} - -TEST(ObjectSamplerTest, NormalizePassesThroughPrimitiveArray) { - const char *signature = "[B"; - const char *out_name = nullptr; - size_t out_len = 0; - EXPECT_TRUE(ObjectSampler::normalizeClassSignature(signature, &out_name, &out_len)); - EXPECT_EQ(out_name, signature); - EXPECT_EQ(out_len, strlen("[B")); -} - -TEST(ObjectSamplerTest, NormalizePassesThroughObjectArray) { - const char *signature = "[Ljava/lang/String;"; - const char *out_name = nullptr; - size_t out_len = 0; - EXPECT_TRUE(ObjectSampler::normalizeClassSignature(signature, &out_name, &out_len)); - // Array signatures are passed through verbatim - the leading 'L' - // strip rule applies only when the very first character is 'L'. - EXPECT_EQ(out_name, signature); - EXPECT_EQ(out_len, strlen("[Ljava/lang/String;")); -} - -// T-01: GetClassSignature returns error with non-NULL sentinel in *signature_ptr. -// Deallocate MUST NOT be called. -TEST_F(ObjectSamplerDeallocateTest, DeallocateNotCalledOnErrorWithNonNullSentinel) { - runAndExpect(mock_GetClassSignature_error_with_sentinel, true, 0); -} - -// T-02: GetClassSignature succeeds with a valid class name. -// Deallocate IS called exactly once (on the success path). -// Note: lookupClass returns -1 because the class map is empty, so the -// method returns without recording — that is the expected behaviour. -TEST_F(ObjectSamplerDeallocateTest, DeallocateCalledOnceOnGetClassSignatureSuccess) { - runAndExpect(mock_GetClassSignature_success, true, 1); -} - -// T-03: GetClassSignature fails and leaves class_name at NULL. -// Deallocate MUST NOT be called. -TEST_F(ObjectSamplerDeallocateTest, DeallocateNotCalledOnErrorWithNullName) { - runAndExpect(mock_GetClassSignature_error_null, true, 0); -} - -// T-04: GetClassSignature succeeds but writes NULL into *signature_ptr. -// Deallocate MUST NOT be called (the NULL guard in the condition fires). -TEST_F(ObjectSamplerDeallocateTest, DeallocateNotCalledWhenSuccessButNullName) { - runAndExpect(mock_GetClassSignature_success_null_name, true, 0); -} - -// T-05: _active is false — recordAllocation returns immediately. -// Deallocate MUST NOT be called. -TEST_F(ObjectSamplerDeallocateTest, DeallocateNotCalledWhenNotActive) { - runAndExpect(mock_GetClassSignature_success, false, 0); -} diff --git a/ddprof-lib/src/test/cpp/park_state_ut.cpp b/ddprof-lib/src/test/cpp/park_state_ut.cpp deleted file mode 100644 index 28da50468..000000000 --- a/ddprof-lib/src/test/cpp/park_state_ut.cpp +++ /dev/null @@ -1,364 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include -#include "thread.h" -#include "threadFilter.h" -#include "wallClock.h" - -namespace { - -struct ProfiledThreadDeleter { - void operator()(ProfiledThread *thread) const { - ProfiledThread::deleteForTest(thread); - } -}; - -using TestProfiledThread = std::unique_ptr; - -TestProfiledThread testThread(int tid) { - return TestProfiledThread(ProfiledThread::forTid(tid)); -} - -} // namespace - -// Tests cover FLAG_PARKED lifecycle and the once-per-run slot filter state transitions. -// The slot state lives in ThreadFilter process-lifetime storage so the wall-clock -// timer can read it without dereferencing per-thread objects from another thread. - -TEST(ProfiledThreadParkStateTest, ParkFlagLifecycle) { - TestProfiledThread thread = testThread(12345); - - u64 park_block_token = 0; - EXPECT_FALSE(thread->parkExit(park_block_token)); // not parked initially - - EXPECT_TRUE(thread->parkEnter()); - thread->setParkBlockToken(0x123400000001ULL); - - EXPECT_TRUE(thread->parkExit(park_block_token)); - EXPECT_EQ(0x123400000001ULL, park_block_token); - - EXPECT_FALSE(thread->parkExit(park_block_token)); // idempotent after clear -} - -TEST(ProfiledThreadParkStateTest, JavaThreadTypeSurvivesParkTransitions) { - TestProfiledThread thread = testThread(12349); - - thread->setJavaThread(true); - EXPECT_EQ(ProfiledThread::TYPE_JAVA_THREAD, thread->threadType()); - - EXPECT_TRUE(thread->parkEnter()); - EXPECT_EQ(ProfiledThread::TYPE_JAVA_THREAD, thread->threadType()); - - thread->setJavaThread(false); - EXPECT_EQ(ProfiledThread::TYPE_NOT_JAVA_THREAD, thread->threadType()); - - u64 park_block_token = 0; - EXPECT_TRUE(thread->parkExit(park_block_token)); - EXPECT_EQ(ProfiledThread::TYPE_NOT_JAVA_THREAD, thread->threadType()); -} - -TEST(ProfiledThreadParkStateTest, ConcurrentTypeAndParkUpdatesKeepValidTypeBits) { - TestProfiledThread thread = testThread(12350); - std::atomic stop{false}; - - std::thread type_toggler([&] { - for (int i = 0; i < 10000; i++) { - thread->setJavaThread((i & 1) == 0); - } - stop.store(true, std::memory_order_release); - }); - - while (!stop.load(std::memory_order_acquire)) { - thread->parkEnter(); - u64 park_block_token = 0; - thread->parkExit(park_block_token); - ProfiledThread::ThreadType type = thread->threadType(); - EXPECT_TRUE(type == ProfiledThread::TYPE_JAVA_THREAD || - type == ProfiledThread::TYPE_NOT_JAVA_THREAD || - type == ProfiledThread::TYPE_UNKNOWN); - } - - type_toggler.join(); - u64 park_block_token = 0; - thread->parkExit(park_block_token); - ProfiledThread::ThreadType type = thread->threadType(); - EXPECT_TRUE(type == ProfiledThread::TYPE_JAVA_THREAD || - type == ProfiledThread::TYPE_NOT_JAVA_THREAD); -} - -TEST(ProfiledThreadParkStateTest, NewThreadStartsNotParked) { - TestProfiledThread thread = testThread(12346); - u64 park_block_token = 0; - EXPECT_FALSE(thread->parkExit(park_block_token)); - // Out-params must not be touched on failed exit. - EXPECT_EQ(0ULL, park_block_token); -} - -TEST(ProfiledThreadParkStateTest, SecondParkEnterPreservesToken) { - TestProfiledThread thread = testThread(12347); - EXPECT_TRUE(thread->parkEnter()); - thread->setParkBlockToken(0xfeed00000001ULL); - EXPECT_FALSE(thread->parkEnter()); - - u64 park_block_token = 0; - EXPECT_TRUE(thread->parkExit(park_block_token)); - EXPECT_EQ(0xfeed00000001ULL, park_block_token); // first owner token wins - - // Flag is now clear; second exit is a no-op. - EXPECT_FALSE(thread->parkExit(park_block_token)); - EXPECT_EQ(0xfeed00000001ULL, park_block_token); -} - -TEST(ProfiledThreadParkStateTest, ParkExitReturnsZeroTokenWhenBlockRunWasNotArmed) { - TestProfiledThread thread = testThread(12348); - EXPECT_TRUE(thread->parkEnter()); - thread->setParkBlockToken(0); - - u64 park_block_token = 42; - EXPECT_TRUE(thread->parkExit(park_block_token)); - EXPECT_EQ(0ULL, park_block_token); -} - -TEST(WallClockOncePerRunFilterTest, SlotStateTransitions) { - ThreadFilter::Slot slot; - - EXPECT_FALSE(slot.sampledThisRun()); - EXPECT_EQ(OSThreadState::UNKNOWN, slot.lastSampledState()); - EXPECT_EQ(OSThreadState::UNKNOWN, slot.activeBlockState()); - - // First signal: arm. - slot.setActiveBlockState(OSThreadState::SLEEPING); - slot.markSampledThisRun(OSThreadState::SLEEPING); - EXPECT_TRUE(slot.sampledThisRun()); - EXPECT_EQ(OSThreadState::SLEEPING, slot.lastSampledState()); - EXPECT_EQ(OSThreadState::SLEEPING, slot.activeBlockState()); - - // Same state again: suppress (flag + state both match). - EXPECT_TRUE(slot.sampledThisRun() && - OSThreadState::SLEEPING == slot.lastSampledState()); - EXPECT_TRUE(slot.sampledThisRun() && - slot.activeBlockState() == slot.lastSampledState()); - - // Transition within skip set (SLEEPING -> CONDVAR_WAIT): state mismatch -> re-arm. - slot.setActiveBlockState(OSThreadState::CONDVAR_WAIT); - EXPECT_FALSE(slot.sampledThisRun() && - OSThreadState::CONDVAR_WAIT == slot.lastSampledState()); - slot.markSampledThisRun(OSThreadState::CONDVAR_WAIT); - EXPECT_TRUE(slot.sampledThisRun()); - EXPECT_EQ(OSThreadState::CONDVAR_WAIT, slot.lastSampledState()); - EXPECT_TRUE(slot.sampledThisRun() && - slot.activeBlockState() == slot.lastSampledState()); - - // Leave skip set: reset -> next blocked entry re-arms. - slot.setActiveBlockState(OSThreadState::UNKNOWN); - slot.resetSampledRun(OSThreadState::RUNNABLE); - EXPECT_FALSE(slot.sampledThisRun()); - EXPECT_EQ(OSThreadState::RUNNABLE, slot.lastSampledState()); - EXPECT_EQ(OSThreadState::UNKNOWN, slot.activeBlockState()); - - slot.setActiveBlockState(OSThreadState::SLEEPING); - slot.markSampledThisRun(OSThreadState::SLEEPING); - EXPECT_TRUE(slot.sampledThisRun()); - EXPECT_EQ(OSThreadState::SLEEPING, slot.lastSampledState()); - EXPECT_EQ(OSThreadState::SLEEPING, slot.activeBlockState()); -} - -TEST(WallClockOncePerRunFilterTest, UnownedBlockedFallbackCarriesWeight) { - ThreadFilter::Slot slot; - - EXPECT_TRUE(slot.shouldRecordUnownedBlockedSample()); - EXPECT_EQ(1ULL, slot.consumeUnownedBlockedWeight()); - - for (u64 i = 1; i < ThreadFilter::Slot::kUnownedBlockedFallbackRatio; i++) { - EXPECT_FALSE(slot.shouldRecordUnownedBlockedSample()); - } - - EXPECT_TRUE(slot.shouldRecordUnownedBlockedSample()); - EXPECT_EQ(ThreadFilter::Slot::kUnownedBlockedFallbackRatio, - slot.consumeUnownedBlockedWeight()); - - slot.restoreUnownedBlockedWeight(4); - EXPECT_EQ(4ULL, slot.consumeUnownedBlockedWeight()); - - slot.resetUnownedBlockedSampling(); - EXPECT_TRUE(slot.shouldRecordUnownedBlockedSample()); - EXPECT_EQ(1ULL, slot.consumeUnownedBlockedWeight()); -} - -TEST(WallClockOncePerRunFilterTest, UnownedBlockedFallbackFlushesTailWeightWithRecordedStack) { - ThreadFilter::Slot slot; - - ASSERT_TRUE(slot.shouldRecordUnownedBlockedSample()); - EXPECT_EQ(1ULL, slot.consumeUnownedBlockedWeight()); - slot.recordUnownedBlockedSample(42, OSThreadState::SLEEPING); - - for (u64 i = 1; i < ThreadFilter::Slot::kUnownedBlockedFallbackRatio; i++) { - EXPECT_FALSE(slot.shouldRecordUnownedBlockedSample()); - } - - u64 call_trace_id = 0; - u64 weight = 0; - OSThreadState state = OSThreadState::UNKNOWN; - EXPECT_TRUE(slot.flushUnownedBlockedTail(call_trace_id, weight, state)); - EXPECT_EQ(42ULL, call_trace_id); - EXPECT_EQ(ThreadFilter::Slot::kUnownedBlockedFallbackRatio - 1, weight); - EXPECT_EQ(OSThreadState::SLEEPING, state); - - EXPECT_FALSE(slot.flushUnownedBlockedTail(call_trace_id, weight, state)); - EXPECT_TRUE(slot.shouldRecordUnownedBlockedSample()); - EXPECT_EQ(1ULL, slot.consumeUnownedBlockedWeight()); -} - -TEST(WallClockOncePerRunFilterTest, UnownedBlockedFallbackDoesNotFlushWithoutRecordedStack) { - ThreadFilter::Slot slot; - - ASSERT_TRUE(slot.shouldRecordUnownedBlockedSample()); - EXPECT_EQ(1ULL, slot.consumeUnownedBlockedWeight()); - - for (u64 i = 1; i < ThreadFilter::Slot::kUnownedBlockedFallbackRatio; i++) { - EXPECT_FALSE(slot.shouldRecordUnownedBlockedSample()); - } - - u64 call_trace_id = 0; - u64 weight = 0; - OSThreadState state = OSThreadState::UNKNOWN; - EXPECT_FALSE(slot.flushUnownedBlockedTail(call_trace_id, weight, state)); - EXPECT_EQ(0ULL, call_trace_id); - EXPECT_EQ(ThreadFilter::Slot::kUnownedBlockedFallbackRatio - 1, weight); - EXPECT_EQ(OSThreadState::UNKNOWN, state); - EXPECT_TRUE(slot.shouldRecordUnownedBlockedSample()); -} - -TEST(WallClockOncePerRunFilterTest, UnownedBlockedFallbackDoesNotFlushWithoutSavedState) { - ThreadFilter::Slot slot; - - ASSERT_TRUE(slot.shouldRecordUnownedBlockedSample()); - EXPECT_EQ(1ULL, slot.consumeUnownedBlockedWeight()); - slot.recordUnownedBlockedSample(42, OSThreadState::SLEEPING); - - for (u64 i = 1; i < ThreadFilter::Slot::kUnownedBlockedFallbackRatio; i++) { - EXPECT_FALSE(slot.shouldRecordUnownedBlockedSample()); - } - - slot.unowned_blocked_state.store(OSThreadState::UNKNOWN, std::memory_order_relaxed); - - u64 call_trace_id = 0; - u64 weight = 0; - OSThreadState state = OSThreadState::SLEEPING; - EXPECT_FALSE(slot.flushUnownedBlockedTail(call_trace_id, weight, state)); - EXPECT_EQ(42ULL, call_trace_id); - EXPECT_EQ(ThreadFilter::Slot::kUnownedBlockedFallbackRatio - 1, weight); - EXPECT_EQ(OSThreadState::UNKNOWN, state); -} - -TEST(WallClockOncePerRunFilterTest, UnownedBlockedTailStateConcurrentStress) { - ThreadFilter::Slot slot; - std::atomic start{false}; - std::atomic invariant_failures{0}; - std::vector workers; - - for (int worker = 0; worker < 4; worker++) { - workers.emplace_back([&, worker] { - while (!start.load(std::memory_order_acquire)) { - } - for (int i = 0; i < 2000; i++) { - int operation = (i + worker) % 4; - if (operation == 0) { - if (slot.shouldRecordUnownedBlockedSample()) { - u64 weight = slot.consumeUnownedBlockedWeight(); - if (weight == 0) { - invariant_failures.fetch_add(1, std::memory_order_relaxed); - } - } - } else if (operation == 1) { - slot.recordUnownedBlockedSample( - 1000 + static_cast(worker), OSThreadState::SLEEPING); - } else if (operation == 2) { - u64 call_trace_id = 0; - u64 weight = 0; - OSThreadState state = OSThreadState::UNKNOWN; - bool flushed = slot.flushUnownedBlockedTail(call_trace_id, weight, state); - if (flushed && - (call_trace_id == 0 || weight == 0 || - state != OSThreadState::SLEEPING)) { - invariant_failures.fetch_add(1, std::memory_order_relaxed); - } - } else { - slot.resetUnownedBlockedSampling(); - } - } - }); - } - - start.store(true, std::memory_order_release); - for (std::thread& worker : workers) { - worker.join(); - } - - EXPECT_EQ(0, invariant_failures.load(std::memory_order_relaxed)); - - slot.resetUnownedBlockedSampling(); - u64 call_trace_id = 0; - u64 weight = 0; - OSThreadState state = OSThreadState::UNKNOWN; - EXPECT_FALSE(slot.flushUnownedBlockedTail(call_trace_id, weight, state)); -} - -TEST(WallClockOncePerRunFilterTest, FilterHelpersManageActiveBlockState) { - ThreadFilter filter; - filter.init("1"); - ThreadFilter::SlotID slot_id = filter.registerThread(); - - filter.enterBlockedRun(slot_id, OSThreadState::CONDVAR_WAIT); - ThreadFilter::Slot *slot = filter.slotForId(slot_id); - ASSERT_NE(nullptr, slot); - EXPECT_EQ(OSThreadState::CONDVAR_WAIT, slot->activeBlockState()); - - slot->markSampledThisRun(OSThreadState::CONDVAR_WAIT); - EXPECT_TRUE(slot->sampledThisRun()); - EXPECT_TRUE(slot->sampledThisRun() && - slot->activeBlockState() == slot->lastSampledState()); - - filter.exitBlockedRun(slot_id); - EXPECT_EQ(OSThreadState::UNKNOWN, slot->activeBlockState()); - EXPECT_FALSE(slot->sampledThisRun()); - EXPECT_EQ(OSThreadState::RUNNABLE, slot->lastSampledState()); -} - -// Slot reuse: stale armed state from the previous owner must be cleared before -// the new thread takes the slot (ThreadFilter::resetSlotRunState does this). -TEST(WallClockOncePerRunFilterTest, ResetClearsArmedFlagOnSlotReuse) { - ThreadFilter filter; - filter.init("1"); - ThreadFilter::SlotID slot_id = filter.registerThread(); - filter.enterBlockedRun(slot_id, OSThreadState::CONDVAR_WAIT); - ThreadFilter::Slot *slot = filter.slotForId(slot_id); - ASSERT_NE(nullptr, slot); - slot->markSampledThisRun(OSThreadState::CONDVAR_WAIT); - EXPECT_TRUE(slot->sampledThisRun()); - EXPECT_EQ(OSThreadState::CONDVAR_WAIT, slot->activeBlockState()); - - filter.resetSlotRunState(slot_id); - - EXPECT_FALSE(slot->sampledThisRun()); - EXPECT_EQ(OSThreadState::UNKNOWN, slot->lastSampledState()); - EXPECT_EQ(OSThreadState::UNKNOWN, slot->activeBlockState()); -} diff --git a/ddprof-lib/src/test/cpp/primeProbing_ut.cpp b/ddprof-lib/src/test/cpp/primeProbing_ut.cpp deleted file mode 100644 index ef4592d72..000000000 --- a/ddprof-lib/src/test/cpp/primeProbing_ut.cpp +++ /dev/null @@ -1,388 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include "primeProbing.h" -#include "../../main/cpp/gtest_crash_handler.h" - -// Test name for crash handler -static constexpr char PRIME_PROBING_TEST_NAME[] = "PrimeProbingTest"; - -// Global crash handler installation -class PrimeProbingGlobalSetup { -public: - PrimeProbingGlobalSetup() { - installGtestCrashHandler(); - } - ~PrimeProbingGlobalSetup() { - restoreDefaultSignalHandlers(); - } -}; - -// Install global crash handler for all tests in this file -static PrimeProbingGlobalSetup prime_probing_global_setup; - -/** - * Test basic HashProbe construction and initial state - */ -TEST(HashProbeTest, BasicConstruction) { - u64 seed = 0x123456789ABCDEF0ULL; - u32 capacity = 16; // Power of 2 - - HashProbe probe(seed, capacity); - - EXPECT_EQ(0, probe.stepCount()); - EXPECT_TRUE(probe.hasNext()); -} - -/** - * Test probe sequence generation and uniqueness - */ -TEST(HashProbeTest, ProbeSequence) { - u64 seed = 42; - u32 capacity = 32; // Power of 2 - - HashProbe probe(seed, capacity); - std::set visited_slots; - - // Test probe sequence - while (probe.hasNext() && visited_slots.size() < capacity) { - u32 current_slot = probe.slot(); - - // Each slot should be unique until we've visited all slots - EXPECT_EQ(0, visited_slots.count(current_slot)) - << "Slot " << current_slot << " visited twice"; - - visited_slots.insert(current_slot); - probe.next(); - } - - // Should be able to visit all slots in the capacity - EXPECT_EQ(capacity, visited_slots.size()); -} - -/** - * Test probe sequence with various capacities (all powers of 2) - */ -TEST(HashProbeTest, VariousCapacities) { - u64 seed = 0xDEADBEEF; - - for (u32 capacity = 4; capacity <= 1024; capacity *= 2) { - HashProbe probe(seed, capacity); - std::set visited_slots; - - // Test that we can visit all slots - while (probe.hasNext() && visited_slots.size() < capacity) { - u32 current_slot = probe.slot(); - EXPECT_LT(current_slot, capacity) << "Slot " << current_slot - << " exceeds capacity " << capacity; - - visited_slots.insert(current_slot); - probe.next(); - } - - EXPECT_EQ(capacity, visited_slots.size()) - << "Failed to visit all slots for capacity " << capacity; - } -} - -/** - * Test different seeds produce different sequences - */ -TEST(HashProbeTest, DifferentSeeds) { - u32 capacity = 64; - u64 seed1 = 12345; - u64 seed2 = 67890; - - HashProbe probe1(seed1, capacity); - HashProbe probe2(seed2, capacity); - - // Different seeds should produce different initial slots (usually) - // or different step patterns - bool sequences_differ = false; - - for (int i = 0; i < 10 && probe1.hasNext() && probe2.hasNext(); i++) { - if (probe1.slot() != probe2.slot()) { - sequences_differ = true; - break; - } - probe1.next(); - probe2.next(); - } - - // At least one difference should occur in the first 10 steps - EXPECT_TRUE(sequences_differ) << "Sequences for different seeds are identical"; -} - -/** - * Test step counting functionality - */ -TEST(HashProbeTest, StepCounting) { - u64 seed = 999; - u32 capacity = 16; - - HashProbe probe(seed, capacity); - - EXPECT_EQ(0, probe.stepCount()); - - for (u32 expected_count = 1; expected_count <= capacity && probe.hasNext(); expected_count++) { - probe.next(); - EXPECT_EQ(expected_count, probe.stepCount()); - } -} - -/** - * Test hasNext() boundary conditions - */ -TEST(HashProbeTest, HasNextBoundary) { - u64 seed = 777; - u32 capacity = 8; - - HashProbe probe(seed, capacity); - - // Should have next for capacity iterations - for (u32 i = 0; i < capacity; i++) { - EXPECT_TRUE(probe.hasNext()) << "hasNext() failed at iteration " << i; - if (i < capacity - 1) { // Don't call next() on the last iteration - probe.next(); - } - } - - // After capacity steps, should not have next - probe.next(); - EXPECT_FALSE(probe.hasNext()); -} - -/** - * Test prime step selection avoids common factors - */ -TEST(HashProbeTest, PrimeStepSelection) { - // Test with capacities that are multiples of small primes - std::vector test_capacities = {16, 32, 64, 128, 256}; - - for (u32 capacity : test_capacities) { - // Try multiple seeds to ensure we get different step values - std::set observed_steps; - - for (u64 seed = 0; seed < 100; seed++) { - HashProbe probe(seed, capacity); - - // Force some probing to observe the step behavior - std::set slots_in_sequence; - u32 iterations = 0; - - while (probe.hasNext() && iterations < capacity && - slots_in_sequence.size() < capacity) { - u32 slot = probe.slot(); - slots_in_sequence.insert(slot); - probe.next(); - iterations++; - } - - // If we visited all slots, the step size was coprime with capacity - if (slots_in_sequence.size() == capacity) { - // This indicates a good step selection - // We can't directly access the step, but this validates the behavior - EXPECT_EQ(capacity, slots_in_sequence.size()); - } - } - } -} - -/** - * Test probe distribution quality - */ -TEST(HashProbeTest, ProbeDistribution) { - u32 capacity = 64; - u32 num_seeds = 100; - - // Count how often each slot is visited first across different seeds - std::vector first_slot_counts(capacity, 0); - - for (u64 seed = 0; seed < num_seeds; seed++) { - HashProbe probe(seed, capacity); - first_slot_counts[probe.slot()]++; - } - - // Check that distribution is reasonably uniform - // Each slot should be visited at least once as the first slot - u32 empty_slots = 0; - for (u32 count : first_slot_counts) { - if (count == 0) { - empty_slots++; - } - } - - // Allow some empty slots, but not too many (less than 25% empty) - EXPECT_LT(empty_slots, capacity / 4) - << "Too many slots never selected as first slot"; -} - -/** - * Test edge case with minimum capacity - */ -TEST(HashProbeTest, MinimalCapacity) { - u64 seed = 42; - u32 capacity = 1; - - HashProbe probe(seed, capacity); - - EXPECT_EQ(0, probe.slot()); // Only slot 0 possible with capacity 1 - EXPECT_EQ(0, probe.stepCount()); - EXPECT_TRUE(probe.hasNext()); - - probe.next(); - EXPECT_EQ(1, probe.stepCount()); - EXPECT_FALSE(probe.hasNext()); -} - -/** - * Test that advance slot operation works correctly - */ -TEST(HashProbeTest, SlotAdvancement) { - u64 seed = 12345; - u32 capacity = 32; - - HashProbe probe(seed, capacity); - u32 initial_slot = probe.slot(); - - probe.next(); - u32 next_slot = probe.slot(); - - // Slots should be different (unless step size is a multiple of capacity) - // and both should be within bounds - EXPECT_LT(initial_slot, capacity); - EXPECT_LT(next_slot, capacity); -} - -/** - * Test full cycle completion for various seed patterns - */ -TEST(HashProbeTest, FullCycleCompletion) { - u32 capacity = 16; - - // Test with seeds that have different bit patterns - std::vector test_seeds = { - 0x0000000000000001ULL, // Minimal seed - 0x5555555555555555ULL, // Alternating bits - 0xAAAAAAAAAAAAAAAAULL, // Alternating bits (inverse) - 0xFFFFFFFFFFFFFFFFULL, // All bits set - 0x123456789ABCDEF0ULL // Mixed pattern - }; - - for (u64 seed : test_seeds) { - HashProbe probe(seed, capacity); - std::unordered_set visited; - - u32 steps = 0; - while (probe.hasNext() && steps < capacity * 2) { // Safety limit - visited.insert(probe.slot()); - probe.next(); - steps++; - } - - // Should visit all slots exactly once - EXPECT_EQ(capacity, visited.size()) - << "Failed full cycle for seed 0x" << std::hex << seed; - EXPECT_EQ(capacity, steps) - << "Wrong step count for seed 0x" << std::hex << seed; - } -} - -/** - * Test enhanced Knuth multiplicative hashing distribution with high-bit extraction - */ -TEST(HashProbeTest, KnuthHashingDistribution) { - u32 capacity = 64; - u32 num_tests = 1000; - - // Test that Knuth hashing with high-bit extraction provides excellent distribution - // The implementation uses: (hash >> (sizeof(size_t) * 8 - 13)) % capacity - // This extracts the high-order bits after Knuth multiplication for better distribution - std::vector slot_counts(capacity, 0); - - for (u64 seed = 1; seed <= num_tests; seed++) { - HashProbe probe(seed, capacity); - slot_counts[probe.slot()]++; - } - - // Check for excellent distribution - high-bit extraction should provide superior results - u32 min_hits = num_tests; - u32 max_hits = 0; - u32 empty_slots = 0; - - for (u32 count : slot_counts) { - if (count == 0) { - empty_slots++; - } else { - min_hits = std::min(min_hits, count); - max_hits = std::max(max_hits, count); - } - } - - // With high-bit extraction after Knuth multiplication, we expect even better distribution: - // - Very few empty slots (less than 5% of capacity due to high-bit extraction) - // - Tighter distribution bounds (max hits shouldn't be more than 3x average) - u32 expected_avg = num_tests / capacity; - - EXPECT_LT(empty_slots, capacity / 20) << "Too many empty slots - high-bit extraction should minimize this"; - EXPECT_LT(max_hits, expected_avg * 3) << "Max hits per slot too high - high-bit extraction should prevent clustering"; - - if (min_hits > 0) { - EXPECT_GT(min_hits * 8, expected_avg) << "Min hits per slot too low for enhanced hashing"; - } -} - -/** - * Test high-bit extraction effectiveness across different capacity sizes - */ -TEST(HashProbeTest, HighBitExtractionEffectiveness) { - // Test multiple power-of-2 capacities to validate high-bit extraction - std::vector test_capacities = {16, 32, 64, 128, 256}; - - for (u32 capacity : test_capacities) { - std::vector slot_counts(capacity, 0); - u32 num_tests = capacity * 10; // 10x capacity for good statistics - - // Use seeds with various bit patterns to stress-test the hash function - for (u64 base_seed = 1; base_seed <= num_tests; base_seed++) { - // Create seeds with different high/low bit patterns - u64 seed = (base_seed << 32) | (base_seed * 0x9E3779B9ULL); - HashProbe probe(seed, capacity); - slot_counts[probe.slot()]++; - } - - // Calculate distribution statistics - u32 min_hits = num_tests, max_hits = 0, empty_slots = 0; - for (u32 count : slot_counts) { - if (count == 0) { - empty_slots++; - } else { - min_hits = std::min(min_hits, count); - max_hits = std::max(max_hits, count); - } - } - - u32 expected_avg = num_tests / capacity; - - // High-bit extraction should provide excellent distribution across all capacity sizes - EXPECT_LT(empty_slots, capacity / 16) - << "Capacity " << capacity << ": too many empty slots with high-bit extraction"; - EXPECT_LT(max_hits, expected_avg * 2.5) - << "Capacity " << capacity << ": clustering detected despite high-bit extraction"; - - if (min_hits > 0) { - EXPECT_GT(min_hits * 6, expected_avg) - << "Capacity " << capacity << ": poor minimum distribution with high-bit extraction"; - } - } -} - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} \ No newline at end of file diff --git a/ddprof-lib/src/test/cpp/profiler_null_calltrace_buffer_ut.cpp b/ddprof-lib/src/test/cpp/profiler_null_calltrace_buffer_ut.cpp deleted file mode 100644 index af6acb2dd..000000000 --- a/ddprof-lib/src/test/cpp/profiler_null_calltrace_buffer_ut.cpp +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - */ - -#include -#include "../../main/cpp/profiler.h" -#include "../../main/cpp/gtest_crash_handler.h" - -// Regression tests for the PROF-14679 null-calltrace-buffer crash. -// -// Profiler::_calltrace_buffer[i] is NULL from the constructor until -// Profiler::start() allocates it. Before the fix, both recordJVMTISample and -// recordExternalSample dereferenced the slot without checking for null, -// producing a SIGSEGV whenever sampling fired before or between starts. -// After the fix both functions null-check the slot under the shard lock and -// return early (incrementing the skip counter) when the buffer is absent. -// -// These tests exercise that early-return path by calling the functions on the -// singleton whose _calltrace_buffer slots are still NULL (no start() was -// called). Pre-fix: SIGSEGV -> test binary killed -> CI failure. -// Post-fix: graceful return -> test passes. - -static constexpr char PROFILER_NULL_BUFFER_TEST_NAME[] = "ProfilerNullCalltraceBufferTest"; -class ProfilerNullBufferGlobalSetup { -public: - ProfilerNullBufferGlobalSetup() { installGtestCrashHandler(); } - ~ProfilerNullBufferGlobalSetup() { restoreDefaultSignalHandlers(); } -}; -static ProfilerNullBufferGlobalSetup profiler_null_buffer_global_setup; - -// thread=nullptr is safe: GetStackTrace is never reached when the buffer is null. -TEST(ProfilerNullCalltraceBufferTest, RecordJvmtiSampleNullBufferDoesNotCrash) { - Profiler::instance()->recordJVMTISample( - /*counter=*/1, /*tid=*/1, /*thread=*/nullptr, - BCI_ALLOC, /*event=*/nullptr, /*deferred=*/false); -} - -TEST(ProfilerNullCalltraceBufferTest, RecordExternalSampleNullBufferDoesNotCrash) { - ASGCT_CallFrame frame{}; - Profiler::instance()->recordExternalSample( - /*weight=*/1, /*tid=*/1, /*num_frames=*/1, - &frame, /*truncated=*/false, BCI_ALLOC, /*event=*/nullptr); -} diff --git a/ddprof-lib/src/test/cpp/refCountGuard_ut.cpp b/ddprof-lib/src/test/cpp/refCountGuard_ut.cpp deleted file mode 100644 index 6fac0bbe2..000000000 --- a/ddprof-lib/src/test/cpp/refCountGuard_ut.cpp +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "refCountGuard.h" -#include -#include -#include -#include -#include - -// ── Activation-window regression tests ──────────────────────────────────── -// -// The non-reentrant RefCountGuard constructor stores active_ptr (RELEASE) -// BEFORE incrementing count (RELEASE). Between those two stores the slot has -// count==0 but active_ptr!=null: the "activation window". -// -// Both wait functions must also check active_ptr when count==0, otherwise they -// return early and the caller can free the resource while the holder is still in -// the window and will access it after the free (heap-use-after-free). -// -// These tests exercise exactly that scenario: -// 1. Holder stores active_ptr, sleeps to keep the window open. -// 2. Cleaner calls the wait function, then frees the resource. -// 3. Holder wakes, increments count (exits window), writes to the resource. -// -// Without the fix: cleaner's wait returns immediately (count==0 → skip slot), -// freeing the resource before holder uses it. -// ASAN reports: heap-use-after-free on holder's write. -// TSAN reports: data race / use-after-free between the delete and the write. -// -// With the fix: cleaner's wait observes active_ptr!=null and blocks until -// holder clears it (after finishing the write), so the free is safe. -// -// NOTE: These tests bypass getThreadRefCountSlot() and manipulate -// refcount_slots[] directly. They use the last slot (MAX_THREADS-1) which -// is unlikely to be claimed by any real thread during the test, and always -// restore it to the clean state on exit. - -class RefCountGuardActivationWindowTest : public ::testing::Test { -protected: - static constexpr int TEST_SLOT = RefCountGuard::MAX_THREADS - 1; - - void SetUp() override { - auto& slot = RefCountGuard::refcount_slots[TEST_SLOT]; - __atomic_store_n(&slot.count, 0u, __ATOMIC_SEQ_CST); - __atomic_store_n(&slot.active_ptr, nullptr, __ATOMIC_SEQ_CST); - for (int i = 0; i < RefCountSlot::OUTER_STACK_DEPTH; ++i) { - __atomic_store_n(&slot.outer_stack[i], nullptr, __ATOMIC_SEQ_CST); - } - } - - void TearDown() override { - // Restore to clean state regardless of test outcome. - auto& slot = RefCountGuard::refcount_slots[TEST_SLOT]; - __atomic_store_n(&slot.count, 0u, __ATOMIC_SEQ_CST); - __atomic_store_n(&slot.active_ptr, nullptr, __ATOMIC_SEQ_CST); - } -}; - -// waitForAllRefCountsToClear must not return while a slot is in the activation window. -TEST_F(RefCountGuardActivationWindowTest, WaitForAllBlocksDuringActivationWindow) { - auto& slot = RefCountGuard::refcount_slots[TEST_SLOT]; - - char* resource = new char[64]; - std::memset(resource, 0xAB, 64); - - // Keep a stable pointer for the holder's write so the compiler cannot fold it - // away even after resource is set to nullptr by the cleaner. - volatile char* stable = resource; - - std::atomic window_entered{false}; - - std::thread holder([&] { - // Activation window start: store active_ptr (RELEASE), count still 0. - __atomic_store_n(&slot.active_ptr, resource, __ATOMIC_RELEASE); - window_entered.store(true, std::memory_order_release); - - // Hold the window open long enough for the cleaner to enter its wait loop. - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - - // Activation window end: count++ (RELEASE). - __atomic_fetch_add(&slot.count, 1u, __ATOMIC_RELEASE); - - // Write to resource. Under the bug this is heap-use-after-free (ASAN/TSAN catch it). - *stable = 0xCD; - - // Destroy guard: count--, active_ptr = null. - __atomic_fetch_sub(&slot.count, 1u, __ATOMIC_RELEASE); - __atomic_store_n(&slot.active_ptr, nullptr, __ATOMIC_RELEASE); - }); - - // Ensure holder is in the activation window before the cleaner proceeds. - while (!window_entered.load(std::memory_order_acquire)) { /* spin */ } - - // With the fix: blocks until holder clears active_ptr (after the write). - // Without the fix: returns immediately because count==0. - RefCountGuard::waitForAllRefCountsToClear(); - - // Free the resource. With the fix the holder has already finished writing. - // Without the fix the holder is still asleep and will write to freed memory. - delete[] resource; - resource = nullptr; - - holder.join(); -} - -// waitForRefCountToClear(ptr) must not return while the target ptr is in the activation window. -TEST_F(RefCountGuardActivationWindowTest, WaitForSpecificBlocksDuringActivationWindow) { - auto& slot = RefCountGuard::refcount_slots[TEST_SLOT]; - - char* resource = new char[64]; - std::memset(resource, 0xAB, 64); - - volatile char* stable = resource; - - std::atomic window_entered{false}; - - std::thread holder([&] { - __atomic_store_n(&slot.active_ptr, resource, __ATOMIC_RELEASE); - window_entered.store(true, std::memory_order_release); - - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - - __atomic_fetch_add(&slot.count, 1u, __ATOMIC_RELEASE); - - *stable = 0xCD; - - __atomic_fetch_sub(&slot.count, 1u, __ATOMIC_RELEASE); - __atomic_store_n(&slot.active_ptr, nullptr, __ATOMIC_RELEASE); - }); - - while (!window_entered.load(std::memory_order_acquire)) { /* spin */ } - - RefCountGuard::waitForRefCountToClear(resource); - - delete[] resource; - resource = nullptr; - - holder.join(); -} diff --git a/ddprof-lib/src/test/cpp/remoteargs_ut.cpp b/ddprof-lib/src/test/cpp/remoteargs_ut.cpp deleted file mode 100644 index 4286cf9a2..000000000 --- a/ddprof-lib/src/test/cpp/remoteargs_ut.cpp +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "arguments.h" -#include "../../main/cpp/gtest_crash_handler.h" - -static constexpr char REMOTE_ARGS_TEST_NAME[] = "RemoteArgsTest"; - -class RemoteArgsGlobalSetup { -public: - RemoteArgsGlobalSetup() { - installGtestCrashHandler(); - } - ~RemoteArgsGlobalSetup() { - restoreDefaultSignalHandlers(); - } -}; - -static RemoteArgsGlobalSetup global_setup; - -class RemoteArgsTest : public ::testing::Test { -protected: - void SetUp() override { - // Test setup - } - - void TearDown() override { - // Test cleanup - } -}; - -TEST_F(RemoteArgsTest, DefaultRemoteSymbolicationDisabled) { - Arguments args; - - // Remote symbolication should be disabled by default - EXPECT_FALSE(args._remote_symbolication); -} - -TEST_F(RemoteArgsTest, EnableRemoteSymbolication) { - Arguments args; - - // Test enabling remote symbolication - Error error = args.parse("remotesym=true"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._remote_symbolication); -} - -TEST_F(RemoteArgsTest, EnableRemoteSymbolicationShort) { - Arguments args; - - // Test short form - Error error = args.parse("remotesym=y"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._remote_symbolication); -} - -TEST_F(RemoteArgsTest, DisableRemoteSymbolication) { - Arguments args; - - // Test explicitly disabling - Error error = args.parse("remotesym=false"); - EXPECT_FALSE(error); - EXPECT_FALSE(args._remote_symbolication); -} - -TEST_F(RemoteArgsTest, MultipleArgsWithRemoteSymbolication) { - Arguments args; - - // Test with multiple arguments - Error error = args.parse("event=cpu,interval=1000000,remotesym=true"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._remote_symbolication); - EXPECT_STREQ(args._event, "cpu"); - EXPECT_EQ(args._interval, 1000000); -} - -TEST_F(RemoteArgsTest, RemoteSymbolicationWithOtherFlags) { - Arguments args; - - // Test interaction with lightweight flag - Error error = args.parse("lightweight=true,remotesym=true"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._lightweight); - EXPECT_TRUE(args._remote_symbolication); -} - -TEST_F(RemoteArgsTest, RemoteSymbolicationNoValue) { - Arguments args; - - // Test no value (should enable) - Error error = args.parse("remotesym"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._remote_symbolication); -} - -TEST_F(RemoteArgsTest, RemoteSymbolicationNumericValues) { - { - Arguments args; - Error error = args.parse("remotesym=1"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._remote_symbolication); - } - { - Arguments args; - Error error = args.parse("remotesym=0"); - EXPECT_FALSE(error); - EXPECT_FALSE(args._remote_symbolication); - } -} - -TEST_F(RemoteArgsTest, RemoteSymbolicationNoVariant) { - { - Arguments args; - Error error = args.parse("remotesym=no"); - EXPECT_FALSE(error); - EXPECT_FALSE(args._remote_symbolication); - } - { - Arguments args; - Error error = args.parse("remotesym=n"); - EXPECT_FALSE(error); - EXPECT_FALSE(args._remote_symbolication); - } -} - -TEST_F(RemoteArgsTest, RemoteSymbolicationInvalidValue) { - Arguments args; - - // Test invalid value that starts with unrecognized char (should enable due to default) - Error error = args.parse("remotesym=invalid"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._remote_symbolication); -} \ No newline at end of file diff --git a/ddprof-lib/src/test/cpp/remotesymbolication_ut.cpp b/ddprof-lib/src/test/cpp/remotesymbolication_ut.cpp deleted file mode 100644 index 4ad98bff1..000000000 --- a/ddprof-lib/src/test/cpp/remotesymbolication_ut.cpp +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include "symbols_linux.h" -#include "profiler.h" -#include "vmEntry.h" -#include "../../main/cpp/gtest_crash_handler.h" - -// RemoteFramePacker tests are platform-independent (pure bit manipulation) -using RFP = Profiler::RemoteFramePacker; - -TEST(RemoteFramePackerTest, RoundTrip) { - uintptr_t pc_offset = 0x123456789AB; - char mark = 5; - uint32_t lib_index = 1000; - - unsigned long packed = RFP::pack(pc_offset, mark, lib_index); - EXPECT_EQ(RFP::unpackPcOffset(packed), pc_offset); - EXPECT_EQ(RFP::unpackMark(packed), mark); - EXPECT_EQ(RFP::unpackLibIndex(packed), lib_index); -} - -TEST(RemoteFramePackerTest, ZeroValues) { - unsigned long packed = RFP::pack(0, 0, 0); - EXPECT_EQ(packed, 0UL); - EXPECT_EQ(RFP::unpackPcOffset(packed), 0UL); - EXPECT_EQ(RFP::unpackMark(packed), 0); - EXPECT_EQ(RFP::unpackLibIndex(packed), 0U); -} - -TEST(RemoteFramePackerTest, MaxValues) { - uintptr_t max_pc = RFP::PC_OFFSET_MASK; // 44 bits all set - char max_mark = (char)RFP::MARK_MASK; // 3 bits all set = 7 - uint32_t max_lib = (uint32_t)RFP::LIB_INDEX_MASK; // 15 bits all set = 32767 - - unsigned long packed = RFP::pack(max_pc, max_mark, max_lib); - EXPECT_EQ(RFP::unpackPcOffset(packed), max_pc); - EXPECT_EQ(RFP::unpackMark(packed), max_mark); - EXPECT_EQ(RFP::unpackLibIndex(packed), max_lib); -} - -TEST(RemoteFramePackerTest, FieldIsolation) { - // Setting only pc_offset should not affect mark or lib_index - unsigned long packed_pc = RFP::pack(0xABCDE, 0, 0); - EXPECT_EQ(RFP::unpackMark(packed_pc), 0); - EXPECT_EQ(RFP::unpackLibIndex(packed_pc), 0U); - - // Setting only mark should not affect pc_offset or lib_index - unsigned long packed_mark = RFP::pack(0, 3, 0); - EXPECT_EQ(RFP::unpackPcOffset(packed_mark), 0UL); - EXPECT_EQ(RFP::unpackLibIndex(packed_mark), 0U); - - // Setting only lib_index should not affect pc_offset or mark - unsigned long packed_lib = RFP::pack(0, 0, 500); - EXPECT_EQ(RFP::unpackPcOffset(packed_lib), 0UL); - EXPECT_EQ(RFP::unpackMark(packed_lib), 0); -} - -TEST(RemoteFramePackerTest, Overflow_PcOffsetTruncated) { - // pc_offset larger than 44 bits should be silently truncated - uintptr_t oversize_pc = (1ULL << 44) | 0x42; - unsigned long packed = RFP::pack(oversize_pc, 0, 0); - EXPECT_EQ(RFP::unpackPcOffset(packed), (uintptr_t)0x42) - << "Bits above 44 should be masked off"; -} - -TEST(RemoteFramePackerTest, UnsymbolizedFramePacksWithZeroMark) { - // The new code path packs unsymbolized frames with mark=0 - uintptr_t pc_offset = 0x1A2B3C; - uint32_t lib_index = 7; - unsigned long packed = RFP::pack(pc_offset, 0, lib_index); - - EXPECT_EQ(RFP::unpackPcOffset(packed), pc_offset); - EXPECT_EQ(RFP::unpackMark(packed), 0); - EXPECT_EQ(RFP::unpackLibIndex(packed), lib_index); - // Packed value is non-zero even with mark=0 (prevents false NULL check) - EXPECT_NE(packed, 0UL); -} - -#ifdef __linux__ - -static constexpr char REMOTE_TEST_NAME[] = "RemoteSymbolicationTest"; - -class RemoteSymbolicationGlobalSetup { -public: - RemoteSymbolicationGlobalSetup() { - installGtestCrashHandler(); - } - ~RemoteSymbolicationGlobalSetup() { - restoreDefaultSignalHandlers(); - } -}; - -static RemoteSymbolicationGlobalSetup global_setup; - -class RemoteSymbolicationTest : public ::testing::Test { -protected: - void SetUp() override { - // Test setup - } - - void TearDown() override { - // Test cleanup - } -}; - -TEST_F(RemoteSymbolicationTest, RemoteFrameInfoConstruction) { - const char* test_build_id = "deadbeefcafebabe"; - uintptr_t test_offset = 0x1234; - short test_lib_index = 5; - - RemoteFrameInfo rfi(test_build_id, test_offset, test_lib_index); - - EXPECT_STREQ(rfi.build_id, test_build_id); - EXPECT_EQ(rfi.pc_offset, test_offset); - EXPECT_EQ(rfi.lib_index, test_lib_index); -} - -TEST_F(RemoteSymbolicationTest, BciFrameTypeConstants) { - // Verify that the new BCI constant is defined - EXPECT_EQ(BCI_NATIVE_FRAME_REMOTE, -19); - - // Verify it doesn't conflict with existing constants - EXPECT_NE(BCI_NATIVE_FRAME_REMOTE, BCI_NATIVE_FRAME); - EXPECT_NE(BCI_NATIVE_FRAME_REMOTE, BCI_ERROR); - EXPECT_NE(BCI_NATIVE_FRAME_REMOTE, BCI_ALLOC); -} - -// Test build-id extraction from a minimal ELF -TEST_F(RemoteSymbolicationTest, BuildIdExtractionBasic) { - // Create a minimal ELF file with a build-id note section - // This test would be more comprehensive with a real ELF file - // For now, just test the function doesn't crash on invalid input - - size_t build_id_len = 0; - char* build_id = SymbolsLinux::extractBuildId("/nonexistent", &build_id_len); - - // Should return null for non-existent file - EXPECT_EQ(build_id, nullptr); - EXPECT_EQ(build_id_len, 0); -} - -TEST_F(RemoteSymbolicationTest, BuildIdExtractionInvalidInput) { - size_t build_id_len = 0; - - // Test null inputs - char* build_id1 = SymbolsLinux::extractBuildId(nullptr, &build_id_len); - EXPECT_EQ(build_id1, nullptr); - - char* build_id2 = SymbolsLinux::extractBuildId("/some/file", nullptr); - EXPECT_EQ(build_id2, nullptr); - - // Test non-ELF file - const char* test_content = "This is not an ELF file"; - char temp_file[] = "/tmp/not_an_elf_XXXXXX"; - - int fd = mkstemp(temp_file); - if (fd >= 0) { - write(fd, test_content, strlen(test_content)); - close(fd); - - char* build_id3 = SymbolsLinux::extractBuildId(temp_file, &build_id_len); - EXPECT_EQ(build_id3, nullptr); - - unlink(temp_file); - } -} - -TEST_F(RemoteSymbolicationTest, BuildIdFromMemoryInvalidInput) { - size_t build_id_len = 0; - - // Test null pointer - char* build_id1 = SymbolsLinux::extractBuildIdFromMemory(nullptr, 100, &build_id_len); - EXPECT_EQ(build_id1, nullptr); - - // Test invalid size - char dummy_data[10] = {0}; - char* build_id2 = SymbolsLinux::extractBuildIdFromMemory(dummy_data, 0, &build_id_len); - EXPECT_EQ(build_id2, nullptr); - - // Test null output parameter - char* build_id3 = SymbolsLinux::extractBuildIdFromMemory(dummy_data, 10, nullptr); - EXPECT_EQ(build_id3, nullptr); -} - -#endif // __linux__ diff --git a/ddprof-lib/src/test/cpp/safefetch_ut.cpp b/ddprof-lib/src/test/cpp/safefetch_ut.cpp deleted file mode 100644 index f1cf4ae7f..000000000 --- a/ddprof-lib/src/test/cpp/safefetch_ut.cpp +++ /dev/null @@ -1,338 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "safeAccess.h" -#include "os.h" -#include "../../main/cpp/gtest_crash_handler.h" - -// Test name for crash handler -static constexpr char SAFEFETCH_TEST_NAME[] = "SafeFetchTest"; - - -static void (*orig_segvHandler)(int signo, siginfo_t *siginfo, void *ucontext); -static void (*orig_busHandler)(int signo, siginfo_t *siginfo, void *ucontext); - - -void signal_handle_wrapper(int signo, siginfo_t* siginfo, void* context) { - if (!SafeAccess::handle_safefetch(signo, context)) { - if (signo == SIGBUS && orig_busHandler != nullptr) { - orig_busHandler(signo, siginfo, context); - } else if (signo == SIGSEGV && orig_segvHandler != nullptr) { - orig_segvHandler(signo, siginfo, context); - } else { - // If no original handler, use crash handler for debugging - gtestCrashHandler(signo, siginfo, context, SAFEFETCH_TEST_NAME); - } - } -} - -class SafeFetchTest : public ::testing::Test { -protected: - void SetUp() override { - orig_segvHandler = OS::replaceSigsegvHandler(signal_handle_wrapper); - orig_busHandler = OS::replaceSigbusHandler(signal_handle_wrapper); - } - - void TearDown() override { - OS::replaceSigsegvHandler(orig_segvHandler); - OS::replaceSigbusHandler(orig_busHandler); - } -}; - - -TEST_F(SafeFetchTest, validAccess32) { - int i = 42; - int* p = &i; - int res = SafeAccess::safeFetch32(p, -1); - EXPECT_EQ(res, i); - i = INT_MAX; - res = SafeAccess::safeFetch32(p, -1); - EXPECT_EQ(res, i); - i = INT_MIN; - res = SafeAccess::safeFetch32(p, 0); - EXPECT_EQ(res, i); -} - - -TEST_F(SafeFetchTest, invalidAccess32) { - int* p = nullptr; - int res = SafeAccess::safeFetch32(p, -1); - EXPECT_EQ(res, -1); - res = SafeAccess::safeFetch32(p, -2); - EXPECT_EQ(res, -2); -} - -TEST_F(SafeFetchTest, validAccess64) { - int64_t i = 42; - int64_t* p = &i; - int64_t res = SafeAccess::safeFetch64(p, -1); - EXPECT_EQ(res, i); - i = LONG_MIN; - res = SafeAccess::safeFetch64(p, -19); - EXPECT_EQ(res, i); - i = LONG_MAX; - res = SafeAccess::safeFetch64(p, -1); - EXPECT_EQ(res, i); -} - -TEST_F(SafeFetchTest, invalidAccess64) { - int64_t* p = nullptr; - int64_t res = SafeAccess::safeFetch64(p, -1); - EXPECT_EQ(res, -1); - res = SafeAccess::safeFetch64(p, -2); - EXPECT_EQ(res, -2); -} - -TEST_F(SafeFetchTest, validAccessPtr) { - char c = 'a'; - void* p = (void*)&c; - void** pp = &p; - void* res = SafeAccess::loadPtr(pp, nullptr); - EXPECT_EQ(res, p); -} - -TEST_F(SafeFetchTest, invalidAccessPtr) { - int a, b; - void* ap = (void*)&a; - void* bp = (void*)&b; - void** pp = nullptr; - void* res = SafeAccess::loadPtr(pp, ap); - EXPECT_EQ(res, ap); - res = SafeAccess::loadPtr(pp, bp); - EXPECT_EQ(res, bp); -} - -TEST_F(SafeFetchTest, isReadable) { - char c = 'x'; - EXPECT_TRUE(SafeAccess::isReadable(&c)); - EXPECT_FALSE(SafeAccess::isReadable(nullptr)); -} - -/** - * Tests that safeFetch32 correctly handles mprotected memory. - * - * This test simulates the musl-specific memory layout where certain memory - * regions are protected. Without the NOINLINE fix, the compiler may inline - * the load under -O3, causing faults to occur at the wrong PC and bypassing - * the safefetch fault handler. - * - * Expected behavior: - * - With NOINLINE fix: Returns error value (-999) - * - Without NOINLINE fix under -O3: Crashes - */ -TEST_F(SafeFetchTest, mprotectedMemory32) { - void* page = mmap(NULL, 4096, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - ASSERT_NE(page, MAP_FAILED); - - int* ptr = static_cast(page); - *ptr = 0xDEADBEEF; - ASSERT_EQ(mprotect(page, 4096, PROT_NONE), 0); - - // This MUST return error value, not crash - int result = SafeAccess::safeFetch32(ptr, -999); - EXPECT_EQ(result, -999); - - munmap(page, 4096); -} - -/** - * Tests that safeFetch64 correctly handles mprotected memory. - * Same rationale as mprotectedMemory32 above. - */ -TEST_F(SafeFetchTest, mprotectedMemory64) { - void* page = mmap(NULL, 4096, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - ASSERT_NE(page, MAP_FAILED); - - int64_t* ptr = static_cast(page); - *ptr = 0xDEADBEEFCAFEBABE; - ASSERT_EQ(mprotect(page, 4096, PROT_NONE), 0); - - int64_t result = SafeAccess::safeFetch64(ptr, -999); - EXPECT_EQ(result, -999); - - munmap(page, 4096); -} - -// --------------------------------------------------------------------------- -// SafeAccess::safeCopy — bulk variant of safeFetch{32,64} that copies a byte -// range via the safefetch trampoline. Must: -// - return true and copy the bytes exactly when src is fully readable, -// including when [src, src+len) sits within a few bytes of an unmapped -// page boundary (aligned-load strategy keeps over-reads in-page) -// - return false (no crash) when the requested range itself crosses into -// an unmapped page -// - handle unaligned src by fetching at the previous 4-byte aligned -// address and discarding the leading 1..3 bytes -// - never write past dst[len-1] even when len is not a multiple of 4 -// - not mis-classify real data as a fault when it equals one sentinel -// --------------------------------------------------------------------------- - -TEST_F(SafeFetchTest, safeCopy_happyPath) { - const char src[] = "java/lang/Object"; - char dst[sizeof(src)] = {0}; - EXPECT_TRUE(SafeAccess::safeCopy(dst, src, sizeof(src) - 1)); - EXPECT_EQ(0, memcmp(dst, src, sizeof(src) - 1)); -} - -TEST_F(SafeFetchTest, safeCopy_zeroLength) { - // Even if src is NULL, len=0 must be a no-op success. - char dst[8] = {0}; - EXPECT_TRUE(SafeAccess::safeCopy(dst, nullptr, 0)); -} - -TEST_F(SafeFetchTest, safeCopy_shortLength_doesNotOverwriteDst) { - // The internal 4-byte fetch must not overflow dst beyond len bytes. - const char src[] = "AB"; - char dst[8]; - memset(dst, 0x5A, sizeof(dst)); - EXPECT_TRUE(SafeAccess::safeCopy(dst, src, 2)); - EXPECT_EQ('A', dst[0]); - EXPECT_EQ('B', dst[1]); - // Sentinel bytes 2..7 must be untouched. - for (int i = 2; i < 8; i++) { - EXPECT_EQ((char)0x5A, dst[i]) << "dst[" << i << "] was overwritten"; - } -} - -TEST_F(SafeFetchTest, safeCopy_unmappedSource_returnsFalse) { - // Map a page, then unmap it: the address is now firmly invalid. safeCopy - // must return false rather than SIGSEGV. - void* page = mmap(NULL, 4096, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - ASSERT_NE(page, MAP_FAILED); - ASSERT_EQ(0, munmap(page, 4096)); - - char dst[64] = {0}; - EXPECT_FALSE(SafeAccess::safeCopy(dst, page, 32)); -} - -TEST_F(SafeFetchTest, safeCopy_protNoneSource_returnsFalse) { - // mprotect-PROT_NONE the page (similar to mprotectedMemory32). safeCopy - // must return false on the first faulting word, not crash. - void* page = mmap(NULL, 4096, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - ASSERT_NE(page, MAP_FAILED); - memcpy(page, "ignored", 7); - ASSERT_EQ(0, mprotect(page, 4096, PROT_NONE)); - - char dst[64] = {0}; - EXPECT_FALSE(SafeAccess::safeCopy(dst, page, 32)); - - // Restore so munmap can run cleanly. - ASSERT_EQ(0, mprotect(page, 4096, PROT_READ | PROT_WRITE)); - munmap(page, 4096); -} - -TEST_F(SafeFetchTest, safeCopy_tailNearUnmappedBoundary_stillSucceeds) { - // Map two adjacent pages, unmap only the second. Place src so the bytes - // we ask for end inside the mapped page but the (over-read of the) next - // 4-byte word would touch the unmapped page. The aligned-load strategy - // must keep the load within the mapped page → success, not fault. - long page_size = sysconf(_SC_PAGESIZE); - ASSERT_GT(page_size, 0); - - void* region = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - ASSERT_NE(region, MAP_FAILED); - ASSERT_EQ(0, munmap((char*)region + page_size, page_size)); - - char* mapped_end = (char*)region + page_size; - char* src = mapped_end - 2; // 2 bytes from page boundary, k = 2 - src[0] = 'X'; - src[1] = 'Y'; - - char dst[16]; - memset(dst, 0, sizeof(dst)); - EXPECT_TRUE(SafeAccess::safeCopy(dst, src, 2)); - EXPECT_EQ('X', dst[0]); - EXPECT_EQ('Y', dst[1]); - - munmap(region, page_size); -} - -TEST_F(SafeFetchTest, safeCopy_requestedRangeCrossesUnmappedPage_returnsFalse) { - // Distinct from the case above: here the *requested* range itself - // crosses into the unmapped page. safeCopy must legitimately fault - // when it can't read all the bytes the caller asked for. - long page_size = sysconf(_SC_PAGESIZE); - ASSERT_GT(page_size, 0); - - void* region = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - ASSERT_NE(region, MAP_FAILED); - ASSERT_EQ(0, munmap((char*)region + page_size, page_size)); - - char* mapped_end = (char*)region + page_size; - char* src = mapped_end - 2; - src[0] = 'X'; - src[1] = 'Y'; - - // Asking for 8 bytes pushes 6 bytes into the unmapped page → must fault. - char dst[16] = {0}; - EXPECT_FALSE(SafeAccess::safeCopy(dst, src, 8)); - - munmap(region, page_size); -} - -TEST_F(SafeFetchTest, safeCopy_unalignedSource_allMisalignments) { - // The front fixup must correctly extract leading bytes from the - // previous-aligned-word fetch for every misalignment k ∈ {1, 2, 3}. - static const char kSentinel[] = "ABCDEFGHIJKLMNOP"; // 16 bytes - // Use a 4-byte-aligned buffer so we can shift src forward by k. - alignas(4) char buf[32]; - memcpy(buf + 4, kSentinel, 16); // place payload at aligned offset 4 - - for (size_t k = 1; k <= 3; k++) { - const char* src = buf + 4 + k; // misaligned by k - size_t len = 16 - k; // copy the rest of the payload - char dst[16]; - memset(dst, 0, sizeof(dst)); - EXPECT_TRUE(SafeAccess::safeCopy(dst, src, len)) << "k=" << k; - EXPECT_EQ(0, memcmp(dst, kSentinel + k, len)) << "k=" << k; - } -} - -TEST_F(SafeFetchTest, safeCopy_unalignedShortAtPageEnd_stillSucceeds) { - // Combine misalignment with proximity to an unmapped boundary: src is - // misaligned AND only a few bytes from the end of the mapped page. - // The front fixup reads backward (into the same page) → success. - long page_size = sysconf(_SC_PAGESIZE); - ASSERT_GT(page_size, 0); - - void* region = mmap(NULL, 2 * page_size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); - ASSERT_NE(region, MAP_FAILED); - ASSERT_EQ(0, munmap((char*)region + page_size, page_size)); - - char* mapped_end = (char*)region + page_size; - // mapped_end is 4-byte aligned (pages are 4 KiB-aligned). Place src - // 3 bytes back from the boundary so k = 1 and only 3 bytes are wanted. - char* src = mapped_end - 3; - src[0] = 'P'; - src[1] = 'Q'; - src[2] = 'R'; - - char dst[8] = {0}; - EXPECT_TRUE(SafeAccess::safeCopy(dst, src, 3)); - EXPECT_EQ('P', dst[0]); - EXPECT_EQ('Q', dst[1]); - EXPECT_EQ('R', dst[2]); - - munmap(region, page_size); -} - -TEST_F(SafeFetchTest, safeCopy_dataMatchingSingleSentinel_stillSucceeds) { - // The two-sentinel pattern must not mis-classify real data that happens - // to equal one of the sentinels. SENT_A is 0x55AA55AA. - uint32_t real_data = 0x55AA55AA; - char dst[4]; - ASSERT_TRUE(SafeAccess::safeCopy(dst, &real_data, 4)); - EXPECT_EQ(0, memcmp(dst, &real_data, 4)); -} diff --git a/ddprof-lib/src/test/cpp/sframe_ut.cpp b/ddprof-lib/src/test/cpp/sframe_ut.cpp deleted file mode 100644 index 31f9a157b..000000000 --- a/ddprof-lib/src/test/cpp/sframe_ut.cpp +++ /dev/null @@ -1,888 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "sframe.h" -#include "../../main/cpp/gtest_crash_handler.h" - -#include -#include -#include - -static constexpr char SFRAME_TEST_NAME[] = "SFrameTest"; - -class SFrameGlobalSetup { - public: - SFrameGlobalSetup() { installGtestCrashHandler(); } - ~SFrameGlobalSetup() { restoreDefaultSignalHandlers(); } -}; -static SFrameGlobalSetup sframe_global_setup; - -#ifdef __linux__ - -// --- Architecture constants --- -#ifdef __x86_64__ -static const uint8_t HOST_ABI = SFRAME_ABI_AMD64_ENDIAN_LITTLE; -static const uint8_t WRONG_ABI = SFRAME_ABI_AARCH64_ENDIAN_LITTLE; -#elif defined(__aarch64__) -static const uint8_t HOST_ABI = SFRAME_ABI_AARCH64_ENDIAN_LITTLE; -static const uint8_t WRONG_ABI = SFRAME_ABI_AMD64_ENDIAN_LITTLE; -#else -static const uint8_t HOST_ABI = 0xFF; -static const uint8_t WRONG_ABI = 0xFE; -#endif - -// --- Binary builder helpers --- -static void put8(std::vector& buf, uint8_t v) { buf.push_back(v); } - -static void put16(std::vector& buf, uint16_t v) { - buf.push_back(static_cast(v)); - buf.push_back(static_cast(v >> 8)); -} - -static void put32(std::vector& buf, uint32_t v) { - buf.push_back(static_cast(v)); - buf.push_back(static_cast(v >> 8)); - buf.push_back(static_cast(v >> 16)); - buf.push_back(static_cast(v >> 24)); -} - -// Appends a valid SFrameHeader (28 bytes, auxhdr_len=0). -static void buildHeader(std::vector& buf, - uint8_t abi_arch, - int8_t cfa_fixed_ra_offset, - uint32_t num_fdes, - uint32_t num_fres, - uint32_t fre_len, - uint32_t fdeoff, - uint32_t freoff) { - put16(buf, SFRAME_MAGIC); - put8(buf, SFRAME_VERSION_2); - put8(buf, SFRAME_F_FDE_SORTED); - put8(buf, abi_arch); - put8(buf, 0); // cfa_fixed_fp_offset - put8(buf, static_cast(cfa_fixed_ra_offset)); - put8(buf, 0); // auxhdr_len - put32(buf, num_fdes); - put32(buf, num_fres); - put32(buf, fre_len); - put32(buf, fdeoff); - put32(buf, freoff); -} - -// Appends an SFrameFDE (20 bytes). -// fre_type: SFRAME_FRE_TYPE_ADDR1/2/4; pcmask: set bit 0 of info for PCMASK type. -static void buildFDE(std::vector& buf, - int32_t start_addr, - uint32_t func_size, - uint32_t fre_off, - uint32_t fre_num, - uint8_t fre_type, - bool pcmask = false) { - put32(buf, static_cast(start_addr)); - put32(buf, func_size); - put32(buf, fre_off); - put32(buf, fre_num); - uint8_t info = (pcmask ? 0x1 : 0x0) | (fre_type << 1); - put8(buf, info); - put8(buf, 0); // rep_size - put16(buf, 0); // padding -} - -// Appends an FRE with 1-byte start address (ADDR1) and 1-byte signed offsets (OFFSET_1B). -// fre_info bits: bit0=CFA_base(0=SP,1=FP), bits1-2=0(1B), bit3=RA_tracked, bit4=FP_tracked. -// SFrame V2 wire order after CFA: RA offset (if tracked), then FP offset (if tracked). -// ra_off and fp_off are written only when the corresponding bit is set in fre_info. -static void buildFRE_1B(std::vector& buf, - uint8_t start_offset, - uint8_t fre_info, - int8_t cfa_off, - int8_t ra_off = 0, - int8_t fp_off = 0) { - put8(buf, start_offset); - put8(buf, fre_info); - put8(buf, static_cast(cfa_off)); - if (SFRAME_FRE_RA_TRACKED(fre_info)) put8(buf, static_cast(ra_off)); - if (SFRAME_FRE_FP_TRACKED(fre_info)) put8(buf, static_cast(fp_off)); -} - -// Appends an FRE with 2-byte start address (ADDR2) and 2-byte signed offsets (OFFSET_2B). -// SFrame V2 wire order after CFA: RA offset (if tracked), then FP offset (if tracked). -static void buildFRE_2B(std::vector& buf, - uint16_t start_offset, - uint8_t fre_info, - int16_t cfa_off, - int16_t ra_off = 0, - int16_t fp_off = 0) { - put16(buf, start_offset); - put8(buf, fre_info); - put16(buf, static_cast(cfa_off)); - if (SFRAME_FRE_RA_TRACKED(fre_info)) put16(buf, static_cast(ra_off)); - if (SFRAME_FRE_FP_TRACKED(fre_info)) put16(buf, static_cast(fp_off)); -} - -// Appends an FRE with 4-byte start address (ADDR4) and 4-byte signed offsets (OFFSET_4B). -// SFrame V2 wire order after CFA: RA offset (if tracked), then FP offset (if tracked). -static void buildFRE_4B(std::vector& buf, - uint32_t start_offset, - uint8_t fre_info, - int32_t cfa_off, - int32_t ra_off = 0, - int32_t fp_off = 0) { - put32(buf, start_offset); - put8(buf, fre_info); - put32(buf, static_cast(cfa_off)); - if (SFRAME_FRE_RA_TRACKED(fre_info)) put32(buf, static_cast(ra_off)); - if (SFRAME_FRE_FP_TRACKED(fre_info)) put32(buf, static_cast(fp_off)); -} - -// ============================================================ -// Header validation tests -// ============================================================ - -TEST(SFrameParser, InvalidMagic) { - std::vector buf(28, 0); // all zeros; magic = 0x0000, not 0xDEE2 - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, UnsupportedVersion) { - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 0, 0, 0, 0, 0); - buf[2] = 3; // overwrite version byte with V3 - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, WrongArch) { - // Build a structurally valid section with WRONG_ABI to confirm arch check fires. - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); // start=0, SP-based, cfa_off=8 - - std::vector buf; - buildHeader(buf, WRONG_ABI, -8, 1, 1, - static_cast(fre_buf.size()), // fre_len - 0, // fdeoff (FDE array right after header) - 20); // freoff (FRE section after FDE) - buildFDE(buf, 0x1000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, TruncatedSection) { - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 0, 0, 0, 0, 0); - buf.resize(10); // shorter than sizeof(SFrameHeader) = 28 - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, AuxhdrLenBoundsCheck) { - // auxhdr_len = 200 pushes data_start past section end - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 0, 0, 0, 0, 0); - buf[7] = 200; // auxhdr_len byte - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, EmptyFDEArray) { - // num_fdes=0 → no records → parse() returns false - std::vector buf; - buildHeader(buf, HOST_ABI, -8, /*num_fdes=*/0, 0, 0, 0, 0); - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, PCMASK_Skipped) { - // A single FDE with PCMASK type (bit 0 set in info) is skipped; no records → false - // - // Layout: header(28) | FDE(20) | FRE(3) - // fdeoff=0 (FDE array starts at data_start), freoff=20 (FRE section after FDE) - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); // valid FRE, but FDE is PCMASK so it's skipped - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), /*fdeoff=*/0, /*freoff=*/20); - buildFDE(buf, 0x100, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1, /*pcmask=*/true); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, EmptyFDE_Skipped) { - // A single FDE with fre_num=0 is skipped; no records → false - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 0, 0, /*fdeoff=*/0, /*freoff=*/20); - buildFDE(buf, 0x100, 64, 0, /*fre_num=*/0, SFRAME_FRE_TYPE_ADDR1); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, SingleFDE_SingleFRE_SPBased) { - // SP-based CFA with fixed RA offset -8 (x86_64 style). - // fre_info = 0x00: SP base (bit0=0), 1B offsets (bits1-2=0), no RA (bit3=0), no FP (bit4=0) - // CFA offset = 8 → cfa = (8 << 8) | DW_REG_SP - // fp_off = DW_SAME_FP (FP not tracked) - // pc_off = -8 (from cfa_fixed_ra_offset) - // - // Layout: header(28) | FDE(20) | FRE(3 bytes: 1 addr + 1 info + 1 cfa_off) - std::vector fre_buf; - buildFRE_1B(fre_buf, /*start_offset=*/0, /*fre_info=*/0x00, /*cfa_off=*/8); - - std::vector buf; - buildHeader(buf, HOST_ABI, /*cfa_fixed_ra_offset=*/-8, 1, 1, - (uint32_t)fre_buf.size(), /*fdeoff=*/0, /*freoff=*/20); - buildFDE(buf, /*start_addr=*/0x1000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - EXPECT_EQ(parser.count(), 1); - - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - // loc = section_offset(0) + start_addr(0x1000) + fre_start(0) = 0x1000 - EXPECT_EQ(table[0].loc, static_cast(0x1000)); - // cfa = (8 << 8) | DW_REG_SP - EXPECT_EQ(table[0].cfa, static_cast((8u << 8) | (unsigned)DW_REG_SP)); - EXPECT_EQ(table[0].fp_off, DW_SAME_FP); - EXPECT_EQ(table[0].pc_off, -8); - free(table); -} - -TEST(SFrameParser, SingleFDE_SingleFRE_FPBased) { - // FP-based CFA with FP tracked, fixed RA offset -8. - // fre_info = 0x11: FP base (bit0=1), 1B offsets (bits1-2=0), no RA (bit3=0), FP tracked (bit4=1) - // = 0b00010001 = 0x11 - // CFA offset = 16 → cfa = (16 << 8) | DW_REG_FP - // fp_off = -16 - // pc_off = -8 - std::vector fre_buf; - buildFRE_1B(fre_buf, 4, /*fre_info=*/0x11, /*cfa_off=*/16, /*ra_off=*/0, /*fp_off=*/-16); - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0x2000, 128, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - EXPECT_EQ(parser.count(), 1); - - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - // loc = 0 + 0x2000 + 4 = 0x2004 - EXPECT_EQ(table[0].loc, static_cast(0x2004)); - EXPECT_EQ(table[0].cfa, static_cast((16u << 8) | (unsigned)DW_REG_FP)); - EXPECT_EQ(table[0].fp_off, -16); - EXPECT_EQ(table[0].pc_off, -8); - free(table); -} - -TEST(SFrameParser, FixedRAOffset) { - // Verify cfa_fixed_ra_offset drives pc_off even when FRE RA bit is set. - // fre_info = 0x08: SP base (bit0=0), 1B offsets (bits1-2=0), RA tracked (bit3=1), no FP (bit4=0) - // The FRE encodes ra_off=-16, but the header's cfa_fixed_ra_offset=-8 must win. - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, /*fre_info=*/0x08, /*cfa_off=*/8, /*ra_off=*/-16); - - std::vector buf; - buildHeader(buf, HOST_ABI, /*cfa_fixed_ra_offset=*/-8, 1, 1, - (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0x500, 32, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - // pc_off must come from the header (-8), not from the FRE RA offset (-16) - EXPECT_EQ(table[0].pc_off, -8); - free(table); -} - -TEST(SFrameParser, PerFRE_RA) { - // aarch64 style: cfa_fixed_ra_offset=0, RA tracked per-FRE. - // fre_info = 0x08: SP base (bit0=0), 1B offsets (bits1-2=0), RA tracked (bit3=1), no FP (bit4=0) - // = 0b00001000 = 0x08 - // RA offset = -8 - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, /*fre_info=*/0x08, /*cfa_off=*/16, /*ra_off=*/-8); - - std::vector buf; - buildHeader(buf, HOST_ABI, /*cfa_fixed_ra_offset=*/0, 1, 1, - (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0x3000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - EXPECT_EQ(table[0].pc_off, -8); - free(table); -} - -TEST(SFrameParser, PerFRE_RA_Untracked) { - // aarch64 leaf function: cfa_fixed_ra_offset=0, RA not tracked → DW_LINK_REGISTER. - // fre_info = 0x00: SP base, 1B offsets, no RA, no FP - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); - - std::vector buf; - buildHeader(buf, HOST_ABI, /*cfa_fixed_ra_offset=*/0, 1, 1, - (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0x4000, 32, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - EXPECT_EQ(table[0].pc_off, DW_LINK_REGISTER); - free(table); -} - -TEST(SFrameParser, OffsetSize_2B) { - // FRE with 2-byte start address (ADDR2) and 2-byte offset encoding (OFFSET_2B). - // fre_info = 0x02: SP base (bit0=0), 2B offsets (bits1-2=01), no RA, no FP - // = 0b00000010 = 0x02 - // CFA offset = 512 (requires 2 bytes) - std::vector fre_buf; - buildFRE_2B(fre_buf, /*start_offset=*/8, /*fre_info=*/0x02, /*cfa_off=*/512); - - // fre_buf size: 2 (addr) + 1 (info) + 2 (cfa) = 5 bytes - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0x5000, 256, 0, 1, SFRAME_FRE_TYPE_ADDR2); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - // loc = 0 + 0x5000 + 8 = 0x5008 - EXPECT_EQ(table[0].loc, static_cast(0x5008)); - // cfa = (512 << 8) | DW_REG_SP - EXPECT_EQ(table[0].cfa, static_cast((512u << 8) | (unsigned)DW_REG_SP)); - free(table); -} - -TEST(SFrameParser, OffsetSize_4B) { - // FRE with 4-byte start address (ADDR4) and 4-byte offset encoding (OFFSET_4B). - // fre_info = 0x04: SP base (bit0=0), 4B offsets (bits1-2=10), no RA, no FP - // = 0b00000100 = 0x04 - // CFA offset = 65536 (requires 4 bytes) - std::vector fre_buf; - buildFRE_4B(fre_buf, /*start_offset=*/0, /*fre_info=*/0x04, /*cfa_off=*/65536); - - // fre_buf size: 4 (addr) + 1 (info) + 4 (cfa) = 9 bytes - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0x6000, 512, 0, 1, SFRAME_FRE_TYPE_ADDR4); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - EXPECT_EQ(table[0].loc, static_cast(0x6000)); - EXPECT_EQ(table[0].cfa, static_cast((65536u << 8) | (unsigned)DW_REG_SP)); - free(table); -} - -TEST(SFrameParser, MultipleFDEs) { - // 3 FDEs with 2 FREs each → 6 total records, verify sorted by loc. - // - // FRE layout (2 FREs per FDE × 3 FDEs, each FRE = 3 bytes: 1addr+1info+1cfa): - // FDE0 FREs at fre_section+0: offset 0 cfa=8, offset 8 cfa=16 - // FDE1 FREs at fre_section+6: offset 0 cfa=8, offset 4 cfa=16 - // FDE2 FREs at fre_section+12: offset 0 cfa=8, offset 2 cfa=16 - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); - buildFRE_1B(fre_buf, 8, 0x00, 16); - buildFRE_1B(fre_buf, 0, 0x00, 8); - buildFRE_1B(fre_buf, 4, 0x00, 16); - buildFRE_1B(fre_buf, 0, 0x00, 8); - buildFRE_1B(fre_buf, 2, 0x00, 16); - // fre_buf.size() = 18 - - std::vector buf; - // freoff = 3 * sizeof(SFrameFDE) = 60 - buildHeader(buf, HOST_ABI, -8, 3, 6, (uint32_t)fre_buf.size(), /*fdeoff=*/0, /*freoff=*/60); - buildFDE(buf, 0xA000, 32, 0, 2, SFRAME_FRE_TYPE_ADDR1); // FDE0: FREs at offset 0 - buildFDE(buf, 0xB000, 32, 6, 2, SFRAME_FRE_TYPE_ADDR1); // FDE1: FREs at offset 6 - buildFDE(buf, 0xC000, 32, 12, 2, SFRAME_FRE_TYPE_ADDR1); // FDE2: FREs at offset 12 - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - EXPECT_EQ(parser.count(), 6); - - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - // Verify sorted ascending - for (int i = 0; i + 1 < 6; i++) { - EXPECT_LT(table[i].loc, table[i + 1].loc) - << "table not sorted at index " << i; - } - // Spot-check locs: FDE0@0xA000, FDE0@0xA008, FDE1@0xB000, ... - EXPECT_EQ(table[0].loc, static_cast(0xA000)); - EXPECT_EQ(table[1].loc, static_cast(0xA008)); - EXPECT_EQ(table[2].loc, static_cast(0xB000)); - free(table); -} - -TEST(SFrameParser, MultipleFDEs_ReverseOrder) { - // 3 FDEs in descending address order: 0xC000, 0xB000, 0xA000. - // The SFRAME_F_FDE_SORTED flag is NOT set so qsort must fire. - // Output must be sorted ascending. - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); // FDE0 (start=0xC000) - buildFRE_1B(fre_buf, 4, 0x00, 16); - buildFRE_1B(fre_buf, 0, 0x00, 8); // FDE1 (start=0xB000) - buildFRE_1B(fre_buf, 4, 0x00, 16); - buildFRE_1B(fre_buf, 0, 0x00, 8); // FDE2 (start=0xA000) - buildFRE_1B(fre_buf, 4, 0x00, 16); - // fre_buf.size() = 18 - - std::vector buf; - // freoff = 3 * 20 = 60; flags = 0 (no SORTED flag) - put16(buf, SFRAME_MAGIC); - put8(buf, SFRAME_VERSION_2); - put8(buf, 0); // flags: NOT SFRAME_F_FDE_SORTED - put8(buf, HOST_ABI); - put8(buf, 0); // cfa_fixed_fp_offset - put8(buf, static_cast(-8)); // cfa_fixed_ra_offset - put8(buf, 0); // auxhdr_len - put32(buf, 3); // num_fdes - put32(buf, 6); // num_fres - put32(buf, (uint32_t)fre_buf.size()); // fre_len - put32(buf, 0); // fdeoff - put32(buf, 60); // freoff - - buildFDE(buf, 0xC000, 32, 0, 2, SFRAME_FRE_TYPE_ADDR1); - buildFDE(buf, 0xB000, 32, 6, 2, SFRAME_FRE_TYPE_ADDR1); - buildFDE(buf, 0xA000, 32, 12, 2, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - EXPECT_EQ(parser.count(), 6); - - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - // Must be sorted ascending after qsort - for (int i = 0; i + 1 < 6; i++) { - EXPECT_LT(table[i].loc, table[i + 1].loc) - << "table not sorted at index " << i; - } - EXPECT_EQ(table[0].loc, static_cast(0xA000)); - EXPECT_EQ(table[5].loc, static_cast(0xC004)); - free(table); -} - -TEST(SFrameParser, AddressTranslation) { - // section_offset=0x1000, FDE start_addr=0x200, FRE start=0x10 - // Expected loc = 0x1000 + 0x200 + 0x10 = 0x1210 - std::vector fre_buf; - buildFRE_1B(fre_buf, /*start_offset=*/0x10, 0x00, 8); - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, /*start_addr=*/0x200, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), - /*section_offset=*/0x1000); - ASSERT_TRUE(parser.parse()); - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - EXPECT_EQ(table[0].loc, static_cast(0x1210)); - free(table); -} - -TEST(SFrameParser, BoundsCheck_FREOverrun) { - // FDE0's fre_off points at fre_end → parseFDE entry-level bounds check fires - // immediately (no records added). FDE1 has a valid FRE → 1 record. - // - // Layout: header(28) | FDE0(20) | FDE1(20) | FRE(3 bytes) - // fre_section = data_start + 40, fre_end = fre_section + 3 - // FDE0: fre_off=3, fre_num=1 → fre_ptr = fre_section+3 = fre_end → fail - // FDE1: fre_off=0, fre_num=1 → fre_ptr = fre_section+0 → reads 3 bytes → OK - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); // FDE1's FRE (3 bytes at offset 0) - // fre_buf.size() = 3; FDE0's fre_off=3 points exactly at fre_end - - std::vector buf; - // freoff = 2 * sizeof(SFrameFDE) = 40 - buildHeader(buf, HOST_ABI, -8, 2, 1, (uint32_t)fre_buf.size(), /*fdeoff=*/0, /*freoff=*/40); - buildFDE(buf, 0xD000, 64, /*fre_off=*/3, /*fre_num=*/1, SFRAME_FRE_TYPE_ADDR1); // overruns - buildFDE(buf, 0xE000, 64, /*fre_off=*/0, /*fre_num=*/1, SFRAME_FRE_TYPE_ADDR1); // OK - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - // parse() returns true because FDE1 produced a record - ASSERT_TRUE(parser.parse()); - EXPECT_EQ(parser.count(), 1); // only FDE1's FRE made it through - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - EXPECT_EQ(table[0].loc, static_cast(0xE000)); - free(table); -} - -TEST(SFrameParser, BoundsCheck_AddrSizeTruncated) { - // ADDR2 FRE: buffer ends after only 1 of the 2 start-address bytes. - // fre_ptr + addr_size(2) > fre_end triggers guard (b). - std::vector fre_buf; - // Write only 1 byte of the 2-byte start address (truncated) - put8(fre_buf, 0x00); // first byte of start address — second byte missing - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0x1000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR2); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - // FDE is skipped due to bounds failure; no records → false - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, BoundsCheck_InfoByteTruncated) { - // ADDR1 FRE: buffer ends exactly after the 1-byte start address, no room for info byte. - // fre_ptr + 1 > fre_end triggers guard (c). - std::vector fre_buf; - put8(fre_buf, 0x00); // start address byte only; info byte missing - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0x1000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, BoundsCheck_FDEOffOutOfBounds) { - // fdeoff set beyond data_len; parse() must return false. - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); - - std::vector buf; - // data_len = buf.size() - 28 (header). Set fdeoff = data_len + 100 (way past end). - uint32_t fake_fdeoff = 9999; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), - /*fdeoff=*/fake_fdeoff, /*freoff=*/20); - buildFDE(buf, 0x1000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, BoundsCheck_FRELenOverflow) { - // fre_len set so that freoff + fre_len > data_len; parse() must return false. - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); - - std::vector buf; - // freoff=20 (valid), fre_len=9999 (way past end) - buildHeader(buf, HOST_ABI, -8, 1, 1, /*fre_len=*/9999, /*fdeoff=*/0, /*freoff=*/20); - buildFDE(buf, 0x1000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, BoundsCheck_FREOffOutOfBounds) { - // fde->fre_off == fre_section_len: points exactly past the FRE sub-section. - // parseFDE must reject before doing pointer arithmetic. - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); // 3 bytes; fre_section_len = 3 - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), /*fdeoff=*/0, /*freoff=*/20); - // fre_off == fre_len (3): exactly at end, out-of-bounds - buildFDE(buf, 0x1000, 64, /*fre_off=*/3, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, ReservedFREAddrSize) { - // FDE info with addr-size bits = 0b11 (value 3) → default:return false in parseFDE. - // FDE is skipped; no records → parse() returns false. - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - // Encode addr-size = 3 (0b11) in bits 1-2 of FDE info byte: info = 0b00000110 = 0x06 - uint8_t fde_info = 0x06; // not pcmask (bit0=0), addr-size=3 (bits1-2=11) - put32(buf, static_cast(0x1000)); // start_addr - put32(buf, 64u); // func_size - put32(buf, 0u); // fre_off - put32(buf, 1u); // fre_num - put8(buf, fde_info); - put8(buf, 0); // rep_size - put16(buf, 0); // padding - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, ReservedFREOffsetSize) { - // FRE info byte with offset-size bits = 0b11 (value 3) → default:return false. - // Build a minimal ADDR1 FRE but with reserved offset-size in the info byte. - // fre_info = 0b00000110 = 0x06: SP base, offset-size=3 (reserved), no RA, no FP - std::vector fre_buf; - put8(fre_buf, 0x00); // start address = 0 - put8(fre_buf, 0x06); // fre_info: offset-size bits = 0b11 → reserved - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0x1000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); -} - -TEST(SFrameParser, AuxhdrLen_NonZero) { - // auxhdr_len=4: 4 extra bytes between header and data_start. - // fdeoff and freoff are relative to data_start (after auxhdr), so no adjustment needed. - std::vector fre_buf; - buildFRE_1B(fre_buf, 0, 0x00, 8); - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - // Overwrite auxhdr_len byte (offset 7 in header) with 4 - buf[7] = 4; - // Append 4 auxhdr bytes - put32(buf, 0xDEADBEEF); // dummy auxhdr content - // Now append FDE and FRE (positions shift by 4, but offsets are relative to data_start) - buildFDE(buf, 0x1000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - EXPECT_EQ(parser.count(), 1); - - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - EXPECT_EQ(table[0].loc, static_cast(0x1000)); - free(table); -} - -TEST(SFrameParser, ParseFailure_FreesTable) { - // parse() fails (wrong magic) → destructor must free _table without crashing. - // With ASAN this also detects leaks. - std::vector buf(28, 0); // all zeros → wrong magic - { - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - EXPECT_FALSE(parser.parse()); - // destructor runs here; _table was malloc'd in constructor and must be freed - } - // Reaching here without crash/ASAN report is the pass condition -} - -TEST(SFrameParser, DefaultFrameDetection) { - // Build a section with one FP-based FRE so _linked_frame_size gets set. - // fre_info = 0x11: FP base (bit0=1), 1B offsets (bits1-2=0), no RA, FP tracked (bit4=1) - // = 0b00010001 = 0x11 - // cfa_offset = LINKED_FRAME_CLANG_SIZE (16 on both x86_64 and aarch64 clang) - std::vector fre_buf; - buildFRE_1B(fre_buf, 4, 0x11, (int8_t)LINKED_FRAME_CLANG_SIZE, /*ra_off=*/0, (int8_t)-LINKED_FRAME_CLANG_SIZE); - - std::vector buf; - buildHeader(buf, HOST_ABI, -8, 1, 1, (uint32_t)fre_buf.size(), 0, 20); - buildFDE(buf, 0xF000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - - const FrameDesc& def = parser.detectedDefaultFrame(); - -#if defined(__aarch64__) - // On aarch64: LINKED_FRAME_CLANG_SIZE(16) != LINKED_FRAME_SIZE(0) → clang frame - EXPECT_EQ(&def, &FrameDesc::default_clang_frame); -#elif defined(__x86_64__) - // On x86_64: LINKED_FRAME_CLANG_SIZE == LINKED_FRAME_SIZE → default_frame - EXPECT_EQ(&def, &FrameDesc::default_frame); -#endif - - free(parser.table()); -} - -// ============================================================ -// T1 — Both RA and FP tracked: correct offset assignment -// ============================================================ - -TEST(SFrameParser, BothRA_and_FP_Tracked_CorrectOrder) { - // fre_info = 0x18: SP base (bit0=0), 1B offsets (bits1-2=0), - // RA tracked (bit3=1), FP tracked (bit4=1) → 0b00011000 - // SFrame V2 wire order: CFA, then RA slot, then FP slot. - // RA slot = -8, FP slot = -16. - // cfa_fixed_ra_offset = 0 so per-FRE RA value drives pc_off. - // - // Expected: pc_off == -8 (from RA slot), fp_off == -16 (from FP slot). - // If the read order is swapped: pc_off == -16, fp_off == -8. - static const uint8_t fre_info = 0x18; - std::vector fre_buf; - buildFRE_1B(fre_buf, /*start_offset=*/0, fre_info, /*cfa_off=*/16, - /*ra_off=*/-8, /*fp_off=*/-16); - - std::vector buf; - buildHeader(buf, HOST_ABI, /*cfa_fixed_ra_offset=*/0, 1, 1, - (uint32_t)fre_buf.size(), /*fdeoff=*/0, /*freoff=*/20); - buildFDE(buf, 0x7000, 64, 0, 1, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - EXPECT_EQ(parser.count(), 1); - - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - EXPECT_EQ(table[0].pc_off, -8) << "RA slot must map to pc_off (read order: RA before FP)"; - EXPECT_EQ(table[0].fp_off, -16) << "FP slot must map to fp_off (read order: RA before FP)"; - free(table); -} - -// ============================================================ -// T2 — Partial FDE rollback: corrupt second FDE leaves no trace -// ============================================================ - -TEST(SFrameParser, PartialFDE_Rollback_CountRestored) { - // FDE0 (valid): one FRE at 0xA000 → adds 1 record. - // FDE1 (corrupt): fre_num=2 but the buffer for the second FRE is missing. - // parseFDE adds the first FRE of FDE1 then returns false on the second. - // With rollback, that partial record must not survive. - // - // Each valid FRE: 1 (start_offset) + 1 (fre_info) + 1 (cfa_off) = 3 bytes. - std::vector fre_buf; - // FDE0's FRE (offset 0 in fre section) - buildFRE_1B(fre_buf, 0, 0x00, 8); - // FDE1's first FRE (offset 3) — valid bytes - buildFRE_1B(fre_buf, 0, 0x00, 8); - // FDE1's second FRE is intentionally absent (truncated buffer). - - // fre_section layout: [FDE0-fre0(3)] [FDE1-fre0(3)] total = 6 bytes - // FDE0: fre_off=0, fre_num=1 - // FDE1: fre_off=3, fre_num=2 → second FRE missing → parseFDE returns false - std::vector buf; - // freoff = 2 * sizeof(SFrameFDE) = 40 - buildHeader(buf, HOST_ABI, -8, 2, /*num_fres=*/3, (uint32_t)fre_buf.size(), - /*fdeoff=*/0, /*freoff=*/40); - buildFDE(buf, 0xA000, 32, /*fre_off=*/0, /*fre_num=*/1, SFRAME_FRE_TYPE_ADDR1); - buildFDE(buf, 0xB000, 32, /*fre_off=*/3, /*fre_num=*/2, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - - // Only FDE0's record must survive; FDE1's partial record must be rolled back. - EXPECT_EQ(parser.count(), 1); - - FrameDesc* table = parser.table(); - ASSERT_NE(table, nullptr); - EXPECT_EQ(table[0].loc, static_cast(0xA000)); - free(table); -} - -// ============================================================ -// T3 — Partial FDE rollback: _linked_frame_size not poisoned -// ============================================================ - -TEST(SFrameParser, PartialFDE_Rollback_LinkedFrameSizeRestored) { - // FDE array order matters: FDE1 (valid, SP-based) is processed FIRST so it - // produces one committed record. FDE0 (FP-based, fre_num=2) is processed - // SECOND: its FRE0 succeeds and sets _linked_frame_size, then FRE1 runs off - // the end of the fre section → parseFDE returns false → rollback restores - // _count to 1 and _linked_frame_size to -1. - // - // fre_info for FP-based FRE with FP tracked: - // 0x11 = 0b00010001: FP base (bit0=1), 1B offsets, no RA (bit3=0), FP tracked (bit4=1) - // Each such FRE: 1(start) + 1(info) + 1(cfa) + 1(fp) = 4 bytes - // fre_info for SP-based FRE, no tracked offsets: - // 0x00: 1(start) + 1(info) + 1(cfa) = 3 bytes - static const uint8_t fp_fre_info = 0x11; - - std::vector fre_buf; - - // Offset 0: FDE1's single FRE (SP-based, 3 bytes). - buildFRE_1B(fre_buf, 0, 0x00, 8); - - // Offset 3: FDE0's FRE0 (FP-based, 4 bytes). Sets _linked_frame_size. - // After this FRE fre_ptr == fre_end, so FRE1 triggers the fre_ptr>=fre_end - // early-exit in parseFDE, returning false. - buildFRE_1B(fre_buf, 0, fp_fre_info, (int8_t)LINKED_FRAME_CLANG_SIZE, - /*ra_off=*/0, (int8_t)-LINKED_FRAME_CLANG_SIZE); - // fre_buf.size() == 7; fre_end == fre_section + 7. - // FDE0's second FRE iteration: fre_ptr (= fre_section + 7) >= fre_end → false. - - std::vector buf; - // freoff = 2 * sizeof(SFrameFDE) = 40 - buildHeader(buf, HOST_ABI, -8, 2, /*num_fres=*/3, (uint32_t)fre_buf.size(), - /*fdeoff=*/0, /*freoff=*/40); - // FDE1 first in array: valid, fre_off=0, fre_num=1. - buildFDE(buf, 0xC000, 32, /*fre_off=*/0, /*fre_num=*/1, SFRAME_FRE_TYPE_ADDR1); - // FDE0 second: FP-based, fre_off=3, fre_num=2 (second FRE runs off fre section). - buildFDE(buf, 0xA000, 64, /*fre_off=*/3, /*fre_num=*/2, SFRAME_FRE_TYPE_ADDR1); - buf.insert(buf.end(), fre_buf.begin(), fre_buf.end()); - - SFrameParser parser("test", reinterpret_cast(buf.data()), buf.size(), 0); - ASSERT_TRUE(parser.parse()); - - // FDE1's record must survive; FDE0's partial record must be rolled back. - EXPECT_EQ(parser.count(), 1); - - // _linked_frame_size was mutated by FDE0's FRE0, then restored by rollback. - // On aarch64 (where LINKED_FRAME_CLANG_SIZE != LINKED_FRAME_SIZE) this asserts - // the clang-frame variant is NOT selected; on x86_64 both sizes are equal so - // detectedDefaultFrame() always returns default_frame regardless — the assertion - // is structurally correct but only diagnostic on aarch64. - const FrameDesc& def = parser.detectedDefaultFrame(); - EXPECT_EQ(&def, &FrameDesc::default_frame) - << "_linked_frame_size must be rolled back when parseFDE returns false"; - - free(parser.table()); -} - -// ============================================================ -// T4 — section_offset_full overflow guard -// ============================================================ - -// The guard `section_offset_full > UINT32_MAX` lives in ElfParser::parseDwarfInfo() -// (symbols_linux.cpp). That function requires a live ELF image and cannot be -// exercised in the SFrameParser unit-test harness without pulling in ElfParser. -// -// Validation approach: the guard is a single arithmetic comparison inserted before -// the SFrame path is entered. When section_base - _base > 0xFFFFFFFF the function -// falls through to the DWARF path rather than constructing an SFrameParser with a -// truncated u32 offset. The correctness of this guard is verified by code review -// of the production change in symbols_linux.cpp:710-711. -// -// The placeholder test below documents the requirement so it appears in the test -// suite and can be extended if an ElfParser integration harness is added later. -TEST(SFrameParser, SectionOffsetOverflow_GuardDocumented) { - // This test serves as a permanent marker that the UINT32_MAX guard in - // parseDwarfInfo() is a required correctness fix (spec requirement #3). - // No SFrameParser API is needed to verify the guard; the production code - // path that is guarded does not reach SFrameParser::parse() when the offset - // exceeds UINT32_MAX. - SUCCEED(); -} - -#endif // __linux__ diff --git a/ddprof-lib/src/test/cpp/sigaction_interception_ut.cpp b/ddprof-lib/src/test/cpp/sigaction_interception_ut.cpp deleted file mode 100644 index 6d45bee2b..000000000 --- a/ddprof-lib/src/test/cpp/sigaction_interception_ut.cpp +++ /dev/null @@ -1,266 +0,0 @@ -/* - * Copyright 2025 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include -#include -#include -#include - -#include "os.h" - -/** - * Test for signal handler chaining to prevent infinite loops. - * - * This test verifies that when we intercept sigaction calls from other libraries - * (like wasmtime), the oldact we return points to the original handler (e.g., JVM's), - * not to our handler. If we return our handler as oldact, the chain becomes: - * Us -> other_lib -> Us -> other_lib -> ... (infinite loop) - * - * The correct chain should be: - * Us -> other_lib -> original_handler (JVM) - */ - -// Type for our sigaction hook function -typedef int (*sigaction_hook_t)(int, const struct sigaction*, struct sigaction*); - -// Counter to detect infinite loops -static std::atomic handler_call_count{0}; -static const int MAX_HANDLER_CALLS = 10; - -// Jump buffer for escaping infinite loops -static sigjmp_buf escape_jmp; -static std::atomic should_escape{false}; - -// The "other library's" handler (simulating wasmtime) -static struct sigaction other_lib_saved_oldact; - -static void other_lib_handler(int signo, siginfo_t* siginfo, void* context) { - handler_call_count++; - - // Detect infinite loop - if (handler_call_count > MAX_HANDLER_CALLS) { - // We're in an infinite loop - escape - if (should_escape) { - siglongjmp(escape_jmp, 1); - } - return; - } - - // Simulate what wasmtime does: if we don't handle it, chain to "previous" handler - // If oldact points back to our handler, this will cause infinite recursion - if (other_lib_saved_oldact.sa_flags & SA_SIGINFO) { - if (other_lib_saved_oldact.sa_sigaction != nullptr) { - other_lib_saved_oldact.sa_sigaction(signo, siginfo, context); - } - } else if (other_lib_saved_oldact.sa_handler != SIG_DFL && - other_lib_saved_oldact.sa_handler != SIG_IGN && - other_lib_saved_oldact.sa_handler != nullptr) { - other_lib_saved_oldact.sa_handler(signo); - } - // If oldact is SIG_DFL, we just return (signal will re-trigger or terminate) -} - -// Our handler (profiler's handler) -static void our_handler(int signo, siginfo_t* siginfo, void* context) { - handler_call_count++; - - // Detect infinite loop - if (handler_call_count > MAX_HANDLER_CALLS) { - if (should_escape) { - siglongjmp(escape_jmp, 1); - } - return; - } - - // We don't handle this signal, chain to the "other library" via chain target - SigAction chain = OS::getSegvChainTarget(); - if (chain != nullptr) { - chain(signo, siginfo, context); - } - // After chain returns (or if no chain), we return -} - -// Original handler (simulating JVM's handler) -static std::atomic original_handler_called{false}; -static void original_handler(int signo, siginfo_t* siginfo, void* context) { - original_handler_called = true; - // The original handler would normally handle the signal or terminate. - // For this test, we just mark that we were called. - if (should_escape) { - siglongjmp(escape_jmp, 1); - } -} - -class SigactionInterceptionTest : public ::testing::Test { -protected: - struct sigaction saved_segv_action; - - void SetUp() override { - // Save current SIGSEGV handler - sigaction(SIGSEGV, nullptr, &saved_segv_action); - - // Reset state - handler_call_count = 0; - original_handler_called = false; - should_escape = false; - memset(&other_lib_saved_oldact, 0, sizeof(other_lib_saved_oldact)); - } - - void TearDown() override { - // Restore original SIGSEGV handler - sigaction(SIGSEGV, &saved_segv_action, nullptr); - // Reset static interception state so tests don't bleed into each other - OS::resetSignalHandlersForTesting(); - } -}; - -/** - * Test that sigaction interception returns the correct oldact. - * - * Setup: - * 1. Install "original" handler (simulating JVM) - * 2. Call protectSignalHandlers with "our" handler - * 3. Install "our" handler - * 4. "Other library" calls sigaction (via the hook) to install its handler - * 5. Verify that oldact returned to "other library" is the original handler, not ours - * - * This ensures that when "other library" chains to oldact, it goes to the original - * handler, not back to us (which would cause infinite loop). - * - * NOTE: We call the sigaction hook directly because in a standalone test binary, - * GOT patching isn't active. This tests the core logic of the hook function. - */ -TEST_F(SigactionInterceptionTest, OldactPointsToOriginalHandler) { - // Get the sigaction hook function - sigaction_hook_t hook = (sigaction_hook_t)OS::getSigactionHook(); -#ifdef __APPLE__ - // Sigaction interception is only implemented on Linux - if (hook == nullptr) { - GTEST_SKIP() << "Sigaction interception not implemented on macOS"; - } -#endif - ASSERT_NE(hook, nullptr) << "getSigactionHook returned nullptr"; - - // Step 1: Install "original" handler (simulating JVM) - struct sigaction original_sa; - memset(&original_sa, 0, sizeof(original_sa)); - original_sa.sa_sigaction = original_handler; - original_sa.sa_flags = SA_SIGINFO; - sigemptyset(&original_sa.sa_mask); - sigaction(SIGSEGV, &original_sa, nullptr); - - // Step 2 & 3: Protect and install our handler - // Note: protectSignalHandlers should save the current (original) handler - OS::protectSignalHandlers(our_handler, nullptr); - OS::replaceSigsegvHandler(our_handler); - - // Step 4: "Other library" calls sigaction via the hook to install its handler - struct sigaction other_lib_sa; - memset(&other_lib_sa, 0, sizeof(other_lib_sa)); - other_lib_sa.sa_sigaction = other_lib_handler; - other_lib_sa.sa_flags = SA_SIGINFO; - sigemptyset(&other_lib_sa.sa_mask); - - // Call the hook directly (simulates what GOT patching does in production) - int result = hook(SIGSEGV, &other_lib_sa, &other_lib_saved_oldact); - ASSERT_EQ(result, 0); - - // Step 5: Verify oldact - // The oldact should point to original_handler, NOT to our_handler - // If it points to our_handler, chaining will cause infinite loop - - // Check that oldact is not our handler - if (other_lib_saved_oldact.sa_flags & SA_SIGINFO) { - EXPECT_NE(other_lib_saved_oldact.sa_sigaction, our_handler) - << "oldact points to our handler - this would cause infinite loop!"; - - // It should be the original handler - EXPECT_EQ(other_lib_saved_oldact.sa_sigaction, original_handler) - << "oldact should be the original (JVM's) handler"; - } else { - // If not SA_SIGINFO, check sa_handler - EXPECT_NE(other_lib_saved_oldact.sa_handler, (void (*)(int))our_handler) - << "oldact points to our handler - this would cause infinite loop!"; - } -} - -/** - * Test that signal chaining doesn't cause infinite loop. - * - * This test actually triggers a SIGSEGV and verifies the chain doesn't loop forever. - * The chain should be: our_handler -> other_lib_handler -> original_handler - * - * NOTE: We use the hook directly to set up the interception since GOT patching - * isn't active in the standalone test binary. - */ -TEST_F(SigactionInterceptionTest, NoInfiniteLoopOnChaining) { - // Get the sigaction hook function - sigaction_hook_t hook = (sigaction_hook_t)OS::getSigactionHook(); -#ifdef __APPLE__ - // Sigaction interception is only implemented on Linux - if (hook == nullptr) { - GTEST_SKIP() << "Sigaction interception not implemented on macOS"; - } -#endif - ASSERT_NE(hook, nullptr) << "getSigactionHook returned nullptr"; - - // Setup: Install original handler (simulating JVM) - struct sigaction original_sa; - memset(&original_sa, 0, sizeof(original_sa)); - original_sa.sa_sigaction = original_handler; - original_sa.sa_flags = SA_SIGINFO; - sigemptyset(&original_sa.sa_mask); - sigaction(SIGSEGV, &original_sa, nullptr); - - // Protect and install our handler - OS::protectSignalHandlers(our_handler, nullptr); - OS::replaceSigsegvHandler(our_handler); - - // "Other library" installs its handler via the hook - struct sigaction other_lib_sa; - memset(&other_lib_sa, 0, sizeof(other_lib_sa)); - other_lib_sa.sa_sigaction = other_lib_handler; - other_lib_sa.sa_flags = SA_SIGINFO; - sigemptyset(&other_lib_sa.sa_mask); - hook(SIGSEGV, &other_lib_sa, &other_lib_saved_oldact); - - // Now trigger a SIGSEGV and see what happens - should_escape = true; - - if (sigsetjmp(escape_jmp, 1) == 0) { - // Trigger SIGSEGV by accessing null pointer - volatile int* p = nullptr; - *p = 42; // This will trigger SIGSEGV - - // Should not reach here - FAIL() << "SIGSEGV was not triggered"; - } else { - // We escaped via siglongjmp - // Check that we didn't loop too many times - EXPECT_LE(handler_call_count.load(), MAX_HANDLER_CALLS) - << "Handler was called too many times - possible infinite loop!"; - - // Ideally, the chain should be: our_handler(1) -> other_lib(2) -> original(3) - // So we expect around 3 calls, definitely less than MAX_HANDLER_CALLS - EXPECT_LE(handler_call_count.load(), 5) - << "Handler chain is longer than expected"; - - // Verify the original handler was actually called (chain completed) - EXPECT_TRUE(original_handler_called) - << "Original handler was not called - chain may not be set up correctly"; - } -} diff --git a/ddprof-lib/src/test/cpp/signalOrigin_bench.cpp b/ddprof-lib/src/test/cpp/signalOrigin_bench.cpp deleted file mode 100644 index 0c6f25bf9..000000000 --- a/ddprof-lib/src/test/cpp/signalOrigin_bench.cpp +++ /dev/null @@ -1,376 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - * - * Microbenchmark for foreign-signal forwarding overhead. - * - * Quantifies the per-signal cost added by the origin-validation gate when - * the kernel delivers a signal that does NOT carry our cookie (so the - * handler rejects it, increments CTIMER_SIGNAL_FOREIGN, and chains via - * forwardForeignSignal). - * - * Three scenarios are measured: - * BASELINE — only the "foreign" handler is installed; raise() goes - * straight to it. Measures the signal-delivery baseline. - * FAST_PATH — our classifier is installed on top; prev handler has - * an empty sa_mask (common case). forwardForeignSignal - * skips rt_sigprocmask syscalls. - * SLOW_PATH — prev handler has a non-empty sa_mask AND - * DDPROF_FORWARD_APPLY_SIGMASK=1 is set (opt-in). - * forwardForeignSignal applies SIG_BLOCK / SIG_SETMASK - * (two syscalls). - * - * The delta between BASELINE and FAST_PATH is the overhead per foreign - * signal when sa_mask chaining is disabled (the default). The delta between - * FAST_PATH and SLOW_PATH is the cost of the two rt_sigprocmask syscalls. - * - * The benchmark runs only when the env var BENCH_SIGNAL_ORIGIN is set, so - * normal CI does not pay for it. - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "os.h" -#include "signalCookie.h" - -#ifdef __linux__ - -namespace { - -// "Foreign" handler — the target of forwardForeignSignal. Kept as lean as -// possible so the timing reflects the forwarding-path cost, not the prev -// handler's own work. -std::atomic g_prev_calls{0}; - -void noopForeignHandler(int /*signo*/, siginfo_t* /*si*/, void* /*uc*/) { - g_prev_calls.fetch_add(1, std::memory_order_relaxed); -} - -int setupForeignHandler(int signo, bool with_mask, struct sigaction* saved_out) { - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_sigaction = noopForeignHandler; - sa.sa_flags = SA_SIGINFO | SA_RESTART; - sigemptyset(&sa.sa_mask); - if (with_mask) { - // Populate with a few signals to force forwardForeignSignal onto the - // slow path (non-empty sa_mask → rt_sigprocmask block + restore). - sigaddset(&sa.sa_mask, SIGUSR2); - sigaddset(&sa.sa_mask, SIGALRM); - sigaddset(&sa.sa_mask, SIGCHLD); - } - return sigaction(signo, &sa, saved_out); -} - -uint64_t nanosNow() { - struct timespec ts; - clock_gettime(CLOCK_MONOTONIC_RAW, &ts); - return (uint64_t)ts.tv_sec * 1000000000ULL + (uint64_t)ts.tv_nsec; -} - -struct Result { - double ns_per_signal; - uint64_t iterations; - uint64_t prev_calls; -}; - -Result measure(int iters, int signo) { - g_prev_calls.store(0); - uint64_t start = nanosNow(); - for (int i = 0; i < iters; ++i) { - raise(signo); - } - uint64_t end = nanosNow(); - return { - (double)(end - start) / (double)iters, - (uint64_t)iters, - g_prev_calls.load() - }; -} - -const int kIterations = 500000; -const int kBenchSignal = SIGUSR1; - -bool benchEnabled() { - const char* v = getenv("BENCH_SIGNAL_ORIGIN"); - return v != nullptr && v[0] != '\0' && strcmp(v, "0") != 0; -} - -// Snapshot/restore an env var — same pattern as signalOrigin_ut.cpp, so the -// developer's own shell exports survive a test run. -class EnvGuard { -public: - explicit EnvGuard(const char* name) - : _name(name), _had_value(false) { - const char* v = getenv(name); - if (v != nullptr) { _had_value = true; _saved = v; } - } - void reset() const { - if (_had_value) setenv(_name, _saved.c_str(), 1); - else unsetenv(_name); - } -private: - const char* _name; - bool _had_value; - std::string _saved; -}; - -// Optimization barrier — prevents the compiler from hoisting / -// constant-folding loop-invariant calls out of the measurement loop. A -// simple `volatile bool sink` forces the STORE but not necessarily the CALL; -// asm volatile with a read-write constraint makes the value `observable` at -// every iteration so the compiler must re-emit the load path. -inline void doNotOptimize(bool& v) { - asm volatile("" : "+r"(v) : : "memory"); -} - -} // namespace - -class SignalOriginBench : public ::testing::Test { -protected: - struct sigaction saved_action; - EnvGuard _guard_mask{"DDPROF_FORWARD_APPLY_SIGMASK"}; - EnvGuard _guard_origin{"DDPROF_SIGNAL_ORIGIN_CHECK"}; - - void SetUp() override { - if (!benchEnabled()) { - GTEST_SKIP() << "Set BENCH_SIGNAL_ORIGIN=1 to run the benchmark"; - } - memset(&saved_action, 0, sizeof(saved_action)); - // Default prime — slow-path tests below override and re-prime. - unsetenv("DDPROF_FORWARD_APPLY_SIGMASK"); - unsetenv("DDPROF_SIGNAL_ORIGIN_CHECK"); - OS::primeSignalOriginCheck(/*forceReload=*/true); - OS::resetSignalHandlersForTesting(); - } - - void TearDown() override { - OS::resetSignalHandlersForTesting(); - if (saved_action.sa_handler != nullptr || saved_action.sa_sigaction != nullptr) { - sigaction(kBenchSignal, &saved_action, nullptr); - } - _guard_mask.reset(); - _guard_origin.reset(); - OS::primeSignalOriginCheck(/*forceReload=*/true); - } - - // Enable sa_mask-respecting chain for the slow-path benchmark variants. - void enableSigmaskChain() { - setenv("DDPROF_FORWARD_APPLY_SIGMASK", "1", 1); - OS::primeSignalOriginCheck(/*forceReload=*/true); - } -}; - -// -------- BASELINE: just the foreign handler, no ddprof classifier -------- - -TEST_F(SignalOriginBench, Baseline_NoClassifier) { - ASSERT_EQ(0, setupForeignHandler(kBenchSignal, /*with_mask=*/false, &saved_action)); - - // Warm up. - for (int i = 0; i < 1000; ++i) raise(kBenchSignal); - - Result r = measure(kIterations, kBenchSignal); - EXPECT_EQ((uint64_t)kIterations, r.prev_calls); - - std::printf("\n [BASELINE] %.1f ns/signal (iters=%llu prev_calls=%llu)\n", - r.ns_per_signal, - (unsigned long long)r.iterations, - (unsigned long long)r.prev_calls); -} - -// -------- FAST PATH: classifier rejects SI_TKILL from raise(), forwards -------- - -static void classifierHandler(int signo, siginfo_t* siginfo, void* ucontext) { - if (!OS::shouldProcessSignal(siginfo, SI_TIMER, SignalCookie::cpu())) { - OS::forwardForeignSignal(signo, siginfo, ucontext); - return; - } - // Unreachable in this benchmark — raise() produces si_code=SI_TKILL on - // Linux (glibc raise() uses tgkill internally), never SI_TIMER. -} - -TEST_F(SignalOriginBench, FastPath_ClassifierPlusEmptyMaskForward) { - // Pre-install the foreign handler so OS::installSignalHandler captures it - // as the oldaction to chain to. - ASSERT_EQ(0, setupForeignHandler(kBenchSignal, /*with_mask=*/false, &saved_action)); - - OS::installSignalHandler(kBenchSignal, classifierHandler); - - for (int i = 0; i < 1000; ++i) raise(kBenchSignal); - - Result r = measure(kIterations, kBenchSignal); - EXPECT_EQ((uint64_t)kIterations, r.prev_calls) << "forwardForeignSignal did not chain"; - - std::printf("\n [FAST_PATH] %.1f ns/signal (iters=%llu prev_calls=%llu)\n", - r.ns_per_signal, - (unsigned long long)r.iterations, - (unsigned long long)r.prev_calls); -} - -// -------- SLOW PATH: classifier rejects, prev handler has non-empty sa_mask -------- - -TEST_F(SignalOriginBench, SlowPath_ClassifierPlusMaskedForward) { - // Slow-path chain is opt-in behind DDPROF_FORWARD_APPLY_SIGMASK. - enableSigmaskChain(); - // Install a "foreign" handler with a non-empty sa_mask so - // forwardForeignSignal has to invoke two rt_sigprocmask syscalls. - ASSERT_EQ(0, setupForeignHandler(kBenchSignal, /*with_mask=*/true, &saved_action)); - - OS::installSignalHandler(kBenchSignal, classifierHandler); - - for (int i = 0; i < 1000; ++i) raise(kBenchSignal); - - Result r = measure(kIterations, kBenchSignal); - EXPECT_EQ((uint64_t)kIterations, r.prev_calls); - - std::printf("\n [SLOW_PATH] %.1f ns/signal (iters=%llu prev_calls=%llu)\n", - r.ns_per_signal, - (unsigned long long)r.iterations, - (unsigned long long)r.prev_calls); -} - -// -------- PURE FUNCTION CALL: no kernel, no signal delivery -------- -// -// Calls OS::shouldProcessSignal + OS::forwardForeignSignal directly with a -// fabricated siginfo_t, measuring only the in-process overhead of the -// origin-check machinery. This isolates the cost we actually added — the -// scenarios above include a constant ~300 ns per iteration for the kernel -// signal-delivery path that is common to ALL three end-to-end variants. -// -// All measurement loops pass the `sink` through `doNotOptimize()` so the -// compiler is forced to re-emit the call on every iteration instead of -// CSE-ing a loop-invariant result. -// NOTE: This file is compiled at -O0 in the test build, so reported ns/call -// include function-call overhead for SignalCookie::cpu() and shouldProcessSignal -// that disappears at -O2 in production (both inline to 3-4 instructions). -// Production overhead is substantially lower than the figures printed here. - -static const int kPureBenchSignal = SIGUSR1; -static const int kPureIterations = 5000000; // more iters — fast path is ~ns - -namespace { - -siginfo_t makeForeignSiginfo() { - siginfo_t si; - memset(&si, 0, sizeof(si)); - si.si_signo = kPureBenchSignal; - si.si_code = SI_USER; // not SI_TIMER → classifier rejects - return si; -} - -siginfo_t makeOwnSiginfo() { - siginfo_t si; - memset(&si, 0, sizeof(si)); - si.si_signo = kPureBenchSignal; - si.si_code = SI_TIMER; - si.si_value.sival_ptr = SignalCookie::cpu(); - return si; -} - -} // namespace - -TEST_F(SignalOriginBench, Pure_ClassifierReject) { - siginfo_t si = makeForeignSiginfo(); - bool sink = false; - - // Warm up. - for (int i = 0; i < 10000; ++i) { - sink = OS::shouldProcessSignal(&si, SI_TIMER, SignalCookie::cpu()); - doNotOptimize(sink); - } - - uint64_t start = nanosNow(); - for (int i = 0; i < kPureIterations; ++i) { - sink = OS::shouldProcessSignal(&si, SI_TIMER, SignalCookie::cpu()); - doNotOptimize(sink); - } - uint64_t end = nanosNow(); - double ns = (double)(end - start) / (double)kPureIterations; - - std::printf("\n [pure_classifier_reject] %.2f ns/call (iters=%d)\n", - ns, kPureIterations); -} - -TEST_F(SignalOriginBench, Pure_ClassifierAccept) { - siginfo_t si = makeOwnSiginfo(); - bool sink = false; - - for (int i = 0; i < 10000; ++i) { - sink = OS::shouldProcessSignal(&si, SI_TIMER, SignalCookie::cpu()); - doNotOptimize(sink); - } - - uint64_t start = nanosNow(); - for (int i = 0; i < kPureIterations; ++i) { - sink = OS::shouldProcessSignal(&si, SI_TIMER, SignalCookie::cpu()); - doNotOptimize(sink); - } - uint64_t end = nanosNow(); - double ns = (double)(end - start) / (double)kPureIterations; - - std::printf("\n [pure_classifier_accept] %.2f ns/call (iters=%d)\n", - ns, kPureIterations); -} - -TEST_F(SignalOriginBench, Pure_ForwardFastPath) { - // Install an empty-mask foreign handler so forwardForeignSignal captures - // it as the oldaction via OS::installSignalHandler. - ASSERT_EQ(0, setupForeignHandler(kPureBenchSignal, /*with_mask=*/false, &saved_action)); - OS::installSignalHandler(kPureBenchSignal, classifierHandler); - - siginfo_t si = makeForeignSiginfo(); - g_prev_calls.store(0); - - for (int i = 0; i < 10000; ++i) { - OS::forwardForeignSignal(kPureBenchSignal, &si, nullptr); - } - - uint64_t start = nanosNow(); - for (int i = 0; i < kPureIterations; ++i) { - OS::forwardForeignSignal(kPureBenchSignal, &si, nullptr); - } - uint64_t end = nanosNow(); - double ns = (double)(end - start) / (double)kPureIterations; - uint64_t chained = g_prev_calls.load(); - - std::printf("\n [pure_forward_fast] %.2f ns/call (iters=%d chained=%llu)\n", - ns, kPureIterations, (unsigned long long)chained); - EXPECT_GE(chained, (uint64_t)kPureIterations); -} - -TEST_F(SignalOriginBench, Pure_ForwardSlowPath) { - enableSigmaskChain(); - // Non-empty mask → rt_sigprocmask twice per call. - ASSERT_EQ(0, setupForeignHandler(kPureBenchSignal, /*with_mask=*/true, &saved_action)); - OS::installSignalHandler(kPureBenchSignal, classifierHandler); - - siginfo_t si = makeForeignSiginfo(); - g_prev_calls.store(0); - - for (int i = 0; i < 10000; ++i) { - OS::forwardForeignSignal(kPureBenchSignal, &si, nullptr); - } - - // Fewer iterations — this path has 2 syscalls per call. - const int slow_iters = kPureIterations / 10; - uint64_t start = nanosNow(); - for (int i = 0; i < slow_iters; ++i) { - OS::forwardForeignSignal(kPureBenchSignal, &si, nullptr); - } - uint64_t end = nanosNow(); - double ns = (double)(end - start) / (double)slow_iters; - uint64_t chained = g_prev_calls.load(); - - std::printf("\n [pure_forward_slow] %.2f ns/call (iters=%d chained=%llu)\n", - ns, slow_iters, (unsigned long long)chained); - EXPECT_GE(chained, (uint64_t)slow_iters); -} - -#endif // __linux__ diff --git a/ddprof-lib/src/test/cpp/signalOrigin_ut.cpp b/ddprof-lib/src/test/cpp/signalOrigin_ut.cpp deleted file mode 100644 index 68a55245c..000000000 --- a/ddprof-lib/src/test/cpp/signalOrigin_ut.cpp +++ /dev/null @@ -1,417 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "guards.h" -#include "os.h" -#include "signalCookie.h" -#include "thread.h" - -#ifdef __linux__ - -/** - * Unit tests for the signal-origin validation helpers: - * OS::shouldProcessSignal(siginfo, expected_si_code, expected_cookie) - * OS::sendSignalWithCookie(tid, signo, cookie) - * OS::forwardForeignSignal(signo, siginfo, ucontext) - * OS::primeSignalOriginCheck(forceReload) - * OS::signalOriginCheckEnabled() - * - * These do not rely on timer_create or the full engine — they build siginfo_t - * fakes directly and exercise the classifier / forwarder in isolation. - */ - -namespace { - -// Snapshot an env var's value at SetUp and restore it at TearDown, so a -// developer who exported DDPROF_* in their shell to reproduce an issue does -// not have their environment wiped by running the test binary. -class EnvGuard { -public: - explicit EnvGuard(const char* name) - : _name(name), _had_value(false) { - const char* v = getenv(name); - if (v != nullptr) { - _had_value = true; - _saved = v; - } - } - void reset() const { - if (_had_value) { - setenv(_name, _saved.c_str(), /*overwrite=*/1); - } else { - unsetenv(_name); - } - } -private: - const char* _name; - bool _had_value; - std::string _saved; -}; - -} // namespace - -class SignalOriginTest : public ::testing::Test { -protected: - // Snapshot/restore env vars the tests mutate so the developer's shell - // exports survive a test run. - EnvGuard _guard_origin{"DDPROF_SIGNAL_ORIGIN_CHECK"}; - EnvGuard _guard_mask{"DDPROF_FORWARD_APPLY_SIGMASK"}; - - void SetUp() override { - // Default: origin check enabled, sigmask chain disabled. - unsetenv("DDPROF_SIGNAL_ORIGIN_CHECK"); - unsetenv("DDPROF_FORWARD_APPLY_SIGMASK"); - OS::primeSignalOriginCheck(/*forceReload=*/true); - - // Wipe any installed_oldaction state from previous tests so chaining - // invariants can be observed in isolation. - OS::resetSignalHandlersForTesting(); - } - - void TearDown() override { - OS::resetSignalHandlersForTesting(); - _guard_origin.reset(); - _guard_mask.reset(); - OS::primeSignalOriginCheck(/*forceReload=*/true); - } - - static siginfo_t makeSiginfo(int code, void* sival) { - siginfo_t si; - memset(&si, 0, sizeof(si)); - si.si_code = code; - si.si_value.sival_ptr = sival; - return si; - } -}; - -TEST_F(SignalOriginTest, AcceptsMatchingCookieAndSiCode) { - siginfo_t si = makeSiginfo(SI_TIMER, SignalCookie::cpu()); - EXPECT_TRUE(OS::shouldProcessSignal(&si, SI_TIMER, SignalCookie::cpu())); -} - -TEST_F(SignalOriginTest, RejectsMismatchedSiCode) { - siginfo_t si = makeSiginfo(SI_USER, SignalCookie::cpu()); - EXPECT_FALSE(OS::shouldProcessSignal(&si, SI_TIMER, SignalCookie::cpu())); -} - -TEST_F(SignalOriginTest, RejectsMismatchedCookie) { - siginfo_t si = makeSiginfo(SI_TIMER, /*foreign=*/(void*)0xF00D); - EXPECT_FALSE(OS::shouldProcessSignal(&si, SI_TIMER, SignalCookie::cpu())); -} - -TEST_F(SignalOriginTest, RejectsNullSiginfo) { - EXPECT_FALSE(OS::shouldProcessSignal(nullptr, SI_TIMER, SignalCookie::cpu())); -} - -TEST_F(SignalOriginTest, WallclockCookieDistinctFromCpu) { - EXPECT_NE(SignalCookie::cpu(), SignalCookie::wallclock()); - siginfo_t si = makeSiginfo(SI_QUEUE, SignalCookie::cpu()); - // CPU cookie with SI_QUEUE / wallclock cookie must not be accepted. - EXPECT_FALSE(OS::shouldProcessSignal(&si, SI_QUEUE, SignalCookie::wallclock())); -} - -TEST_F(SignalOriginTest, FeatureFlagDisableAcceptsEverything) { - setenv("DDPROF_SIGNAL_ORIGIN_CHECK", "false", /*overwrite=*/1); - OS::primeSignalOriginCheck(/*forceReload=*/true); - EXPECT_FALSE(OS::signalOriginCheckEnabled()); - - // Any siginfo should be accepted when the flag is off. - siginfo_t si = makeSiginfo(SI_USER, (void*)0xBADBAD); - EXPECT_TRUE(OS::shouldProcessSignal(&si, SI_TIMER, SignalCookie::cpu())); -} - -TEST_F(SignalOriginTest, FeatureFlagAcceptsCommonDisableSpellings) { - for (const char* v : {"false", "0", "off", "no", "OFF", "No"}) { - setenv("DDPROF_SIGNAL_ORIGIN_CHECK", v, 1); - OS::primeSignalOriginCheck(/*forceReload=*/true); - EXPECT_FALSE(OS::signalOriginCheckEnabled()) << "for value " << v; - } -} - -TEST_F(SignalOriginTest, FeatureFlagAcceptsCommonEnableSpellings) { - for (const char* v : {"true", "1", "on", "yes", "ON", "Yes"}) { - setenv("DDPROF_SIGNAL_ORIGIN_CHECK", v, 1); - OS::primeSignalOriginCheck(/*forceReload=*/true); - EXPECT_TRUE(OS::signalOriginCheckEnabled()) << "for value " << v; - } -} - -TEST_F(SignalOriginTest, FeatureFlagDefaultOn) { - unsetenv("DDPROF_SIGNAL_ORIGIN_CHECK"); - OS::primeSignalOriginCheck(/*forceReload=*/true); - EXPECT_TRUE(OS::signalOriginCheckEnabled()); -} - -TEST_F(SignalOriginTest, FeatureFlagUnknownValueKeepsDefault) { - // Unknown values should not disable the origin check (default ON). - // A warning is emitted via Log::warn — not asserted here, just sanity. - for (const char* v : {"disable", "maybe", "2", " "}) { - setenv("DDPROF_SIGNAL_ORIGIN_CHECK", v, 1); - OS::primeSignalOriginCheck(/*forceReload=*/true); - EXPECT_TRUE(OS::signalOriginCheckEnabled()) - << "unknown value " << v << " should keep default"; - } -} - -// -------- sendSignalWithCookie + receive on SIGUSR2 -------- - -static std::atomic g_received_si_code{0}; -static std::atomic g_received_sival{nullptr}; - -static void captureHandler(int /*signo*/, siginfo_t* si, void* /*uc*/) { - g_received_si_code.store(si != nullptr ? si->si_code : 0); - g_received_sival.store(si != nullptr ? si->si_value.sival_ptr : nullptr); -} - -TEST_F(SignalOriginTest, QueueSignalDeliversCookieAndSiCode) { - struct sigaction prev; - struct sigaction sa; - memset(&sa, 0, sizeof(sa)); - sa.sa_sigaction = captureHandler; - sa.sa_flags = SA_SIGINFO; - sigemptyset(&sa.sa_mask); - ASSERT_EQ(0, sigaction(SIGUSR2, &sa, &prev)); - - g_received_si_code.store(0); - g_received_sival.store(nullptr); - - // Block SIGUSR2 so the pending signal is delivered deterministically - // when we unblock — avoids a racy spin-wait and guarantees the handler - // has run before the EXPECT checks below. - sigset_t block_mask, old_mask; - sigemptyset(&block_mask); - sigaddset(&block_mask, SIGUSR2); - ASSERT_EQ(0, pthread_sigmask(SIG_BLOCK, &block_mask, &old_mask)); - - void* cookie = (void*)0xCAFEBABE; - ASSERT_TRUE(OS::sendSignalWithCookie(OS::threadId(), SIGUSR2, cookie)); - - // Restoring the old mask unblocks SIGUSR2; the kernel delivers the pending - // signal before pthread_sigmask returns. - ASSERT_EQ(0, pthread_sigmask(SIG_SETMASK, &old_mask, nullptr)); - - EXPECT_EQ(SI_QUEUE, g_received_si_code.load()); - EXPECT_EQ(cookie, g_received_sival.load()); - - sigaction(SIGUSR2, &prev, nullptr); -} - -// -------- forwardForeignSignal chains to previous handler -------- - -static std::atomic g_chained_calls{0}; - -static void chainedHandler(int /*signo*/, siginfo_t* /*si*/, void* /*uc*/) { - g_chained_calls.fetch_add(1); -} - -static void secondChainedHandler(int /*signo*/, siginfo_t* /*si*/, void* /*uc*/) { - // Distinct from chainedHandler so installSignalHandler will not detect - // this as a self-install on a second call. - g_chained_calls.fetch_add(100); -} - -TEST_F(SignalOriginTest, ForwardForeignSignalChainsToPrevious) { - // Install a "previous" handler on SIGUSR1 using raw sigaction (not via - // OS::installSignalHandler — we want this to be the "pre-existing" state). - struct sigaction saved; - struct sigaction prev_action; - memset(&prev_action, 0, sizeof(prev_action)); - prev_action.sa_sigaction = chainedHandler; - prev_action.sa_flags = SA_SIGINFO; - sigemptyset(&prev_action.sa_mask); - ASSERT_EQ(0, sigaction(SIGUSR1, &prev_action, &saved)); - - // Now install OUR handler via OS::installSignalHandler — this should - // capture the previous action for forwarding. - OS::installSignalHandler(SIGUSR1, captureHandler); - - g_chained_calls.store(0); - - // Build a fake siginfo and call forwardForeignSignal directly. - siginfo_t si = makeSiginfo(SI_USER, (void*)0); - si.si_signo = SIGUSR1; - OS::forwardForeignSignal(SIGUSR1, &si, nullptr); - - EXPECT_EQ(1, g_chained_calls.load()); - - sigaction(SIGUSR1, &saved, nullptr); -} - -TEST_F(SignalOriginTest, ForwardForeignSignalSilentOnUninstalledSignal) { - // No previous handler has been captured for SIGUSR2 by OS::installSignalHandler. - // forwardForeignSignal should be a no-op rather than crash. - siginfo_t si = makeSiginfo(SI_USER, (void*)0); - OS::forwardForeignSignal(SIGUSR2, &si, nullptr); - SUCCEED(); -} - -// -------- store-exactly-once invariant -------- -// -// Regression test for the "installed_oldaction is captured once, never -// overwritten" invariant. If a foreign library installs its handler over -// ours after profiler start, and the profiler then re-installs (restart), -// the captured prev must still be the ORIGINAL handler, not the foreign -// one that briefly owned the slot. - -TEST_F(SignalOriginTest, ReinstallPreservesFirstCapturedPrev) { - struct sigaction saved; - sigaction(SIGUSR1, nullptr, &saved); - - // Install the "original" (e.g. JVM-like) handler. - struct sigaction original; - memset(&original, 0, sizeof(original)); - original.sa_sigaction = chainedHandler; - original.sa_flags = SA_SIGINFO; - sigemptyset(&original.sa_mask); - ASSERT_EQ(0, sigaction(SIGUSR1, &original, nullptr)); - - // First profiler install — captures original as prev. - OS::installSignalHandler(SIGUSR1, captureHandler); - - // A foreign library sneaks in and overwrites our handler. - struct sigaction foreign; - memset(&foreign, 0, sizeof(foreign)); - foreign.sa_sigaction = secondChainedHandler; - foreign.sa_flags = SA_SIGINFO; - sigemptyset(&foreign.sa_mask); - ASSERT_EQ(0, sigaction(SIGUSR1, &foreign, nullptr)); - - // Profiler re-installs (simulates restart). Before the fix, this would - // have overwritten our captured prev with `foreign` — losing the - // original chain target. With store-exactly-once, the original is kept. - OS::installSignalHandler(SIGUSR1, captureHandler); - - g_chained_calls.store(0); - siginfo_t si = makeSiginfo(SI_USER, (void*)0); - si.si_signo = SIGUSR1; - OS::forwardForeignSignal(SIGUSR1, &si, nullptr); - - // Expect the ORIGINAL chainedHandler (+1), not secondChainedHandler (+100). - EXPECT_EQ(1, g_chained_calls.load()) - << "forwardForeignSignal chained to the wrong handler — " - "store-exactly-once invariant violated"; - - sigaction(SIGUSR1, &saved, nullptr); -} - -// -------- resetSignalHandlersForTesting clears oldaction state -------- - -TEST_F(SignalOriginTest, ResetForTestingClearsOldactionCache) { - struct sigaction saved; - struct sigaction prev_action; - memset(&prev_action, 0, sizeof(prev_action)); - prev_action.sa_sigaction = chainedHandler; - prev_action.sa_flags = SA_SIGINFO; - sigemptyset(&prev_action.sa_mask); - ASSERT_EQ(0, sigaction(SIGUSR1, &prev_action, &saved)); - - OS::installSignalHandler(SIGUSR1, captureHandler); - - // Confirm chaining is armed. - g_chained_calls.store(0); - siginfo_t si = makeSiginfo(SI_USER, (void*)0); - si.si_signo = SIGUSR1; - OS::forwardForeignSignal(SIGUSR1, &si, nullptr); - EXPECT_EQ(1, g_chained_calls.load()); - - // Reset and confirm the chain is cleared (forward becomes a no-op). - OS::resetSignalHandlersForTesting(); - g_chained_calls.store(0); - OS::forwardForeignSignal(SIGUSR1, &si, nullptr); - EXPECT_EQ(0, g_chained_calls.load()) - << "resetSignalHandlersForTesting did not clear installed_oldaction state"; - - sigaction(SIGUSR1, &saved, nullptr); -} - -// -------- WallclockGuardContract: predicate contract for WallClockJvmti guard -------- -// -// WallClockJvmti::sharedSignalHandler gates on -// OS::shouldProcessSignal(siginfo, SI_QUEUE, SignalCookie::wallclock()) -// These cases cover the three branches the guard must handle: own send, -// bare tgkill/kill (no cookie), and a queued signal with a foreign cookie. - -TEST_F(SignalOriginTest, WallclockGuardContract_OwnSignalAccepted) { - siginfo_t si = makeSiginfo(SI_QUEUE, SignalCookie::wallclock()); - EXPECT_TRUE(OS::shouldProcessSignal(&si, SI_QUEUE, SignalCookie::wallclock())); -} - -TEST_F(SignalOriginTest, WallclockGuardContract_BareKillRejected) { - // tgkill/pthread_kill delivers SI_TKILL; kill/raise delivers SI_USER. - for (int code : {SI_TKILL, SI_USER}) { - siginfo_t si = makeSiginfo(code, nullptr); - EXPECT_FALSE(OS::shouldProcessSignal(&si, SI_QUEUE, SignalCookie::wallclock())) - << "bare signal with si_code " << code << " must be rejected"; - } -} - -TEST_F(SignalOriginTest, WallclockGuardContract_ForeignCookieRejected) { - siginfo_t si = makeSiginfo(SI_QUEUE, (void*)0xF00D); - EXPECT_FALSE(OS::shouldProcessSignal(&si, SI_QUEUE, SignalCookie::wallclock())); -} - -// Regression test for the fix: when a foreign signal is handled, -// SIGNAL_HANDLER_GUARD_RELEASE() must be called before forwardForeignSignal so -// that a chained handler escaping via siglongjmp cannot leave _signal_depth -// permanently incremented. Verifies the SIGNAL_HANDLER_GUARD / release -// contract directly: depth is 0 after an early release, and the destructor is -// a no-op. -TEST_F(SignalOriginTest, WallclockGuardContract_ForeignSignalReleasesGuard) { - ProfiledThread::initCurrentThread(); - EXPECT_EQ(0, getInSignalDepth()); - { - SIGNAL_HANDLER_GUARD(); - EXPECT_EQ(1, getInSignalDepth()); - // Mirrors the fix: release before forwarding so a non-returning - // chained handler cannot leave depth > 0. - SIGNAL_HANDLER_GUARD_RELEASE(); - EXPECT_EQ(0, getInSignalDepth()); - // Destructor runs here; must not double-decrement. - } - EXPECT_EQ(0, getInSignalDepth()); - ProfiledThread::release(); -} - -TEST_F(SignalOriginTest, ForwardAppliesSigmaskWhenEnabled) { - // Verify that the slow path (DDPROF_FORWARD_APPLY_SIGMASK=1) still - // chains to the previous SA_SIGINFO handler correctly. - EnvGuard guard_mask("DDPROF_FORWARD_APPLY_SIGMASK"); - setenv("DDPROF_FORWARD_APPLY_SIGMASK", "1", /*overwrite=*/1); - OS::primeSignalOriginCheck(/*forceReload=*/true); - - // Install a previous handler with a non-empty sa_mask. - struct sigaction prev_sa, saved; - memset(&prev_sa, 0, sizeof(prev_sa)); - prev_sa.sa_sigaction = chainedHandler; - prev_sa.sa_flags = SA_SIGINFO; - sigemptyset(&prev_sa.sa_mask); - sigaddset(&prev_sa.sa_mask, SIGUSR1); // non-empty sa_mask - ASSERT_EQ(0, sigaction(SIGUSR1, &prev_sa, &saved)); - - OS::installSignalHandler(SIGUSR1, captureHandler); - - g_chained_calls.store(0); - siginfo_t si = makeSiginfo(SI_USER, (void*)0); - si.si_signo = SIGUSR1; - OS::forwardForeignSignal(SIGUSR1, &si, nullptr); - - EXPECT_EQ(1, g_chained_calls.load()) - << "forwardForeignSignal with DDPROF_FORWARD_APPLY_SIGMASK=1 did not chain"; - - sigaction(SIGUSR1, &saved, nullptr); - guard_mask.reset(); - OS::primeSignalOriginCheck(/*forceReload=*/true); -} - -#endif // __linux__ diff --git a/ddprof-lib/src/test/cpp/signalSafety_ut.cpp b/ddprof-lib/src/test/cpp/signalSafety_ut.cpp deleted file mode 100644 index a451e7439..000000000 --- a/ddprof-lib/src/test/cpp/signalSafety_ut.cpp +++ /dev/null @@ -1,144 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include "signalSafety.h" -#include "thread.h" -#include - -class SignalSafetyTest : public ::testing::Test { -protected: - void SetUp() override { - // SignalHandlerScope reads/writes ProfiledThread::_signal_depth — the - // tests need a thread context to exist on the gtest thread, otherwise - // every scope is a no-op (which is the intended production behavior - // on uninstrumented threads, but not what these unit tests assert). - ProfiledThread::initCurrentThread(); - } - - void TearDown() override { - // Catch leaked depth from a failed test so subsequent tests start clean. - ASSERT_EQ(0, getInSignalDepth()) << "depth not zero after test — check for leaked SignalHandlerScope"; - ProfiledThread::release(); - } -}; - -TEST_F(SignalSafetyTest, DepthStartsAtZero) { - EXPECT_EQ(0, getInSignalDepth()); -} - -TEST_F(SignalSafetyTest, DepthSymmetry) { - EXPECT_EQ(0, getInSignalDepth()); - { - SignalHandlerScope scope; - EXPECT_EQ(1, getInSignalDepth()); - } - EXPECT_EQ(0, getInSignalDepth()); -} - -TEST_F(SignalSafetyTest, NestedDepth) { - EXPECT_EQ(0, getInSignalDepth()); - { - SignalHandlerScope outer; - EXPECT_EQ(1, getInSignalDepth()); - { - SignalHandlerScope inner; - EXPECT_EQ(2, getInSignalDepth()); - } - EXPECT_EQ(1, getInSignalDepth()); - } - EXPECT_EQ(0, getInSignalDepth()); -} - -TEST_F(SignalSafetyTest, TrackedSignalContextRequiresScope) { - EXPECT_FALSE(isInTrackedSignalContext()); - { - SignalHandlerScope scope; - EXPECT_TRUE(isInTrackedSignalContext()); - } - EXPECT_FALSE(isInTrackedSignalContext()); -} - -// SIGNAL_HANDLER_GUARD_RELEASE path: explicit early release inside a scope, -// destructor must not double-decrement. -TEST_F(SignalSafetyTest, ReleaseDecrementsAndDestructorIsNoOp) { - EXPECT_EQ(0, getInSignalDepth()); - { - SignalHandlerScope scope; - EXPECT_EQ(1, getInSignalDepth()); - scope.release(); - EXPECT_EQ(0, getInSignalDepth()); - // Destructor runs at end of scope and must NOT decrement again. - } - EXPECT_EQ(0, getInSignalDepth()); -} - -// SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP path: simulate a longjmp that bypasses -// SignalHandlerScope's destructor. signalHandlerUnwindAfterLongjmp() must -// decrement the depth even when no live scope object is around. -// -// Sequence (longjmp simulated by an extra block-scope guard that we don't -// destroy): -// outer scope ctor → depth=1 -// inner scope ctor → depth=2 -// imagine longjmp: inner scope's dtor is skipped -// signalHandlerUnwindAfterLongjmp() at landing → depth=1 -// outer scope dtor → depth=0 -TEST_F(SignalSafetyTest, SignalHandlerUnwindAfterLongjmpDecrementsOnce) { - EXPECT_EQ(0, getInSignalDepth()); - { - SignalHandlerScope outer; - EXPECT_EQ(1, getInSignalDepth()); - - // Heap-allocate the inner scope so we can drop it without running its - // destructor — emulating a longjmp that bypasses the C++ stack unwind. - SignalHandlerScope *leaked_inner = new SignalHandlerScope(); - EXPECT_EQ(2, getInSignalDepth()); - - // Pretend the longjmp landed here. Compensate for the bypassed dtor. - SIGNAL_HANDLER_UNWIND_AFTER_LONGJMP(); - EXPECT_EQ(1, getInSignalDepth()); - - // Outer scope dtor runs next and brings depth back to 0. - // Free leaked_inner without calling its destructor (placement-new - // wasn't used so a plain operator delete would invoke the destructor; - // route the storage through ::operator delete to skip dtor). - ::operator delete(leaked_inner); - } - EXPECT_EQ(0, getInSignalDepth()); -} - -// Safety property: signalHandlerUnwindAfterLongjmp() saturates at zero; -// double calls do not underflow. -TEST_F(SignalSafetyTest, SignalHandlerUnwindAfterLongjmpSaturatesAtZero) { - EXPECT_EQ(0, getInSignalDepth()); - signalHandlerUnwindAfterLongjmp(); - signalHandlerUnwindAfterLongjmp(); - EXPECT_EQ(0, getInSignalDepth()); -} - -TEST(SignalSafetyTestNoContext, NullProfiledThreadIsNotTrackedSignal) { - // isInTrackedSignalContext() returns false on null because the - // SignalHandlerScope never ran — used by Profiler::dlopen_hook so - // uninstrumented JVM threads doing a normal dlopen take the fast - // (synchronous refresh) path instead of deferring. - EXPECT_FALSE(isInTrackedSignalContext()); -} - -TEST(SignalSafetyTestNoContext, NullProfiledThreadDepthIsZero) { - // Mutation guard: getInSignalDepth() must return 0 on null PT, not a - // sentinel that happens to equal zero today. - EXPECT_EQ(0, getInSignalDepth()); -} diff --git a/ddprof-lib/src/test/cpp/spinLock_ut.cpp b/ddprof-lib/src/test/cpp/spinLock_ut.cpp deleted file mode 100644 index 5b7ebceeb..000000000 --- a/ddprof-lib/src/test/cpp/spinLock_ut.cpp +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "spinLock.h" -#include "../../main/cpp/gtest_crash_handler.h" -#include -#include - -static constexpr char SPINLOCK_TEST_NAME[] = "SpinLockTest"; - -class SpinLockTest : public ::testing::Test { -protected: - void SetUp() override { - installGtestCrashHandler(); - } - void TearDown() override { - restoreDefaultSignalHandlers(); - } - - SpinLock lock; -}; - -// OptionalSharedLockGuard acquires on a free lock and releases on destruction. -TEST_F(SpinLockTest, OptionalGuard_UncontendedAcquire) { - { - OptionalSharedLockGuard g(&lock); - EXPECT_TRUE(g.ownsLock()); - } - // After destruction the lock must be back to 0 (unlocked). - // Verify by taking an exclusive lock — would spin forever if still shared. - EXPECT_TRUE(lock.tryLock()); - lock.unlock(); -} - -// When an exclusive lock is held, tryLockShared returns false immediately -// (first load sees _lock == 1 > 0, exits without spinning). -TEST_F(SpinLockTest, OptionalGuard_ExclusiveHeld_ImmediateReturn) { - lock.lock(); - OptionalSharedLockGuard g(&lock, 1000000); - EXPECT_FALSE(g.ownsLock()); - lock.unlock(); -} - -// The spin budget must be honoured: even with a tiny budget the constructor -// returns (does not hang) when readers are continuously racing the CAS. -TEST_F(SpinLockTest, OptionalGuard_SpinBudgetBound) { - // A background contender thread rapidly locks/unlocks shared to create - // CAS contention on _lock; the guard must return without hanging. - std::atomic stop{false}; - - // Background thread hammers shared lock/unlock to create CAS contention. - std::thread contender([&] { - while (!stop.load(std::memory_order_relaxed)) { - lock.lockShared(); - lock.unlockShared(); - } - }); - - // With a very small budget the guard must return promptly, not hang. - for (int i = 0; i < 1000; ++i) { - OptionalSharedLockGuard g(&lock, 8); - // ownsLock() may be true or false depending on timing — we only assert - // the constructor returned (i.e. we reach here without hanging). - (void)g.ownsLock(); - } - - stop.store(true, std::memory_order_relaxed); - contender.join(); -} - -// Verifies that the budget is enforced: with an exclusive lock held, -// tryLockShared(N) must return false regardless of N. -TEST_F(SpinLockTest, OptionalGuard_BudgetEnforced_ExclusivePath) { - lock.lock(); - // With any budget, exclusive lock causes immediate false return. - EXPECT_FALSE(lock.tryLockShared(1)); - EXPECT_FALSE(lock.tryLockShared(1000)); - EXPECT_FALSE(lock.tryLockShared(SpinLock::DEFAULT_SHARED_SPIN_BUDGET)); - lock.unlock(); -} - -// Multiple shared guards can be held simultaneously (readers don't starve each other). -TEST_F(SpinLockTest, OptionalGuard_SharedReentrancy) { - OptionalSharedLockGuard g1(&lock); - OptionalSharedLockGuard g2(&lock); - EXPECT_TRUE(g1.ownsLock()); - EXPECT_TRUE(g2.ownsLock()); - // Exclusive try must fail while shared locks are held. - EXPECT_FALSE(lock.tryLock()); -} - -// tryLockShared() (unbounded) still works correctly alongside the bounded overload. -TEST_F(SpinLockTest, TryLockShared_ExclusiveHeld_ReturnsFalse) { - lock.lock(); - EXPECT_FALSE(lock.tryLockShared()); - lock.unlock(); -} - -TEST_F(SpinLockTest, TryLockShared_Free_ReturnsTrue) { - EXPECT_TRUE(lock.tryLockShared()); - lock.unlockShared(); -} diff --git a/ddprof-lib/src/test/cpp/stackWalker_ut.cpp b/ddprof-lib/src/test/cpp/stackWalker_ut.cpp deleted file mode 100644 index b0230bd2a..000000000 --- a/ddprof-lib/src/test/cpp/stackWalker_ut.cpp +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - */ - -#include -#include "../../main/cpp/stackWalker.h" -#include "../../main/cpp/gtest_crash_handler.h" - -static constexpr char STACKWALKER_TEST_NAME[] = "StackWalkerTest"; - -class StackWalkerTest : public ::testing::Test { -protected: - void SetUp() override { - installGtestCrashHandler(); - } - - void TearDown() override { - restoreDefaultSignalHandlers(); - } - - // Helper to create a frame with a non-NULL method_id - static ASGCT_CallFrame knownFrame(int id) { - ASGCT_CallFrame f = {}; - f.bci = 0; - f.method_id = (jmethodID)(uintptr_t)(id + 1); // non-NULL - return f; - } - - // Helper to create a frame with NULL method_id (unknown) - static ASGCT_CallFrame unknownFrame() { - ASGCT_CallFrame f = {}; - f.bci = 0; - f.method_id = NULL; - return f; - } -}; - -TEST_F(StackWalkerTest, dropUnknownLeaf_empty_trace) { - ASGCT_CallFrame frames[1]; - int depth = StackWalkValidation::dropUnknownLeaf(frames, 0); - EXPECT_EQ(0, depth); -} - -TEST_F(StackWalkerTest, dropUnknownLeaf_single_unknown_frame) { - ASGCT_CallFrame frames[1] = { unknownFrame() }; - int depth = StackWalkValidation::dropUnknownLeaf(frames, 1); - EXPECT_EQ(0, depth); -} - -TEST_F(StackWalkerTest, dropUnknownLeaf_single_known_frame) { - ASGCT_CallFrame frames[1] = { knownFrame(1) }; - int depth = StackWalkValidation::dropUnknownLeaf(frames, 1); - EXPECT_EQ(1, depth); - EXPECT_NE(nullptr, frames[0].method_id); -} - -TEST_F(StackWalkerTest, dropUnknownLeaf_unknown_leaf_with_known_callers) { - // frames[0] is the leaf (top of stack), frames[1..2] are callers - ASGCT_CallFrame frames[3] = { unknownFrame(), knownFrame(1), knownFrame(2) }; - int depth = StackWalkValidation::dropUnknownLeaf(frames, 3); - EXPECT_EQ(2, depth); - // The former frames[1] and frames[2] should now be at [0] and [1] - EXPECT_EQ((jmethodID)(uintptr_t)2, frames[0].method_id); - EXPECT_EQ((jmethodID)(uintptr_t)3, frames[1].method_id); -} - -TEST_F(StackWalkerTest, dropUnknownLeaf_known_leaf_not_dropped) { - ASGCT_CallFrame frames[3] = { knownFrame(1), knownFrame(2), knownFrame(3) }; - int depth = StackWalkValidation::dropUnknownLeaf(frames, 3); - EXPECT_EQ(3, depth); - EXPECT_EQ((jmethodID)(uintptr_t)2, frames[0].method_id); - EXPECT_EQ((jmethodID)(uintptr_t)3, frames[1].method_id); - EXPECT_EQ((jmethodID)(uintptr_t)4, frames[2].method_id); -} - -TEST_F(StackWalkerTest, dropUnknownLeaf_unknown_non_leaf_not_dropped) { - // Only the leaf (index 0) should be checked — unknown at other positions stays - ASGCT_CallFrame frames[3] = { knownFrame(1), unknownFrame(), knownFrame(2) }; - int depth = StackWalkValidation::dropUnknownLeaf(frames, 3); - EXPECT_EQ(3, depth); - EXPECT_NE(nullptr, frames[0].method_id); - EXPECT_EQ(nullptr, frames[1].method_id); - EXPECT_NE(nullptr, frames[2].method_id); -} - -// ---- isValidFP ---- - -TEST_F(StackWalkerTest, isValidFP_null_is_invalid) { - EXPECT_FALSE(StackWalkValidation::isValidFP(0)); -} - -TEST_F(StackWalkerTest, isValidFP_low_address_is_invalid) { - EXPECT_FALSE(StackWalkValidation::isValidFP(0x100)); // below DEAD_ZONE (0x1000) - EXPECT_FALSE(StackWalkValidation::isValidFP(0xfff)); -} - -TEST_F(StackWalkerTest, isValidFP_high_address_is_invalid) { - // Within DEAD_ZONE of UINTPTR_MAX (i.e. >= -DEAD_ZONE) - EXPECT_FALSE(StackWalkValidation::isValidFP(~(uintptr_t)0)); // UINTPTR_MAX - EXPECT_FALSE(StackWalkValidation::isValidFP(~(uintptr_t)0 - 0x100)); // still in dead zone -} - -TEST_F(StackWalkerTest, isValidFP_misaligned_is_invalid) { - // Aligned address in valid range but with low bits set - EXPECT_FALSE(StackWalkValidation::isValidFP(0x10001)); // odd - EXPECT_FALSE(StackWalkValidation::isValidFP(0x10002)); // 2-byte aligned but not pointer-aligned -} - -TEST_F(StackWalkerTest, isValidFP_valid_aligned_address) { - // Aligned addresses well within valid range should pass - EXPECT_TRUE(StackWalkValidation::isValidFP(0x10000)); - EXPECT_TRUE(StackWalkValidation::isValidFP(0x7fff0000)); -} - -// ---- isValidSP ---- - -TEST_F(StackWalkerTest, isValidSP_must_be_strictly_above_lo) { - uintptr_t lo = 0x1000; - uintptr_t hi = 0x5000; - EXPECT_FALSE(StackWalkValidation::isValidSP(lo, lo, hi)); // sp == lo: not strictly above - EXPECT_FALSE(StackWalkValidation::isValidSP(lo - 8, lo, hi)); // sp < lo -} - -TEST_F(StackWalkerTest, isValidSP_must_be_strictly_below_hi) { - uintptr_t lo = 0x1000; - uintptr_t hi = 0x5000; - EXPECT_FALSE(StackWalkValidation::isValidSP(hi, lo, hi)); // sp == hi: not strictly below - EXPECT_FALSE(StackWalkValidation::isValidSP(hi + 8, lo, hi)); // sp > hi -} - -TEST_F(StackWalkerTest, isValidSP_misaligned_is_invalid) { - uintptr_t lo = 0x1000; - uintptr_t hi = 0x5000; - EXPECT_FALSE(StackWalkValidation::isValidSP(0x2001, lo, hi)); // in range but misaligned -} - -TEST_F(StackWalkerTest, isValidSP_valid_aligned_in_range) { - uintptr_t lo = 0x1000; - uintptr_t hi = 0x5000; - EXPECT_TRUE(StackWalkValidation::isValidSP(0x2000, lo, hi)); - EXPECT_TRUE(StackWalkValidation::isValidSP(lo + 8, lo, hi)); - EXPECT_TRUE(StackWalkValidation::isValidSP(hi - 8, lo, hi)); -} diff --git a/ddprof-lib/src/test/cpp/stress_callTraceStorage.cpp b/ddprof-lib/src/test/cpp/stress_callTraceStorage.cpp deleted file mode 100644 index a8706967d..000000000 --- a/ddprof-lib/src/test/cpp/stress_callTraceStorage.cpp +++ /dev/null @@ -1,2421 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "gtest/gtest.h" -#include "callTraceStorage.h" -#include "callTraceHashTable.h" -#include "guards.h" -#include "common.h" // TSAN_ENABLED (toolchain-agnostic sanitizer detection) -#include -#include -#include -#include -#include -#include -#include -#include "arch.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "../../main/cpp/gtest_crash_handler.h" - -// Test name for crash handler -static constexpr const char STRESS_TEST_NAME[] = "StressCallTraceStorage"; - -// Expansion fires when the table reaches 75 % of its initial 65536-slot capacity. -// Any stress test that exercises the _prev chain (multi-node table) must insert -// strictly more than this many unique traces between processTraces() calls. -static constexpr int CALLTRACE_EXPANSION_THRESHOLD = 65536 * 3 / 4; // 49152 - -// Helper function to find a CallTrace by trace_id in an unordered_set -CallTrace* findTraceById(const std::unordered_set& traces, u64 trace_id) { - for (CallTrace* trace : traces) { - if (trace && trace != CallTraceSample::PREPARING && trace->trace_id == trace_id) { - return trace; - } - } - return nullptr; -} - -// Optimized batch lookup for multiple trace IDs -void findMultipleTracesById(const std::unordered_set& traces, - const std::vector& trace_ids, - size_t& found_count) { - // Create a lookup set for O(1) lookups instead of O(n) per trace - std::unordered_set target_ids(trace_ids.begin(), trace_ids.end()); - found_count = 0; - - for (CallTrace* trace : traces) { - if (trace && trace != CallTraceSample::PREPARING) { - if (target_ids.find(trace->trace_id) != target_ids.end()) { - found_count++; - // Early termination - found all traces - if (found_count == trace_ids.size()) { - break; - } - } - } - } -} - -// Thread-safe random number generator for deterministic testing -class ThreadSafeRandom { -private: - std::mt19937 gen_; - std::mutex mutex_; - -public: - explicit ThreadSafeRandom(uint32_t seed = std::random_device{}()) : gen_(seed) {} - - uint64_t next(uint64_t max_val = UINT64_MAX) { - std::lock_guard lock(mutex_); - std::uniform_int_distribution dis(0, max_val); - return dis(gen_); - } -}; - -// Guarded buffer for detecting memory corruption -class GuardedBuffer { -private: - static constexpr uint32_t GUARD_PATTERN = 0xDEADBEEF; - static constexpr size_t GUARD_SIZE = sizeof(uint32_t); - static constexpr size_t ALIGNMENT = 8; // 8-byte alignment for ASGCT_CallFrame - - void* buffer_; - size_t size_; - void* aligned_data_; - - void setGuards() { - uint32_t* front_guard = reinterpret_cast(buffer_); - uint32_t* back_guard = reinterpret_cast( - static_cast(aligned_data_) + size_ - ); - *front_guard = GUARD_PATTERN; - *back_guard = GUARD_PATTERN; - } - - // Calculate the next properly aligned address - static void* align_pointer(void* ptr, size_t alignment) { - uintptr_t addr = reinterpret_cast(ptr); - uintptr_t aligned = (addr + alignment - 1) & ~(alignment - 1); - return reinterpret_cast(aligned); - } - -public: - explicit GuardedBuffer(size_t size) : size_(size) { - // Allocate extra space for guards + alignment padding - size_t total_size = GUARD_SIZE + (ALIGNMENT - 1) + size + GUARD_SIZE; - buffer_ = malloc(total_size); - if (buffer_ == nullptr) { - throw std::bad_alloc(); - } - - // Calculate aligned data pointer after front guard - void* after_front_guard = static_cast(buffer_) + GUARD_SIZE; - aligned_data_ = align_pointer(after_front_guard, ALIGNMENT); - - setGuards(); - } - - ~GuardedBuffer() { - if (buffer_) { - free(buffer_); - } - } - - void* data() { - return aligned_data_; - } - - bool checkCorruption() const { - uint32_t* front_guard = reinterpret_cast(buffer_); - uint32_t* back_guard = reinterpret_cast( - static_cast(aligned_data_) + size_ - ); - return (*front_guard != GUARD_PATTERN) || (*back_guard != GUARD_PATTERN); - } -}; - -class StressTestSuite : public ::testing::Test { -public: - // Single shared CallTraceStorage instance - matches production usage pattern - static std::unique_ptr shared_storage; - // Mutex for processTraces calls - ensures single-threaded access as in production - static std::mutex process_traces_mutex; - -protected: - - void SetUp() override { - // Install crash handler for detailed debugging - installGtestCrashHandler(); - - // Initialize shared storage if not already done - if (!shared_storage) { - shared_storage = std::make_unique(); - } - - // Clear any traces from previous tests to start fresh - shared_storage->clear(); - } - - void TearDown() override { - // Restore default signal handlers - restoreDefaultSignalHandlers(); - - // Clear storage for next test but don't destroy it - if (shared_storage) { - shared_storage->clear(); - } - } - - static void TearDownTestSuite() { - // Clean up shared resources after all tests - shared_storage.reset(); - } -}; - -// Static member definitions -std::unique_ptr StressTestSuite::shared_storage; -std::mutex StressTestSuite::process_traces_mutex; - -// Test 1: SwapStormTest - Double-buffered call-trace storage under rapid swapping -TEST_F(StressTestSuite, SwapStormTest) { - const int NUM_THREADS = 8; - const int OPERATIONS_PER_THREAD = 5000; - const int SWAP_FREQUENCY_MS = 10; - - std::atomic test_running{true}; - std::atomic test_failed{false}; - std::atomic total_operations{0}; - std::atomic successful_puts{0}; - std::atomic swap_count{0}; - - // Use shared storage instance - matches production pattern - CallTraceStorage* storage = shared_storage.get(); - ThreadSafeRandom random_gen(12345); - - // Worker threads continuously adding traces - std::vector workers; - for (int i = 0; i < NUM_THREADS; ++i) { - workers.emplace_back([&, i]() { - std::mt19937 local_gen(random_gen.next(UINT32_MAX)); - std::uniform_int_distribution bci_dis(1, 1000); - std::uniform_int_distribution method_dis(0x1000, 0x9999); - - for (int op = 0; op < OPERATIONS_PER_THREAD && test_running.load(); ++op) { - try { - ASGCT_CallFrame frame; - frame.bci = bci_dis(local_gen); - frame.method_id = reinterpret_cast(method_dis(local_gen)); - - u64 trace_id = storage->put(1, &frame, false, 1); - if (trace_id > 0) { - successful_puts.fetch_add(1, std::memory_order_relaxed); - } - - total_operations.fetch_add(1, std::memory_order_relaxed); - - // Occasional yield to allow swaps - if (op % 100 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - test_failed.store(true); - break; - } - } - }); - } - - // Rapid swapping thread - std::thread swapper([&]() { - while (test_running.load() && !test_failed.load()) { - std::this_thread::sleep_for(std::chrono::milliseconds(SWAP_FREQUENCY_MS)); - - try { - // Use mutex to ensure single-threaded processTraces access - matches production - { - std::lock_guard lock(process_traces_mutex); - storage->processTraces([](const std::unordered_set& traces) { - // Process traces (simulating JFR serialization) - (void)traces.size(); - }); - } - swap_count.fetch_add(1, std::memory_order_relaxed); - } catch (...) { - test_failed.store(true); - break; - } - } - }); - - // Let the stress test run for a reasonable duration - std::this_thread::sleep_for(std::chrono::seconds(2)); - test_running.store(false); - - // Wait for all threads - for (auto& worker : workers) { - worker.join(); - } - swapper.join(); - - // Verify results - EXPECT_FALSE(test_failed.load()) << "Stress test encountered failures"; - EXPECT_GT(swap_count.load(), 0) << "No swaps occurred during test"; - EXPECT_GT(successful_puts.load(), 0) << "No successful trace insertions"; - EXPECT_EQ(total_operations.load(), NUM_THREADS * OPERATIONS_PER_THREAD) - << "Not all operations completed"; - - std::cout << "SwapStorm completed: " << total_operations.load() << " ops, " - << swap_count.load() << " swaps, " << successful_puts.load() << " successful puts" << std::endl; -} - -// Test 2: HashTableContentionTest - Concurrent hash table operations -TEST_F(StressTestSuite, HashTableContentionTest) { - const int NUM_THREADS = 6; - const int TRACES_PER_THREAD = 3000; - - // Use heap allocation with proper alignment to avoid ASAN alignment issues - // Stack allocation with high alignment requirements (64 bytes) is problematic under ASAN - void* aligned_memory = std::aligned_alloc(alignof(CallTraceHashTable), sizeof(CallTraceHashTable)); - ASSERT_NE(aligned_memory, nullptr) << "Failed to allocate aligned memory for CallTraceHashTable"; - - auto hash_table_ptr = std::unique_ptr( - new(aligned_memory) CallTraceHashTable(), - [](CallTraceHashTable* ptr) { - ptr->~CallTraceHashTable(); - std::free(ptr); - } - ); - CallTraceHashTable& hash_table = *hash_table_ptr; - hash_table.setInstanceId(42); - - std::atomic test_failed{false}; - std::atomic successful_operations{0}; - std::atomic expansion_triggers{0}; - std::vector threads; - - // Create diverse stack traces to force table expansion - for (int t = 0; t < NUM_THREADS; ++t) { - threads.emplace_back([&, t]() { - std::mt19937 gen(std::random_device{}() + t); - std::uniform_int_distribution bci_dis(1, 10000); - std::uniform_int_distribution method_dis(0x1000, 0xFFFF); - - for (int i = 0; i < TRACES_PER_THREAD; ++i) { - try { - ASGCT_CallFrame frame; - frame.bci = t * 10000 + bci_dis(gen); // Ensure uniqueness - frame.method_id = reinterpret_cast(t * 0x10000 + method_dis(gen)); - - u64 trace_id = hash_table.put(1, &frame, false, 1); - - if (trace_id == 0) { - // Sample was dropped - acceptable under high contention - continue; - } - - if (trace_id == 0x7fffffffffffffffULL) { - // Overflow trace - also acceptable - continue; - } - - successful_operations.fetch_add(1, std::memory_order_relaxed); - - // Detect expansion events (approximate) - if (i > 0 && i % 1000 == 0) { - expansion_triggers.fetch_add(1, std::memory_order_relaxed); - } - - // Yield occasionally to increase contention - if (i % 500 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - test_failed.store(true); - return; - } - } - }); - } - - // Wait for all threads - for (auto& thread : threads) { - thread.join(); - } - - EXPECT_FALSE(test_failed.load()) << "Hash table contention test failed"; - EXPECT_GT(successful_operations.load(), 0) << "No successful hash table operations"; - - // Verify table still functions after stress - ASGCT_CallFrame test_frame; - test_frame.bci = 99999; - test_frame.method_id = reinterpret_cast(0x99999); - u64 final_trace_id = hash_table.put(1, &test_frame, false, 1); - EXPECT_GT(final_trace_id, 0) << "Hash table non-functional after stress test"; - - std::cout << "HashTable contention completed: " << successful_operations.load() - << " successful operations" << std::endl; -} - -// Test 3: TraceIdFuzzTest - 64-bit TraceId bit-packing validation -TEST_F(StressTestSuite, TraceIdFuzzTest) { - const int NUM_THREADS = 4; - const int OPERATIONS_PER_THREAD = 50000; - - std::atomic test_failed{false}; - std::atomic total_operations{0}; - std::atomic sign_extension_violations{0}; - std::vector threads; - - // Helper functions for TraceId manipulation - auto extract_slot = [](u64 trace_id) -> u64 { - return trace_id & 0xFFFFFFFFULL; - }; - - auto extract_instance_id = [](u64 trace_id) -> u64 { - return trace_id >> 32; - }; - - auto create_trace_id = [](u64 instance_id, u64 slot) -> u64 { - return (instance_id << 32) | (slot & 0xFFFFFFFFULL); - }; - - for (int t = 0; t < NUM_THREADS; ++t) { - threads.emplace_back([&, t]() { - std::mt19937 gen(std::random_device{}() + t); - std::uniform_int_distribution dis(0, 0xFFFFFFFFULL); - - for (int i = 0; i < OPERATIONS_PER_THREAD; ++i) { - try { - u64 instance_id = dis(gen); - u64 slot = dis(gen); - - u64 trace_id = create_trace_id(instance_id, slot); - u64 extracted_instance = extract_instance_id(trace_id); - u64 extracted_slot = extract_slot(trace_id); - - // Verify bit-packing correctness - if (extracted_instance != instance_id || extracted_slot != slot) { - test_failed.store(true); - return; - } - - // Check for potential sign-extension issues - int32_t slot_as_int32 = static_cast(slot); - if (slot_as_int32 < 0) { - sign_extension_violations.fetch_add(1, std::memory_order_relaxed); - } - - // Test with extreme values - if (i % 1000 == 0) { - std::vector extreme_values = { - 0x0000000000000000ULL, - 0xFFFFFFFFFFFFFFFFULL, - 0x7FFFFFFFFFFFFFFFULL, - 0x8000000000000000ULL, - 0x00000000FFFFFFFFULL, - 0xFFFFFFFF00000000ULL, - }; - - for (u64 extreme_trace_id : extreme_values) { - u64 e_slot = extract_slot(extreme_trace_id); - u64 e_instance = extract_instance_id(extreme_trace_id); - u64 reconstructed = create_trace_id(e_instance, e_slot); - - if (reconstructed != extreme_trace_id) { - test_failed.store(true); - return; - } - } - } - - total_operations.fetch_add(1, std::memory_order_relaxed); - - } catch (...) { - test_failed.store(true); - return; - } - } - }); - } - - // Wait for all threads - for (auto& thread : threads) { - thread.join(); - } - - EXPECT_FALSE(test_failed.load()) << "TraceId bit-packing test failed"; - EXPECT_EQ(total_operations.load(), NUM_THREADS * OPERATIONS_PER_THREAD) - << "Not all TraceId operations completed"; - - std::cout << "TraceId fuzz test completed: " << total_operations.load() - << " operations, " << sign_extension_violations.load() - << " sign extension cases detected" << std::endl; -} - -// Test 4: AsgctBoundsTest - ASGCT frame handling bounds checking -TEST_F(StressTestSuite, AsgctBoundsTest) { - const int NUM_THREADS = 4; - const int FRAMES_PER_THREAD = 10000; - const size_t MAX_FRAMES = 1024; - - std::atomic test_failed{false}; - std::atomic guard_violations{0}; - std::atomic bounds_checks{0}; - std::vector threads; - - // Pre-allocated guarded buffers for each thread - std::vector> buffers; - for (int t = 0; t < NUM_THREADS; ++t) { - buffers.push_back(std::make_unique(MAX_FRAMES * sizeof(ASGCT_CallFrame))); - } - - for (int t = 0; t < NUM_THREADS; ++t) { - threads.emplace_back([&, t]() { - ASGCT_CallFrame* frames = static_cast(buffers[t]->data()); - std::mt19937 gen(std::random_device{}() + t); - std::uniform_int_distribution bci_dis(0, UINT32_MAX); - std::uniform_int_distribution method_dis(0x1000, 0xFFFFF); - std::uniform_int_distribution frame_count_dis(1, MAX_FRAMES); - - for (int i = 0; i < FRAMES_PER_THREAD; ++i) { - try { - size_t num_frames = frame_count_dis(gen); - - // Fill frames with random data - for (size_t f = 0; f < num_frames; ++f) { - frames[f].bci = bci_dis(gen); - frames[f].method_id = reinterpret_cast(method_dis(gen)); - } - - // Simulate bounds checking that might occur in actual profiler - for (size_t f = 0; f < num_frames; ++f) { - if (frames[f].bci == static_cast(-1)) { - // Native frame marker - acceptable - continue; - } - - // Check for reasonable BCI values - if (frames[f].bci > 0x7FFFFFFF) { - bounds_checks.fetch_add(1, std::memory_order_relaxed); - } - - // Verify method_id is not null (would be problematic) - if (frames[f].method_id == nullptr) { - bounds_checks.fetch_add(1, std::memory_order_relaxed); - } - } - - // Check for buffer corruption - if (buffers[t]->checkCorruption()) { - guard_violations.fetch_add(1, std::memory_order_relaxed); - test_failed.store(true); - return; - } - - // Yield occasionally - if (i % 1000 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - test_failed.store(true); - return; - } - } - }); - } - - // Wait for all threads - for (auto& thread : threads) { - thread.join(); - } - - EXPECT_FALSE(test_failed.load()) << "ASGCT bounds test failed"; - EXPECT_EQ(guard_violations.load(), 0) << "Buffer corruption detected"; - - std::cout << "ASGCT bounds test completed: " << bounds_checks.load() - << " bounds checks performed" << std::endl; -} - -// Test 5: JfrTinyBufferTest - JFR serialization with minimal buffers -TEST_F(StressTestSuite, JfrTinyBufferTest) { - const int NUM_THREADS = 4; - const int OPERATIONS_PER_THREAD = 5000; - const size_t TINY_BUFFER_SIZE = 64; // Deliberately small - - std::atomic test_failed{false}; - std::atomic buffer_overruns{0}; - std::atomic successful_writes{0}; - std::vector threads; - - for (int t = 0; t < NUM_THREADS; ++t) { - threads.emplace_back([&, t]() { - auto buffer = std::make_unique(TINY_BUFFER_SIZE); - char* write_ptr = static_cast(buffer->data()); - std::mt19937 gen(std::random_device{}() + t); - std::uniform_int_distribution write_size_dis(1, TINY_BUFFER_SIZE + 10); - - for (int i = 0; i < OPERATIONS_PER_THREAD; ++i) { - try { - size_t write_size = write_size_dis(gen); - - // Simulate JFR buffer write with bounds checking - if (write_size <= TINY_BUFFER_SIZE) { - // Safe write - std::memset(write_ptr, static_cast(0xAA + (i % 16)), write_size); - successful_writes.fetch_add(1, std::memory_order_relaxed); - } else { - // Would overflow - record but don't actually overflow - buffer_overruns.fetch_add(1, std::memory_order_relaxed); - } - - // Check for corruption - if (buffer->checkCorruption()) { - test_failed.store(true); - return; - } - - // Yield occasionally - if (i % 500 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - test_failed.store(true); - return; - } - } - }); - } - - // Wait for all threads - for (auto& thread : threads) { - thread.join(); - } - - EXPECT_FALSE(test_failed.load()) << "JFR tiny buffer test failed"; - EXPECT_GT(successful_writes.load(), 0) << "No successful buffer writes"; - EXPECT_GT(buffer_overruns.load(), 0) << "No buffer overrun cases detected"; - - std::cout << "JFR tiny buffer test completed: " << successful_writes.load() - << " successful writes, " << buffer_overruns.load() << " overruns detected" << std::endl; -} - -// Test 6: LivenessPurityTest - Liveness callback purity validation -TEST_F(StressTestSuite, LivenessPurityTest) { - const int NUM_ITERATIONS = 500; // Reduced from 1000 for better performance - const int TRACES_PER_ITERATION = 50; - - std::atomic test_failed{false}; - std::atomic callback_invocations{0}; - std::atomic preserved_traces{0}; - - // Use shared storage instance - matches production pattern - CallTraceStorage* storage = shared_storage.get(); - ThreadSafeRandom random_gen(54321); - - for (int iteration = 0; iteration < NUM_ITERATIONS; ++iteration) { - try { - std::vector trace_ids; - - // Add traces - for (int t = 0; t < TRACES_PER_ITERATION; ++t) { - ASGCT_CallFrame frame; - frame.bci = static_cast(random_gen.next(10000)); - frame.method_id = reinterpret_cast(random_gen.next(0xFFFF) + 0x1000); - - u64 trace_id = storage->put(1, &frame, false, 1); - if (trace_id > 0) { - trace_ids.push_back(trace_id); - } - } - - if (trace_ids.empty()) { - continue; - } - - // Register liveness checker - should be pure and deterministic - size_t preserve_count = trace_ids.size() / 2; - std::vector to_preserve(trace_ids.begin(), trace_ids.begin() + preserve_count); - - storage->registerLivenessChecker([to_preserve](std::unordered_set& buffer) { - // Pure callback - no side effects, deterministic output - for (u64 trace_id : to_preserve) { - buffer.insert(trace_id); - } - }); - - callback_invocations.fetch_add(1, std::memory_order_relaxed); - - // Process traces and verify preservation - size_t actual_preserved = 0; - { - std::lock_guard lock(process_traces_mutex); - storage->processTraces([&](const std::unordered_set& traces) { - findMultipleTracesById(traces, to_preserve, actual_preserved); - }); - } - - preserved_traces.fetch_add(actual_preserved, std::memory_order_relaxed); - - // Verify deterministic behavior - re-register same callback - storage->registerLivenessChecker([to_preserve](std::unordered_set& buffer) { - for (u64 trace_id : to_preserve) { - buffer.insert(trace_id); - } - }); - - // Second process should have consistent results - size_t second_preserved = 0; - { - std::lock_guard lock(process_traces_mutex); - storage->processTraces([&](const std::unordered_set& traces) { - findMultipleTracesById(traces, to_preserve, second_preserved); - }); - } - - // Yield periodically - if (iteration % 100 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - test_failed.store(true); - break; - } - } - - EXPECT_FALSE(test_failed.load()) << "Liveness purity test failed"; - EXPECT_GT(callback_invocations.load(), 0) << "No liveness callbacks invoked"; - EXPECT_GT(preserved_traces.load(), 0) << "No traces preserved"; - - std::cout << "Liveness purity test completed: " << callback_invocations.load() - << " callback invocations, " << preserved_traces.load() << " traces preserved" << std::endl; -} - -// TLS-focused stress tests - -// TLS canary pattern for detecting buffer corruption -struct TLSCanary { - static constexpr uint64_t CANARY_PATTERN = 0xDEADBEEFCAFEBABEULL; - static constexpr size_t BUFFER_SIZE = 8192; - static constexpr size_t CANARY_COUNT = 4; - - uint64_t front_canary[CANARY_COUNT]; - char buffer[BUFFER_SIZE]; - uint64_t back_canary[CANARY_COUNT]; - - TLSCanary() { - for (size_t i = 0; i < CANARY_COUNT; ++i) { - front_canary[i] = CANARY_PATTERN + i; - back_canary[i] = CANARY_PATTERN + i + CANARY_COUNT; - } - std::memset(buffer, 0xAA, BUFFER_SIZE); - } - - bool checkCanaries() const { - for (size_t i = 0; i < CANARY_COUNT; ++i) { - if (front_canary[i] != CANARY_PATTERN + i || - back_canary[i] != CANARY_PATTERN + i + CANARY_COUNT) { - return false; - } - } - return true; - } - - void simulateLogWrite(const std::string& message) { - // Simulate writing log data with potential for overrun - size_t write_size = std::min(message.length(), BUFFER_SIZE - 1); - std::memcpy(buffer, message.c_str(), write_size); - buffer[write_size] = '\0'; - } - - void simulatePathWrite(const std::string& path) { - // Simulate long path name writes - size_t path_len = std::min(path.length(), BUFFER_SIZE / 2); - std::memcpy(buffer, path.c_str(), path_len); - - // Add some stack frame simulation - char stack_info[512]; - snprintf(stack_info, sizeof(stack_info), - "|frame:%p|method:%s|bci:%d", - (void*)0x12345678, "someMethod", (int)(path_len % 1000)); - - size_t remaining = BUFFER_SIZE - path_len - 1; - size_t stack_len = std::min(strlen(stack_info), remaining); - std::memcpy(buffer + path_len, stack_info, stack_len); - } -}; - -// Thread-local storage for TLS tests -thread_local TLSCanary* tls_canary = nullptr; - -// Test 7: TLS Overrun Canary Test -TEST_F(StressTestSuite, TLSOverrunCanaryTest) { - const int NUM_THREADS = 6; - const int OPERATIONS_PER_THREAD = 10000; - const int SWAP_FREQUENCY_MS = 5; // More aggressive swapping - - std::atomic test_running{true}; - std::atomic canary_corruption{false}; - std::atomic total_operations{0}; - std::atomic canary_checks{0}; - std::atomic swap_count{0}; - - // Use shared storage instance - matches production pattern - CallTraceStorage* storage = shared_storage.get(); - ThreadSafeRandom random_gen(99999); - - // Worker threads hammering TLS buffers while doing storage operations - std::vector workers; - for (int i = 0; i < NUM_THREADS; ++i) { - workers.emplace_back([&, i]() { - // Initialize TLS canary for this thread - tls_canary = new TLSCanary(); - - std::mt19937 local_gen(random_gen.next(UINT32_MAX)); - std::uniform_int_distribution size_dis(100, 4000); - std::uniform_int_distribution operation_dis(0, 2); - - for (int op = 0; op < OPERATIONS_PER_THREAD && test_running.load(); ++op) { - try { - // Check canary at start of each operation - if (!tls_canary->checkCanaries()) { - canary_corruption.store(true); - break; - } - - // Simulate various TLS buffer stress operations - int operation = operation_dis(local_gen); - switch (operation) { - case 0: { - // Large log line simulation - size_t log_size = size_dis(local_gen); - std::string large_log(log_size, 'L'); - large_log += std::to_string(op) + "_thread_" + std::to_string(i); - tls_canary->simulateLogWrite(large_log); - break; - } - case 1: { - // Deep path simulation - std::string deep_path = "/very/deep/file/system/path/that/could/be/very/long/"; - for (int depth = 0; depth < 20; ++depth) { - deep_path += "subdir" + std::to_string(depth) + "/"; - } - deep_path += "filename_" + std::to_string(op); - tls_canary->simulatePathWrite(deep_path); - break; - } - case 2: { - // Stack stringification simulation - std::ostringstream stack_trace; - for (int frame = 0; frame < 50; ++frame) { - stack_trace << "Frame" << frame - << ":Method" << (frame * 123 + op) - << ":BCI" << (frame * 456 + i) << ";"; - } - tls_canary->simulateLogWrite(stack_trace.str()); - break; - } - } - - // Also do some storage operations to create interference - ASGCT_CallFrame frame; - frame.bci = static_cast(op % 10000); - frame.method_id = reinterpret_cast(0x1000 + i * 1000 + op); - storage->put(1, &frame, false, 1); - - // Check canary after operations - canary_checks.fetch_add(1, std::memory_order_relaxed); - if (!tls_canary->checkCanaries()) { - canary_corruption.store(true); - break; - } - - total_operations.fetch_add(1, std::memory_order_relaxed); - - // Yield occasionally to allow swaps - if (op % 200 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - canary_corruption.store(true); - break; - } - } - - // Final canary check and cleanup - if (tls_canary && !tls_canary->checkCanaries()) { - canary_corruption.store(true); - } - delete tls_canary; - tls_canary = nullptr; - }); - } - - // Aggressive swapping thread - std::thread swapper([&]() { - while (test_running.load() && !canary_corruption.load()) { - std::this_thread::sleep_for(std::chrono::milliseconds(SWAP_FREQUENCY_MS)); - - try { - { - std::lock_guard lock(process_traces_mutex); - storage->processTraces([](const std::unordered_set& traces) { - // Aggressive processing to stress TLS during swaps - volatile size_t count = traces.size(); - (void)count; - }); - } - swap_count.fetch_add(1, std::memory_order_relaxed); - } catch (...) { - canary_corruption.store(true); - break; - } - } - }); - - // Run stress test - std::this_thread::sleep_for(std::chrono::seconds(3)); - test_running.store(false); - - // Wait for threads - for (auto& worker : workers) { - worker.join(); - } - swapper.join(); - - // Verify results - EXPECT_FALSE(canary_corruption.load()) << "TLS canary corruption detected"; - EXPECT_GT(canary_checks.load(), 0) << "No canary checks performed"; - EXPECT_GT(swap_count.load(), 0) << "No storage swaps occurred"; - - std::cout << "TLS canary test completed: " << total_operations.load() << " ops, " - << canary_checks.load() << " canary checks, " << swap_count.load() - << " swaps, corruption=" << (canary_corruption.load() ? "YES" : "NO") << std::endl; -} - -// Test 8: TCMalloc A/B Runner -TEST_F(StressTestSuite, TCMallocABRunner) { - const int NUM_ITERATIONS = 1000; - const int ALLOCATION_SIZE = 1024; - - std::atomic test_failed{false}; - std::atomic normal_crashes{0}; - std::atomic preload_crashes{0}; - std::atomic fence_crashes{0}; - - // Helper to run workload and detect crashes - auto run_workload = [&](const std::string& env_setup) -> bool { - pid_t pid = fork(); - if (pid == 0) { - // Child process - run the workload - if (!env_setup.empty()) { - std::system(("export " + env_setup).c_str()); - } - - try { - // Simulate the exact workload from other tests - std::vector allocations; - allocations.reserve(NUM_ITERATIONS); - - for (int i = 0; i < NUM_ITERATIONS; ++i) { - void* ptr = malloc(ALLOCATION_SIZE + (i % 100)); - if (ptr) { - std::memset(ptr, 0xAB + (i % 16), ALLOCATION_SIZE + (i % 100)); - allocations.push_back(ptr); - } - - // Some allocations freed immediately, others kept - if (i % 3 == 0 && !allocations.empty()) { - free(allocations.back()); - allocations.pop_back(); - } - - // Simulate some storage work - if (i % 100 == 0) { - // Use heap allocation to avoid ASAN alignment issues with stack objects - void* aligned_mem = std::aligned_alloc(alignof(CallTraceHashTable), sizeof(CallTraceHashTable)); - if (aligned_mem) { - auto test_table_ptr = std::unique_ptr( - new(aligned_mem) CallTraceHashTable(), - [](CallTraceHashTable* ptr) { - ptr->~CallTraceHashTable(); - std::free(ptr); - } - ); - CallTraceHashTable& test_table = *test_table_ptr; - test_table.setInstanceId(42); - ASGCT_CallFrame frame; - frame.bci = i; - frame.method_id = reinterpret_cast(0x1000 + i); - test_table.put(1, &frame, false, 1); - } - } - } - - // Cleanup - for (void* ptr : allocations) { - free(ptr); - } - - _exit(0); // Success - } catch (...) { - _exit(1); // Failure - } - } else if (pid > 0) { - // Parent process - wait for child - int status; - waitpid(pid, &status, 0); - - if (WIFEXITED(status)) { - return WEXITSTATUS(status) == 0; - } else { - // Child crashed - return false; - } - } else { - // Fork failed - return false; - } - }; - - // Test 1: Normal run (baseline) - for (int run = 0; run < 3; ++run) { - if (!run_workload("")) { - normal_crashes.fetch_add(1, std::memory_order_relaxed); - } - } - - // Test 2: With tcmalloc LD_PRELOAD (if available) - std::string tcmalloc_path; - std::vector possible_paths = { - "/usr/lib/x86_64-linux-gnu/libtcmalloc.so.4", - "/usr/lib/libtcmalloc.so", - "/opt/homebrew/lib/libtcmalloc.so", - "/usr/local/lib/libtcmalloc.so" - }; - - for (const std::string& path : possible_paths) { - if (access(path.c_str(), R_OK) == 0) { - tcmalloc_path = path; - break; - } - } - - if (!tcmalloc_path.empty()) { - for (int run = 0; run < 3; ++run) { - if (!run_workload("LD_PRELOAD=" + tcmalloc_path)) { - preload_crashes.fetch_add(1, std::memory_order_relaxed); - } - } - - // Test 3: With TCMALLOC_PAGE_FENCE=1 if available - for (int run = 0; run < 3; ++run) { - if (!run_workload("LD_PRELOAD=" + tcmalloc_path + " TCMALLOC_PAGE_FENCE=1")) { - fence_crashes.fetch_add(1, std::memory_order_relaxed); - } - } - } - - // Record results (crashes are not necessarily test failures - they're data points) - std::cout << "TCMalloc A/B test completed:" << std::endl; - std::cout << " Normal runs: " << normal_crashes.load() << " crashes out of 3" << std::endl; - if (!tcmalloc_path.empty()) { - std::cout << " TCMalloc preload: " << preload_crashes.load() << " crashes out of 3" << std::endl; - std::cout << " TCMalloc fence: " << fence_crashes.load() << " crashes out of 3" << std::endl; - std::cout << " TCMalloc path: " << tcmalloc_path << std::endl; - } else { - std::cout << " TCMalloc not found - skipped preload tests" << std::endl; - } - - // Test passes if we collected data (crashes are informational) - EXPECT_FALSE(test_failed.load()) << "TCMalloc A/B test infrastructure failed"; -} - -// Global state for signal pressure test -static std::atomic signal_pressure_active{false}; -static std::atomic signals_delivered{0}; -static std::atomic signal_corruption_detected{false}; -thread_local volatile uint32_t tls_write_counter = 0; - -// Global state for realistic signal test -static std::atomic realistic_test_running{false}; -static std::atomic realistic_handler_corruption{false}; -static std::atomic realistic_signals_handled{0}; -static std::atomic realistic_storage_operations{0}; -static CallTraceStorage* realistic_shared_storage = nullptr; - -// Signal handler for pressure test -void pressure_signal_handler(int sig) { - if (!signal_pressure_active.load()) { - return; - } - CriticalSection cs; - - if (!cs.entered()) { - // behave like the real-life signal handler - return; - } - - signals_delivered.fetch_add(1, std::memory_order_relaxed); - - // Simulate lightweight profiling work in signal handler - // Check TLS consistency - uint32_t expected = tls_write_counter; - if (expected != tls_write_counter) { - signal_corruption_detected.store(true); - } - - // Tiny bit of work (signal-safe) - volatile uint64_t dummy = 0; - for (int i = 0; i < 10; ++i) { - dummy += i; - } - (void)dummy; -} - -// Realistic signal handler for profiler stress test -void realistic_profiler_signal_handler(int sig) { - if (!realistic_test_running.load()) return; - CriticalSection cs; - - // Critical: Check if critical section is active (storage swap in progress) - if (!cs.entered()) { - return; // Skip this signal - storage operation in progress - } - - realistic_signals_handled.fetch_add(1, std::memory_order_relaxed); - - try { - // Simulate what the real profiler does in signal context - // 1. Get thread ID (potential race with thread destruction) - pthread_t current_thread = pthread_self(); - - // 2. Try to record a sample (this should be signal-safe) - ASGCT_CallFrame frame; - frame.bci = static_cast(realistic_signals_handled.load() % 10000); - frame.method_id = reinterpret_cast(0x1000 + (uintptr_t)current_thread); - - // 3. This is where real bugs occur - storage operations in signal context - if (realistic_shared_storage) { - u64 trace_id = realistic_shared_storage->put(1, &frame, false, 1); - if (trace_id > 0) { - realistic_storage_operations.fetch_add(1, std::memory_order_relaxed); - } - } - - // 4. Simulate some work that might cause corruption - static thread_local volatile uint64_t signal_work_counter = 0; - signal_work_counter++; - - // Check for corruption pattern - if we're accessing destroyed TLS - if (signal_work_counter > 20000) { - realistic_handler_corruption.store(true); - } - - } catch (...) { - realistic_handler_corruption.store(true); - } -} - -// Test 9: Signal Pressure Test -TEST_F(StressTestSuite, SignalPressureTest) { - const int SIGNAL_FREQUENCY_HZ = 1000; // 1000 Hz profiling signals - const int TEST_DURATION_MS = 2000; - const int NUM_WORKER_THREADS = 3; - - std::atomic test_running{true}; - std::atomic deadlock_detected{false}; - std::atomic tls_writes_completed{0}; - std::vector workers; - - // Install signal handler - struct sigaction old_action; - struct sigaction new_action; - new_action.sa_handler = pressure_signal_handler; - sigemptyset(&new_action.sa_mask); - new_action.sa_flags = SA_RESTART; - - if (sigaction(SIGUSR1, &new_action, &old_action) != 0) { - GTEST_SKIP() << "Could not install signal handler"; - return; - } - - signal_pressure_active.store(true); - signals_delivered.store(0); - signal_corruption_detected.store(false); - - // Worker threads doing TLS writes - for (int t = 0; t < NUM_WORKER_THREADS; ++t) { - workers.emplace_back([&, t]() { - tls_write_counter = 0; - const size_t TINY_WRITE_SIZE = 64; - char tls_buffer[TINY_WRITE_SIZE]; - - // Setup sigaltstack for this thread (test both with and without) - bool use_altstack = (t % 2 == 0); - stack_t alt_stack; - stack_t old_stack; - - if (use_altstack) { - alt_stack.ss_sp = malloc(SIGSTKSZ); - alt_stack.ss_size = SIGSTKSZ; - alt_stack.ss_flags = 0; - - if (alt_stack.ss_sp && sigaltstack(&alt_stack, &old_stack) == 0) { - // Successfully installed alt stack - } else { - use_altstack = false; - } - } - - auto start_time = std::chrono::steady_clock::now(); - uint32_t write_count = 0; - - while (test_running.load()) { - try { - // Tiny TLS writes with counter increments - tls_write_counter = ++write_count; - - // Simulate small TLS buffer operations - snprintf(tls_buffer, TINY_WRITE_SIZE, "t%d_w%u", t, write_count); - - // Verify consistency - if (tls_write_counter != write_count) { - signal_corruption_detected.store(true); - break; - } - - tls_writes_completed.fetch_add(1, std::memory_order_relaxed); - - // Very short yield to allow signal delivery - if (write_count % 100 == 0) { - std::this_thread::yield(); - } - - // Deadlock detection - if we're stuck too long, bail - auto now = std::chrono::steady_clock::now(); - if (std::chrono::duration_cast(now - start_time).count() > TEST_DURATION_MS * 2) { - deadlock_detected.store(true); - break; - } - - } catch (...) { - signal_corruption_detected.store(true); - break; - } - } - - // Cleanup alt stack - if (use_altstack && alt_stack.ss_sp) { - sigaltstack(&old_stack, nullptr); - free(alt_stack.ss_sp); - } - }); - } - - // Signal delivery thread - std::thread signaller([&]() { - auto signal_interval = std::chrono::microseconds(1000000 / SIGNAL_FREQUENCY_HZ); - auto start_time = std::chrono::steady_clock::now(); - - while (test_running.load()) { - for (std::thread& worker : workers) { - pthread_kill(worker.native_handle(), SIGUSR1); - } - - std::this_thread::sleep_for(signal_interval); - - // Check for test timeout - auto now = std::chrono::steady_clock::now(); - if (std::chrono::duration_cast(now - start_time).count() > TEST_DURATION_MS) { - test_running.store(false); - break; - } - } - }); - - // Wait for test completion - signaller.join(); - - // Stop signal pressure - signal_pressure_active.store(false); - - // Wait for workers - for (auto& worker : workers) { - worker.join(); - } - - // Restore signal handler - sigaction(SIGUSR1, &old_action, nullptr); - - // Verify results - EXPECT_FALSE(signal_corruption_detected.load()) << "Signal pressure caused TLS corruption"; - EXPECT_FALSE(deadlock_detected.load()) << "Deadlock detected during signal pressure test"; - EXPECT_GT(signals_delivered.load(), 0) << "No signals were delivered"; - EXPECT_GT(tls_writes_completed.load(), 0) << "No TLS writes completed"; - - std::cout << "Signal pressure test completed: " << signals_delivered.load() - << " signals delivered, " << tls_writes_completed.load() << " TLS writes, " - << "corruption=" << (signal_corruption_detected.load() ? "YES" : "NO") << std::endl; -} - -// Test 10: Teardown Fuzz Test -TEST_F(StressTestSuite, TeardownFuzzTest) { - const int NUM_THREAD_CYCLES = 1000; - const int CONCURRENT_THREADS = 8; - - std::atomic teardown_corruption{false}; - std::atomic threads_created{0}; - std::atomic threads_completed{0}; - std::atomic agent_work_completed{0}; - - // Use the class shared storage for thread lifecycle testing - CallTraceStorage* test_storage = shared_storage.get(); - ThreadSafeRandom cycle_random(77777); - - for (int cycle = 0; cycle < NUM_THREAD_CYCLES / CONCURRENT_THREADS; ++cycle) { - std::vector native_threads; - - // Create batch of native threads - for (int t = 0; t < CONCURRENT_THREADS; ++t) { - native_threads.emplace_back([&, cycle, t]() { - threads_created.fetch_add(1, std::memory_order_relaxed); - - try { - // Initialize thread-local agent data - thread_local bool tls_initialized = false; - thread_local uint64_t tls_agent_id = 0; - - if (!tls_initialized) { - tls_agent_id = cycle_random.next(UINT32_MAX); - tls_initialized = true; - } - - // Simulate small amount of agent work - std::vector trace_ids; - for (int work = 0; work < 10; ++work) { - ASGCT_CallFrame frame; - frame.bci = static_cast(cycle * 1000 + t * 100 + work); - frame.method_id = reinterpret_cast(tls_agent_id + work); - - u64 trace_id = test_storage->put(1, &frame, false, 1); - if (trace_id > 0) { - trace_ids.push_back(trace_id); - } - - // Verify TLS is still valid - if (!tls_initialized || tls_agent_id == 0) { - teardown_corruption.store(true); - return; - } - } - - agent_work_completed.fetch_add(trace_ids.size(), std::memory_order_relaxed); - - // Simulate thread doing work after "TLS cleanup" - // This is the dangerous case we're testing for - tls_initialized = false; // Simulate TLS being cleared - - // Try to do more agent work (this should be safe or fail gracefully) - for (int post_work = 0; post_work < 3; ++post_work) { - try { - ASGCT_CallFrame frame; - frame.bci = static_cast(-1); // Native frame - frame.method_id = reinterpret_cast(0x999999); - - // This might fail, but shouldn't crash - u64 result = test_storage->put(1, &frame, false, 1); - (void)result; - - // Check if we can still access TLS safely - if (tls_agent_id != 0) { - // TLS still accessible after "cleanup" - record this - agent_work_completed.fetch_add(1, std::memory_order_relaxed); - } - - } catch (...) { - // Exceptions during post-cleanup work are acceptable - // as long as they don't crash the process - } - } - - threads_completed.fetch_add(1, std::memory_order_relaxed); - - } catch (...) { - teardown_corruption.store(true); - } - }); - } - - // Wait for this batch of threads to complete - for (auto& thread : native_threads) { - thread.join(); - } - - // Periodic cleanup of storage to simulate real usage patterns - if (cycle % 10 == 0) { - std::lock_guard lock(process_traces_mutex); - test_storage->processTraces([](const std::unordered_set& traces) { - // Simulate processing collected traces - volatile size_t count = traces.size(); - (void)count; - }); - } - - // Break early if corruption detected - if (teardown_corruption.load()) { - break; - } - } - - // Final cleanup handled by TearDown() - - // Verify results - EXPECT_FALSE(teardown_corruption.load()) << "Teardown corruption detected"; - EXPECT_EQ(threads_created.load(), threads_completed.load()) << "Thread creation/completion mismatch"; - EXPECT_GT(agent_work_completed.load(), 0) << "No agent work completed"; - - std::cout << "Teardown fuzz test completed: " << threads_created.load() - << " threads created, " << threads_completed.load() << " completed, " - << agent_work_completed.load() << " work units, " - << "corruption=" << (teardown_corruption.load() ? "YES" : "NO") << std::endl; -} - -// REALISTIC STRESS TESTS - Target actual profiler code paths -// These tests are designed to catch real bugs by exercising actual production code - -// CRASH-SAFE TEST EXECUTION FRAMEWORK -// This allows us to continue testing even after individual tests crash - -// Helper function for crash-safe test execution using process isolation -bool executeCrashSafeTest(const std::string& test_name, std::function test_func) { - std::cout << "\n=== Executing crash-safe test: " << test_name << " ===" << std::endl; - - pid_t pid = fork(); - if (pid == 0) { - // Child process - run the test in isolation - try { - test_func(); - std::cout << "Test " << test_name << " completed successfully" << std::endl; - _exit(0); // Success - } catch (const std::exception& e) { - std::cout << "Test " << test_name << " threw exception: " << e.what() << std::endl; - _exit(1); // Exception - } catch (...) { - std::cout << "Test " << test_name << " threw unknown exception" << std::endl; - _exit(2); // Unknown exception - } - } else if (pid > 0) { - // Parent process - wait and analyze result - int status; - pid_t result = waitpid(pid, &status, 0); - - if (result == -1) { - std::cout << "Test " << test_name << " - waitpid failed: " << strerror(errno) << std::endl; - return false; - } - - if (WIFEXITED(status)) { - int exit_code = WEXITSTATUS(status); - if (exit_code == 0) { - std::cout << "✅ Test " << test_name << " - PASSED" << std::endl; - return true; - } else { - std::cout << "❌ Test " << test_name << " - FAILED with exit code " << exit_code << std::endl; - return false; - } - } else if (WIFSIGNALED(status)) { - int sig = WTERMSIG(status); - std::cout << "💥 Test " << test_name << " - CRASHED with signal " << sig; - switch (sig) { - case SIGSEGV: std::cout << " (SIGSEGV - segmentation fault - memory bug found!)"; break; - case SIGABRT: std::cout << " (SIGABRT - abort - assertion failure)"; break; - case SIGBUS: std::cout << " (SIGBUS - bus error - alignment issue)"; break; - case SIGFPE: std::cout << " (SIGFPE - floating point exception)"; break; - case SIGTRAP: std::cout << " (SIGTRAP - debug trap)"; break; - case SIGILL: std::cout << " (SIGILL - illegal instruction)"; break; - default: std::cout << " (signal " << sig << ")"; break; - } - std::cout << std::endl; - return false; - } else { - std::cout << "❓ Test " << test_name << " - UNKNOWN termination (status=" << status << ")" << std::endl; - return false; - } - } else { - std::cout << "💀 Test " << test_name << " - fork failed: " << strerror(errno) << std::endl; - return false; - } -} - -// Test Results Collector -struct TestSuiteResults { - int total_tests = 0; - int passed_tests = 0; - int failed_tests = 0; - int crashed_tests = 0; - std::vector crashes_found; - std::vector failures_found; - - void recordPass(const std::string& test_name) { - total_tests++; - passed_tests++; - } - - void recordFailure(const std::string& test_name) { - total_tests++; - failed_tests++; - failures_found.push_back(test_name); - } - - void recordCrash(const std::string& test_name) { - total_tests++; - crashed_tests++; - crashes_found.push_back(test_name); - } - - void printSummary() const { - std::cout << "\n" << std::string(60, '=') << std::endl; - std::cout << "STRESS TEST SUITE SUMMARY" << std::endl; - std::cout << std::string(60, '=') << std::endl; - std::cout << "Total tests run: " << total_tests << std::endl; - std::cout << "✅ Passed: " << passed_tests << std::endl; - std::cout << "❌ Failed: " << failed_tests << std::endl; - std::cout << "💥 Crashed: " << crashed_tests << " (BUGS FOUND!)" << std::endl; - - if (!crashes_found.empty()) { - std::cout << "\nCrashes found in:" << std::endl; - for (const auto& crash : crashes_found) { - std::cout << " 💥 " << crash << std::endl; - } - } - - if (!failures_found.empty()) { - std::cout << "\nFailures in:" << std::endl; - for (const auto& failure : failures_found) { - std::cout << " ❌ " << failure << std::endl; - } - } - - std::cout << std::string(60, '=') << std::endl; - } -}; - -// Implementation function for signal stress (isolated for crash safety) -static void realProfilerSignalStressImpl(int signal_barrage_count, int num_worker_threads) { - std::atomic test_running{true}; - std::atomic handler_corruption{false}; - std::atomic signals_handled{0}; - std::atomic storage_operations{0}; - - // Use the single shared storage that will be hammered during signal handling - CallTraceStorage* signal_storage = StressTestSuite::shared_storage.get(); - - // Set up global state for signal handler - realistic_test_running.store(true); - realistic_handler_corruption.store(false); - realistic_signals_handled.store(0); - realistic_storage_operations.store(0); - realistic_shared_storage = signal_storage; - - // Install realistic signal handler - struct sigaction new_action, old_action; - new_action.sa_handler = realistic_profiler_signal_handler; - sigemptyset(&new_action.sa_mask); - new_action.sa_flags = SA_RESTART; - - if (sigaction(SIGUSR2, &new_action, &old_action) != 0) { - throw std::runtime_error("Could not install signal handler"); - } - - // Worker threads doing normal profiler operations while signals fire - std::vector workers; - for (int t = 0; t < num_worker_threads; ++t) { - workers.emplace_back([&, t]() { - while (test_running.load()) { - try { - // Simulate normal application work that profiler samples - for (int work = 0; work < 50; ++work) { - ASGCT_CallFrame frame; - frame.bci = work; - frame.method_id = reinterpret_cast(0x2000 + t * 100 + work); - - u64 trace_id = realistic_shared_storage->put(1, &frame, false, 1); - // Small delay to allow signal interference - if (work % 10 == 0) { - std::this_thread::yield(); - } - - storage_operations.fetch_add(1, std::memory_order_relaxed); - } - } catch (...) { - realistic_handler_corruption.store(true); - break; - } - } - }); - } - - // Single dump thread - represents realistic JFR dump operations - // In production, this would be protected by mutex and only one thread does dumps - std::thread dump_thread([&]() { - int dump_count = 0; - while (test_running.load() && dump_count < 3) { // Only do a few dumps - try { - // Wait a bit to let some traces accumulate - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - - // Single-threaded processTraces call - matches production pattern - { - std::lock_guard lock(StressTestSuite::process_traces_mutex); - signal_storage->processTraces([](const std::unordered_set& traces) { - volatile size_t count = traces.size(); - (void)count; - }); - } - - dump_count++; - std::this_thread::sleep_for(std::chrono::milliseconds(20)); - } catch (...) { - realistic_handler_corruption.store(true); - break; - } - } - }); - - // Signal barrage thread - this is where crashes typically occur - std::thread signaller([&]() { - for (int i = 0; i < signal_barrage_count && test_running.load(); ++i) { - // Send signals to all worker threads simultaneously - for (std::thread& worker : workers) { - pthread_kill(worker.native_handle(), SIGUSR2); - } - - // Brief pause to let signals get handled - std::this_thread::sleep_for(std::chrono::microseconds(100)); - - // Break early if we detect issues - if (realistic_handler_corruption.load()) { - break; - } - } - realistic_test_running.store(false); - test_running.store(false); - }); - - // Wait for test completion - signaller.join(); - dump_thread.join(); - for (auto& worker : workers) { - worker.join(); - } - - // Clean up global state - realistic_shared_storage = nullptr; - realistic_test_running.store(false); - - // Restore signal handler - sigaction(SIGUSR2, &old_action, nullptr); - - // Report results - std::cout << "Signal stress (" << signal_barrage_count << " signals, " << num_worker_threads - << " threads): " << realistic_signals_handled.load() << " signals handled, " - << realistic_storage_operations.load() << " storage ops, " - << "corruption=" << (realistic_handler_corruption.load() ? "YES" : "NO") << std::endl; - - if (realistic_handler_corruption.load()) { - throw std::runtime_error("Signal handler corruption detected"); - } -} - -// Test 11: Instance ID and Trace ID Generation Stress Test -TEST_F(StressTestSuite, InstanceIdTraceIdStressTest) { - const int NUM_THREADS = 12; // High contention on instance ID generation - const int NUM_STORAGE_INSTANCES = 8; // Multiple CallTraceStorage instances - const int OPERATIONS_PER_THREAD = 10000; - const int RAPID_SWAPS_COUNT = 1000; // Frequent table swaps to stress instance ID assignment - - std::atomic test_failed{false}; - std::atomic collision_detected{false}; - std::atomic overflow_detected{false}; - std::atomic invalid_trace_id_detected{false}; - std::atomic total_trace_ids_generated{0}; - std::atomic duplicate_trace_ids{0}; - std::atomic zero_trace_ids{0}; - std::atomic max_instance_id_seen{0}; - - // Set to track all generated trace IDs and stack trace hashes for analysis - std::mutex trace_id_mutex; - std::unordered_set all_trace_ids; - std::unordered_set all_stack_hashes; // Track unique stack trace hashes - - // Use single shared storage instance - matches production pattern - // Note: NUM_THREADS threads will contend on the same storage instance - CallTraceStorage* storage_instance = shared_storage.get(); - - std::cout << "Testing instance ID and trace ID generation under extreme concurrency..." << std::endl; - - // Worker threads that hammer trace ID generation across multiple storage instances - std::vector workers; - for (int t = 0; t < NUM_THREADS; ++t) { - workers.emplace_back([&, t]() { - for (int op = 0; op < OPERATIONS_PER_THREAD && !test_failed.load(); ++op) { - try { - // Use the single shared storage instance - CallTraceStorage* storage = storage_instance; - - // Create a DETERMINISTIC unique frame - no randomness - // Each (thread, operation) pair generates a unique frame - ASGCT_CallFrame frame; - frame.bci = t * 1000000 + op; // Deterministic: thread_id * 1M + op_index - frame.method_id = reinterpret_cast(0x10000000ULL + (u64)t * 10000000ULL + (u64)op); - - // Calculate stack trace hash for analysis (simplified hash of frame data) - u64 stack_hash = (u64)frame.bci ^ ((u64)frame.method_id << 32); - - // Generate trace ID - u64 trace_id = storage->put(1, &frame, false, 1); - - if (trace_id == 0) { - zero_trace_ids.fetch_add(1, std::memory_order_relaxed); - continue; // Dropped trace is acceptable - } - - if (trace_id == CallTraceStorage::DROPPED_TRACE_ID) { - continue; // Also acceptable - } - - // Extract instance ID and slot from trace ID - u64 instance_id = trace_id >> 32; - u64 slot = trace_id & 0xFFFFFFFFULL; - - // Validate trace ID structure - if (instance_id == 0) { - invalid_trace_id_detected.store(true); - test_failed.store(true); - return; - } - - // Check for slot overflow (should fit in 32 bits) - if (slot > 0xFFFFFFFFULL) { - overflow_detected.store(true); - test_failed.store(true); - return; - } - - // Track maximum instance ID to detect counter behavior - uint64_t current_max = max_instance_id_seen.load(); - while (instance_id > current_max) { - if (max_instance_id_seen.compare_exchange_weak(current_max, instance_id)) { - break; // Successfully updated - } - // CAS failed - current_max now contains the actual current value - // Loop continues if instance_id is still greater than the updated current_max - } - - // Check for trace ID collisions and track stack hashes - { - std::lock_guard lock(trace_id_mutex); - all_stack_hashes.insert(stack_hash); // Track all stack hashes - - if (all_trace_ids.find(trace_id) != all_trace_ids.end()) { - duplicate_trace_ids.fetch_add(1, std::memory_order_relaxed); - } else { - all_trace_ids.insert(trace_id); - } - } - - total_trace_ids_generated.fetch_add(1, std::memory_order_relaxed); - - // Occasionally trigger rapid table swaps to stress instance ID assignment - if (op % 100 == 0 && t == 0) { // Only one thread does swaps - for (int swap = 0; swap < 3; ++swap) { - std::lock_guard lock(process_traces_mutex); - storage->processTraces([](const std::unordered_set& traces) { - volatile size_t count = traces.size(); - (void)count; - }); - } - } - - // Yield periodically to increase contention - if (op % 500 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - test_failed.store(true); - return; - } - } - }); - } - - // Additional thread that does rapid processTraces() calls to stress instance ID assignment - std::thread rapid_swapper([&]() { - for (int swap = 0; swap < RAPID_SWAPS_COUNT && !test_failed.load(); ++swap) { - try { - // Use single shared storage instance for swap - { - std::lock_guard lock(process_traces_mutex); - shared_storage->processTraces([](const std::unordered_set& traces) { - // Process traces - this triggers new instance ID assignment - volatile size_t count = traces.size(); - (void)count; - }); - } - - // Brief pause - std::this_thread::sleep_for(std::chrono::microseconds(100)); - - } catch (...) { - test_failed.store(true); - return; - } - } - }); - - // Wait for all threads - for (auto& worker : workers) { - worker.join(); - } - rapid_swapper.join(); - - // Analyze results - u64 unique_trace_ids = 0; - u64 unique_stack_hashes = 0; - { - std::lock_guard lock(trace_id_mutex); - unique_trace_ids = all_trace_ids.size(); - unique_stack_hashes = all_stack_hashes.size(); - } - - std::cout << "Instance ID/Trace ID stress test completed:" << std::endl; - std::cout << " Total trace IDs generated: " << total_trace_ids_generated.load() << std::endl; - std::cout << " Unique stack traces: " << unique_stack_hashes << std::endl; - std::cout << " Unique trace IDs: " << unique_trace_ids << std::endl; - std::cout << " Duplicate trace IDs: " << duplicate_trace_ids.load() << std::endl; - std::cout << " Zero trace IDs: " << zero_trace_ids.load() << std::endl; - std::cout << " Max instance ID seen: " << max_instance_id_seen.load() << std::endl; - std::cout << " Overflow detected: " << (overflow_detected.load() ? "YES" : "NO") << std::endl; - std::cout << " Invalid trace ID detected: " << (invalid_trace_id_detected.load() ? "YES" : "NO") << std::endl; - - // Verify results - EXPECT_FALSE(test_failed.load()) << "Instance ID/Trace ID stress test failed"; - EXPECT_FALSE(overflow_detected.load()) << "Slot overflow detected"; - EXPECT_FALSE(invalid_trace_id_detected.load()) << "Invalid trace ID structure detected"; - EXPECT_GT(total_trace_ids_generated.load(), 0) << "No trace IDs generated"; - EXPECT_GT(max_instance_id_seen.load(), 0) << "No valid instance IDs seen"; - - // Calculate duplication metrics - double duplication_rate = (double)duplicate_trace_ids.load() / total_trace_ids_generated.load(); - double stack_uniqueness_rate = (double)unique_stack_hashes / total_trace_ids_generated.load(); - - std::cout << " Duplication rate: " << (duplication_rate * 100.0) << "%" << std::endl; - std::cout << " Stack trace uniqueness: " << (stack_uniqueness_rate * 100.0) << "%" << std::endl; - - // Only fail if trace IDs are more duplicated than stack traces (indicates a bug) - // If stack traces themselves have duplicates, then trace ID duplicates are expected - EXPECT_GE(unique_trace_ids, unique_stack_hashes) - << "Trace IDs less unique than stack traces - indicates trace ID generation bug"; - - // Allow legitimate deduplication but warn if uniqueness is surprisingly low - if (stack_uniqueness_rate < 0.9) { - std::cout << " WARNING: Low stack trace uniqueness suggests frame generation issues" << std::endl; - } -} - -// Test 12: Hash Table Spin-Wait Edge Cases Stress Test -TEST_F(StressTestSuite, HashTableSpinWaitEdgeCasesTest) { - const int NUM_THREADS = 16; // High contention to trigger spin-waits - const int OPERATIONS_PER_THREAD = 5000; - const int HASH_COLLISION_GROUPS = 50; // Force hash collisions to trigger spin-wait paths - const int SLOW_ALLOCATION_FREQUENCY = 10; // Simulate slow allocations - - std::atomic test_failed{false}; - std::atomic timeout_detected{false}; - std::atomic preparing_deadlock{false}; - std::atomic allocation_failure_cascade{false}; - std::atomic spin_wait_events{0}; - std::atomic timeout_recoveries{0}; - std::atomic allocation_failures{0}; - std::atomic successful_insertions{0}; - std::atomic dropped_traces{0}; - std::atomic hash_collisions_detected{0}; - - // Single hash table to maximize contention - // Use heap allocation with proper alignment to avoid ASAN alignment issues - void* aligned_memory = std::aligned_alloc(alignof(CallTraceHashTable), sizeof(CallTraceHashTable)); - ASSERT_NE(aligned_memory, nullptr) << "Failed to allocate aligned memory for CallTraceHashTable"; - - auto hash_table_ptr = std::unique_ptr( - new(aligned_memory) CallTraceHashTable(), - [](CallTraceHashTable* ptr) { - ptr->~CallTraceHashTable(); - std::free(ptr); - } - ); - CallTraceHashTable& hash_table = *hash_table_ptr; - hash_table.setInstanceId(42); - - std::cout << "Testing hash table spin-wait logic under extreme edge cases..." << std::endl; - - // Create controlled hash collision groups to force same-slot contention - std::vector> collision_groups(HASH_COLLISION_GROUPS); - for (int g = 0; g < HASH_COLLISION_GROUPS; ++g) { - // Generate frames that will likely hash to similar slots - for (int f = 0; f < 20; ++f) { - ASGCT_CallFrame frame; - frame.bci = g * 1000 + f; // Group-based BCI to encourage collisions - frame.method_id = reinterpret_cast(0x100000 + g * 100 + f); - collision_groups[g].push_back(frame); - } - } - - std::vector workers; - for (int t = 0; t < NUM_THREADS; ++t) { - workers.emplace_back([&, t]() { - std::mt19937 gen(12345 + t); // Fixed seed to increase collision probability - std::uniform_int_distribution group_dis(0, HASH_COLLISION_GROUPS - 1); - std::uniform_int_distribution frame_dis(0, 19); - std::uniform_int_distribution slow_dis(1, 100); - - for (int op = 0; op < OPERATIONS_PER_THREAD && !test_failed.load(); ++op) { - try { - // Pick a frame from collision groups to maximize slot contention - int group = group_dis(gen); - int frame_idx = frame_dis(gen); - ASGCT_CallFrame frame = collision_groups[group][frame_idx]; - - // Add some uniqueness to prevent exact duplicates while preserving hash patterns - frame.bci += t * 100000 + op; - - // Simulate slow allocation periodically to stress the spin-wait logic - if (slow_dis(gen) <= SLOW_ALLOCATION_FREQUENCY) { - // Brief delay to simulate memory allocation pressure - std::this_thread::sleep_for(std::chrono::microseconds(100)); - } - - // This should trigger the spin-wait paths due to hash collisions - u64 trace_id = hash_table.put(1, &frame, false, 1); - - if (trace_id == 0) { - dropped_traces.fetch_add(1, std::memory_order_relaxed); - continue; - } - - if (trace_id == CallTraceStorage::DROPPED_TRACE_ID) { - allocation_failures.fetch_add(1, std::memory_order_relaxed); - continue; - } - - if (trace_id == 0x7fffffffffffffffULL) { // OVERFLOW_TRACE_ID - continue; - } - - successful_insertions.fetch_add(1, std::memory_order_relaxed); - - // Every successful insertion in the same collision group indicates potential spin-wait - spin_wait_events.fetch_add(1, std::memory_order_relaxed); - - // Yield occasionally to increase interleaving and contention - if (op % 50 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - test_failed.store(true); - return; - } - } - }); - } - - // Monitor thread to detect potential deadlocks in spin-wait logic - std::atomic monitor_running{true}; - std::thread monitor([&]() { - auto start_time = std::chrono::steady_clock::now(); - u64 last_insertions = 0; - - while (monitor_running.load()) { - std::this_thread::sleep_for(std::chrono::milliseconds(500)); - - u64 current_insertions = successful_insertions.load(); - auto now = std::chrono::steady_clock::now(); - auto elapsed = std::chrono::duration_cast(now - start_time).count(); - - // Check for progress stall (potential deadlock in spin-wait) - if (elapsed > 5 && current_insertions == last_insertions) { - // No progress for too long - possible deadlock - preparing_deadlock.store(true); - test_failed.store(true); - break; - } - - // Check for excessive timeout recoveries - if (timeout_recoveries.load() > successful_insertions.load() / 10) { - timeout_detected.store(true); - } - - // Check for allocation failure cascade - if (allocation_failures.load() > successful_insertions.load()) { - allocation_failure_cascade.store(true); - } - - last_insertions = current_insertions; - } - }); - - // Wait for all workers - for (auto& worker : workers) { - worker.join(); - } - monitor_running.store(false); - monitor.join(); - - // Analyze results - double failure_rate = (double)allocation_failures.load() / (successful_insertions.load() + allocation_failures.load()); - double drop_rate = (double)dropped_traces.load() / (successful_insertions.load() + dropped_traces.load()); - - std::cout << "Hash table spin-wait stress test completed:" << std::endl; - std::cout << " Successful insertions: " << successful_insertions.load() << std::endl; - std::cout << " Allocation failures: " << allocation_failures.load() << std::endl; - std::cout << " Dropped traces: " << dropped_traces.load() << std::endl; - std::cout << " Spin-wait events: " << spin_wait_events.load() << std::endl; - std::cout << " Timeout recoveries: " << timeout_recoveries.load() << std::endl; - std::cout << " Hash collisions detected: " << hash_collisions_detected.load() << std::endl; - std::cout << " Failure rate: " << (failure_rate * 100.0) << "%" << std::endl; - std::cout << " Drop rate: " << (drop_rate * 100.0) << "%" << std::endl; - std::cout << " Preparing deadlock: " << (preparing_deadlock.load() ? "YES" : "NO") << std::endl; - std::cout << " Timeout detected: " << (timeout_detected.load() ? "YES" : "NO") << std::endl; - std::cout << " Allocation cascade: " << (allocation_failure_cascade.load() ? "YES" : "NO") << std::endl; - - // Verify results - EXPECT_FALSE(test_failed.load()) << "Hash table spin-wait test failed"; - EXPECT_FALSE(preparing_deadlock.load()) << "Deadlock detected in PREPARING state spin-wait"; - EXPECT_GT(successful_insertions.load(), 0) << "No successful hash table insertions"; - - // Some failures are expected under extreme contention, but not excessive - EXPECT_LT(failure_rate, 0.8) << "Excessive allocation failure rate: " << failure_rate; - EXPECT_LT(drop_rate, 0.5) << "Excessive trace drop rate: " << drop_rate; -} - -// Regression test: TSan data race in findCallTrace() — keys[] plain reads -// raced with atomic CAS writes from concurrent put(). When a table is -// promoted to prev after expansion, threads that loaded the old table -// pointer before the CAS swap still write into it while new threads call -// findCallTrace(prev, hash) — the read must be atomic. -// -// Memory-ordering races on 8-byte aligned values are benign on x86_64 (the -// CPU guarantees naturally-atomic loads), so the race is only reliably -// detectable under ThreadSanitizer. The test is therefore skipped outside -// TSan builds to avoid giving false confidence that the regression is covered. -// -// The dedup assertion (dedup_mismatches == 0) validates findCallTrace -// correctness across expansion boundaries independently of memory ordering; -// it catches logic regressions in all build configurations. -TEST_F(StressTestSuite, FindCallTraceAtomicReadRaceTest) { -#if !defined(TSAN_ENABLED) - GTEST_SKIP() << "TSan-only race regression: re-run with -fsanitize=thread"; -#endif - - // Fill the table to just below the 75% expansion threshold - // (INITIAL_CAPACITY = 65536; 75% = 49152). - const int FILL_TARGET = 48800; - const int NUM_THREADS = 16; - const int OPS_PER_THREAD = 400; - // Number of pre-filled entries to re-insert concurrently as a dedup check. - // After expansion these will be looked up via findCallTrace(prev, hash). - const int VERIFY_COUNT = 200; - - void* aligned_memory = std::aligned_alloc(alignof(CallTraceHashTable), sizeof(CallTraceHashTable)); - ASSERT_NE(aligned_memory, nullptr); - auto hash_table_ptr = std::unique_ptr( - new(aligned_memory) CallTraceHashTable(), - [](CallTraceHashTable* ptr) { - ptr->~CallTraceHashTable(); - std::free(ptr); - } - ); - CallTraceHashTable& hash_table = *hash_table_ptr; - hash_table.setInstanceId(42); - - // Pre-fill single-threaded up to just below the expansion threshold, - // recording trace_ids for the last VERIFY_COUNT entries. - std::vector expected_ids(VERIFY_COUNT); - std::vector verify_frames(VERIFY_COUNT); - - for (int i = 0; i < FILL_TARGET; ++i) { - ASGCT_CallFrame frame; - frame.bci = i + 1; - frame.method_id = reinterpret_cast(0x10000 + i); - u64 id = hash_table.put(1, &frame, false, 1); - if (i >= FILL_TARGET - VERIFY_COUNT) { - int vi = i - (FILL_TARGET - VERIFY_COUNT); - expected_ids[vi] = id; - verify_frames[vi] = frame; - } - } - - std::atomic go{false}; - std::atomic successes{0}; - // Counts put() calls that returned a trace_id different from the originally - // recorded id for a known pre-filled entry. A non-zero value means - // findCallTrace(prev, hash) returned nullptr when it should have returned - // the entry — either a logic bug or a memory-ordering failure. - std::atomic dedup_mismatches{0}; - std::vector workers; - - // Half the threads insert new distinct frames to trigger expansion. They - // load the old table pointer before the CAS swap and write keys[] in prev, - // creating the concurrent-write side of the race window. - for (int t = 0; t < NUM_THREADS / 2; ++t) { - workers.emplace_back([&, t]() { - while (!go.load(std::memory_order_acquire)) { /* spin */ } - for (int i = 0; i < OPS_PER_THREAD; ++i) { - ASGCT_CallFrame frame; - frame.bci = FILL_TARGET + 1 + t * OPS_PER_THREAD + i; - frame.method_id = reinterpret_cast(0x80000 + t * OPS_PER_THREAD + i); - u64 id = hash_table.put(1, &frame, false, 1); - if (id != 0 && id != CallTraceStorage::DROPPED_TRACE_ID) { - successes.fetch_add(1, std::memory_order_relaxed); - } - } - }); - } - - // The other half re-insert pre-filled frames. After expansion the new - // table is empty for those hashes, so put() claims a new slot and calls - // findCallTrace(prev, hash) — the read side of the race window. A correct - // findCallTrace must return the original trace so dedup produces the same - // trace_id. - for (int t = NUM_THREADS / 2; t < NUM_THREADS; ++t) { - workers.emplace_back([&, t]() { - while (!go.load(std::memory_order_acquire)) { /* spin */ } - for (int i = 0; i < VERIFY_COUNT; ++i) { - u64 id = hash_table.put(1, &verify_frames[i], false, 1); - if (id != 0 && id != CallTraceStorage::DROPPED_TRACE_ID && - id != expected_ids[i]) { - dedup_mismatches.fetch_add(1, std::memory_order_relaxed); - } - } - }); - } - - go.store(true, std::memory_order_release); - for (auto& w : workers) { - w.join(); - } - - EXPECT_GT(successes.load(), 0u) << "No successful insertions after expansion"; - EXPECT_EQ(dedup_mismatches.load(), 0u) - << "findCallTrace returned wrong trace_id for pre-filled entries after expansion " - "(findCallTrace failed to locate entry in prev table)"; -} - -// Test 13: Hash Table Memory Allocation Failure Stress Test -TEST_F(StressTestSuite, HashTableAllocationFailureStressTest) { - const int NUM_THREADS = 8; - const int OPERATIONS_PER_THREAD = 2000; - const int LARGE_FRAME_COUNT = 500; // Large stack traces to stress allocator - - std::atomic test_failed{false}; - std::atomic corruption_detected{false}; - std::atomic inconsistent_state{false}; - std::atomic allocation_failures{0}; - std::atomic successful_large_traces{0}; - std::atomic key_cleanup_events{0}; - std::atomic preparing_state_leaks{0}; - - // Use heap allocation with proper alignment to avoid ASAN alignment issues - void* aligned_memory = std::aligned_alloc(alignof(CallTraceHashTable), sizeof(CallTraceHashTable)); - ASSERT_NE(aligned_memory, nullptr) << "Failed to allocate aligned memory for CallTraceHashTable"; - - auto hash_table_ptr = std::unique_ptr( - new(aligned_memory) CallTraceHashTable(), - [](CallTraceHashTable* ptr) { - ptr->~CallTraceHashTable(); - std::free(ptr); - } - ); - CallTraceHashTable& hash_table = *hash_table_ptr; - hash_table.setInstanceId(77); - - std::cout << "Testing hash table allocation failure recovery..." << std::endl; - - std::vector workers; - for (int t = 0; t < NUM_THREADS; ++t) { - workers.emplace_back([&, t]() { - std::mt19937 gen(std::random_device{}() + t); - std::uniform_int_distribution frame_count_dis(1, LARGE_FRAME_COUNT); - std::uniform_int_distribution bci_dis(1, 1000000); - std::uniform_int_distribution method_dis(0x100000, 0xFFFFFF); - - for (int op = 0; op < OPERATIONS_PER_THREAD && !test_failed.load(); ++op) { - try { - // Create large stack traces to increase allocation pressure - int num_frames = frame_count_dis(gen); - std::vector frames(num_frames); - - for (int f = 0; f < num_frames; ++f) { - frames[f].bci = bci_dis(gen) + t * 10000000 + op * 1000 + f; - frames[f].method_id = reinterpret_cast(method_dis(gen) + f); - } - - // This should sometimes fail allocation due to large size - u64 trace_id = hash_table.put(num_frames, frames.data(), false, 1); - - if (trace_id == CallTraceStorage::DROPPED_TRACE_ID) { - allocation_failures.fetch_add(1, std::memory_order_relaxed); - // Verify that the slot was properly cleaned up after allocation failure - key_cleanup_events.fetch_add(1, std::memory_order_relaxed); - } else if (trace_id != 0 && trace_id != 0x7fffffffffffffffULL) { - successful_large_traces.fetch_add(1, std::memory_order_relaxed); - - // Verify trace ID structure for large traces - u64 instance_id = trace_id >> 32; - u64 slot = trace_id & 0xFFFFFFFFULL; - - if (instance_id != 77 || slot >= 1048576) { // LARGE_TABLE_CAPACITY - inconsistent_state.store(true); - test_failed.store(true); - return; - } - } - - // Periodically check for leaked PREPARING states - if (op % 100 == 0) { - // This is a heuristic - we can't directly inspect internal state - // but if we see extreme allocation failures, it might indicate leaks - if (allocation_failures.load() > successful_large_traces.load() * 3) { - preparing_state_leaks.fetch_add(1, std::memory_order_relaxed); - } - } - - // Yield to allow other threads to interfere during allocation - if (op % 50 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - test_failed.store(true); - return; - } - } - }); - } - - // Wait for completion - for (auto& worker : workers) { - worker.join(); - } - - // Analyze results - u64 total_operations = successful_large_traces.load() + allocation_failures.load(); - double allocation_failure_rate = (double)allocation_failures.load() / total_operations; - - std::cout << "Hash table allocation failure stress test completed:" << std::endl; - std::cout << " Total operations: " << total_operations << std::endl; - std::cout << " Successful large traces: " << successful_large_traces.load() << std::endl; - std::cout << " Allocation failures: " << allocation_failures.load() << std::endl; - std::cout << " Key cleanup events: " << key_cleanup_events.load() << std::endl; - std::cout << " Preparing state leaks: " << preparing_state_leaks.load() << std::endl; - std::cout << " Allocation failure rate: " << (allocation_failure_rate * 100.0) << "%" << std::endl; - std::cout << " Corruption detected: " << (corruption_detected.load() ? "YES" : "NO") << std::endl; - std::cout << " Inconsistent state: " << (inconsistent_state.load() ? "YES" : "NO") << std::endl; - - // Verify results - EXPECT_FALSE(test_failed.load()) << "Allocation failure stress test failed"; - EXPECT_FALSE(corruption_detected.load()) << "Memory corruption detected"; - EXPECT_FALSE(inconsistent_state.load()) << "Inconsistent internal state detected"; - EXPECT_GT(total_operations, 0) << "No operations completed"; - - // Some allocation failures are expected with large traces - EXPECT_GT(successful_large_traces.load(), 0) << "No large traces successfully stored"; - - // But not excessive leaks of PREPARING states - EXPECT_LT(preparing_state_leaks.load(), total_operations / 100) << "Excessive PREPARING state leaks"; -} - -// Test 14: Real Profiler Signal Handler Stress - Now crash-safe with progressive difficulty -TEST_F(StressTestSuite, RealProfilerSignalStressSafe) { - TestSuiteResults results; - - // Test with progressively more aggressive parameters to find the breaking point - // macOS is more resource-constrained than Linux, so use conservative limits - std::vector> test_configs; - -#ifdef __APPLE__ - // macOS-specific conservative limits to avoid false positive crashes - test_configs = { - {50, 1}, // Very gentle - should always pass - {200, 1}, // Moderate - likely to pass - {300, 1}, // Single-threaded stress - avoids macOS multi-thread signal issues - {500, 1}, // Higher single-threaded load - {100, 2}, // Conservative multi-thread test - {200, 2}, // Moderate multi-thread - real bugs should still manifest - {300, 2}, // Push macOS limits a bit - real memory bugs should still show - {1000, 1}, // High single-threaded - tests signal coalescing limits - }; - std::cout << "Running macOS-optimized signal stress tests..." << std::endl; -#else - // Linux can handle higher stress levels - test_configs = { - {50, 1}, // Very gentle - should always pass - {200, 1}, // Moderate - likely to pass - {500, 2}, // Aggressive - may pass or fail - {1000, 2}, // Very aggressive - likely to find issues - {2000, 3}, // Extreme - very likely to crash - {5000, 3}, // Extreme stress - ultimate test of critical section fixes - }; - std::cout << "Running Linux-optimized signal stress tests..." << std::endl; -#endif - - std::cout << "Running progressive signal stress tests to find breaking points..." << std::endl; - - for (size_t i = 0; i < test_configs.size(); ++i) { - int signal_count = test_configs[i].first; - int thread_count = test_configs[i].second; - - std::string test_name = "SignalStress_" + std::to_string(signal_count) + "_signals_" - + std::to_string(thread_count) + "_threads"; - - auto test_func = [signal_count, thread_count]() { - realProfilerSignalStressImpl(signal_count, thread_count); - }; - - bool test_passed = executeCrashSafeTest(test_name, test_func); - - if (test_passed) { - results.recordPass(test_name); - } else { - // Determine if it was a crash or just a failure - // We'll assume crashes for now since that's our main concern - results.recordCrash(test_name); - std::cout << "⚠️ Configuration " << test_name << " failed - bug found at this stress level!" << std::endl; - } - - // Small pause between tests - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - // Print comprehensive results - results.printSummary(); - - // Test always "passes" from a gtest perspective - we report bugs instead of failing - EXPECT_GT(results.passed_tests, 0) << "No signal stress configurations passed - complete system failure"; - - if (results.crashed_tests > 0) { - std::cout << "\n🎯 SUCCESS: Found " << results.crashed_tests << " stress levels that expose memory safety bugs!" << std::endl; - std::cout << "These crashes indicate real vulnerabilities in the profiler's signal handling." << std::endl; - } else { - std::cout << "\n🛡️ Signal handling appears robust under all tested stress levels." << std::endl; - } -} - -// Regression guard for defects B and C in CallTraceHashTable: -// Defect B: non-atomic _table read in collect()/clearTableOnly() races with the -// ACQ_REL CAS expansion path in put() on aarch64 (weak ordering). -// Defect C: chain-clearing loop in clearTableOnly() skips all but the first node -// in an expanded (multi-node) _prev chain, leaving dangling pointers. -// -// Both defects are only reachable when the table grows past CALLTRACE_EXPANSION_THRESHOLD. -// No prior stress test crossed that boundary; this test explicitly does so. -TEST_F(StressTestSuite, ConcurrentExpansionAndCollectStressTest) { - static constexpr int TARGET_TRACES = CALLTRACE_EXPANSION_THRESHOLD + 4096; - static constexpr int FILL_THREADS = 8; - - CallTraceStorage* storage = shared_storage.get(); - storage->clear(); - - // --- Phase 1: fill past expansion threshold with unique traces --- - std::atomic total_inserted{0}; - { - std::vector fillers; - int per_thread = (TARGET_TRACES / FILL_THREADS) + 1; - for (int t = 0; t < FILL_THREADS; t++) { - fillers.emplace_back([&, t]() { - for (int i = 0; i < per_thread; i++) { - ASGCT_CallFrame frame; - // Unique (bci, method_id) across all threads: multiply thread - // index by a large prime to prevent aliasing between threads. - frame.bci = (t * 100003 + i) % 1000000; - frame.method_id = reinterpret_cast( - static_cast(0x100000ULL + (u64)t * 100003ULL + (u64)i)); - u64 id = storage->put(1, &frame, false, 1); - if (id > 0 && id != CallTraceStorage::DROPPED_TRACE_ID) { - total_inserted.fetch_add(1, std::memory_order_relaxed); - } - } - }); - } - for (auto& th : fillers) th.join(); - } - - ASSERT_GT(total_inserted.load(), CALLTRACE_EXPANSION_THRESHOLD) - << "Table did not fill past expansion threshold (" - << CALLTRACE_EXPANSION_THRESHOLD << "); _prev chain not created. " - "Adjust FILL_THREADS or per_thread count."; - - // --- Phase 2: processTraces() while concurrent puts continue --- - // The concurrent putter keeps the put() -> CAS expansion path hot so TSan - // can observe the racy plain _table load in collect() / clearTableOnly(). - std::atomic running{true}; - std::atomic phase2_failed{false}; - - std::thread concurrent_putter([&]() { - int seq = 0; - while (running.load()) { - ASGCT_CallFrame frame; - frame.bci = 999000 + (seq % 1000); - frame.method_id = reinterpret_cast( - static_cast(0xFFFF0000ULL + (u64)(seq & 0xFFFF))); - u64 id = storage->put(1, &frame, false, 1); - (void)id; - seq++; - if (seq % 1000 == 0) std::this_thread::yield(); - } - }); - - // Rotate three times so clearTableOnly() runs on an expanded table each time. - for (int cycle = 0; cycle < 3 && !phase2_failed.load(); cycle++) { - { - std::lock_guard lock(process_traces_mutex); - storage->processTraces([&](const std::unordered_set& traces) { - // Sanity: first cycle must contain all traces from Phase 1. - if (cycle == 0 && - static_cast(traces.size()) < total_inserted.load()) { - phase2_failed.store(true); - } - }); - } - // Brief pause so the concurrent putter can insert a few more entries - // before the next processTraces() call. - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - } - - running.store(false); - concurrent_putter.join(); - - EXPECT_FALSE(phase2_failed.load()) << "collect() missed traces from Phase 1 on first cycle"; -} diff --git a/ddprof-lib/src/test/cpp/stress_stringDictionary.cpp b/ddprof-lib/src/test/cpp/stress_stringDictionary.cpp deleted file mode 100644 index b9ff4ba63..000000000 --- a/ddprof-lib/src/test/cpp/stress_stringDictionary.cpp +++ /dev/null @@ -1,651 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "stringDictionary.h" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// ── helpers ─────────────────────────────────────────────────────────────── - -static std::vector makeKeys(int thread_id, int count) { - std::vector keys; - keys.reserve(count); - for (int i = 0; i < count; i++) { - keys.push_back("t" + std::to_string(thread_id) + "_k" + std::to_string(i)); - } - return keys; -} - -// ── StringArena concurrent chunk growth ─────────────────────────────────── -// -// 8 threads each insert 1 000 large keys (≈88 bytes each). -// Total ≈ 8 × 1000 × 88 = 704 KB > 512 KB chunk size, so multiple threads -// will hit the chunk boundary simultaneously and race inside grow(). -// -// Failure mode: if grow()'s CAS spinlock has a race, two threads could both -// create a new chunk and only one is linked — the other's allocations land in -// an unreachable chunk, corrupting those key strings and causing lookup misses. -// ASan/TSan also catch data races between concurrent alloc() and grow(). -TEST(StressStringArena, ConcurrentChunkGrowthNoCorruption) { - StringDictionaryBuffer buf; - constexpr int N_THREADS = 8; - constexpr int KEYS_PER_THREAD = 1000; - const std::string pad(68, 'z'); - - std::vector threads; - // Store per-thread insert results to verify after join. - std::vector> got(N_THREADS, - std::vector(KEYS_PER_THREAD, 0)); - for (int t = 0; t < N_THREADS; t++) { - threads.emplace_back([&buf, &got, t, &pad]() { - for (int i = 0; i < KEYS_PER_THREAD; i++) { - std::string key = "cg_" + std::to_string(t) - + "_" + std::to_string(i) + "_" + pad; - u32 id = static_cast(t * KEYS_PER_THREAD + i + 1); - got[t][i] = buf.insert_with_id(key.c_str(), key.size(), id); - } - }); - } - for (auto& th : threads) th.join(); - - // Every key was unique so every insert must have succeeded. - // If a key's arena region was clobbered by a concurrent alloc, the - // stored string is corrupted and lookup returns 0 or the wrong id. - for (int t = 0; t < N_THREADS; t++) { - for (int i = 0; i < KEYS_PER_THREAD; i++) { - u32 expected = static_cast(t * KEYS_PER_THREAD + i + 1); - EXPECT_EQ(expected, got[t][i]) - << "insert failed for thread " << t << " key " << i; - std::string key = "cg_" + std::to_string(t) - + "_" + std::to_string(i) + "_" + pad; - EXPECT_EQ(expected, buf.lookup(key.c_str(), key.size())) - << "lookup failed for thread " << t << " key " << i; - } - } -} - -// ── StringDictionaryBuffer concurrent insert + read ──────────────────────── - -TEST(StressStringDictionaryBuffer, ConcurrentInsertNoCorruption) { - StringDictionaryBuffer buf; - constexpr int N_THREADS = 8; - constexpr int KEYS_PER_THREAD = 500; - - std::vector threads; - for (int t = 0; t < N_THREADS; t++) { - threads.emplace_back([&buf, t]() { - auto keys = makeKeys(t, KEYS_PER_THREAD); - for (int i = 0; i < (int)keys.size(); i++) { - u32 id = static_cast(t * KEYS_PER_THREAD + i + 1); - buf.insert_with_id(keys[i].c_str(), keys[i].size(), id); - } - }); - } - for (auto& th : threads) th.join(); - - EXPECT_LE(buf.size(), N_THREADS * KEYS_PER_THREAD); - EXPECT_GT(buf.size(), 0); -} - -TEST(StressStringDictionaryBuffer, ConcurrentInsertAndLookupNoCorruption) { - StringDictionaryBuffer buf; - constexpr int N_WRITERS = 4; - constexpr int N_READERS = 4; - constexpr int OPS = 1000; - std::atomic stop{false}; - - std::vector writers; - for (int t = 0; t < N_WRITERS; t++) { - writers.emplace_back([&buf, &stop, t]() { - auto keys = makeKeys(t, OPS); - for (int i = 0; i < OPS && !stop.load(std::memory_order_relaxed); i++) { - buf.insert_with_id(keys[i].c_str(), keys[i].size(), - static_cast(t * OPS + i + 1)); - } - }); - } - - std::vector readers; - for (int t = 0; t < N_READERS; t++) { - readers.emplace_back([&buf, &stop, t]() { - while (!stop.load(std::memory_order_relaxed)) { - std::string key = "t0_k" + std::to_string(t % OPS); - buf.lookup(key.c_str(), key.size()); - } - }); - } - - for (auto& th : writers) th.join(); - stop.store(true); - for (auto& th : readers) th.join(); - - SUCCEED(); -} - -// Same key inserted by multiple threads: exactly one ID must survive and -// all threads must return that same ID. -TEST(StressStringDictionaryBuffer, ConcurrentSameKeyInsertReturnsConsistentId) { - StringDictionaryBuffer buf; - constexpr int N_THREADS = 16; - std::vector results(N_THREADS, 0); - - std::vector threads; - for (int t = 0; t < N_THREADS; t++) { - threads.emplace_back([&buf, &results, t]() { - // All threads try to insert the same key with different ids. - results[t] = buf.insert_with_id("shared/Key", 10, static_cast(t + 1)); - }); - } - for (auto& th : threads) th.join(); - - // All results must be the same value (the winner's id). - u32 expected = results[0]; - EXPECT_GT(expected, 0u); - for (int t = 0; t < N_THREADS; t++) { - EXPECT_EQ(expected, results[t]) << "thread " << t << " got different id"; - } -} - -// ── StringDictionary concurrent stress ──────────────────────────────────── - -// Invariant: once a key is assigned an id, every subsequent bounded_lookup -// must return the same id, across any number of rotations. -TEST(StressStringDictionary, IdStabilityUnderConcurrentRotation) { - StringDictionary dict; - constexpr int N_INSERTERS = 4; - constexpr int KEYS_PER_THREAD = 200; - - // Phase 1: insert all keys and record their ids. - std::vector> recorded(N_INSERTERS); - { - std::vector inserters; - for (int t = 0; t < N_INSERTERS; t++) { - inserters.emplace_back([&dict, &recorded, t]() { - for (auto& key : makeKeys(t, KEYS_PER_THREAD)) { - u32 id = dict.lookup(key.c_str(), key.size()); - recorded[t][key] = id; - } - }); - } - for (auto& th : inserters) th.join(); - } - - // Phase 2: rotate many times; ids must remain stable. - constexpr int N_ROTATIONS = 20; - for (int r = 0; r < N_ROTATIONS; r++) { - dict.rotate(); - dict.clearStandby(); - - for (int t = 0; t < N_INSERTERS; t++) { - for (auto& kv : recorded[t]) { - u32 current = dict.bounded_lookup(kv.first.c_str(), kv.first.size()); - EXPECT_EQ(kv.second, current) - << "id changed for key '" << kv.first << "' at rotation " << r; - } - } - } -} - -// Concurrent inserts AND rotations simultaneously. -TEST(StressStringDictionary, ConcurrentInsertAndRotateNoCorruption) { - StringDictionary dict; - constexpr int N_INSERTERS = 6; - constexpr int KEYS_PER_THREAD = 300; - std::atomic done{false}; - - std::thread rotator([&dict, &done]() { - while (!done.load(std::memory_order_relaxed)) { - dict.rotate(); - dict.clearStandby(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - }); - - std::vector inserters; - for (int t = 0; t < N_INSERTERS; t++) { - inserters.emplace_back([&dict, t]() { - for (auto& key : makeKeys(t, KEYS_PER_THREAD)) { - dict.lookup(key.c_str(), key.size()); - } - }); - } - for (auto& th : inserters) th.join(); - done.store(true); - rotator.join(); - - SUCCEED(); -} - -// bounded_lookup (simulating signal handlers) concurrent with inserts and rotation. -TEST(StressStringDictionary, BoundedLookupSafeUnderConcurrentRotation) { - StringDictionary dict; - constexpr int N_INSERTERS = 4; - constexpr int N_READERS = 4; - constexpr int KEYS_PER_THREAD = 200; - std::atomic done{false}; - - // Pre-insert known keys for readers to probe. - auto base_keys = makeKeys(99, 50); - for (auto& key : base_keys) dict.lookup(key.c_str(), key.size()); - - std::thread rotator([&dict, &done]() { - while (!done.load(std::memory_order_relaxed)) { - dict.rotate(); - dict.clearStandby(); - std::this_thread::sleep_for(std::chrono::milliseconds(2)); - } - }); - - std::vector inserters; - for (int t = 0; t < N_INSERTERS; t++) { - inserters.emplace_back([&dict, t]() { - for (auto& key : makeKeys(t, KEYS_PER_THREAD)) { - dict.lookup(key.c_str(), key.size()); - } - }); - } - - std::vector readers; - for (int t = 0; t < N_READERS; t++) { - readers.emplace_back([&dict, &base_keys, &done]() { - while (!done.load(std::memory_order_relaxed)) { - for (auto& key : base_keys) { - dict.bounded_lookup(key.c_str(), key.size()); - } - } - }); - } - - for (auto& th : inserters) th.join(); - done.store(true); - rotator.join(); - for (auto& th : readers) th.join(); - - SUCCEED(); -} - -// lookupDuringDump called concurrently with inserts into active. -// Invariant: all keys found or inserted by lookupDuringDump must appear in standby. -TEST(StressStringDictionary, LookupDuringDumpSafeUnderConcurrentInserts) { - StringDictionary dict; - constexpr int N_INSERTERS = 4; - constexpr int KEYS_PER_THREAD = 100; - - // Pre-populate and rotate so lookupDuringDump has a dump buffer. - for (auto& key : makeKeys(99, 20)) dict.lookup(key.c_str(), key.size()); - dict.rotate(); - - // Start inserters FIRST so lookupDuringDump races with active inserts. - std::atomic done{false}; - std::vector inserters; - for (int t = 0; t < N_INSERTERS; t++) { - inserters.emplace_back([&dict, &done, t]() { - for (auto& key : makeKeys(t, KEYS_PER_THREAD)) { - if (done.load(std::memory_order_relaxed)) break; - dict.lookup(key.c_str(), key.size()); - } - }); - } - - // Dump thread probes pre-populated keys concurrently with active inserts. - std::vector> dump_results; - for (auto& key : makeKeys(99, 20)) { - u32 id = dict.lookupDuringDump(key.c_str(), key.size()); - EXPECT_GT(id, 0u) << "lookupDuringDump returned 0 for pre-populated key"; - dump_results.push_back({key, id}); - } - - done.store(true); - for (auto& th : inserters) th.join(); - - // All keys returned by lookupDuringDump must be in the dump buffer (standby). - std::map snap; - dict.standby()->collect(snap); - for (auto& kv : dump_results) { - EXPECT_EQ(1u, snap.count(kv.second)) - << "key '" << kv.first << "' with id " << kv.second << " not in standby"; - } -} - -// ── Multi-dictionary atomic rotation ────────────────────────────────────── -// -// Mirrors the production pattern in Profiler::dump(): three independent -// StringDictionaries are rotated atomically under a single critical section -// while inserters and signal-style readers run concurrently against all three. -// -// Invariants asserted: -// - Seed keys recorded before the rotator starts retain stable ids in every -// dictionary across many rotation cycles (concurrent inserts must not -// shift them). -// - After each atomic rotate-of-all-three, every seed id is present in the -// standby buffer of its dictionary — i.e. the rotation snapshot is -// consistent across the three dictionaries simultaneously. -TEST(StressStringDictionary, MultiDictionaryAtomicRotation) { - StringDictionary d1, d2, d3; - StringDictionary* dicts[3] = {&d1, &d2, &d3}; - - constexpr int N_INSERTERS_PER_DICT = 2; - constexpr int N_READERS = 3; - constexpr int SEED_KEY_COUNT = 40; - constexpr int SOAK_MS = 1500; - - // Pre-insert seed keys into all three dicts and record their ids. - auto seed_keys = makeKeys(99, SEED_KEY_COUNT); - std::unordered_map> seed_ids; - for (auto& k : seed_keys) { - seed_ids[k] = { - d1.lookup(k.c_str(), k.size()), - d2.lookup(k.c_str(), k.size()), - d3.lookup(k.c_str(), k.size()) - }; - } - - std::atomic done{false}; - std::mutex rotate_mutex; // simulates Profiler::lockAll() - - // Rotator: rotate all three atomically, then verify the rotation snapshot. - std::thread rotator([&]() { - while (!done.load(std::memory_order_relaxed)) { - { - std::lock_guard lk(rotate_mutex); - d1.rotate(); - d2.rotate(); - d3.rotate(); - - // Atomic snapshot: every seed id is present in every standby. - std::map s1, s2, s3; - d1.standby()->collect(s1); - d2.standby()->collect(s2); - d3.standby()->collect(s3); - for (auto& kv : seed_ids) { - EXPECT_EQ(1u, s1.count(kv.second[0])) - << "d1 standby missing seed id " << kv.second[0] << " for '" << kv.first << "'"; - EXPECT_EQ(1u, s2.count(kv.second[1])) - << "d2 standby missing seed id " << kv.second[1] << " for '" << kv.first << "'"; - EXPECT_EQ(1u, s3.count(kv.second[2])) - << "d3 standby missing seed id " << kv.second[2] << " for '" << kv.first << "'"; - } - } - d1.clearStandby(); - d2.clearStandby(); - d3.clearStandby(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - }); - - // Inserters: per dict, hammer a per-thread key set. Each inserter loops - // its key list to keep memory bounded while exercising the lookup path - // (first iteration inserts; subsequent iterations are hot-hit lookups). - std::vector inserters; - for (int d = 0; d < 3; d++) { - for (int t = 0; t < N_INSERTERS_PER_DICT; t++) { - inserters.emplace_back([dicts, d, t, &done]() { - auto keys = makeKeys(d * 10 + t, 100); - while (!done.load(std::memory_order_relaxed)) { - for (auto& k : keys) { - dicts[d]->lookup(k.c_str(), k.size()); - } - } - }); - } - } - - // Readers: signal-style bounded_lookup of seed keys on all three dicts. - // Ids must remain stable across rotations. - std::vector readers; - for (int r = 0; r < N_READERS; r++) { - readers.emplace_back([&]() { - while (!done.load(std::memory_order_relaxed)) { - for (auto& kv : seed_ids) { - u32 i1 = d1.bounded_lookup(kv.first.c_str(), kv.first.size()); - u32 i2 = d2.bounded_lookup(kv.first.c_str(), kv.first.size()); - u32 i3 = d3.bounded_lookup(kv.first.c_str(), kv.first.size()); - EXPECT_EQ(kv.second[0], i1) << "d1 id drifted for '" << kv.first << "'"; - EXPECT_EQ(kv.second[1], i2) << "d2 id drifted for '" << kv.first << "'"; - EXPECT_EQ(kv.second[2], i3) << "d3 id drifted for '" << kv.first << "'"; - } - } - }); - } - - std::this_thread::sleep_for(std::chrono::milliseconds(SOAK_MS)); - done.store(true); - rotator.join(); - for (auto& th : inserters) th.join(); - for (auto& th : readers) th.join(); -} - -// ── rotate() without external lock ──────────────────────────────────────── -// -// Production change: rotate() is called before lockAll() in rotateDictsAndRun(). -// This test verifies that the standby snapshot is complete and correct when -// rotate() runs with NO external mutex while concurrent inserts are live. -// -// Failure mode: if rotate() needed lockAll() for correctness, concurrent inserts -// during Phase 1→Phase 2 would produce a standby missing entries or with wrong -// IDs, and the EXPECT_EQ assertions below would fire. -// -// Also run under TSan to catch ordering violations between the rotator and the -// concurrent inserters that rotate()'s internal protocol is supposed to handle. -TEST(StressStringDictionary, RotateWithoutMutexPreservesStandbySnapshot) { - StringDictionary dict; - constexpr int N_INSERTERS = 6; - constexpr int SEED_KEYS = 50; - constexpr int N_ROTATIONS = 30; - - // Phase 1: pre-insert seed keys and record their ids. - auto seed_keys = makeKeys(99, SEED_KEYS); - std::unordered_map seed_ids; - for (auto& k : seed_keys) seed_ids[k] = dict.lookup(k.c_str(), k.size()); - - std::atomic done{false}; - - // Concurrent inserters run the whole time — no mutex protecting rotate(). - std::vector inserters; - for (int t = 0; t < N_INSERTERS; t++) { - inserters.emplace_back([&dict, &done, t]() { - auto keys = makeKeys(t, 200); - int idx = 0; - while (!done.load(std::memory_order_relaxed)) { - auto& k = keys[idx++ % (int)keys.size()]; - dict.lookup(k.c_str(), k.size()); - } - }); - } - - // Rotator: rotate without holding any external mutex, then verify standby. - for (int r = 0; r < N_ROTATIONS; r++) { - dict.rotate(); // no lockAll() — this is the invariant under test - - std::map snap; - dict.standby()->collect(snap); - for (auto& kv : seed_ids) { - EXPECT_EQ(1u, snap.count(kv.second)) - << "rotation " << r << ": standby missing id " << kv.second - << " for key '" << kv.first << "'"; - } - - dict.clearStandby(); - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - } - - done.store(true); - for (auto& th : inserters) th.join(); -} - -// ── clearAll() without external lock ────────────────────────────────────── -// -// Production change: clearAll() is called without lockAll() wrapper in -// Profiler::start(). This test verifies that clearAll()'s internal protocol -// (_accepting=false + RefCountGuard drain) is sufficient on its own. -// -// Failure mode: if clearAll() needed lockAll() to be safe, concurrent -// bounded_lookup() or lookup() calls that slip through the _accepting gate -// could dereference freed key memory — caught by ASan as a use-after-free. -// Without ASan the test would still catch it via crash or assertion failure. -// -// Contrast with ClearAllUnderConcurrentReaders, which uses a shared_mutex to -// model the *caller's* quiescing protocol. This test exercises the dictionary -// entirely lock-free, proving the internal mechanism is self-contained. -TEST(StressStringDictionary, ClearAllWithoutExternalLockIsSafe) { - StringDictionary dict; - constexpr int N_INSERTERS = 4; - constexpr int N_READERS = 4; - constexpr int N_CLEAR_OPS = 25; - constexpr int SOAK_MS = 1200; - - std::atomic done{false}; - - std::vector inserters; - for (int t = 0; t < N_INSERTERS; t++) { - inserters.emplace_back([&dict, &done, t]() { - auto keys = makeKeys(t, 80); - int idx = 0; - while (!done.load(std::memory_order_relaxed)) { - auto& k = keys[idx++ % (int)keys.size()]; - dict.lookup(k.c_str(), k.size()); - } - }); - } - - std::vector readers; - for (int t = 0; t < N_READERS; t++) { - readers.emplace_back([&dict, &done, t]() { - auto keys = makeKeys(t + 100, 40); - int idx = 0; - while (!done.load(std::memory_order_relaxed)) { - auto& k = keys[idx++ % (int)keys.size()]; - dict.bounded_lookup(k.c_str(), k.size()); - } - }); - } - - // Clearer: call clearAll() with no external lock — internal mechanism only. - std::thread clearer([&dict, &done]() { - for (int i = 0; i < N_CLEAR_OPS && !done.load(std::memory_order_relaxed); i++) { - std::this_thread::sleep_for(std::chrono::milliseconds(40)); - dict.clearAll(); // no lockAll() wrapper — this is the invariant under test - } - }); - - std::this_thread::sleep_for(std::chrono::milliseconds(SOAK_MS)); - done.store(true); - clearer.join(); - for (auto& th : inserters) th.join(); - for (auto& th : readers) th.join(); - - SUCCEED(); -} - -// ── clearAll under concurrent readers ───────────────────────────────────── -// -// StringDictionary::clearAll() frees every malloc'd key in all three buffers. -// Its contract is that the caller must quiesce signal handlers first -// (cf. RefCountGuard::waitForAllRefCountsToClear in the production callsite). -// This test models that protocol using a std::shared_mutex barrier: -// readers/inserters acquire it shared, the clearer acquires it exclusive. -// -// Under ASan/TSan, this test catches: UAF on freed keys, lost stores from a -// torn clearAll, or any heap corruption from clear-and-reinsert cycles. -// -// Invariants asserted: -// - No use-after-free, no heap corruption (caught by sanitizers). -// - Each clearAll-then-reinsert cycle yields a self-consistent id mapping: -// the same key resolves to one id through all reads in that epoch. -TEST(StressStringDictionary, ClearAllUnderConcurrentReaders) { - StringDictionary dict; - - constexpr int N_READERS = 4; - constexpr int N_INSERTERS = 2; - constexpr int N_CLEAR_OPS = 20; - constexpr int SOAK_MS = 1500; - - auto seed_keys = makeKeys(99, 30); - - // shared_mutex: shared = workers; unique = clearer. Mirrors the production - // requirement that clearAll runs only after all signal handlers are quiesced. - std::shared_mutex epoch_mtx; - std::atomic epoch{0}; // bumped every clearAll; readers re-snapshot ids per epoch - - // Re-seed under exclusive lock. Returns the per-key id map for this epoch. - auto reseed = [&](std::unordered_map& out) { - out.clear(); - for (auto& k : seed_keys) { - out[k] = dict.lookup(k.c_str(), k.size()); - } - }; - - std::unordered_map seed_ids; - reseed(seed_ids); // initial epoch 0 - - std::atomic done{false}; - - // Clearer: bounded number of clearAll cycles, then exits. - std::thread clearer([&]() { - for (int i = 0; i < N_CLEAR_OPS && !done.load(std::memory_order_relaxed); i++) { - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - std::unordered_map new_ids; - { - std::unique_lock lk(epoch_mtx); - dict.clearAll(); - reseed(new_ids); - seed_ids = std::move(new_ids); - epoch.fetch_add(1, std::memory_order_release); - } - } - }); - - // Inserters: arbitrary keys, just to stress the malloc/CAS paths under - // contention with clearAll cycles. - std::vector inserters; - for (int t = 0; t < N_INSERTERS; t++) { - inserters.emplace_back([&, t]() { - auto ks = makeKeys(t, 60); - while (!done.load(std::memory_order_relaxed)) { - std::shared_lock lk(epoch_mtx); - for (auto& k : ks) { - dict.lookup(k.c_str(), k.size()); - } - } - }); - } - - // Readers: snapshot the current-epoch id map, then verify lookups within - // the epoch are consistent. An epoch bump invalidates the snapshot, so - // re-read it under shared lock on each loop. - std::vector readers; - for (int t = 0; t < N_READERS; t++) { - readers.emplace_back([&]() { - while (!done.load(std::memory_order_relaxed)) { - std::shared_lock lk(epoch_mtx); - u64 my_epoch = epoch.load(std::memory_order_acquire); - auto local = seed_ids; // snapshot under lock - for (auto& kv : local) { - u32 id = dict.bounded_lookup(kv.first.c_str(), kv.first.size()); - // While the shared lock is held, no clearAll can run, so - // the id must equal what reseed recorded for this epoch. - EXPECT_EQ(kv.second, id) - << "epoch " << my_epoch << " key '" << kv.first - << "' expected " << kv.second << " got " << id; - } - } - }); - } - - std::this_thread::sleep_for(std::chrono::milliseconds(SOAK_MS)); - done.store(true); - clearer.join(); - for (auto& th : inserters) th.join(); - for (auto& th : readers) th.join(); -} diff --git a/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp b/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp deleted file mode 100644 index 05acb2e4f..000000000 --- a/ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * Layer-1 reproducer for the logs-backend crash (Java 25, thread-churn × - * recording-dump). Drives the profiler's own thread-lifecycle and dump data - * structures concurrently, with NO JVM in the process, so ASan/TSan can flag - * the use-after-free / race at its origin. - * - * See doc/plans/2026-05-29-logs-backend-crash-simulation-design.md - */ -#include "gtest/gtest.h" - -#ifdef __linux__ - -#include "callTraceStorage.h" -#include "callTraceHashTable.h" -#include "threadFilter.h" -#include "thread.h" -#include "arch.h" -#include "spinLock.h" - -#include -#include -#include -#include -#include -#include -#include "../../main/cpp/gtest_crash_handler.h" - -// Crash handler test name (installed in each multithreaded test below). -static constexpr const char STRESS_TEST_NAME[] = "StressThreadLifecycle"; - -// Number of churn workers and iterations per worker. -static constexpr int kChurnWorkers = 16; -static constexpr int kChurnIterations = 2000; - -// Shared dump-side storage. Churn workers write it through the same shard-lock -// protocol as profiler sample paths; the dump thread processes it under -// lock_all(), matching Profiler::rotateDictsAndRun(). -static CallTraceStorage g_storage; -static std::atomic g_run{false}; - -// Mirrors Profiler::_locks. Sample writers take one shard lock with tryLock(); -// dump takes all shard locks before processing/clearing shared dump-side state. -static constexpr int kLockCount = 16; -static SpinLock g_locks[kLockCount]; - -static u32 lock_index_for_tid(int tid) { - u32 lock_index = tid; - lock_index ^= lock_index >> 8; - lock_index ^= lock_index >> 4; - return lock_index % kLockCount; -} - -static bool try_record_lock(int tid, u32* lock_index) { - *lock_index = lock_index_for_tid(tid); - if (g_locks[*lock_index].tryLock()) { - return true; - } - *lock_index = (*lock_index + 1) % kLockCount; - if (g_locks[*lock_index].tryLock()) { - return true; - } - *lock_index = (*lock_index + 1) % kLockCount; - return g_locks[*lock_index].tryLock(); -} - -static void lock_all() { - for (int i = 0; i < kLockCount; i++) { - g_locks[i].lock(); - } -} - -static void unlock_all() { - for (int i = 0; i < kLockCount; i++) { - g_locks[i].unlock(); - } -} - -// Record a small fake call trace, mirroring profiler sample paths that hold a -// shard lock while writing to CallTraceStorage. ASGCT_CallFrame uses `bci` -// (jint) and the `method_id` union member (see vmEntry.h). -static void record_trace(int salt, int tid) { - u32 lock_index; - if (!try_record_lock(tid, &lock_index)) { - return; - } - - ASGCT_CallFrame frames[4]; - std::memset(frames, 0, sizeof(frames)); - for (int i = 0; i < 4; i++) { - frames[i].bci = i + salt; - frames[i].method_id = - reinterpret_cast(static_cast(0x1000 + i + salt)); - } - g_storage.put(4, frames, false, 1); - - g_locks[lock_index].unlock(); -} - -// onThreadStart -> work -> onThreadEnd loop, mirroring the profiler's per-thread -// lifecycle: initCurrentThread / current / register filter slot / setFilterSlotId -// / (work) / remove + unregister / release. -// -// Thread-name / ThreadInfo coverage: this native reproducer does not call -// ThreadInfo::set() or updateJavaThreadNames(). That path is covered by the -// JVM-level DumpStormAntagonist antagonist (Layer 2). A clean ASan/TSan run -// here is not conclusive for the thread-name path. -static void churn_worker(ThreadFilter* filter, bool with_dump) { - while (!g_run.load(std::memory_order_acquire)) { } - for (int i = 0; i < kChurnIterations && g_run.load(std::memory_order_relaxed); i++) { - ProfiledThread::initCurrentThread(); - ProfiledThread* self = ProfiledThread::current(); - EXPECT_NE(nullptr, self); - if (!self) return; - - ThreadFilter::SlotID slot = filter->registerThread(); - if (slot >= 0) { - self->setFilterSlotId(slot); - filter->add(self->tid(), slot); - } - - if (with_dump) { - record_trace(i, self->tid()); - } - std::this_thread::yield(); - - if (slot >= 0) { - filter->remove(slot); - filter->unregisterThread(slot); - } - self->setFilterSlotId(-1); - ProfiledThread::release(); - } -} - -// Continuously processes the trace storage under lock_all(), matching the JFR -// dump path where Profiler::rotateDictsAndRun() holds all shard locks while -// writeStackTraces() calls processCallTraces(). -static void dump_thread() { - while (g_run.load(std::memory_order_relaxed)) { - lock_all(); - g_storage.processTraces([](const std::unordered_set& traces) { - volatile size_t n = 0; - for (CallTrace* t : traces) { - if (t && t != CallTraceSample::PREPARING) { - n += 1; - } - } - (void)n; - }); - unlock_all(); - } -} - -TEST(StressThreadLifecycle, Smoke) { - CallTraceStorage storage; - storage.clear(); - SUCCEED(); -} - -TEST(StressThreadLifecycle, ChurnOnly) { - installGtestCrashHandler(); - g_storage.clear(); - - ThreadFilter filter; - filter.init("*"); - ASSERT_TRUE(filter.enabled()); - - g_run.store(true, std::memory_order_release); - std::vector ts; - for (int t = 0; t < kChurnWorkers; t++) { - ts.emplace_back(churn_worker, &filter, false); - } - for (auto& t : ts) { - t.join(); - } - g_run.store(false); - - restoreDefaultSignalHandlers(); - SUCCEED(); -} - -TEST(StressThreadLifecycle, ChurnDuringDump) { - installGtestCrashHandler(); - g_storage.clear(); - - ThreadFilter filter; - filter.init("*"); - ASSERT_TRUE(filter.enabled()); - - g_run.store(true, std::memory_order_release); - std::thread dumper(dump_thread); - std::vector ts; - for (int t = 0; t < kChurnWorkers; t++) { - ts.emplace_back(churn_worker, &filter, true); - } - for (auto& t : ts) { - t.join(); - } - g_run.store(false); - dumper.join(); - - restoreDefaultSignalHandlers(); - SUCCEED(); -} - -#endif // __linux__ diff --git a/ddprof-lib/src/test/cpp/stringDictionary_ut.cpp b/ddprof-lib/src/test/cpp/stringDictionary_ut.cpp deleted file mode 100644 index aaad96243..000000000 --- a/ddprof-lib/src/test/cpp/stringDictionary_ut.cpp +++ /dev/null @@ -1,273 +0,0 @@ -#include -#include "stringDictionary.h" -#include -#include -#include -#include -#include - -// ── StringDictionaryBuffer ───────────────────────────────────────────────── - -TEST(StringDictionaryBufferTest, InsertWithIdReturnsSameIdForSameKey) { - StringDictionaryBuffer buf; - u32 id = buf.insert_with_id("hello", 5, 42); - EXPECT_EQ(42u, id); - EXPECT_EQ(42u, buf.insert_with_id("hello", 5, 42)); -} - -TEST(StringDictionaryBufferTest, InsertPreservesExistingIdOnDuplicate) { - StringDictionaryBuffer buf; - buf.insert_with_id("key", 3, 7); - // Second insert of same key must return 7, not some other value - EXPECT_EQ(7u, buf.insert_with_id("key", 3, 99)); -} - -TEST(StringDictionaryBufferTest, LookupReturnZeroOnMiss) { - StringDictionaryBuffer buf; - EXPECT_EQ(0u, buf.lookup("absent", 6)); -} - -TEST(StringDictionaryBufferTest, LookupFindsInsertedKey) { - StringDictionaryBuffer buf; - buf.insert_with_id("java/lang/String", 16, 1); - EXPECT_EQ(1u, buf.lookup("java/lang/String", 16)); -} - -TEST(StringDictionaryBufferTest, LookupDoesNotInsert) { - StringDictionaryBuffer buf; - buf.lookup("ghost", 5); - std::map out; - buf.collect(out); - EXPECT_EQ(0u, out.size()); -} - -TEST(StringDictionaryBufferTest, CollectReturnsAllInsertedEntries) { - StringDictionaryBuffer buf; - buf.insert_with_id("a", 1, 1); - buf.insert_with_id("b", 1, 2); - buf.insert_with_id("c", 1, 3); - std::map out; - buf.collect(out); - ASSERT_EQ(3u, out.size()); - EXPECT_STREQ("a", out[1]); - EXPECT_STREQ("b", out[2]); - EXPECT_STREQ("c", out[3]); -} - -TEST(StringDictionaryBufferTest, CopyFromPreservesAllEntriesWithIds) { - StringDictionaryBuffer src; - src.insert_with_id("java/lang/String", 16, 10); - src.insert_with_id("java/lang/Integer", 17, 20); - - StringDictionaryBuffer dst; - dst.copyFrom(src); - - EXPECT_EQ(10u, dst.lookup("java/lang/String", 16)); - EXPECT_EQ(20u, dst.lookup("java/lang/Integer", 17)); -} - -TEST(StringDictionaryBufferTest, ClearResetsToEmpty) { - StringDictionaryBuffer buf; - buf.insert_with_id("x", 1, 5); - buf.clear(); - EXPECT_EQ(0u, buf.lookup("x", 1)); - std::map out; - buf.collect(out); - EXPECT_EQ(0u, out.size()); -} - -// ── StringArena behaviour (via StringDictionaryBuffer) ──────────────────── -// -// StringArena is a private implementation detail; these tests exercise its -// observable effects through StringDictionaryBuffer's public API. - -// Inserting N distinct keys must not corrupt any of them: if arena regions -// overlapped, some key strings would be overwritten and lookups would fail. -TEST(StringArenaTest, AllocsAreNonOverlapping) { - StringDictionaryBuffer buf; - constexpr int N = 2000; - for (int i = 0; i < N; i++) { - std::string key = "nooverlap_" + std::to_string(i); - u32 id = static_cast(i + 1); - ASSERT_EQ(id, buf.insert_with_id(key.c_str(), key.size(), id)) - << "insert failed at key " << i; - } - for (int i = 0; i < N; i++) { - std::string key = "nooverlap_" + std::to_string(i); - EXPECT_EQ(static_cast(i + 1), buf.lookup(key.c_str(), key.size())) - << "lookup failed at key " << i; - } -} - -// Each key is ~88 bytes aligned; 6 000 keys ≈ 528 KB > the 512 KB chunk size, -// forcing at least one new chunk to be allocated. All keys must remain -// accessible across the chunk boundary. -TEST(StringArenaTest, GrowsAcrossChunkBoundary) { - StringDictionaryBuffer buf; - constexpr int N = 6000; - const std::string pad(70, 'x'); - std::vector keys; - keys.reserve(N); - for (int i = 0; i < N; i++) { - keys.push_back("chunk_" + std::to_string(i) + "_" + pad); - } - for (int i = 0; i < N; i++) { - ASSERT_NE(0u, buf.insert_with_id(keys[i].c_str(), keys[i].size(), - static_cast(i + 1))) - << "insert failed at key " << i << " (unexpected arena OOM)"; - } - for (int i = 0; i < N; i++) { - EXPECT_EQ(static_cast(i + 1), - buf.lookup(keys[i].c_str(), keys[i].size())) - << "lookup failed at key " << i << " after chunk growth"; - } -} - -// After clear() the arena is reset: old keys are gone and the arena space is -// reused for new inserts. -TEST(StringArenaTest, ClearResetsArena) { - StringDictionaryBuffer buf; - buf.insert_with_id("alpha", 5, 1); - buf.insert_with_id("beta", 4, 2); - buf.clear(); - EXPECT_EQ(0u, buf.lookup("alpha", 5)) << "stale key visible after clear"; - EXPECT_EQ(0u, buf.lookup("beta", 4)) << "stale key visible after clear"; - // Reinsertion into the recycled arena must work. - EXPECT_EQ(10u, buf.insert_with_id("alpha", 5, 10)); - EXPECT_EQ(20u, buf.insert_with_id("gamma", 5, 20)); - EXPECT_EQ(10u, buf.lookup("alpha", 5)); - EXPECT_EQ(20u, buf.lookup("gamma", 5)); - EXPECT_EQ(0u, buf.lookup("beta", 4)) << "beta must still be absent"; -} - -// After filling multiple chunks and then calling clear(), the extra chunks are -// freed and subsequent inserts succeed — verifying the arena is fully recycled. -TEST(StringArenaTest, ClearAfterChunkGrowthRecyclesExtraChunks) { - StringDictionaryBuffer buf; - constexpr int N = 6000; - const std::string pad(70, 'y'); - for (int i = 0; i < N; i++) { - std::string k = "recycle_" + std::to_string(i) + "_" + pad; - buf.insert_with_id(k.c_str(), k.size(), static_cast(i + 1)); - } - buf.clear(); - // Fresh inserts must succeed; the arena must be back to a usable state. - for (int i = 0; i < 200; i++) { - std::string k = "fresh_" + std::to_string(i); - u32 id = static_cast(i + 1000); - EXPECT_EQ(id, buf.insert_with_id(k.c_str(), k.size(), id)) - << "insert failed after clear+chunk-recycle at " << i; - } - // Verify all fresh keys are readable. - for (int i = 0; i < 200; i++) { - std::string k = "fresh_" + std::to_string(i); - EXPECT_EQ(static_cast(i + 1000), buf.lookup(k.c_str(), k.size())) - << "lookup failed after clear+chunk-recycle at " << i; - } -} - -// ── StringDictionary (persistent, global IDs) ───────────────────────────── - -class StringDictionaryTest : public ::testing::Test { -protected: - StringDictionary dict; -}; - -TEST_F(StringDictionaryTest, LookupAssignsGlobalId) { - u32 id = dict.lookup("java/lang/String", 16); - EXPECT_GT(id, 0u); - EXPECT_EQ(id, dict.lookup("java/lang/String", 16)); -} - -TEST_F(StringDictionaryTest, BoundedLookupFindsActiveEntry) { - u32 id = dict.lookup("Foo", 3); - EXPECT_EQ(id, dict.bounded_lookup("Foo", 3)); -} - -TEST_F(StringDictionaryTest, BoundedLookupReturnsZeroOnMiss) { - EXPECT_EQ(0u, dict.bounded_lookup("Absent", 6)); -} - -TEST_F(StringDictionaryTest, IdStableAcrossRotations) { - u32 id = dict.lookup("java/lang/String", 16); - for (int cycle = 0; cycle < 10; cycle++) { - dict.rotate(); - dict.clearStandby(); - EXPECT_EQ(id, dict.bounded_lookup("java/lang/String", 16)) - << "id changed at cycle " << cycle; - } -} - -TEST_F(StringDictionaryTest, AllEntriesPresentInStandbyAfterRotate) { - u32 id1 = dict.lookup("a", 1); - u32 id2 = dict.lookup("b", 1); - dict.rotate(); - - std::map snap; - dict.standby()->collect(snap); - ASSERT_EQ(2u, snap.size()); - EXPECT_EQ(snap[id1], std::string("a")); - EXPECT_EQ(snap[id2], std::string("b")); -} - -TEST_F(StringDictionaryTest, NewEntryAfterRotateIsInNewActive) { - dict.lookup("early", 5); - dict.rotate(); - u32 id = dict.lookup("late", 4); - - EXPECT_EQ(id, dict.bounded_lookup("late", 4)); - - dict.rotate(); - std::map snap; - dict.standby()->collect(snap); - bool found = false; - for (auto& kv : snap) if (strcmp(kv.second, "late") == 0) { found = true; break; } - EXPECT_TRUE(found); -} - -TEST_F(StringDictionaryTest, LookupDuringDumpFindsPreregisteredKey) { - u32 id = dict.lookup("java/lang/String", 16); - dict.rotate(); - EXPECT_EQ(id, dict.lookupDuringDump("java/lang/String", 16)); -} - -TEST_F(StringDictionaryTest, LookupDuringDumpAlsoAddsToStandby) { - dict.rotate(); - u32 id = dict.lookup("late/Class", 10); - - u32 found = dict.lookupDuringDump("late/Class", 10); - EXPECT_EQ(id, found); - - std::map snap; - dict.standby()->collect(snap); - EXPECT_EQ(1u, snap.count(id)); -} - -TEST_F(StringDictionaryTest, ClearAllResetsEverything) { - u32 id = dict.lookup("x", 1); - (void)id; - dict.rotate(); - dict.clearAll(); - EXPECT_EQ(0u, dict.bounded_lookup("x", 1)); - dict.rotate(); - std::map snap; - dict.standby()->collect(snap); - EXPECT_EQ(0u, snap.size()); - u32 new_id = dict.lookup("x", 1); - EXPECT_EQ(1u, new_id); -} - -TEST_F(StringDictionaryTest, LookupDuringDumpInsertsNewKeyIntoActiveAndStandby) { - dict.rotate(); // empty active becomes dump, fresh active - // Key is not in dump and not in active — lookupDuringDump must insert into both. - u32 id = dict.lookupDuringDump("brand/New", 9); - EXPECT_GT(id, 0u); - - // Must be in dump (standby) - std::map snap; - dict.standby()->collect(snap); - EXPECT_EQ(1u, snap.count(id)); - - // Must be in active (bounded_lookup is a probe of active) - EXPECT_EQ(id, dict.bounded_lookup("brand/New", 9)); -} diff --git a/ddprof-lib/src/test/cpp/test_callTraceStorage.cpp b/ddprof-lib/src/test/cpp/test_callTraceStorage.cpp deleted file mode 100644 index 2b2c96b9b..000000000 --- a/ddprof-lib/src/test/cpp/test_callTraceStorage.cpp +++ /dev/null @@ -1,876 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "gtest/gtest.h" -#include "callTraceStorage.h" -#include -#include -#include -#include -#include -#include -#include -#include "callTraceHashTable.h" -#include "gtest_crash_handler.h" -#include "arch.h" - -// Test name for crash handler -static constexpr char TEST_NAME[] = "CallTraceStorageTest"; - -// Helper function to find a CallTrace by trace_id in an unordered_set -CallTrace* findTraceById(const std::unordered_set& traces, u64 trace_id) { - for (CallTrace* trace : traces) { - if (trace && trace->trace_id == trace_id) { - return trace; - } - } - return nullptr; -} - -class CallTraceStorageTest : public ::testing::Test { -protected: - void SetUp() override { - // Install crash handler for debugging potential issues - installGtestCrashHandler(); - storage = std::make_unique(); - } - - void TearDown() override { - storage.reset(); - // Restore default signal handlers - restoreDefaultSignalHandlers(); - } - - std::unique_ptr storage; -}; - -TEST_F(CallTraceStorageTest, BasicFunctionality) { - // Create a simple call frame - ASGCT_CallFrame frame; - frame.bci = 10; - frame.method_id = (jmethodID)0x1234; - - // Store a trace - u64 trace_id = storage->put(1, &frame, false, 1); - EXPECT_GT(trace_id, 0); - - // Process traces to verify storage - bool found_traces = false; - storage->processTraces([&found_traces](const std::unordered_set& traces) { - found_traces = traces.size() > 0; - }); - EXPECT_TRUE(found_traces); -} - -TEST_F(CallTraceStorageTest, LivenessCheckerRegistration) { - // Store multiple traces first - ASGCT_CallFrame frames[4]; - frames[0].bci = 10; frames[0].method_id = (jmethodID)0x1111; - frames[1].bci = 20; frames[1].method_id = (jmethodID)0x2222; - frames[2].bci = 30; frames[2].method_id = (jmethodID)0x3333; - frames[3].bci = 40; frames[3].method_id = (jmethodID)0x4444; - - u64 trace_id1 = storage->put(1, &frames[0], false, 1); - u64 trace_id2 = storage->put(1, &frames[1], false, 1); - u64 trace_id3 = storage->put(1, &frames[2], false, 1); - u64 trace_id4 = storage->put(1, &frames[3], false, 1); - - // Register a liveness checker that preserves only trace_id2 and trace_id4 - u64 preserved_trace_id2 = trace_id2; - u64 preserved_trace_id4 = trace_id4; - storage->registerLivenessChecker([&preserved_trace_id2, &preserved_trace_id4](std::unordered_set& buffer) { - buffer.insert(preserved_trace_id2); - buffer.insert(preserved_trace_id4); - }); - - // processTraces should preserve trace_id2 and trace_id4 but not trace_id1 and trace_id3 - size_t traces_collected = 0; - storage->processTraces([&traces_collected](const std::unordered_set& traces) { - // Should have all 4 traces from the collection plus the dropped trace - traces_collected = traces.size(); - EXPECT_EQ(traces.size(), 5); - }); - - // After processTraces, only preserved traces should remain in new active storage - size_t traces_after_preserve = 0; - CallTrace* found_trace2 = nullptr; - CallTrace* found_trace4 = nullptr; - - storage->processTraces([&](const std::unordered_set& traces) { - traces_after_preserve = traces.size(); - found_trace2 = findTraceById(traces, preserved_trace_id2); - found_trace4 = findTraceById(traces, preserved_trace_id4); - }); - - // Should have exactly two traces (the preserved ones) plus the dropped trace - EXPECT_EQ(traces_after_preserve, 3); - - // The preserved trace IDs should be valid (content-based IDs are deterministic) - EXPECT_GT(preserved_trace_id2, 0); - EXPECT_GT(preserved_trace_id4, 0); - - // Verify both traces were actually preserved - EXPECT_TRUE(found_trace2 != nullptr); - EXPECT_TRUE(found_trace4 != nullptr); -} - -TEST_F(CallTraceStorageTest, MultipleLivenessCheckers) { - // Store multiple traces with more variety - ASGCT_CallFrame frames[5]; - frames[0].bci = 10; frames[0].method_id = (jmethodID)0x1111; - frames[1].bci = 20; frames[1].method_id = (jmethodID)0x2222; - frames[2].bci = 30; frames[2].method_id = (jmethodID)0x3333; - frames[3].bci = 40; frames[3].method_id = (jmethodID)0x4444; - frames[4].bci = 50; frames[4].method_id = (jmethodID)0x5555; - - u64 trace_id1 = storage->put(1, &frames[0], false, 1); - u64 trace_id2 = storage->put(1, &frames[1], false, 1); - u64 trace_id3 = storage->put(1, &frames[2], false, 1); - u64 trace_id4 = storage->put(1, &frames[3], false, 1); - u64 trace_id5 = storage->put(1, &frames[4], false, 1); - - u64 preserved_id1 = trace_id1; - u64 preserved_id4 = trace_id4; - - // Register two liveness checkers that preserve non-consecutive traces - storage->registerLivenessChecker([&preserved_id1](std::unordered_set& buffer) { - buffer.insert(preserved_id1); - }); - - storage->registerLivenessChecker([&preserved_id4](std::unordered_set& buffer) { - buffer.insert(preserved_id4); - }); - - // processTraces should preserve specified traces and swap storages - storage->processTraces([](const std::unordered_set& traces) { - // Should have all 5 traces from the collection plus the dropped trace - EXPECT_EQ(traces.size(), 6); - }); - - // After processTraces, only preserved traces should remain in new active storage - CallTrace* found_trace1 = nullptr; - CallTrace* found_trace4 = nullptr; - size_t preserved_count = 0; - - storage->processTraces([&](const std::unordered_set& traces) { - preserved_count = traces.size(); - found_trace1 = findTraceById(traces, preserved_id1); - found_trace4 = findTraceById(traces, preserved_id4); - }); - - // Should have exactly 2 traces (the preserved ones) - EXPECT_EQ(preserved_count, 3); - - // Both preserved IDs should still be valid - EXPECT_GT(preserved_id1, 0); - EXPECT_GT(preserved_id4, 0); - - // Verify both traces were actually preserved - EXPECT_TRUE(found_trace1 != nullptr); - EXPECT_TRUE(found_trace4 != nullptr); -} - -TEST_F(CallTraceStorageTest, TraceIdPreservation) { - // Create a simple frame - ASGCT_CallFrame frame; - frame.bci = 10; - frame.method_id = (jmethodID)0x1234; - - // Add trace to storage - u64 original_trace_id = storage->put(1, &frame, false, 1); - EXPECT_GT(original_trace_id, 0); - - // Register liveness checker to preserve this trace - u64 preserved_id = original_trace_id; - storage->registerLivenessChecker([&preserved_id](std::unordered_set& buffer) { - buffer.insert(preserved_id); - }); - - // First process should contain the original trace - u64 first_trace_id = 0; - storage->processTraces([&](const std::unordered_set& traces) { - EXPECT_EQ(traces.size(), 2); - CallTrace* first_trace = findTraceById(traces, original_trace_id); - EXPECT_NE(first_trace, nullptr); - first_trace_id = first_trace->trace_id; - EXPECT_EQ(first_trace->trace_id, original_trace_id); - }); - - // Second process should still contain the preserved trace with SAME ID - u64 preserved_trace_id = 0; - storage->processTraces([&](const std::unordered_set& traces) { - EXPECT_EQ(traces.size(), 2); - CallTrace* preserved_trace = findTraceById(traces, original_trace_id); - EXPECT_NE(preserved_trace, nullptr); - preserved_trace_id = preserved_trace->trace_id; - // Critical test: trace ID must be exactly the same after preservation - EXPECT_EQ(preserved_trace->trace_id, original_trace_id); - // Regression test: access frame content to detect use-after-free (ASan will crash if bug exists) - EXPECT_EQ(preserved_trace->frames[0].bci, 10); - EXPECT_EQ(preserved_trace->frames[0].method_id, (jmethodID)0x1234); - }); - - printf("Original trace ID: %llu, Preserved trace ID: %llu\n", - original_trace_id, preserved_trace_id); -} - -TEST_F(CallTraceStorageTest, ClearMethod) { - // Store a trace - ASGCT_CallFrame frame; - frame.bci = 10; - frame.method_id = (jmethodID)0x1234; - u64 trace_id = storage->put(1, &frame, false, 1); - - // Register a liveness checker (should be ignored by clear()) - u64 preserved_id = trace_id; - storage->registerLivenessChecker([&preserved_id](std::unordered_set& buffer) { - buffer.insert(preserved_id); - }); - - // clear() should completely clear both storages, ignoring liveness checkers - storage->clear(); - - // Should have no traces after clear, except for the dropped trace - size_t traces_after_clear = 0; - storage->processTraces([&](const std::unordered_set& traces) { - traces_after_clear = traces.size(); - }); - EXPECT_EQ(traces_after_clear, 1); -} - -TEST_F(CallTraceStorageTest, ConcurrentClearAndPut) { - // Test concurrent access patterns that might cause NULL dereferences - ASGCT_CallFrame frame; - frame.bci = 10; - frame.method_id = (jmethodID)0x1234; - - // Store initial trace - u64 trace_id = storage->put(1, &frame, false, 1); - EXPECT_GT(trace_id, 0); - - // Simulate what happens when clear() races with put() - // Clear the storage - storage->clear(); - - // Immediately try to put - should handle cleared state gracefully - u64 result_after_clear = storage->put(1, &frame, false, 1); - // This should either succeed (if new table allocated) or return 0 (drop sample) - // Either way, it shouldn't crash - - // Verify system is still functional - storage->processTraces([](const std::unordered_set& traces) { - // No assertion on size since behavior during concurrent operations can vary - // The key test is that we don't crash - }); -} - -TEST_F(CallTraceStorageTest, ConcurrentTableExpansionRegression) { - // Regression test for the crash during table expansion in CallTraceHashTable::put - // The crash occurred at __sync_bool_compare_and_swap(&_current_table, table, new_table) - // when multiple threads triggered table expansion simultaneously - - // Use heap allocation with proper alignment to avoid ASAN alignment issues - // Stack allocation with high alignment requirements (64 bytes) is problematic under ASAN - void* aligned_memory = std::aligned_alloc(alignof(CallTraceHashTable), sizeof(CallTraceHashTable)); - ASSERT_NE(aligned_memory, nullptr) << "Failed to allocate aligned memory for CallTraceHashTable"; - - auto hash_table_ptr = std::unique_ptr( - new(aligned_memory) CallTraceHashTable(), - [](CallTraceHashTable* ptr) { - ptr->~CallTraceHashTable(); - std::free(ptr); - } - ); - CallTraceHashTable& hash_table = *hash_table_ptr; - hash_table.setInstanceId(42); - - const int num_threads = 4; // Reduced from 8 to avoid excessive contention - const int traces_per_thread = 2000; // Reduced from 10000 to avoid livelock - std::atomic crash_counter{0}; - std::atomic completed_threads{0}; - std::vector threads; - - // Create many different stack traces to trigger table expansion - auto worker = [&](int thread_id) { - int successful_puts = 0; - int dropped_samples = 0; - - for (int i = 0; i < traces_per_thread; i++) { - try { - ASGCT_CallFrame frame; - frame.bci = thread_id * 1000 + i; // Unique BCI per trace - frame.method_id = (jmethodID)(0x1000 + thread_id * 1000 + i); - - // This will trigger table expansion multiple times concurrently - u64 trace_id = hash_table.put(1, &frame, false, 1); - - if (trace_id == 0) { - // Sample was dropped - acceptable under high contention - dropped_samples++; - continue; - } - - // Verify trace ID is valid - if (trace_id == 0x7fffffffffffffffULL) { - // Overflow trace - also acceptable - continue; - } - - successful_puts++; - - // Add small yield to reduce contention and prevent livelock - if (i % 100 == 0) { - std::this_thread::yield(); - } - - } catch (...) { - // Any exception indicates a problem - crash_counter++; - } - } - - completed_threads++; - }; - - // Start all threads simultaneously to maximize contention during table expansion - for (int t = 0; t < num_threads; t++) { - threads.emplace_back(worker, t); - } - - // Wait for all threads to complete with timeout to avoid hanging tests - bool all_completed = true; - for (auto& thread : threads) { - thread.join(); - } - - // The main test is that we don't crash during concurrent table expansion - EXPECT_EQ(crash_counter.load(), 0); - EXPECT_EQ(completed_threads.load(), num_threads); - - // Verify the hash table is still functional after all the expansion - ASGCT_CallFrame test_frame; - test_frame.bci = 99999; - test_frame.method_id = (jmethodID)0x99999; - u64 final_trace_id = hash_table.put(1, &test_frame, false, 1); - EXPECT_GT(final_trace_id, 0); -} - -/** - * Test RefCountGuard synchronization during storage swap. - * This test ensures that waitForRefCountToClear() actually prevents - * collection from original_active while there are still pending put() operations - * to the original_active after the active area swap. - */ -TEST_F(CallTraceStorageTest, RefCountGuardSynchronizationDuringSwap) { - // Synchronization primitives for coordinating the test - std::atomic swap_can_proceed{false}; - std::atomic put_threads_ready{false}; - std::atomic put_threads_ready_count{0}; - std::atomic put_operation_started{false}; - std::atomic put_operation_completed{false}; - std::atomic collection_started{false}; - std::atomic collection_completed{false}; - - std::condition_variable ready_cv; - std::mutex ready_mutex; - - std::condition_variable swap_cv; - std::mutex swap_mutex; - - // Test outcome tracking - std::atomic put_trace_id{0}; - std::atomic delayed_trace_id{0}; - - // Create initial traces to populate the storage - ASGCT_CallFrame initial_frame; - initial_frame.bci = 100; - initial_frame.method_id = (jmethodID)0x1000; - - // Add several traces to ensure there's content to process - for (int i = 0; i < 5; i++) { - initial_frame.bci = 100 + i; - storage->put(1, &initial_frame, false, 1); - } - - // Thread that will perform a put() operation right after storage swap - std::thread put_thread([&]() { - // Signal that this thread is ready - { - std::lock_guard lock(ready_mutex); - put_threads_ready_count.fetch_add(1); - } - ready_cv.notify_all(); - - // Wait for permission to proceed with put operation - std::unique_lock lock(swap_mutex); - swap_cv.wait(lock, [&] { return swap_can_proceed.load(); }); - lock.unlock(); - - // This put() should target the new active area - ASGCT_CallFrame put_frame; - put_frame.bci = 999; - put_frame.method_id = (jmethodID)0x999; - - put_operation_started = true; - u64 trace_id = storage->put(1, &put_frame, false, 1); - put_trace_id = trace_id; - put_operation_completed = true; - }); - - // Thread that simulates a longer-running put() operation - std::thread delayed_put_thread([&]() { - // Signal that this thread is ready - { - std::lock_guard lock(ready_mutex); - put_threads_ready_count.fetch_add(1); - } - ready_cv.notify_all(); - - // Wait for permission to proceed - std::unique_lock lock(swap_mutex); - swap_cv.wait(lock, [&] { return swap_can_proceed.load(); }); - lock.unlock(); - - // Simulate a put operation during swap - ASGCT_CallFrame delayed_frame; - delayed_frame.bci = 777; - delayed_frame.method_id = (jmethodID)0x777; - - // Small delay to simulate ongoing operation - std::this_thread::sleep_for(std::chrono::microseconds(50)); - - u64 trace_id = storage->put(1, &delayed_frame, false, 1); - delayed_trace_id = trace_id; - }); - - // Wait for both put threads to be ready before starting process thread - { - std::unique_lock lock(ready_mutex); - ready_cv.wait(lock, [&] { return put_threads_ready_count.load() == 2; }); - put_threads_ready = true; - } - - // Give threads a moment to enter their wait state - std::this_thread::sleep_for(std::chrono::milliseconds(10)); - - // Perform processTraces in separate thread to trigger the storage swap - std::thread process_thread([&]() { - // Start processing - this will swap storage - storage->processTraces([&](const std::unordered_set& traces) { - collection_started = true; - - // Verify we have traces from the initial population - int initial_trace_count = 0; - for (CallTrace* trace : traces) { - if (trace && trace->num_frames > 0 && trace->frames[0].bci >= 100 && trace->frames[0].bci <= 104) { - initial_trace_count++; - } - } - EXPECT_GE(initial_trace_count, 5) << "Should find initial traces"; - }); - - collection_completed = true; - - // Now that swap is complete and refcount wait has finished, - // signal put threads to proceed - { - std::lock_guard lock(swap_mutex); - swap_can_proceed = true; - } - swap_cv.notify_all(); - }); - - // Wait for all threads to complete with timeout to detect deadlock - auto wait_with_timeout = [](std::thread& t, int timeout_seconds, const char* name) -> bool { - auto start = std::chrono::steady_clock::now(); - auto timeout_duration = std::chrono::seconds(timeout_seconds); - - while (std::chrono::steady_clock::now() - start < timeout_duration) { - if (t.joinable()) { - t.join(); - return true; - } - std::this_thread::sleep_for(std::chrono::milliseconds(100)); - } - - // Timeout occurred - ADD_FAILURE() << "Thread " << name << " timed out after " << timeout_seconds << " seconds (possible deadlock)"; - return false; - }; - - EXPECT_TRUE(wait_with_timeout(process_thread, 30, "process_thread")); - EXPECT_TRUE(wait_with_timeout(put_thread, 30, "put_thread")); - EXPECT_TRUE(wait_with_timeout(delayed_put_thread, 30, "delayed_put_thread")); - - // Verification of test outcomes - EXPECT_TRUE(put_threads_ready.load()) << "Put threads should have been ready"; - EXPECT_TRUE(collection_started.load()) << "Collection should have started"; - EXPECT_TRUE(collection_completed.load()) << "Collection should have completed"; - EXPECT_TRUE(put_operation_started.load()) << "Put operation should have started"; - EXPECT_TRUE(put_operation_completed.load()) << "Put operation should have completed"; - EXPECT_GT(put_trace_id.load(), 0ULL) << "Put operation should have returned valid trace ID"; - EXPECT_GT(delayed_trace_id.load(), 0ULL) << "Delayed put operation should have returned valid trace ID"; - - // The key test: if RefCountGuard synchronization works correctly, - // the collection should not interfere with concurrent put operations - // This test primarily validates that we don't crash or corrupt data - - // Additional verification: ensure storage is still functional - ASGCT_CallFrame verification_frame; - verification_frame.bci = 5555; - verification_frame.method_id = (jmethodID)0x5555; - u64 verification_trace = storage->put(1, &verification_frame, false, 1); - EXPECT_GT(verification_trace, 0ULL) << "Storage should remain functional after RefCountGuard synchronization"; - - // Final verification: ensure we can still process traces - std::atomic final_trace_count{0}; - storage->processTraces([&](const std::unordered_set& traces) { - final_trace_count = static_cast(traces.size()); - }); - - EXPECT_GT(final_trace_count.load(), 0) << "Storage should still contain traces after synchronization test"; -} - -/** - * Reproducer test for use-after-free bug in processTraces(). - * - * BUG DESCRIPTION: - * In processTraces(), traces are collected from standby into _traces_buffer (raw pointers), - * then standby->clear() frees all memory, but processor(_traces_buffer) is called AFTER - * the clear, accessing freed memory. - * - * Timeline of the bug: - * 1. original_standby->collect(_traces_buffer, ...) - collects raw pointers from STANDBY - * 2. original_standby->clear() - FREES ALL MEMORY including CallTrace objects! - * 3. processor(_traces_buffer) - accesses freed memory (USE-AFTER-FREE) - * - * KEY INSIGHT: The bug only affects traces from STANDBY, not ACTIVE. - * - Active traces are cleared AFTER the processor runs (safe) - * - Standby traces are cleared BEFORE the processor runs (BUG!) - * - * To trigger the bug, we need traces IN STANDBY at the start of processTraces(). - * Traces get into standby via the liveness preservation mechanism: - * 1. Register a liveness checker that marks traces as "live" - * 2. During processTraces(), live traces are copied to SCRATCH - * 3. After rotation, SCRATCH becomes the new STANDBY - * 4. Next processTraces() will have those preserved traces in STANDBY - * 5. Those traces are collected, then FREED, then ACCESSED → USE-AFTER-FREE! - * - * This test should FAIL (crash or ASan error) before the fix and PASS after. - */ -TEST_F(CallTraceStorageTest, UseAfterFreeInProcessTraces) { - // Create multiple traces with varying frame counts to increase memory footprint - const int NUM_TRACES = 100; - const int MAX_FRAMES = 20; - - std::vector trace_ids; - trace_ids.reserve(NUM_TRACES); - - // Create traces with multiple frames to use more memory - for (int i = 0; i < NUM_TRACES; i++) { - std::vector frames(MAX_FRAMES); - for (int j = 0; j < MAX_FRAMES; j++) { - frames[j].bci = i * 1000 + j; - frames[j].method_id = (jmethodID)(0x10000 + i * 100 + j); - } - - u64 trace_id = storage->put(MAX_FRAMES, frames.data(), false, 1); - ASSERT_GT(trace_id, 0) << "Failed to store trace " << i; - trace_ids.push_back(trace_id); - } - - // CRITICAL: Register a liveness checker that preserves ALL traces. - // This causes traces to be copied to SCRATCH during processTraces(). - // After rotation, SCRATCH becomes STANDBY, so the NEXT processTraces() - // will have these traces in STANDBY where the bug manifests. - storage->registerLivenessChecker([&trace_ids](std::unordered_set& buffer) { - for (u64 id : trace_ids) { - buffer.insert(id); - } - }); - - // First processTraces: traces are in ACTIVE, get collected and preserved to SCRATCH. - // After rotation: SCRATCH becomes STANDBY (now contains preserved traces) - int first_count = 0; - storage->processTraces([&first_count](const std::unordered_set& traces) { - first_count = traces.size(); - printf("First processTraces: %d traces collected\n", first_count); - }); - EXPECT_GT(first_count, NUM_TRACES) << "First processTraces should collect all traces"; - - // Second processTraces: THIS IS WHERE THE BUG OCCURS! - // 1. STANDBY now contains the preserved traces (from first call's scratch) - // 2. Standby traces are collected into _traces_buffer (raw pointers) - // 3. original_standby->clear() - FREES the trace memory! - // 4. processor(_traces_buffer) - accesses FREED memory (USE-AFTER-FREE!) - storage->processTraces([&](const std::unordered_set& traces) { - int total_frames = 0; - int total_bci_sum = 0; - int trace_count = 0; - - for (CallTrace* trace : traces) { - if (trace == nullptr) continue; - if (trace->trace_id == CallTraceStorage::DROPPED_TRACE_ID) continue; - - trace_count++; - - // Deep access to detect use-after-free: - // After standby->clear(), this memory is FREED but we're accessing it! - // ASan will catch this. Without ASan, we might read garbage. - EXPECT_GE(trace->num_frames, 1) << "Corrupted num_frames (use-after-free?)"; - EXPECT_LE(trace->num_frames, MAX_FRAMES) << "Corrupted num_frames (use-after-free?)"; - EXPECT_FALSE(trace->truncated) << "Corrupted truncated flag (use-after-free?)"; - EXPECT_GT(trace->trace_id, 0) << "Corrupted trace_id (use-after-free?)"; - - total_frames += trace->num_frames; - - // Access every frame - this maximizes chance of detecting corruption - for (int i = 0; i < trace->num_frames; i++) { - int bci = trace->frames[i].bci; - total_bci_sum += bci; - EXPECT_GE(bci, 0) << "Corrupted BCI at frame " << i << " (use-after-free?)"; - - jmethodID method = trace->frames[i].method_id; - EXPECT_NE(method, nullptr) << "Null method_id at frame " << i << " (use-after-free?)"; - } - } - - printf("Second processTraces: %d traces, %d total frames, bci_sum=%d\n", - trace_count, total_frames, total_bci_sum); - - // This is the key assertion: we expect to find the preserved traces - // If the bug exists, we're reading freed memory here! - EXPECT_GE(trace_count, NUM_TRACES) << "Should find preserved traces from standby"; - }); - - // Third processTraces: traces should still be preserved (copied to new scratch) - // This further exercises the use-after-free if the bug exists - storage->processTraces([&](const std::unordered_set& traces) { - int trace_count = 0; - for (CallTrace* trace : traces) { - if (trace == nullptr) continue; - if (trace->trace_id == CallTraceStorage::DROPPED_TRACE_ID) continue; - trace_count++; - - // Access all frame data - volatile int sum = 0; - for (int i = 0; i < trace->num_frames; i++) { - sum += trace->frames[i].bci; - } - } - printf("Third processTraces: %d traces\n", trace_count); - EXPECT_GE(trace_count, NUM_TRACES) << "Should still find preserved traces"; - }); -} - -/** - * Regression test for the putWithExistingId infinite-loop bug. - * - * Before the fix: when all INITIAL_CAPACITY (65536) slots were occupied the - * probe loop called probe.next() without checking probe.hasNext(), so the - * prime-probe step cycled forever through already-occupied slots. - * - * After the fix: the hasNext() guard breaks the loop and the call returns, - * silently dropping the trace that could not be inserted. - * - */ -TEST_F(CallTraceStorageTest, PutWithExistingIdNoInfiniteLoopWhenFull) { - static constexpr u32 INITIAL_CAPACITY = 65536; - - // Heap-allocated so the worker's shared_ptr copy keeps it alive if we detach - // before the thread writes completed=true (avoids UAF on slow machines). - auto completed = std::make_shared>(false); - std::thread worker([completed] { // capture by value — shared ownership - void* mem = std::aligned_alloc(alignof(CallTraceHashTable), sizeof(CallTraceHashTable)); - if (mem == nullptr) { - completed->store(true); // Let the join path handle this; EXPECT below will report. - return; - } - - auto tbl = std::unique_ptr( - new (mem) CallTraceHashTable(), - [](CallTraceHashTable* p) { p->~CallTraceHashTable(); std::free(p); }); - tbl->setInstanceId(1); - - // Each iteration uses a distinct (bci, method_id) pair so calcHash produces - // a distinct hash, filling INITIAL_CAPACITY unique slots. Iterations past - // that point find no empty slot and must exit the probe via the hasNext() - // guard rather than cycling forever. - for (u32 i = 0; i < INITIAL_CAPACITY + 128; ++i) { - // Stack-allocate source trace; putWithExistingId copies the payload. - alignas(alignof(CallTrace)) char buf[sizeof(CallTrace)]; - CallTrace* src = new (buf) CallTrace(false, 1, static_cast(i) + 1); - src->frames[0].bci = static_cast(i); - src->frames[0].method_id = reinterpret_cast(static_cast(0x10000 + i)); - tbl->putWithExistingId(src, 1); - } - - completed->store(true); - }); - - // 10-second deadline; an un-fixed infinite loop would never set `completed`. - auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); - while (!completed->load() && std::chrono::steady_clock::now() < deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } - - bool ok = completed->load(); - if (!ok) { - worker.detach(); - } else { - worker.join(); - } - EXPECT_TRUE(ok) - << "putWithExistingId infinite-loop regression: did not terminate within 10 s " - "when the scratch table was full"; -} - -/** - * Integration test: processTraces preserves live traces across rotation cycles - * without hanging. - * - * Each processTraces() cycle copies preserved traces into the scratch table via - * putWithExistingId. This test verifies: - * 1. Every cycle completes promptly (no infinite loop via putWithExistingId). - * 2. Every trace flagged by the liveness checker survives to the next cycle. - * 3. The trace_id of a preserved trace is unchanged after preservation - * (putWithExistingId must keep the original ID, not generate a new one). - * 4. Frame content is intact after copying (detects use-after-free). - */ -TEST_F(CallTraceStorageTest, LivenessPreservationAcrossMultipleCycles) { - const int N = 200; - - // Insert N unique traces and record their IDs and frame values for later checks. - std::vector ids; - std::vector bcis; - ids.reserve(N); - bcis.reserve(N); - for (int i = 0; i < N; i++) { - ASGCT_CallFrame frame; - frame.bci = i + 1000; - frame.method_id = reinterpret_cast(static_cast(0x30000 + i)); - u64 id = storage->put(1, &frame, false, 1); - ASSERT_NE(id, CallTraceStorage::DROPPED_TRACE_ID) << "put() failed for trace " << i; - ids.push_back(id); - bcis.push_back(frame.bci); - } - - // Liveness checker marks every stored trace as live. - storage->registerLivenessChecker([&ids](std::unordered_set& buf) { - for (u64 id : ids) buf.insert(id); - }); - - const int CYCLES = 5; - for (int cycle = 0; cycle < CYCLES; cycle++) { - std::atomic done{false}; - std::thread t([&] { - storage->processTraces([&](const std::unordered_set& traces) { - // All N preserved traces must be present, plus the dropped sentinel. - EXPECT_GE(traces.size(), static_cast(N + 1)) - << "cycle " << cycle << ": too few traces"; - - for (int j = 0; j < N; j++) { - CallTrace* found = findTraceById(traces, ids[j]); - EXPECT_NE(found, nullptr) - << "cycle " << cycle << ": trace_id " << ids[j] << " not preserved"; - if (found == nullptr) continue; - - // Trace ID must be unchanged (putWithExistingId preserves the original ID). - EXPECT_EQ(found->trace_id, ids[j]) - << "cycle " << cycle << ": trace_id mutated during preservation"; - - // Frame content must be intact (detects use-after-free of freed chunks). - EXPECT_EQ(found->frames[0].bci, bcis[j]) - << "cycle " << cycle << ": frame bci corrupted for trace " << ids[j]; - } - }); - done = true; - }); - - auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); - while (!done.load() && std::chrono::steady_clock::now() < deadline) { - std::this_thread::sleep_for(std::chrono::milliseconds(50)); - } - - bool ok = done.load(); - if (!ok) { - // Cannot safely detach: the lambda captures local stack variables by reference. - // If we detach and return, the thread will access destroyed locals (UAF). - // Instead, terminate the process immediately to fail the test cleanly. - std::cerr << "FATAL: processTraces hung on cycle " << cycle - << " (possible infinite-loop regression in putWithExistingId)" << std::endl; - std::abort(); - } - t.join(); - } -} -// Regression for defect C: clearTableOnly() must disconnect the full _prev chain, -// not only the first node. Fill the table past the 75 % expansion threshold so -// it grows to at least two LongHashTable nodes, then call clearTableOnly() and -// confirm the returned fresh table has no _prev chain. -TEST_F(CallTraceStorageTest, ClearTableOnlyDisconnectsFullChain) { - // 65536 initial capacity; expansion triggers at 75 % = 49152 entries. - // Insert 50000 distinct single-frame traces to force at least one expansion. - const int NUM_TRACES = 50000; - std::vector ids; - ids.reserve(NUM_TRACES); - - for (int i = 0; i < NUM_TRACES; i++) { - ASGCT_CallFrame frame; - frame.bci = i % 1000; // Reuse BCI values; uniqueness comes from method_id - frame.method_id = reinterpret_cast(static_cast(i + 1)); - u64 id = storage->put(1, &frame, false, 1); - EXPECT_GT(id, 0u) << "put() dropped trace at i=" << i; - ids.push_back(id); - } - - // processTraces() performs the rotation including clearTableOnly(); run it once - // to expose defect C. - int count = 0; - storage->processTraces([&](const std::unordered_set& traces) { - count = static_cast(traces.size()); - }); - // At least NUM_TRACES + the static dropped-trace sentinel should be present. - EXPECT_GE(count, NUM_TRACES); - // Second processTraces() verifies the fresh table is clean: no new puts occurred - // after rotation, so only the static dropped-trace sentinel should be present. - // This deterministically detects defect C — if clearTableOnly() left stale entries - // in freed memory that somehow end up in the new table, count2 would be wrong. - int count2 = 0; - storage->processTraces([&](const std::unordered_set& traces) { - count2 = static_cast(traces.size()); - }); - EXPECT_EQ(count2, 1); // only the dropped-trace sentinel; no stale entries -} - -// Regression for defect B: collect() must see all traces including those in -// older nodes of an expanded chain. Fill past expansion threshold, run -// processTraces(), and assert all inserted trace IDs are present. -TEST_F(CallTraceStorageTest, CollectFindsAllTracesAcrossExpandedChain) { - const int NUM_TRACES = 50000; - std::unordered_set inserted_ids; - - for (int i = 0; i < NUM_TRACES; i++) { - ASGCT_CallFrame frame; - frame.bci = i % 1000; // reuse bci values; uniqueness comes from method_id - frame.method_id = reinterpret_cast(static_cast(i + 1)); - u64 id = storage->put(1, &frame, false, 1); - EXPECT_GT(id, 0u) << "put() dropped trace at i=" << i; - inserted_ids.insert(id); - } - - std::unordered_set seen_ids; - storage->processTraces([&](const std::unordered_set& traces) { - for (CallTrace* t : traces) { - if (t) seen_ids.insert(t->trace_id); - } - }); - - // Every inserted ID must be visible in the snapshot. - for (u64 id : inserted_ids) { - EXPECT_TRUE(seen_ids.count(id) > 0) - << "Trace ID " << id << " was lost across expansion boundary"; - } -} diff --git a/ddprof-lib/src/test/cpp/threadFilter_ut.cpp b/ddprof-lib/src/test/cpp/threadFilter_ut.cpp deleted file mode 100644 index 4608e1169..000000000 --- a/ddprof-lib/src/test/cpp/threadFilter_ut.cpp +++ /dev/null @@ -1,554 +0,0 @@ -/* - * Copyright 2025 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "threadFilter.h" -#include "../../main/cpp/gtest_crash_handler.h" -#include -#include -#include -#include -#include -#include - -// Test name for crash handler -static constexpr char THREAD_FILTER_TEST_NAME[] = "ThreadFilterTest"; - -class ThreadFilterTest : public ::testing::Test { -protected: - void SetUp() override { - // Install crash handler for debugging potential issues - installGtestCrashHandler(); - filter = std::make_unique(); - filter->init("enabled"); // Enable filtering with non-empty string - } - - void TearDown() override { - filter.reset(); - // Restore default signal handlers - restoreDefaultSignalHandlers(); - } - - std::unique_ptr filter; -}; - -// Basic functionality tests -TEST_F(ThreadFilterTest, BasicRegisterAndAccept) { - EXPECT_TRUE(filter->enabled()); - - int slot_id = filter->registerThread(); - EXPECT_GE(slot_id, 0); - - // Initially should not accept (no tid added) - EXPECT_FALSE(filter->accept(slot_id)); - - // Add tid and test accept - filter->add(1234, slot_id); - EXPECT_TRUE(filter->accept(slot_id)); - - // Remove and test - filter->remove(slot_id); - EXPECT_FALSE(filter->accept(slot_id)); -} - -TEST_F(ThreadFilterTest, DisabledFilterAcceptsAll) { - ThreadFilter disabled_filter; - disabled_filter.init(nullptr); // Disabled with nullptr - - EXPECT_FALSE(disabled_filter.enabled()); - EXPECT_TRUE(disabled_filter.accept(-1)); - EXPECT_TRUE(disabled_filter.accept(0)); - EXPECT_TRUE(disabled_filter.accept(999999)); -} - -TEST_F(ThreadFilterTest, EmptyStringDisablesFilter) { - // Empty string should disable the filter (same as nullptr) - // This allows 'no-thread-filter' when 'filter' argument is provided without value - ThreadFilter empty_filter; - empty_filter.init(""); // Disabled with empty string - - EXPECT_FALSE(empty_filter.enabled()); - EXPECT_TRUE(empty_filter.accept(-1)); - EXPECT_TRUE(empty_filter.accept(0)); - EXPECT_TRUE(empty_filter.accept(999999)); - - // When disabled, registerThread() blocks new registrations - EXPECT_EQ(empty_filter.registerThread(), -1); -} - -TEST_F(ThreadFilterTest, InvalidSlotHandling) { - // Test invalid slot IDs for accept() - still safe due to negative check - EXPECT_FALSE(filter->accept(-1)); - EXPECT_FALSE(filter->accept(-999)); - - // These should not crash - filter->add(1234, -1); - filter->remove(-1); - filter->unregisterThread(-1); - filter->unregisterThread(-999); -} - -TEST_F(ThreadFilterTest, ValidSlotIDContract) { - // Verify that all slot IDs returned by registerThread() are valid - std::vector slot_ids; - - for (int i = 0; i < 100; i++) { - int slot_id = filter->registerThread(); - ASSERT_GE(slot_id, 0) << "registerThread() returned invalid slot_id: " << slot_id; - ASSERT_LT(slot_id, ThreadFilter::kMaxThreads) << "slot_id out of range: " << slot_id; - - slot_ids.push_back(slot_id); - - // These operations should always be safe with slot_ids from registerThread() - filter->add(i + 10000, slot_id); - EXPECT_TRUE(filter->accept(slot_id)); - filter->remove(slot_id); - EXPECT_FALSE(filter->accept(slot_id)); - } - - // Verify slot IDs are unique (no duplicates) - std::set unique_slots(slot_ids.begin(), slot_ids.end()); - EXPECT_EQ(unique_slots.size(), slot_ids.size()) << "registerThread() returned duplicate slot IDs"; -} - -// Edge case: Maximum capacity -TEST_F(ThreadFilterTest, MaxCapacityReached) { - std::vector slot_ids; - - // Register up to the maximum - for (int i = 0; i < ThreadFilter::kMaxThreads; i++) { - int slot_id = filter->registerThread(); - if (slot_id >= 0) { - slot_ids.push_back(slot_id); - filter->add(i + 1000, slot_id); // Use unique tids - } - } - - fprintf(stderr, "Successfully registered %zu slots (max=%d)\n", - slot_ids.size(), ThreadFilter::kMaxThreads); - - // Should have registered all slots - EXPECT_EQ(slot_ids.size(), ThreadFilter::kMaxThreads); - - // Next registration should fail - int overflow_slot = filter->registerThread(); - EXPECT_EQ(overflow_slot, -1); - - // Verify all registered slots work - std::vector collected_tids; - filter->collect(collected_tids); - EXPECT_EQ(collected_tids.size(), ThreadFilter::kMaxThreads); - - // Verify all tids are unique - std::set unique_tids(collected_tids.begin(), collected_tids.end()); - EXPECT_EQ(unique_tids.size(), ThreadFilter::kMaxThreads); -} - -// Edge case: Recovery after max capacity -TEST_F(ThreadFilterTest, RecoveryAfterMaxCapacity) { - std::vector slot_ids; - - // Fill to capacity - for (int i = 0; i < ThreadFilter::kMaxThreads; i++) { - int slot_id = filter->registerThread(); - ASSERT_GE(slot_id, 0); - slot_ids.push_back(slot_id); - filter->add(i + 2000, slot_id); - } - - // Should fail to register more - EXPECT_EQ(filter->registerThread(), -1); - - // Unregister half the slots - int slots_to_free = ThreadFilter::kMaxThreads / 2; - for (int i = 0; i < slots_to_free; i++) { - filter->unregisterThread(slot_ids[i]); - slot_ids[i] = -1; // Mark as freed - } - - // Should be able to register new slots again - std::vector new_slot_ids; - for (int i = 0; i < slots_to_free; i++) { - int slot_id = filter->registerThread(); - EXPECT_GE(slot_id, 0) << "Failed to register slot " << i << " after freeing"; - new_slot_ids.push_back(slot_id); - filter->add(i + 3000, slot_id); - } - - // Verify we can still register up to capacity - EXPECT_EQ(new_slot_ids.size(), slots_to_free); - - // Should fail again when at capacity - EXPECT_EQ(filter->registerThread(), -1); - - // Verify collect works correctly - std::vector collected_tids; - filter->collect(collected_tids); - EXPECT_EQ(collected_tids.size(), ThreadFilter::kMaxThreads); -} - -// Free list stress test -TEST_F(ThreadFilterTest, FreeListStressTest) { - const int iterations = 1000; - const int batch_size = 100; - - for (int iter = 0; iter < iterations; iter++) { - std::vector slot_ids; - - // Register a batch - for (int i = 0; i < batch_size; i++) { - int slot_id = filter->registerThread(); - ASSERT_GE(slot_id, 0); - slot_ids.push_back(slot_id); - filter->add(iter * batch_size + i, slot_id); - } - - // Verify all work - for (int slot_id : slot_ids) { - EXPECT_TRUE(filter->accept(slot_id)); - } - - // Unregister all - for (int slot_id : slot_ids) { - filter->unregisterThread(slot_id); - } - - // Verify cleanup - std::vector tids; - filter->collect(tids); - EXPECT_EQ(tids.size(), 0) << "Iteration " << iter << " left " << tids.size() << " tids"; - } -} - -// Multi-threaded edge case testing -TEST_F(ThreadFilterTest, ConcurrentMaxCapacityStress) { - const int num_threads = 8; - const int slots_per_thread = ThreadFilter::kMaxThreads / num_threads; - - std::vector threads; - std::atomic successful_registrations{0}; - std::atomic failed_registrations{0}; - std::vector> thread_slots(num_threads); - - // Each thread tries to register its share of slots - for (int t = 0; t < num_threads; t++) { - threads.emplace_back([&, t]() { - for (int i = 0; i < slots_per_thread + 10; i++) { // Try to over-register - int slot_id = filter->registerThread(); - if (slot_id >= 0) { - thread_slots[t].push_back(slot_id); - filter->add(t * 1000 + i, slot_id); - successful_registrations++; - } else { - failed_registrations++; - } - } - }); - } - - // Wait for all threads - for (auto& t : threads) { - t.join(); - } - - fprintf(stderr, "Successful: %d, Failed: %d, Total attempted: %d\n", - successful_registrations.load(), failed_registrations.load(), - num_threads * (slots_per_thread + 10)); - - // Should have registered exactly kMaxThreads - EXPECT_EQ(successful_registrations.load(), ThreadFilter::kMaxThreads); - EXPECT_GT(failed_registrations.load(), 0); // Some should have failed - - // Verify collect works - std::vector collected_tids; - filter->collect(collected_tids); - EXPECT_EQ(collected_tids.size(), ThreadFilter::kMaxThreads); -} - -// Chunk boundary testing -TEST_F(ThreadFilterTest, ChunkBoundaryBehavior) { - std::vector slot_ids; - - // Register enough slots to span multiple chunks - int slots_to_register = ThreadFilter::kChunkSize * 3 + 10; // 3+ chunks - - for (int i = 0; i < slots_to_register; i++) { - int slot_id = filter->registerThread(); - ASSERT_GE(slot_id, 0) << "Failed at slot " << i; - slot_ids.push_back(slot_id); - filter->add(i + 5000, slot_id); - } - - // Verify all chunks work correctly - for (int i = 0; i < slots_to_register; i++) { - EXPECT_TRUE(filter->accept(slot_ids[i])) << "Slot " << slot_ids[i] << " (index " << i << ") not accepted"; - } - - // Test collect across chunks - std::vector collected_tids; - filter->collect(collected_tids); - EXPECT_EQ(collected_tids.size(), slots_to_register); - - // Verify tids are correct - std::sort(collected_tids.begin(), collected_tids.end()); - for (int i = 0; i < slots_to_register; i++) { - EXPECT_EQ(collected_tids[i], i + 5000) << "TID mismatch at position " << i; - } -} - -// Race condition testing for add/remove/accept -TEST_F(ThreadFilterTest, ConcurrentAddRemoveAccept) { - const int num_threads = 4; - const int operations_per_thread = 10000; - - // Pre-register slots for each thread - std::vector slot_ids(num_threads); - for (int i = 0; i < num_threads; i++) { - slot_ids[i] = filter->registerThread(); - ASSERT_GE(slot_ids[i], 0); - } - - std::vector threads; - std::atomic total_operations{0}; - - for (int t = 0; t < num_threads; t++) { - threads.emplace_back([&, t]() { - int slot_id = slot_ids[t]; - int tid = t + 6000; - - for (int i = 0; i < operations_per_thread; i++) { - // Add - filter->add(tid, slot_id); - - // Should accept - bool accepted = filter->accept(slot_id); - if (!accepted) { - fprintf(stderr, "Thread %d: accept failed after add (op %d)\n", t, i); - } - - // Remove - filter->remove(slot_id); - - // Should not accept - accepted = filter->accept(slot_id); - if (accepted) { - fprintf(stderr, "Thread %d: accept succeeded after remove (op %d)\n", t, i); - } - - total_operations++; - } - }); - } - - // Wait for all threads - for (auto& t : threads) { - t.join(); - } - - EXPECT_EQ(total_operations.load(), num_threads * operations_per_thread); - - // Final state should be empty - std::vector final_tids; - filter->collect(final_tids); - EXPECT_EQ(final_tids.size(), 0); -} - -// Free list exhaustion and recovery -TEST_F(ThreadFilterTest, FreeListExhaustionRecovery) { - // Fill up the free list by registering and unregistering - std::vector slot_ids; - - // Register many slots - for (int i = 0; i < ThreadFilter::kFreeListSize + 100; i++) { - int slot_id = filter->registerThread(); - if (slot_id >= 0) { - slot_ids.push_back(slot_id); - filter->add(i + 7000, slot_id); - } - } - - fprintf(stderr, "Registered %zu slots\n", slot_ids.size()); - - // Unregister all (this should fill the free list) - for (int slot_id : slot_ids) { - filter->unregisterThread(slot_id); - } - - // Try to register new slots - should reuse from free list - std::vector new_slot_ids; - for (int i = 0; i < 100; i++) { - int slot_id = filter->registerThread(); - EXPECT_GE(slot_id, 0) << "Failed to reuse slot " << i; - new_slot_ids.push_back(slot_id); - filter->add(i + 8000, slot_id); - } - - // Verify reused slots work - for (int slot_id : new_slot_ids) { - EXPECT_TRUE(filter->accept(slot_id)); - } - - std::vector final_tids; - filter->collect(final_tids); - EXPECT_EQ(final_tids.size(), new_slot_ids.size()); -} - -// Performance regression test - only run in release builds -#ifdef NDEBUG -TEST_F(ThreadFilterTest, PerformanceRegression) { - const int num_operations = 100000; - - // Pre-register slots - std::vector slot_ids; - for (int i = 0; i < 100; i++) { - int slot_id = filter->registerThread(); - ASSERT_GE(slot_id, 0); - slot_ids.push_back(slot_id); - } - - auto start = std::chrono::high_resolution_clock::now(); - - // Perform many add/accept/remove operations - for (int i = 0; i < num_operations; i++) { - int slot_id = slot_ids[i % slot_ids.size()]; - filter->add(i, slot_id); - bool accepted = filter->accept(slot_id); - EXPECT_TRUE(accepted); - filter->remove(slot_id); - } - - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - - fprintf(stderr, "Performance: %d operations in %ld microseconds (%.2f ns/op)\n", - num_operations, duration.count(), - (double)duration.count() * 1000.0 / num_operations); - - // Should be fast - less than 200ns per operation is reasonable for this complex test - EXPECT_LT(duration.count() * 1000.0 / num_operations, 200.0); // 200ns per op max -} -#endif // NDEBUG - -// Collect behavior with mixed states -TEST_F(ThreadFilterTest, CollectMixedStates) { - std::vector slot_ids; - std::vector expected_tids; - - // Register slots and add some tids, leave others empty - for (int i = 0; i < 50; i++) { - int slot_id = filter->registerThread(); - ASSERT_GE(slot_id, 0); - slot_ids.push_back(slot_id); - - if (i % 3 == 0) { // Add tid to every 3rd slot - filter->add(i + 9000, slot_id); - expected_tids.push_back(i + 9000); - } - // Leave other slots empty - } - - // Collect should only return slots with tids - std::vector collected_tids; - filter->collect(collected_tids); - - std::sort(expected_tids.begin(), expected_tids.end()); - std::sort(collected_tids.begin(), collected_tids.end()); - - EXPECT_EQ(collected_tids.size(), expected_tids.size()); - for (size_t i = 0; i < expected_tids.size(); i++) { - EXPECT_EQ(collected_tids[i], expected_tids[i]); - } -} - -TEST_F(ThreadFilterTest, ClearActiveDropsPreviousRecordingMembership) { - int stale_slot = filter->registerThread(); - int current_slot = filter->registerThread(); - ASSERT_GE(stale_slot, 0); - ASSERT_GE(current_slot, 0); - - filter->add(1111, stale_slot); - filter->add(2222, current_slot); - filter->enterBlockedRun(stale_slot, OSThreadState::SLEEPING); - ThreadFilter::Slot *stale = filter->slotForId(stale_slot); - ASSERT_NE(nullptr, stale); - stale->markSampledThisRun(OSThreadState::SLEEPING); - - filter->clearActive(); - - std::vector collected_tids; - filter->collect(collected_tids); - EXPECT_TRUE(collected_tids.empty()); - EXPECT_FALSE(filter->accept(stale_slot)); - EXPECT_FALSE(filter->accept(current_slot)); - EXPECT_FALSE(stale->sampledThisRun()); - EXPECT_EQ(OSThreadState::UNKNOWN, stale->lastSampledState()); - EXPECT_EQ(OSThreadState::UNKNOWN, stale->activeBlockState()); - - filter->add(2222, current_slot); - filter->collect(collected_tids); - ASSERT_EQ(1u, collected_tids.size()); - EXPECT_EQ(2222, collected_tids[0]); -} - -TEST_F(ThreadFilterTest, GenerationCheckedExitDoesNotClearAnotherOwner) { - int slot_id = filter->registerThread(); - ASSERT_GE(slot_id, 0); - - u64 first_token = filter->enterBlockedRun(slot_id, OSThreadState::SLEEPING); - ASSERT_NE(0ULL, first_token); - EXPECT_EQ(0ULL, filter->enterBlockedRun(slot_id, OSThreadState::CONDVAR_WAIT)); - - ThreadFilter::Slot *slot = filter->slotForId(slot_id); - ASSERT_NE(nullptr, slot); - EXPECT_EQ(OSThreadState::SLEEPING, slot->activeBlockState()); - - EXPECT_FALSE(filter->exitBlockedRun(slot_id, ThreadFilter::tokenGeneration(first_token) + 1)); - EXPECT_EQ(OSThreadState::SLEEPING, slot->activeBlockState()); - - EXPECT_TRUE(filter->exitBlockedRun(slot_id, ThreadFilter::tokenGeneration(first_token))); - EXPECT_EQ(OSThreadState::UNKNOWN, slot->activeBlockState()); -} - -TEST_F(ThreadFilterTest, NewGenerationRejectsStaleToken) { - int slot_id = filter->registerThread(); - ASSERT_GE(slot_id, 0); - - u64 stale_token = filter->enterBlockedRun(slot_id, OSThreadState::SLEEPING); - ASSERT_NE(0ULL, stale_token); - EXPECT_TRUE(filter->exitBlockedRun(slot_id, ThreadFilter::tokenGeneration(stale_token))); - - u64 current_token = filter->enterBlockedRun(slot_id, OSThreadState::CONDVAR_WAIT); - ASSERT_NE(0ULL, current_token); - EXPECT_NE(ThreadFilter::tokenGeneration(stale_token), - ThreadFilter::tokenGeneration(current_token)); - - ThreadFilter::Slot *slot = filter->slotForId(slot_id); - ASSERT_NE(nullptr, slot); - EXPECT_FALSE(filter->exitBlockedRun(slot_id, ThreadFilter::tokenGeneration(stale_token))); - EXPECT_EQ(OSThreadState::CONDVAR_WAIT, slot->activeBlockState()); - EXPECT_TRUE(filter->exitBlockedRun(slot_id, ThreadFilter::tokenGeneration(current_token))); -} - -TEST_F(ThreadFilterTest, TokenRoundTripPreservesHighGenerationBit) { - ThreadFilter::SlotID slot_id = 7; - u32 generation = 0x80000001u; - u64 token = ThreadFilter::encodeBlockRunToken(slot_id, generation); - int64_t java_token = static_cast(token); - - EXPECT_LT(java_token, 0); - EXPECT_EQ(slot_id, ThreadFilter::tokenSlotId(static_cast(java_token))); - EXPECT_EQ(generation, ThreadFilter::tokenGeneration(static_cast(java_token))); -} diff --git a/ddprof-lib/src/test/cpp/threadIdTable_ut.cpp b/ddprof-lib/src/test/cpp/threadIdTable_ut.cpp deleted file mode 100644 index 2a0edd817..000000000 --- a/ddprof-lib/src/test/cpp/threadIdTable_ut.cpp +++ /dev/null @@ -1,271 +0,0 @@ -/* - * Copyright 2025 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "threadIdTable.h" -#include "../../main/cpp/gtest_crash_handler.h" -#include -#include -#include -#include -#include -#include - -// Test name for crash handler -static constexpr char THREAD_ID_TABLE_TEST_NAME[] = "ThreadIdTableTest"; - -class ThreadIdTableTest : public ::testing::Test { -protected: - void SetUp() override { - // Install crash handler for debugging potential issues - installGtestCrashHandler(); - table = std::make_unique(); - } - - void TearDown() override { - table.reset(); - // Restore default signal handlers - restoreDefaultSignalHandlers(); - } - - std::unique_ptr table; -}; - -// Basic functionality tests -TEST_F(ThreadIdTableTest, BasicInsertAndCollect) { - // Insert some thread IDs - table->insert(1001); - table->insert(1002); - table->insert(1003); - - // Collect and verify - std::unordered_set result; - table->collect(result); - - EXPECT_EQ(result.size(), 3); - EXPECT_TRUE(result.count(1001)); - EXPECT_TRUE(result.count(1002)); - EXPECT_TRUE(result.count(1003)); -} - -TEST_F(ThreadIdTableTest, InvalidThreadIdHandling) { - // Invalid thread ID (0) should be ignored - table->insert(0); - - std::unordered_set result; - table->collect(result); - EXPECT_EQ(result.size(), 0); - - // Negative thread IDs should still work (they're valid Linux thread IDs) - table->insert(-1); - table->collect(result); - EXPECT_EQ(result.size(), 1); - EXPECT_TRUE(result.count(-1)); -} - -TEST_F(ThreadIdTableTest, DuplicateInsertions) { - // Insert same thread ID multiple times - table->insert(2001); - table->insert(2001); - table->insert(2001); - - std::unordered_set result; - table->collect(result); - - // Should only appear once - EXPECT_EQ(result.size(), 1); - EXPECT_TRUE(result.count(2001)); -} - -TEST_F(ThreadIdTableTest, ClearFunctionality) { - // Insert some thread IDs - table->insert(3001); - table->insert(3002); - table->insert(3003); - - // Verify they're there - std::unordered_set result; - table->collect(result); - EXPECT_EQ(result.size(), 3); - - // Clear and verify empty - table->clear(); - result.clear(); - table->collect(result); - EXPECT_EQ(result.size(), 0); -} - -// Hash collision testing -TEST_F(ThreadIdTableTest, HashCollisions) { - // Create thread IDs that will hash to the same slot - // TABLE_SIZE = 256, so tids with same (tid % 256) will collide - std::vector colliding_tids; - int base_tid = 1000; - - // Generate 10 thread IDs that hash to the same slot - for (int i = 0; i < 10; i++) { - int tid = base_tid + (i * 256); // All will hash to same slot - colliding_tids.push_back(tid); - table->insert(tid); - } - - // All should be stored (linear probing should handle collisions) - std::unordered_set result; - table->collect(result); - - EXPECT_EQ(result.size(), colliding_tids.size()); - for (int tid : colliding_tids) { - EXPECT_TRUE(result.count(tid)) << "Missing tid: " << tid; - } -} - -// Capacity testing -TEST_F(ThreadIdTableTest, TableCapacityLimits) { - std::vector inserted_tids; - - // Try to insert more than TABLE_SIZE (256) unique thread IDs - for (int i = 1; i <= 300; i++) { // More than TABLE_SIZE - table->insert(i + 10000); // Use high numbers to avoid conflicts - inserted_tids.push_back(i + 10000); - } - - // Collect and see how many were actually stored - std::unordered_set result; - table->collect(result); - - fprintf(stderr, "Inserted %zu tids, collected %zu tids\n", - inserted_tids.size(), result.size()); - - // Should have stored at most TABLE_SIZE (256) - EXPECT_LE(result.size(), 256); - - // All collected tids should be from our inserted set - for (int tid : result) { - EXPECT_TRUE(std::find(inserted_tids.begin(), inserted_tids.end(), tid) != inserted_tids.end()) - << "Unexpected tid in result: " << tid; - } -} - -// Concurrent access testing (signal safety) -TEST_F(ThreadIdTableTest, ConcurrentInsertions) { - const int num_threads = 8; - const int tids_per_thread = 50; - - std::vector threads; - std::atomic successful_insertions{0}; - std::vector> thread_tids(num_threads); - - // Each thread inserts its own set of thread IDs - for (int t = 0; t < num_threads; t++) { - threads.emplace_back([&, t]() { - for (int i = 0; i < tids_per_thread; i++) { - int tid = t * 1000 + i + 20000; // Unique per thread - thread_tids[t].push_back(tid); - table->insert(tid); - successful_insertions++; - } - }); - } - - // Wait for all threads - for (auto& t : threads) { - t.join(); - } - - EXPECT_EQ(successful_insertions.load(), num_threads * tids_per_thread); - - // Collect and verify - std::unordered_set result; - table->collect(result); - - // Should have all unique thread IDs (or at least most of them) - std::set all_expected_tids; - for (const auto& thread_tids_vec : thread_tids) { - for (int tid : thread_tids_vec) { - all_expected_tids.insert(tid); - } - } - - fprintf(stderr, "Expected %zu unique tids, collected %zu tids\n", - all_expected_tids.size(), result.size()); - - // Table has fixed capacity of 256, so with 400 unique tids, we expect exactly 256 - EXPECT_EQ(result.size(), std::min(all_expected_tids.size(), (size_t)256)); - - // All collected tids should be valid - for (int tid : result) { - EXPECT_TRUE(all_expected_tids.count(tid)) << "Unexpected tid: " << tid; - } -} - -// Edge case: Realistic thread ID patterns -TEST_F(ThreadIdTableTest, RealisticThreadIds) { - // Linux thread IDs are typically large numbers - std::vector realistic_tids = { - 12345, 12346, 12347, // Sequential - 98765, 98766, 98767, // Another sequence - 1234567, 1234568, // Large numbers - 2147483647, // Max int - -1, -2, -3 // Negative (valid in some contexts) - }; - - for (int tid : realistic_tids) { - table->insert(tid); - } - - std::unordered_set result; - table->collect(result); - - // Should have all except tid=0 if any - size_t expected_size = realistic_tids.size(); - EXPECT_EQ(result.size(), expected_size); - - for (int tid : realistic_tids) { - if (tid != 0) { // 0 is invalid and ignored - EXPECT_TRUE(result.count(tid)) << "Missing realistic tid: " << tid; - } - } -} - -// Performance test (release builds only) -#ifdef NDEBUG -TEST_F(ThreadIdTableTest, PerformanceRegression) { - const int num_operations = 100000; - std::vector tids; - - // Pre-generate thread IDs - for (int i = 0; i < 100; i++) { - tids.push_back(i + 30000); - } - - auto start = std::chrono::high_resolution_clock::now(); - - // Perform many insertions - for (int i = 0; i < num_operations; i++) { - table->insert(tids[i % tids.size()]); - } - - auto end = std::chrono::high_resolution_clock::now(); - auto duration = std::chrono::duration_cast(end - start); - - fprintf(stderr, "ThreadIdTable Performance: %d operations in %ld microseconds (%.2f ns/op)\n", - num_operations, duration.count(), - (double)duration.count() * 1000.0 / num_operations); - - // Should be very fast for signal-safe operations - EXPECT_LT(duration.count() * 1000.0 / num_operations, 50.0); // 50ns per op max -} -#endif // NDEBUG \ No newline at end of file diff --git a/ddprof-lib/src/test/cpp/threadInfo_ut.cpp b/ddprof-lib/src/test/cpp/threadInfo_ut.cpp deleted file mode 100644 index 50b4c6b3d..000000000 --- a/ddprof-lib/src/test/cpp/threadInfo_ut.cpp +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "../../main/cpp/threadInfo.h" - -// Covers ThreadInfo::updateThreadName, whose resolver now runs OUTSIDE the -// _ti_lock (PROF-15139). The contract the refactor must preserve: -// - the resolver is NOT invoked when the tid is already named, -// - an empty resolver result is not inserted, -// - a non-empty result is inserted and retrievable. -class ThreadInfoTest : public ::testing::Test {}; - -TEST_F(ThreadInfoTest, resolverSkippedWhenNameKnown) { - ThreadInfo ti; - ti.set(42, "known", 7); - - bool resolver_called = false; - ti.updateThreadName(42, [&](int) { - resolver_called = true; - return std::string("replacement"); - }); - - EXPECT_FALSE(resolver_called); - auto info = ti.get(42); - ASSERT_NE(info.first, nullptr); - EXPECT_EQ(*info.first, "known"); -} - -TEST_F(ThreadInfoTest, emptyResolverResultNotInserted) { - ThreadInfo ti; - ti.updateThreadName(99, [](int) { return std::string(); }); - - auto info = ti.get(99); - EXPECT_EQ(info.first, nullptr); -} - -TEST_F(ThreadInfoTest, resolvedNameInsertedAndRetrievable) { - ThreadInfo ti; - int seen_tid = -1; - ti.updateThreadName(100, [&](int tid) { - seen_tid = tid; - return std::string("C2 CompilerThread0"); - }); - - EXPECT_EQ(seen_tid, 100); - auto info = ti.get(100); - ASSERT_NE(info.first, nullptr); - EXPECT_EQ(*info.first, "C2 CompilerThread0"); -} - -// Exercises the contract introduced by resolving OUTSIDE the lock: if another -// writer inserts the tid during the unlocked resolve window, the subsequent -// emplace must be a no-op so the authoritative name (e.g. the JVMTI name set -// via set()) wins. We deterministically simulate that race by performing the -// competing set() from inside the resolver itself. -TEST_F(ThreadInfoTest, racingSetDuringResolveWins) { - ThreadInfo ti; - ti.updateThreadName(100, [&](int tid) { - // Stands in for a concurrent set() landing in the unlocked window. - ti.set(tid, "jvmti-name", 9); - return std::string("proc-name"); - }); - - auto info = ti.get(100); - ASSERT_NE(info.first, nullptr); - EXPECT_EQ(*info.first, "jvmti-name"); - EXPECT_EQ(info.second, 9u); -} diff --git a/ddprof-lib/src/test/cpp/threadLocal_ut.cpp b/ddprof-lib/src/test/cpp/threadLocal_ut.cpp deleted file mode 100644 index edf963e20..000000000 --- a/ddprof-lib/src/test/cpp/threadLocal_ut.cpp +++ /dev/null @@ -1,291 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include "threadLocal.h" -#include "gtest_crash_handler.h" -#include -#include -#include -#include - -static constexpr char THREADLOCAL_TEST_NAME[] = "ThreadLocalTest"; - -// NOTE on the instances below being namespace-scope `static`: -// Keep the ThreadLocal instances alive for the duration of the test binary so -// their pthread keys are not repeatedly created/deleted across tests. -// This mirrors production usage where ThreadLocal instances are typically static -// and live until process exit. - -// ---- generic pointer specialization: plain set/get, no create/clean ---- -static ThreadLocal g_int_tl; - -// ---- lazy-create + cleanup instrumentation ---- -static std::atomic g_created{0}; -static std::atomic g_freed{0}; - -static void *create_tracked() { - g_created.fetch_add(1, std::memory_order_relaxed); - return new int(1234); -} - -static void free_tracked(void *p) { - g_freed.fetch_add(1, std::memory_order_relaxed); - delete static_cast(p); -} - -static ThreadLocal g_tracked_tl; - -// ---- tracked without auto-create: for clear() tests ---- -static ThreadLocal g_nocreate_tracked_tl; - -// ---- double specialization ---- -static ThreadLocal g_double_tl; - -class ThreadLocalTest : public ::testing::Test { -protected: - void SetUp() override { - installGtestCrashHandler(); - } - void TearDown() override { - restoreDefaultSignalHandlers(); - } -}; - -// set() then get() round-trips a value on the same thread. -TEST_F(ThreadLocalTest, Generic_SetGetRoundTrip) { - g_int_tl.set(42); - EXPECT_EQ(42, g_int_tl.get()); - g_int_tl.set(-7); - EXPECT_EQ(-7, g_int_tl.get()); -} - -// Each thread sees only its own value: storage is per-thread, not shared. -TEST_F(ThreadLocalTest, Generic_PerThreadIsolation) { - constexpr int kThreads = 8; - std::atomic ready{0}; - std::atomic go{false}; - std::vector threads; - std::atomic mismatches{0}; - - for (int i = 0; i < kThreads; ++i) { - threads.emplace_back([&, i] { - // Fresh thread: storage must start empty. - if (g_int_tl.get() != 0) { - mismatches.fetch_add(1, std::memory_order_relaxed); - } - g_int_tl.set(i + 1); - - // Barrier: every thread writes before any thread reads back, so a - // shared (buggy) slot would be observably clobbered. - ready.fetch_add(1, std::memory_order_relaxed); - while (!go.load(std::memory_order_acquire)) { - } - - if (g_int_tl.get() != static_cast(i + 1)) { - mismatches.fetch_add(1, std::memory_order_relaxed); - } - }); - } - - while (ready.load(std::memory_order_relaxed) != kThreads) { - } - go.store(true, std::memory_order_release); - - for (auto &t : threads) { - t.join(); - } - EXPECT_EQ(0, mismatches.load()); -} - -// A fresh thread that never called set() reads the zero-initialized default. -TEST_F(ThreadLocalTest, Generic_UnsetIsZero) { - intptr_t observed = -1; - std::thread t([&] { observed = g_int_tl.get(); }); - t.join(); - EXPECT_EQ(0, observed); -} - -// The create function lazily initializes storage on first get() and is invoked -// exactly once per thread; subsequent get()s return the same pointer. -TEST_F(ThreadLocalTest, Lazy_CreateOncePerThread) { - g_created.store(0, std::memory_order_relaxed); - g_freed.store(0, std::memory_order_relaxed); - - int *first = nullptr; - int *second = nullptr; - int value = 0; - std::thread t([&] { - first = g_tracked_tl.get(); - second = g_tracked_tl.get(); - // Read the payload here: free_tracked() deletes it on thread exit, so - // dereferencing first/second after join() would be use-after-free. - value = *first; - }); - t.join(); - - ASSERT_NE(nullptr, first); - EXPECT_EQ(first, second); // same instance reused (pointer compare only) - EXPECT_EQ(1234, value); // created via create_tracked() - EXPECT_EQ(1, g_created.load()); // created exactly once - EXPECT_EQ(1, g_freed.load()); -} - -// The clean function runs when the owning thread exits, freeing per-thread state. -TEST_F(ThreadLocalTest, Lazy_CleanupOnThreadExit) { - g_created.store(0, std::memory_order_relaxed); - g_freed.store(0, std::memory_order_relaxed); - - std::thread t([&] { - // Touch storage so a value exists to be cleaned up on exit. - ASSERT_NE(nullptr, g_tracked_tl.get()); - }); - t.join(); - // After join the thread has fully terminated, so its TSD destructor - // (free_tracked) must have run. - EXPECT_EQ(1, g_created.load()); - EXPECT_EQ(1, g_freed.load()); -} - -// Independent threads each create and free their own value. -TEST_F(ThreadLocalTest, Lazy_CleanupAcrossManyThreads) { - g_created.store(0, std::memory_order_relaxed); - g_freed.store(0, std::memory_order_relaxed); - - constexpr int kThreads = 16; - std::vector threads; - for (int i = 0; i < kThreads; ++i) { - threads.emplace_back([] { (void)g_tracked_tl.get(); }); - } - for (auto &t : threads) { - t.join(); - } - EXPECT_EQ(kThreads, g_created.load()); - EXPECT_EQ(kThreads, g_freed.load()); -} - -// The double specialization preserves the exact bit pattern through the -// u64<->void* round-trip (on 64-bit targets, where a double fits in a pointer). -TEST_F(ThreadLocalTest, Double_RoundTripPreservesValue) { - static_assert(sizeof(void *) >= sizeof(double), - "ThreadLocal requires pointer >= double width"); - - const double values[] = { - 0.0, - 1.0, - -1.0, - 3.141592653589793, - -2.718281828459045, - 1.7976931348623157e308, // near DBL_MAX - 2.2250738585072014e-308, // near DBL_MIN (smallest normal) - 4.9e-324, // smallest subnormal - }; - - std::atomic mismatches{0}; - std::thread t([&] { - for (double v : values) { - g_double_tl.set(v); - if (g_double_tl.get() != v) { - mismatches.fetch_add(1, std::memory_order_relaxed); - } - } - }); - t.join(); - EXPECT_EQ(0, mismatches.load()); -} - -// An unset double reads back as 0.0 (matches the original `thread_local double = 0`). -TEST_F(ThreadLocalTest, Double_UnsetIsZero) { - double observed = -1.0; - std::thread t([&] { observed = g_double_tl.get(); }); - t.join(); - EXPECT_EQ(0.0, observed); -} - -// clear() zeros the slot and invokes the destructor exactly once. -TEST_F(ThreadLocalTest, Generic_ClearRunsDestructorAndZerosSlot) { - g_created.store(0, std::memory_order_relaxed); - g_freed.store(0, std::memory_order_relaxed); - - int *ptr_before = nullptr; - int *ptr_after = reinterpret_cast(1); // sentinel – must become nullptr - std::thread t([&] { - // Manually place a tracked object; C == nullptr so get() won't re-create. - g_nocreate_tracked_tl.set(new int(55)); - ptr_before = g_nocreate_tracked_tl.get(); - - g_nocreate_tracked_tl.clear(); - - ptr_after = g_nocreate_tracked_tl.get(); // no auto-create → must be nullptr - }); - t.join(); - - EXPECT_NE(nullptr, ptr_before); // value was present before clear() - EXPECT_EQ(nullptr, ptr_after); // slot is zeroed after clear() - EXPECT_EQ(1, g_freed.load()); // destructor ran exactly once -} - -// ThreadLocal::clear() must make the next get() return 0.0. -TEST_F(ThreadLocalTest, Double_ClearReturnsZero) { - double after_clear = -1.0; - std::thread t([&] { - g_double_tl.set(3.141592653589793); - g_double_tl.clear(); - after_clear = g_double_tl.get(); - }); - t.join(); - EXPECT_EQ(0.0, after_clear); -} - -// clear() must zero the slot even when F == nullptr (no destructor registered). -TEST_F(ThreadLocalTest, Generic_ClearWithNoDestructorZerosSlot) { - intptr_t before = 0; - intptr_t after_clear = -1; - std::thread t([&] { - g_int_tl.set(42); - before = g_int_tl.get(); - g_int_tl.clear(); - after_clear = g_int_tl.get(); - }); - t.join(); - EXPECT_EQ(42, before); // value was set - EXPECT_EQ(0, after_clear); // slot is zeroed after clear() despite F == nullptr -} - -// Per-thread accumulation mirrors LivenessTracker's `skipped` usage: each thread -// keeps its own running sum, isolated from the others. -TEST_F(ThreadLocalTest, Double_PerThreadAccumulation) { - constexpr int kThreads = 8; - constexpr int kIters = 1000; - std::atomic mismatches{0}; - std::vector threads; - - for (int i = 0; i < kThreads; ++i) { - threads.emplace_back([&, i] { - const double step = static_cast(i + 1); - for (int k = 0; k < kIters; ++k) { - g_double_tl.set(g_double_tl.get() + step); - } - if (g_double_tl.get() != step * kIters) { - mismatches.fetch_add(1, std::memory_order_relaxed); - } - }); - } - for (auto &t : threads) { - t.join(); - } - EXPECT_EQ(0, mismatches.load()); -} diff --git a/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp b/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp deleted file mode 100644 index d7a371671..000000000 --- a/ddprof-lib/src/test/cpp/thread_teardown_safety_ut.cpp +++ /dev/null @@ -1,468 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * Acceptance tests for the PROF-14603 signal-race fix in thread teardown. - * - * Root cause: SIGVTALRM delivered between ProfiledThread::release() clearing TLS - * (pthread_setspecific(NULL)) and deleting the ProfiledThread object, causing - * a signal handler to call currentSignalSafe() and dereference a freed pointer. - * - * Fix coverage: - * - Signal-safe TLS accessor returns null in the race window (no crash). - * - SignalBlocker properly guards the unregister+release sequence. - * - Thread lifecycle (init/release) is race-free under concurrent signal load. - */ - -#include - -#ifdef __linux__ - -#include "guards.h" -#include "thread.h" - -#include -#include -#include -#include -#include -#include - -#ifdef __GLIBC__ -// declares these only inside the C (non-C++) conditional. -// Redeclare with extern "C" so we can call them directly from C++ code. -extern "C" { - extern void __pthread_register_cancel(__pthread_unwind_buf_t*); - extern void __pthread_unregister_cancel(__pthread_unwind_buf_t*); - [[noreturn]] extern void __pthread_unwind_next(__pthread_unwind_buf_t*); -} -#endif - -// Sentinel value meaning "handler has not run yet" — distinct from both nullptr -// (not registered) and any real ProfiledThread address. -static ProfiledThread* const kNotYetRun = reinterpret_cast(1); - -// Use sigaction() instead of signal() so the handler persists across platforms; -// signal() has implementation-defined reset-on-deliver (SA_RESETHAND) behaviour. -static inline void install_handler(int sig, void (*handler)(int)) { - struct sigaction sa{}; - sa.sa_handler = handler; - sigemptyset(&sa.sa_mask); - sa.sa_flags = 0; - sigaction(sig, &sa, nullptr); -} - -// RAII helper: saves the current sigaction for `sig` and restores it on -// destruction, preventing signal-disposition leaks across tests. -struct SigGuard { - int sig; - struct sigaction saved; - explicit SigGuard(int s) : sig(s) { sigaction(s, nullptr, &saved); } - ~SigGuard() { sigaction(sig, &saved, nullptr); } - SigGuard(const SigGuard&) = delete; - SigGuard& operator=(const SigGuard&) = delete; -}; - -// ── T-01: currentSignalSafe() is non-null while live, null after release ───── - -static std::atomic g_t01_seen{nullptr}; - -static void t01_handler(int) { - g_t01_seen.store(ProfiledThread::currentSignalSafe(), std::memory_order_relaxed); -} - -static void *t01_body(void *) { - ProfiledThread::initCurrentThread(); - - SigGuard guard(SIGVTALRM); - install_handler(SIGVTALRM, t01_handler); - g_t01_seen.store(kNotYetRun, std::memory_order_relaxed); - pthread_kill(pthread_self(), SIGVTALRM); - ProfiledThread *t01_pre = g_t01_seen.load(std::memory_order_relaxed); - if (t01_pre == kNotYetRun) { - ADD_FAILURE() << "SIGVTALRM handler must have run before release() (handler did not execute)"; - return nullptr; - } - EXPECT_NE(nullptr, t01_pre) - << "currentSignalSafe() must return non-null while ProfiledThread is live"; - - ProfiledThread::release(); - - g_t01_seen.store(kNotYetRun, std::memory_order_relaxed); - pthread_kill(pthread_self(), SIGVTALRM); - ProfiledThread *t01_post = g_t01_seen.load(std::memory_order_relaxed); - if (t01_post == kNotYetRun) { - ADD_FAILURE() << "SIGVTALRM handler must have run after release() (handler did not execute)"; - return nullptr; - } - EXPECT_EQ(nullptr, t01_post) - << "currentSignalSafe() must return null after release()"; - - return nullptr; -} - -// Verifies the post-release null guarantee seen from a signal handler. -TEST(ThreadTeardownSafetyTest, TLSAccessibleDuringLifetimeNullAfterRelease) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, t01_body, nullptr)); - pthread_join(t, nullptr); -} - -// ── T-02: Signal in the TLS-clear/delete window does not crash ─────────────── - -static std::atomic g_t02_seen{nullptr}; - -static void t02_handler(int) { - g_t02_seen.store(ProfiledThread::currentSignalSafe(), std::memory_order_relaxed); -} - -static void *t02_body(void *) { - ProfiledThread::initCurrentThread(); - - SigGuard guard(SIGVTALRM); - install_handler(SIGVTALRM, t02_handler); - g_t02_seen.store(kNotYetRun, std::memory_order_relaxed); - - // Simulate the race window: TLS cleared but object not yet freed. - ProfiledThread *detached = ProfiledThread::clearCurrentThreadTLS(); - - // Signal delivered in the race window must see null, not a dangling pointer. - pthread_kill(pthread_self(), SIGVTALRM); - EXPECT_EQ(nullptr, g_t02_seen.load(std::memory_order_relaxed)) - << "currentSignalSafe() must return null in the TLS-clear/delete window"; - - // release() with TLS already null must not double-free. - ProfiledThread::release(); - // Complete the simulated teardown: delete the object (mirrors what freeKey - // would do). Destructor is private so we need the test helper. - ProfiledThread::deleteForTest(detached); - return nullptr; -} - -// Regression for the primary crash path: signal fires between clearTLS and delete. -TEST(ThreadTeardownSafetyTest, SignalInTLSClearDeleteWindowDoesNotCrash) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, t02_body, nullptr)); - pthread_join(t, nullptr); -} - -// ── T-03: Double release() is idempotent ───────────────────────────────────── - -static void *t03_body(void *) { - ProfiledThread::initCurrentThread(); - ProfiledThread::release(); - ProfiledThread::release(); // must not crash or double-free - return nullptr; -} - -TEST(ThreadTeardownSafetyTest, DoubleReleaseIsIdempotent) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, t03_body, nullptr)); - pthread_join(t, nullptr); -} - -// ── T-04: Signal-safe accessor returns null without initCurrentThread() ────── - -static std::atomic g_t04_seen{nullptr}; - -static void t04_handler(int) { - g_t04_seen.store(ProfiledThread::currentSignalSafe(), std::memory_order_relaxed); -} - -static void *t04_body(void *) { - // Intentionally no initCurrentThread(). - SigGuard guard(SIGVTALRM); - install_handler(SIGVTALRM, t04_handler); - g_t04_seen.store(kNotYetRun, std::memory_order_relaxed); - pthread_kill(pthread_self(), SIGVTALRM); - EXPECT_EQ(nullptr, g_t04_seen.load(std::memory_order_relaxed)) - << "currentSignalSafe() must return null for unregistered threads"; - return nullptr; -} - -TEST(ThreadTeardownSafetyTest, SignalSafeAccessorReturnsNullWithoutInit) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, t04_body, nullptr)); - pthread_join(t, nullptr); -} - -// ── T-05: Concurrent signals during teardown stress ────────────────────────── - -static std::atomic g_t05_signal_count{0}; - -static void t05_handler(int) { - (void)ProfiledThread::currentSignalSafe(); - g_t05_signal_count.fetch_add(1, std::memory_order_relaxed); -} - -static void *t05_worker(void *) { - ProfiledThread::initCurrentThread(); - for (int i = 0; i < 10; ++i) { - pthread_kill(pthread_self(), SIGVTALRM); - } - ProfiledThread::release(); - return nullptr; -} - -// 20 threads × 5 rounds with signal spray; handler invocation is verified. -// Signal dispositions are process-wide — install once from the test body so -// no worker can race to restore SIG_DFL (terminate) mid-flight. -TEST(ThreadTeardownSafetyTest, ConcurrentSignalsDuringTeardownStress) { - SigGuard gVtalrm(SIGVTALRM); - SigGuard gProf(SIGPROF); - g_t05_signal_count.store(0, std::memory_order_relaxed); - install_handler(SIGVTALRM, t05_handler); - install_handler(SIGPROF, SIG_IGN); - for (int round = 0; round < 5; ++round) { - std::vector threads(20); - for (auto &tid : threads) { - ASSERT_EQ(0, pthread_create(&tid, nullptr, t05_worker, nullptr)); - } - for (auto &tid : threads) { - pthread_join(tid, nullptr); - } - } - EXPECT_GT(g_t05_signal_count.load(std::memory_order_relaxed), 0) - << "SIGVTALRM handler must have been invoked at least once"; -} - -// ── T-06: SignalBlocker masks SIGPROF + SIGVTALRM and restores on exit ──────── - -static void *t06_body(void *) { - sigset_t before, during, after; - - pthread_sigmask(SIG_SETMASK, nullptr, &before); - - { - SignalBlocker blocker; - pthread_sigmask(SIG_SETMASK, nullptr, &during); - EXPECT_TRUE(sigismember(&during, SIGVTALRM)) - << "SignalBlocker must block SIGVTALRM"; - EXPECT_TRUE(sigismember(&during, SIGPROF)) - << "SignalBlocker must block SIGPROF"; - } - - pthread_sigmask(SIG_SETMASK, nullptr, &after); - EXPECT_EQ(sigismember(&before, SIGVTALRM), sigismember(&after, SIGVTALRM)) - << "SignalBlocker must restore SIGVTALRM to its initial state on exit"; - EXPECT_EQ(sigismember(&before, SIGPROF), sigismember(&after, SIGPROF)) - << "SignalBlocker must restore SIGPROF to its initial state on exit"; - return nullptr; -} - -TEST(ThreadTeardownSafetyTest, SignalBlockerMasksAndRestoresProfSignals) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, t06_body, nullptr)); - pthread_join(t, nullptr); -} - -// ── T-07: Forced unwind with concurrent signal does not crash ───────────────── -// Cancellation mechanism differs between glibc (abi::__forced_unwind via C++ -// unwinder) and musl (C-style pthread_cleanup_push callbacks). - -static std::atomic g_t07_cleanup_ran{false}; -static std::atomic g_t07_release_ran{false}; - -#ifdef __GLIBC__ - -// t07 uses __pthread_register_cancel for cleanup (same as run_with_cleanup in -// libraryPatcher_linux.cpp). Two constraints follow from the static-libgcc / -// libgcc_s.so.1 incompatibility: -// -// 1. No C++ RAII objects with destructors in this frame. After the longjmp -// from glibc's stop function, __pthread_unwind_next calls _Unwind_ForcedUnwind -// again to continue unwinding through this frame. Any LSDA cleanup entry -// (e.g. SigGuard, std::unique_ptr) would make __gxx_personality_v0 call -// _Unwind_SetGR with a cross-version context → abort(). -// -// 2. No try/catch: handler frames also add LSDA entries, same problem. -// -// Signal handler save/restore is done manually (plain struct sigaction, no -// destructor) so there is nothing in the LSDA for this frame. -__attribute__((noinline, no_stack_protector)) static void *t07_body(void *) { - g_t07_cleanup_ran.store(false, std::memory_order_relaxed); - g_t07_release_ran.store(false, std::memory_order_relaxed); - - // Save and install the signal handler WITHOUT RAII (no destructor = no LSDA - // cleanup entry for this frame). - struct sigaction old_sa_vtalrm; - sigaction(SIGVTALRM, nullptr, &old_sa_vtalrm); - install_handler(SIGVTALRM, SIG_IGN); - - ProfiledThread::initCurrentThread(); - - __pthread_unwind_buf_t cancel_buf = {}; - if (__builtin_expect( - __sigsetjmp((struct __jmp_buf_tag*)(void*)cancel_buf.__cancel_jmp_buf, 0), 0)) { - // Restore handler before continuing forced unwind (no RAII to do it for us). - sigaction(SIGVTALRM, &old_sa_vtalrm, nullptr); - g_t07_cleanup_ran.store(true, std::memory_order_relaxed); - ProfiledThread::release(); - g_t07_release_ran.store(true, std::memory_order_relaxed); - __pthread_unwind_next(&cancel_buf); - } - __pthread_register_cancel(&cancel_buf); - // No matching __pthread_unregister_cancel: the only legal exit from this body - // is the forced unwind raised by pthread_cancel, which longjmps into the - // __sigsetjmp branch above and continues via __pthread_unwind_next. The loop - // below never returns normally; if it ever did, cancel_buf would be left - // registered against a destroyed frame. - // Inject a signal before the cancellation point to exercise the combined path. - pthread_kill(pthread_self(), SIGVTALRM); - while (true) { - pthread_testcancel(); - usleep(100); - } - // Must only exit via pthread_cancel above. If control reaches here, the - // frame was not cancelled as expected and cancel_buf is now a dangling - // registration — abort rather than corrupt glibc's cleanup list. - __builtin_unreachable(); -} - -#else // !__GLIBC__ — musl: cancellation runs C cleanup callbacks - -static void t07_cleanup_fn(void *) { - g_t07_cleanup_ran.store(true, std::memory_order_relaxed); - ProfiledThread::release(); - g_t07_release_ran.store(true, std::memory_order_relaxed); -} - -static void *t07_body(void *) { - g_t07_cleanup_ran.store(false, std::memory_order_relaxed); - g_t07_release_ran.store(false, std::memory_order_relaxed); - - SigGuard guard(SIGVTALRM); - install_handler(SIGVTALRM, SIG_IGN); - ProfiledThread::initCurrentThread(); - - pthread_cleanup_push(t07_cleanup_fn, nullptr); - // Inject a signal before the cancellation point to exercise the combined path. - pthread_kill(pthread_self(), SIGVTALRM); - while (true) { - pthread_testcancel(); - usleep(100); - } - pthread_cleanup_pop(0); - ProfiledThread::release(); - return nullptr; -} - -#endif // __GLIBC__ - -TEST(ThreadTeardownSafetyTest, ForcedUnwindWithConcurrentSignalDoesNotCrash) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, t07_body, nullptr)); - usleep(5000); - pthread_cancel(t); - void *retval; - ASSERT_EQ(0, pthread_join(t, &retval)); - EXPECT_TRUE(g_t07_cleanup_ran.load()); - EXPECT_TRUE(g_t07_release_ran.load()); - EXPECT_EQ(PTHREAD_CANCELED, retval); -} - -// ── T-08: Double initCurrentThread() is idempotent ─────────────────────────── - -static void *t08_body(void *) { - ProfiledThread::initCurrentThread(); - ProfiledThread *first = ProfiledThread::currentSignalSafe(); - EXPECT_NE(nullptr, first); - - ProfiledThread::initCurrentThread(); // second call on the same thread - ProfiledThread *second = ProfiledThread::currentSignalSafe(); - EXPECT_NE(nullptr, second); - EXPECT_EQ(first, second) << "double init must not allocate a second ProfiledThread"; - - ProfiledThread::release(); - return nullptr; -} - -TEST(ThreadTeardownSafetyTest, DoubleInitIsIdempotent) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, t08_body, nullptr)); - pthread_join(t, nullptr); -} - -// ── T-09: High-frequency signals during thread churn ───────────────────────── - -static std::atomic g_t09_stop{false}; -static std::atomic g_t09_signal_count{0}; - -static void t09_handler(int) { - (void)ProfiledThread::currentSignalSafe(); - g_t09_signal_count.fetch_add(1, std::memory_order_relaxed); -} - -static void *t09_injector(void *) { - while (!g_t09_stop.load(std::memory_order_relaxed)) { - kill(getpid(), SIGVTALRM); - usleep(500); // ~2 kHz - } - return nullptr; -} - -static void *t09_worker(void *) { - ProfiledThread::initCurrentThread(); - usleep(100); - ProfiledThread::release(); - return nullptr; -} - -// Mirrors DumpWhileChurningThreadsTest at native level: 100 short-lived threads -// under continuous SIGVTALRM injection must complete without crash; handler -// invocation is verified. -TEST(ThreadTeardownSafetyTest, HighFrequencySignalsDuringThreadChurn) { - SigGuard testGuard(SIGVTALRM); - g_t09_signal_count.store(0, std::memory_order_relaxed); - install_handler(SIGVTALRM, t09_handler); - - g_t09_stop.store(false, std::memory_order_relaxed); - pthread_t injector; - ASSERT_EQ(0, pthread_create(&injector, nullptr, t09_injector, nullptr)); - - for (int i = 0; i < 100; ++i) { - pthread_t worker; - ASSERT_EQ(0, pthread_create(&worker, nullptr, t09_worker, nullptr)); - pthread_join(worker, nullptr); - } - - g_t09_stop.store(true, std::memory_order_relaxed); - pthread_join(injector, nullptr); - EXPECT_GT(g_t09_signal_count.load(std::memory_order_relaxed), 0) - << "SIGVTALRM handler must have been invoked at least once"; -} - -// ── T-10: CriticalSection reentrancy guard prevents double-entry ────────────── - -static void *t10_body(void *) { - ProfiledThread::initCurrentThread(); - ProfiledThread *pt = ProfiledThread::currentSignalSafe(); - if (pt == nullptr) { - ADD_FAILURE() << "currentSignalSafe() returned nullptr"; - return nullptr; - } - - // Outer critical section must succeed. - EXPECT_TRUE(pt->tryEnterCriticalSection()); - - // Simulated reentrancy from the same thread (e.g. nested signal handler). - EXPECT_FALSE(pt->tryEnterCriticalSection()) - << "reentrancy must be rejected while outer critical section is active"; - - pt->exitCriticalSection(); - - // After exit, entry succeeds again. - EXPECT_TRUE(pt->tryEnterCriticalSection()); - pt->exitCriticalSection(); - - ProfiledThread::release(); - return nullptr; -} - -TEST(ThreadTeardownSafetyTest, CriticalSectionReentrancyGuard) { - pthread_t t; - ASSERT_EQ(0, pthread_create(&t, nullptr, t10_body, nullptr)); - pthread_join(t, nullptr); -} - -#endif // __linux__ diff --git a/ddprof-lib/src/test/cpp/utils_ut.cpp b/ddprof-lib/src/test/cpp/utils_ut.cpp deleted file mode 100644 index 211b8fc15..000000000 --- a/ddprof-lib/src/test/cpp/utils_ut.cpp +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - */ - -#include -#include -#include "../../main/cpp/utils.h" -#include "../../main/cpp/gtest_crash_handler.h" - -static constexpr char UTILS_TEST_NAME[] = "UtilsTest"; - -class GlobalSetup { -public: - GlobalSetup() { - installGtestCrashHandler(); - } - ~GlobalSetup() { - restoreDefaultSignalHandlers(); - } -}; - -static GlobalSetup global_setup; - -TEST(UtilsTest, IsAlignedNullPointer) { - // NULL should be aligned to any power-of-2 - EXPECT_TRUE(is_aligned((void*)0x0, 1)); - EXPECT_TRUE(is_aligned((void*)0x0, 2)); - EXPECT_TRUE(is_aligned((void*)0x0, 4)); - EXPECT_TRUE(is_aligned((void*)0x0, 8)); - EXPECT_TRUE(is_aligned((void*)0x0, 16)); -} - -TEST(UtilsTest, IsAlignedValidPointers) { - // Aligned addresses - EXPECT_TRUE(is_aligned((void*)0x1000, 8)) << "0x1000 should be 8-byte aligned"; - EXPECT_TRUE(is_aligned((void*)0x1008, 8)) << "0x1008 should be 8-byte aligned"; - EXPECT_TRUE(is_aligned((void*)0x2000, 16)) << "0x2000 should be 16-byte aligned"; - EXPECT_TRUE(is_aligned((void*)0x1000, 4)) << "0x1000 should be 4-byte aligned"; - - // Common heap addresses are typically aligned to sizeof(void*) - void* heap_mem = malloc(64); - ASSERT_NE(heap_mem, nullptr); - EXPECT_TRUE(is_aligned(heap_mem, sizeof(void*))) - << "malloc() should return pointer aligned to sizeof(void*)"; - free(heap_mem); -} - -TEST(UtilsTest, IsAlignedUnalignedPointers) { - // Unaligned addresses - EXPECT_FALSE(is_aligned((void*)0x1001, 8)) << "0x1001 is not 8-byte aligned (off by 1)"; - EXPECT_FALSE(is_aligned((void*)0x1002, 8)) << "0x1002 is not 8-byte aligned (off by 2)"; - EXPECT_FALSE(is_aligned((void*)0x1007, 8)) << "0x1007 is not 8-byte aligned (off by 7)"; - EXPECT_FALSE(is_aligned((void*)0x100F, 16)) << "0x100F is not 16-byte aligned (off by 15)"; - EXPECT_FALSE(is_aligned((void*)0x2001, 2)) << "0x2001 is not 2-byte aligned"; -} - -TEST(UtilsTest, IsAlignedPowerOf2Sizes) { - // Test various power-of-2 alignments on the same address - void* addr = (void*)0x1000; - EXPECT_TRUE(is_aligned(addr, 1)); - EXPECT_TRUE(is_aligned(addr, 2)); - EXPECT_TRUE(is_aligned(addr, 4)); - EXPECT_TRUE(is_aligned(addr, 8)); - EXPECT_TRUE(is_aligned(addr, 16)); - EXPECT_TRUE(is_aligned(addr, 256)); - EXPECT_TRUE(is_aligned(addr, 4096)); -} - -TEST(UtilsTest, IsAlignedCriticalBugCheck) { - // This test specifically catches the inverted logic bug where - // is_aligned() was incorrectly using ~(alignment-1) instead of (alignment-1) - - void* aligned_ptr = (void*)0x1000; - void* unaligned_ptr = (void*)0x1001; - - // With correct logic: - // aligned: (0x1000 & 0x7) == 0 → true ✓ - // unaligned: (0x1001 & 0x7) == 1 → false ✓ - - // With INVERTED logic (the bug): - // aligned: (0x1000 & ~0x7) == 0x1000 != 0 → false ✗ - // unaligned: (0x1001 & ~0x7) == 0x1000 != 0 → false ✗ - - ASSERT_TRUE(is_aligned(aligned_ptr, 8)) - << "CRITICAL BUG: is_aligned() returns false for aligned pointer 0x1000! " - << "This suggests the alignment check logic is inverted. " - << "Expected: (ptr & (alignment-1)) == 0, " - << "Got: (ptr & ~(alignment-1)) == 0"; - - ASSERT_FALSE(is_aligned(unaligned_ptr, 8)) - << "is_aligned() should return false for unaligned pointer 0x1001"; -} - -TEST(UtilsTest, AlignUpBasic) { - EXPECT_EQ(align_up(0, 8), 0); - EXPECT_EQ(align_up(1, 8), 8); - EXPECT_EQ(align_up(7, 8), 8); - EXPECT_EQ(align_up(8, 8), 8); - EXPECT_EQ(align_up(9, 8), 16); - EXPECT_EQ(align_up(15, 8), 16); - EXPECT_EQ(align_up(16, 8), 16); -} - -TEST(UtilsTest, AlignDownBasic) { - EXPECT_EQ(align_down(0, 8), 0); - EXPECT_EQ(align_down(1, 8), 0); - EXPECT_EQ(align_down(7, 8), 0); - EXPECT_EQ(align_down(8, 8), 8); - EXPECT_EQ(align_down(9, 8), 8); - EXPECT_EQ(align_down(15, 8), 8); - EXPECT_EQ(align_down(16, 8), 16); -} - -TEST(UtilsTest, IsPowerOf2) { - EXPECT_FALSE(is_power_of_2(0)); - EXPECT_TRUE(is_power_of_2(1)); - EXPECT_TRUE(is_power_of_2(2)); - EXPECT_FALSE(is_power_of_2(3)); - EXPECT_TRUE(is_power_of_2(4)); - EXPECT_FALSE(is_power_of_2(5)); - EXPECT_TRUE(is_power_of_2(8)); - EXPECT_TRUE(is_power_of_2(16)); - EXPECT_TRUE(is_power_of_2(256)); - EXPECT_FALSE(is_power_of_2(255)); -} diff --git a/ddprof-lib/src/test/cpp/wallClockCounters_ut.cpp b/ddprof-lib/src/test/cpp/wallClockCounters_ut.cpp deleted file mode 100644 index c908b84fc..000000000 --- a/ddprof-lib/src/test/cpp/wallClockCounters_ut.cpp +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "wallClockCounters.h" - -#include - -class WallClockCountersTest : public ::testing::Test { -protected: - void SetUp() override { - WallClockCounters::reset(); - } - - void TearDown() override { - WallClockCounters::reset(); - } -}; - -TEST_F(WallClockCountersTest, DrainReturnsAndClearsSuppressedSampledRun) { - WallClockCounters::incrementSuppressedSampledRun(); - WallClockCounters::incrementSuppressedSampledRun(); - - EXPECT_EQ(2ULL, WallClockCounters::drainSuppressedSampledRun()); - EXPECT_EQ(0ULL, WallClockCounters::drainSuppressedSampledRun()); -} - -TEST_F(WallClockCountersTest, ResetClearsPendingSuppressedSampledRun) { - WallClockCounters::incrementSuppressedSampledRun(); - - WallClockCounters::reset(); - - EXPECT_EQ(0ULL, WallClockCounters::drainSuppressedSampledRun()); -} - -TEST_F(WallClockCountersTest, ResetIsIdempotent) { - WallClockCounters::reset(); - WallClockCounters::reset(); - - EXPECT_EQ(0ULL, WallClockCounters::drainSuppressedSampledRun()); -} diff --git a/ddprof-lib/src/test/cpp/wallprecheck_args_ut.cpp b/ddprof-lib/src/test/cpp/wallprecheck_args_ut.cpp deleted file mode 100644 index 5579e354f..000000000 --- a/ddprof-lib/src/test/cpp/wallprecheck_args_ut.cpp +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include "arguments.h" - -TEST(WallPrecheckArgsTest, DefaultsToDisabled) { - Arguments args; - - EXPECT_FALSE(args._wall_precheck); -} - -TEST(WallPrecheckArgsTest, BareFlagEnablesPrecheck) { - Arguments args; - Error error = args.parse("wallprecheck"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._wall_precheck); -} - -TEST(WallPrecheckArgsTest, ExplicitBooleanValues) { - Arguments args; - - Error error = args.parse("wallprecheck=true"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._wall_precheck); - - // parse() frees the previous _buf before allocating a new one, so reusing - // the same args object is correct and avoids leaking the intermediate buffers. - error = args.parse("wallprecheck=false"); - EXPECT_FALSE(error); - EXPECT_FALSE(args._wall_precheck); - - error = args.parse("wallprecheck=1"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._wall_precheck); - - error = args.parse("wallprecheck=0"); - EXPECT_FALSE(error); - EXPECT_FALSE(args._wall_precheck); -} - -TEST(WallPrecheckArgsTest, UnknownValueTreatedAsTrue) { - // Any value that is not "false" or "0" enables the precheck (strcmp semantics). - Arguments args; - Error error = args.parse("wallprecheck=yes"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._wall_precheck); -} - -TEST(WallPrecheckArgsTest, EnabledWithinLongerArgString) { - Arguments args; - Error error = args.parse("wall=1ms,wallprecheck=true"); - EXPECT_FALSE(error); - EXPECT_TRUE(args._wall_precheck); -} - diff --git a/ddprof-lib/src/test/fuzz/README.md b/ddprof-lib/src/test/fuzz/README.md deleted file mode 100644 index 5714df7b7..000000000 --- a/ddprof-lib/src/test/fuzz/README.md +++ /dev/null @@ -1,202 +0,0 @@ -# Fuzz Testing for ddprof-lib - -This directory contains libFuzzer-based fuzz targets for discovering bugs in the -Datadog Java Profiler's native C++ code. - -## Prerequisites - -- **clang** compiler with `-fsanitize=fuzzer` support (clang 6.0+) -- **libFuzzer** (bundled with clang) -- **AddressSanitizer** support (for memory error detection) - -### Installation Instructions - -#### Ubuntu/Debian (apt-based) -```bash -# Install clang and compiler-rt (includes libFuzzer) -sudo apt-get update -sudo apt-get install clang llvm - -# For specific version (e.g., clang-15) -sudo apt-get install clang-15 llvm-15 -``` - -#### RHEL/CentOS/Fedora (yum/dnf-based) -```bash -# Fedora -sudo dnf install clang compiler-rt llvm - -# RHEL/CentOS (requires EPEL) -sudo yum install epel-release -sudo yum install clang compiler-rt llvm -``` - -#### Alpine Linux (apk-based) -```bash -# Install clang and compiler-rt -sudo apk add clang compiler-rt llvm - -# Note: libFuzzer may not be available in older Alpine versions -# Check with: clang -fsanitize=fuzzer 2>&1 | grep -i fuzzer -``` - -#### macOS -```bash -# IMPORTANT: Xcode's clang does not include libFuzzer -# You MUST install LLVM via Homebrew: -brew install llvm - -# The build system will automatically detect and use Homebrew LLVM -# No environment variables needed! -``` - -#### Verification - -After installation, verify libFuzzer support: -```bash -# Test if libFuzzer is available -echo 'extern "C" int LLVMFuzzerTestOneInput(const char*, long) { return 0; }' | \ - clang++ -fsanitize=fuzzer -x c++ - -o /tmp/fuzz_test && \ - echo "✓ libFuzzer is available" || \ - echo "✗ libFuzzer not found" - -# Clean up -rm -f /tmp/fuzz_test -``` - -### Troubleshooting - -**"library 'libclang_rt.fuzzer_*.a' not found"**: -- On Ubuntu/Debian: Install `libclang-rt-*-dev` package -- On macOS: Use Homebrew LLVM instead of Xcode clang -- Check available sanitizer libraries: - ```bash - find /usr/lib /usr/local/lib $(brew --prefix 2>/dev/null)/lib -name "*clang_rt.fuzzer*" 2>/dev/null - ``` - -**Build skips fuzz targets**: -- Run `./gradlew :ddprof-lib:fuzz:listFuzzTargets` to check if fuzzer is detected -- Check logs for "libFuzzer not available" warning - -## Running Fuzz Tests - -### Quick Start - -Run all fuzz targets for 60 seconds each: -```bash -./gradlew :ddprof-lib:fuzz:fuzz -``` - -### Run Individual Targets - -```bash -# DWARF parser fuzzer -./gradlew :ddprof-lib:fuzz:fuzz_dwarf - -# Arguments parser fuzzer -./gradlew :ddprof-lib:fuzz:fuzz_arguments - -# Buffer serialization fuzzer -./gradlew :ddprof-lib:fuzz:fuzz_buffer -``` - -### Configure Duration - -```bash -# Run for 5 minutes per target -./gradlew :ddprof-lib:fuzz:fuzz -Pfuzz-duration=300 -``` - -### List Available Targets - -```bash -./gradlew :ddprof-lib:fuzz:listFuzzTargets -``` - -## Fuzz Targets - -### fuzz_dwarf.cpp -**Target**: `DwarfParser::parse()` - DWARF exception frame parser - -Parses `.eh_frame_hdr` sections from ELF files to build stack unwind tables. -This code runs in signal handler context, so crashes are JVM-fatal. - -**Expected bugs**: Buffer over-reads, integer overflows in LEB128 decoding, -infinite loops from malformed structures. - -### fuzz_arguments.cpp -**Target**: `Arguments::parse()` - Profiler argument parser - -Processes configuration strings from JVM agent options and attach API. -User-controlled input with environment variable expansion. - -**Expected bugs**: Buffer overflows in `expandFilePattern()`, hash collisions, -memory corruption from malformed strings. - -### fuzz_buffer.cpp -**Target**: `Buffer::put*()` methods - JFR serialization primitives - -**CRITICAL**: All bounds checks in Buffer are `assert()` only - disabled in -release builds! This fuzzer tests the assertions to catch overflow conditions -that would silently corrupt memory in production. - -**Expected bugs**: Buffer overflows when assertions disabled, incorrect varint -encoding for edge-case values. - -### fuzz_callTraceStorage.cpp -**Target**: `CallTraceStorage::put()` / `processTraces()` / `clear()` — call-trace hash table lifecycle - -Drives the table through `put` / `processTraces` / `clear` sequences including inputs that exceed -the 49152-entry expansion threshold, forcing the multi-node `_prev` chain that exposes: -- Heap-use-after-free from dangling `_prev` pointers after `clearTableOnly()` frees expanded nodes -- Non-atomic `_table` reads racing with CAS expansion in `put()` - -**Regression guard — detects if these bugs are reintroduced**: heap-use-after-free (ASan), data race on `_table` (TSan), null-deref in -`processTraces` from use-after-free when `_prev` chain is not fully disconnected. - -## Corpus - -Seed corpus files are in `corpus//`. These provide starting points -for the fuzzer to understand the expected input format. - -During fuzzing, libFuzzer will add new interesting inputs to the corpus -directory. These additions are machine-generated and should not be committed. - -## Crash Artifacts - -When the fuzzer finds a crash, it saves the triggering input to: -``` -ddprof-lib/fuzz/build/fuzz-crashes/- -``` - -To reproduce a crash: -```bash -# Build the fuzzer -./gradlew :ddprof-lib:fuzz:linkFuzz_dwarf - -# Run with the crash file -ddprof-lib/fuzz/build/bin/fuzz/dwarf/dwarf -``` - -## Adding New Fuzz Targets - -1. Create a new `.cpp` file in this directory (e.g., `fuzz_newfeature.cpp`) -2. Implement the `LLVMFuzzerTestOneInput` function: - ```cpp - extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - // Parse or process the fuzzed data - return 0; - } - ``` -3. Optionally add seed corpus files in `corpus/fuzz_newfeature/` -4. The build system will automatically detect the new target - -## CI Integration - -Fuzz tests can be run in CI with a time limit: -```yaml -- name: Run fuzz tests - run: ./gradlew :ddprof-lib:fuzz:fuzz -Pfuzz-duration=60 -``` - -For continuous fuzzing, consider using OSS-Fuzz or ClusterFuzz. diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/cstack_dwarf b/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/cstack_dwarf deleted file mode 100644 index 9615dbdb0..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/cstack_dwarf +++ /dev/null @@ -1 +0,0 @@ -start,cstack=dwarf,jstackdepth=512,filter=com.example.* \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/start_alloc b/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/start_alloc deleted file mode 100644 index a1b87d936..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/start_alloc +++ /dev/null @@ -1 +0,0 @@ -start,event=alloc,memory=512k,liveness=true \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/start_cpu b/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/start_cpu deleted file mode 100644 index 4fd2fc673..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/start_cpu +++ /dev/null @@ -1 +0,0 @@ -start,event=cpu,interval=10ms \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/start_wall_file b/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/start_wall_file deleted file mode 100644 index dc5cbced6..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/start_wall_file +++ /dev/null @@ -1 +0,0 @@ -start,event=wall,interval=50ms,file=/tmp/profile.jfr \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/stop b/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/stop deleted file mode 100644 index a132e899b..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/stop +++ /dev/null @@ -1 +0,0 @@ -stop \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_buffer/put8_sequence b/ddprof-lib/src/test/fuzz/corpus/fuzz_buffer/put8_sequence deleted file mode 100644 index 09ab1f0a1..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_buffer/put8_sequence +++ /dev/null @@ -1 +0,0 @@ -\x00A\x00B\x00C\x00D\x00E \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_buffer/utf8_string b/ddprof-lib/src/test/fuzz/corpus/fuzz_buffer/utf8_string deleted file mode 100644 index 51f410e37..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_buffer/utf8_string +++ /dev/null @@ -1 +0,0 @@ -\x60\x0bHello World \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_buffer/varint_max b/ddprof-lib/src/test/fuzz/corpus/fuzz_buffer/varint_max deleted file mode 100644 index 3bb0566cf..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_buffer/varint_max +++ /dev/null @@ -1 +0,0 @@ -\x40\xff\xff\xff\xff\x50\xff\xff\xff\xff\xff\xff\xff\xff \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_callTraceStorage/seed0 b/ddprof-lib/src/test/fuzz/corpus/fuzz_callTraceStorage/seed0 deleted file mode 100644 index 887213d8f..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_callTraceStorage/seed0 +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_callTraceStorage/seed_expansion b/ddprof-lib/src/test/fuzz/corpus/fuzz_callTraceStorage/seed_expansion deleted file mode 100644 index 008c80393..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_callTraceStorage/seed_expansion +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_dwarf/minimal_eh_frame_hdr b/ddprof-lib/src/test/fuzz/corpus/fuzz_dwarf/minimal_eh_frame_hdr deleted file mode 100644 index 199699d24..000000000 --- a/ddprof-lib/src/test/fuzz/corpus/fuzz_dwarf/minimal_eh_frame_hdr +++ /dev/null @@ -1 +0,0 @@ -\x01\x1b\x03\x3b\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00 \ No newline at end of file diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug1_section_header_oob_1 b/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug1_section_header_oob_1 deleted file mode 100644 index 44f43e459..000000000 Binary files a/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug1_section_header_oob_1 and /dev/null differ diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug1_section_header_oob_2 b/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug1_section_header_oob_2 deleted file mode 100644 index 6aaa68de4..000000000 Binary files a/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug1_section_header_oob_2 and /dev/null differ diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug2_symtab_size_oob b/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug2_symtab_size_oob deleted file mode 100644 index db44f53f9..000000000 Binary files a/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug2_symtab_size_oob and /dev/null differ diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug3_buildid_note_overflow b/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug3_buildid_note_overflow deleted file mode 100644 index f079ff52f..000000000 Binary files a/ddprof-lib/src/test/fuzz/corpus/fuzz_elf/bug3_buildid_note_overflow and /dev/null differ diff --git a/ddprof-lib/src/test/fuzz/corpus/fuzz_stringDictionary/basic_rotation b/ddprof-lib/src/test/fuzz/corpus/fuzz_stringDictionary/basic_rotation deleted file mode 100644 index e1545539b..000000000 Binary files a/ddprof-lib/src/test/fuzz/corpus/fuzz_stringDictionary/basic_rotation and /dev/null differ diff --git a/ddprof-lib/src/test/fuzz/fuzz_arguments.cpp b/ddprof-lib/src/test/fuzz/fuzz_arguments.cpp deleted file mode 100644 index 46b59bfa6..000000000 --- a/ddprof-lib/src/test/fuzz/fuzz_arguments.cpp +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * libFuzzer fuzz target for the profiler argument parser. - * - * The Arguments::parse() function processes profiler configuration strings - * from various sources (JVM agent options, attach API, etc.). This is a - * potential attack surface because: - * - Input comes from user-controlled sources - * - expandFilePattern() uses environment variable expansion with snprintf - * - The hash function may overflow on very long inputs - * - strtok-based parsing with many delimiters could cause issues - * - * Expected bug classes: - * - Buffer overflows in expandFilePattern() with large env vars - * - Integer overflow in hash() with very long strings - * - Memory corruption from malformed argument strings - * - Denial of service via pathological input patterns - */ - -#include -#include -#include -#include - -#include "arguments.h" - -// Set up environment variables with controlled content for fuzzing. -// This allows the fuzzer to trigger env var expansion code paths. -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -namespace { - struct EnvSetup { - EnvSetup() { - // Set some environment variables that the fuzzer can reference - // via %{VAR} patterns in the input - setenv("FUZZ_TEST_VAR", "fuzz_value", 1); - setenv("FUZZ_LONG_VAR", std::string(256, 'A').c_str(), 1); - setenv("HOME", "/tmp/fuzz_home", 1); - } - }; - static EnvSetup env_setup; -} -#endif - -/** - * LLVMFuzzerTestOneInput - The libFuzzer entry point. - * - * This function parses the fuzzer-generated data as a profiler argument string. - * The Arguments class supports various configuration options like: - * - start,event=cpu,interval=10ms - * - stop - * - file=/path/to/output.jfr - * - filter=MyClass::myMethod - * - * @param data Pointer to fuzzer-generated input bytes - * @param size Size of the input data in bytes - * @return 0 (required by libFuzzer interface) - */ -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - // Empty input is valid but uninteresting - if (size == 0) { - return 0; - } - - // Cap size to prevent excessive memory consumption. - // Real argument strings should be < 64KB - if (size > 64 * 1024) { - size = 64 * 1024; - } - - // Create a null-terminated copy of the input data. - // Arguments::parse() expects a null-terminated C string. - char *args = new char[size + 1]; - memcpy(args, data, size); - args[size] = '\0'; - - // Replace any embedded null bytes with spaces to ensure we test - // the full input length (embedded nulls would truncate parsing) - for (size_t i = 0; i < size; i++) { - if (args[i] == '\0') { - args[i] = ' '; - } - } - - // Create an Arguments instance and attempt to parse the fuzzed input - Arguments arguments; - - try { - Error error = arguments.parse(args); - // We don't care about the error result - we're testing for crashes, - // not correct error handling. Invalid arguments should return an - // error, not crash. - (void)error; - } catch (...) { - // Unexpected exceptions indicate a bug - the parser should handle - // all malformed input gracefully without throwing - } - - delete[] args; - return 0; -} - -/** - * Optional: Provide initial corpus seeds to guide fuzzing. - * - * libFuzzer can use these to understand the expected input format. - * Place seed files in ddprof-lib/src/test/fuzz/corpus/fuzz_arguments/ - */ -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -// Example valid argument strings for initial corpus: -// "start,event=cpu,interval=10ms" -// "start,event=wall,interval=50ms,file=/tmp/profile.jfr" -// "start,event=alloc,memory=512k" -// "stop" -// "status" -// "start,cstack=dwarf,jstackdepth=512" -#endif diff --git a/ddprof-lib/src/test/fuzz/fuzz_buffer.cpp b/ddprof-lib/src/test/fuzz/fuzz_buffer.cpp deleted file mode 100644 index ae96a51db..000000000 --- a/ddprof-lib/src/test/fuzz/fuzz_buffer.cpp +++ /dev/null @@ -1,202 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * libFuzzer fuzz target for the JFR buffer serialization code. - * - * The Buffer class provides serialization primitives for JFR recording: - * - putVar32/putVar64: Variable-length integer encoding (LEB128-like) - * - putUtf8: String serialization with length prefix - * - put8/put16/put32/put64: Fixed-size integer writes - * - * CRITICAL FINDING: All bounds checks in Buffer are assert() only! - * This means in release builds (NDEBUG defined), there is NO runtime - * validation of buffer boundaries. This fuzzer tests with assertions - * enabled to catch overflow conditions that would silently corrupt - * memory in production. - * - * Expected bug classes: - * - Buffer overflows when assertions are disabled - * - Incorrect varint encoding for edge-case values - * - String truncation at MAX_STRING_LENGTH boundary - * - Memory corruption from sequences that fill buffer exactly - */ - -#include -#include -#include - -#include "buffers.h" - -// We need common.h for type definitions (u32, u64) -#include "common.h" - -/** - * Interpret fuzzer input as a sequence of operations to perform on a Buffer. - * - * Input format: - * Each byte is interpreted as an operation code: - * - 0x00-0x0F: put8 (next byte is the value) - * - 0x10-0x1F: put16 (next 2 bytes are the value) - * - 0x20-0x2F: put32 (next 4 bytes are the value) - * - 0x30-0x3F: put64 (next 8 bytes are the value) - * - 0x40-0x4F: putVar32 (next 4 bytes interpreted as u32) - * - 0x50-0x5F: putVar64 (next 8 bytes interpreted as u64) - * - 0x60-0x6F: putUtf8 (length from next byte, then string data) - * - 0x70-0x7F: reset buffer - * - 0x80-0xFF: skip (advance by lower 4 bits) - */ -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - if (size == 0) { - return 0; - } - - // Use a RecordingBuffer which has the larger size used in production - // This is where the "overflow space" workaround lives - class TestBuffer : public Buffer { - public: - // Extend the data array like RecordingBuffer does - char _extended_data[RECORDING_BUFFER_SIZE + RECORDING_BUFFER_OVERFLOW - BUFFER_SIZE + sizeof(int)]; - - TestBuffer() : Buffer() { - memset(_extended_data, 0xCD, sizeof(_extended_data)); // Canary pattern - } - - int limit() const override { - return RECORDING_BUFFER_SIZE; - } - - // Check if the canary was overwritten (indicates buffer overflow) - bool checkOverflow() const { - // Check last few bytes of overflow space - const char *end = _extended_data + sizeof(_extended_data) - 16; - for (int i = 0; i < 16; i++) { - if ((unsigned char)end[i] != 0xCD) { - return true; // Overflow detected - } - } - return false; - } - }; - - TestBuffer buffer; - - size_t pos = 0; - while (pos < size) { - uint8_t op = data[pos++]; - - // Check for buffer overflow after each operation - if (buffer.checkOverflow()) { - // Overflow detected - this is a bug! - // In fuzzing mode, this will be caught by ASAN - __builtin_trap(); - } - - if (op < 0x10) { - // put8 - if (pos >= size) break; - char v = (char)data[pos++]; - if (buffer.offset() + 1 < buffer.limit()) { - buffer.put8(v); - } - } else if (op < 0x20) { - // put16 - if (pos + 1 >= size) break; - short v = (short)((data[pos] << 8) | data[pos + 1]); - pos += 2; - if (buffer.offset() + 2 < buffer.limit()) { - buffer.put16(v); - } - } else if (op < 0x30) { - // put32 - if (pos + 3 >= size) break; - int v = (data[pos] << 24) | (data[pos + 1] << 16) | - (data[pos + 2] << 8) | data[pos + 3]; - pos += 4; - if (buffer.offset() + 4 < buffer.limit()) { - buffer.put32(v); - } - } else if (op < 0x40) { - // put64 - if (pos + 7 >= size) break; - u64 v = 0; - for (int i = 0; i < 8; i++) { - v = (v << 8) | data[pos + i]; - } - pos += 8; - if (buffer.offset() + 8 < buffer.limit()) { - buffer.put64(v); - } - } else if (op < 0x50) { - // putVar32 - this is where bugs are likely! - // The assertion checks for 5 bytes, but encoding can write more - if (pos + 3 >= size) break; - u32 v = (data[pos] << 24) | (data[pos + 1] << 16) | - (data[pos + 2] << 8) | data[pos + 3]; - pos += 4; - // putVar32 writes at most 5 bytes - if (buffer.offset() + 5 < buffer.limit()) { - buffer.putVar32(v); - } - } else if (op < 0x60) { - // putVar64 - highest risk for overflow - // Assertion checks for 9 bytes but the loop structure is complex - if (pos + 7 >= size) break; - u64 v = 0; - for (int i = 0; i < 8; i++) { - v = (v << 8) | data[pos + i]; - } - pos += 8; - // putVar64 writes at most 10 bytes (9 asserted but loop can write more) - if (buffer.offset() + 10 < buffer.limit()) { - buffer.putVar64(v); - } - } else if (op < 0x70) { - // putUtf8 - string serialization - if (pos >= size) break; - size_t len = data[pos++]; - // Clamp length to available data - if (pos + len > size) { - len = size - pos; - } - // putUtf8 adds 1 byte type + varint length + data - // Check we have enough space (conservative estimate) - if (buffer.offset() + 1 + 5 + len < (size_t)buffer.limit()) { - buffer.putUtf8((const char *)(data + pos), (u32)len); - } - pos += len; - } else if (op < 0x80) { - // reset - start over with fresh buffer - buffer.reset(); - } else { - // skip - advance buffer position - int skip_amount = (op & 0x0F) + 1; - if (buffer.offset() + skip_amount < buffer.limit()) { - buffer.skip(skip_amount); - } - } - } - - return 0; -} - -/** - * Test specific edge cases that are known to be risky. - * These can be used as seed corpus entries. - */ -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -// Interesting values for putVar32/putVar64: -// - 0: single byte -// - 0x7F: maximum single byte -// - 0x80: minimum two bytes -// - 0x3FFF: maximum two bytes -// - 0x1FFFFF: maximum three bytes -// - UINT32_MAX: maximum u32 -// - UINT64_MAX: maximum u64 -// -// Edge cases for strings: -// - Empty string -// - MAX_STRING_LENGTH exactly -// - MAX_STRING_LENGTH + 1 (should be truncated) -// - Very long strings (should be clamped) -#endif diff --git a/ddprof-lib/src/test/fuzz/fuzz_callTraceStorage.cpp b/ddprof-lib/src/test/fuzz/fuzz_callTraceStorage.cpp deleted file mode 100644 index 091780f6d..000000000 --- a/ddprof-lib/src/test/fuzz/fuzz_callTraceStorage.cpp +++ /dev/null @@ -1,114 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * libFuzzer target for CallTraceStorage / CallTraceHashTable. - * - * Input bytes are consumed as a stream of operations: - * op byte < 0x80 → put(1 frame): next byte is bci (0-127); method_id derived - * from a monotone counter so every call produces a unique trace, - * letting the corpus naturally drive past the 49152-entry - * expansion threshold and create a multi-node _prev chain. - * 0x80 ≤ op < 0xC0 → processTraces() - * op ≥ 0xC0 → clear() - * - * Invariants verified (violation → __builtin_trap() → ASan/fuzzer crash): - * I1. Every trace successfully returned by put() since the last processTraces() - * or clear() must appear in the next processTraces() callback. - * I2. Immediately after clear(), processTraces() must see exactly 1 trace - * (the static DROPPED_TRACE_ID sentinel). - * - * ASan+UBSan are enabled by the fuzzer build config; heap-use-after-free from - * a dangling _prev pointer (defect C) and non-atomic _table accesses (defect B) - * will be caught automatically if regressions are introduced. - */ - -#include -#include -#include - -#include "callTraceStorage.h" - -// Reuse the storage across fuzz calls to amortise the mmap cost of the initial -// 65536-slot LongHashTable allocation. clear() resets logical state without -// releasing the underlying allocator pages, so subsequent runs recycle memory -// and exercise the reuse path that is most likely to surface UAF bugs. -static CallTraceStorage* g_storage = nullptr; - -extern "C" int LLVMFuzzerInitialize(int* /*argc*/, char*** /*argv*/) { - g_storage = new CallTraceStorage(); - return 0; -} - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t* data, size_t size) { - if (size < 2) return 0; - - g_storage->clear(); - - // Shadow set: trace IDs put() successfully since the last processTraces()/clear(). - std::unordered_set pending; - bool just_cleared = true; // true immediately after clear() or at start - int put_seq = 0; // monotone counter → unique method_id per call - - size_t pos = 0; - while (pos < size) { - uint8_t op = data[pos++]; - - if (op < 0x80) { - // put() — consume one bci byte - if (pos >= size) break; - int bci = (int)(data[pos++] & 0x7F); - - ASGCT_CallFrame frame; - frame.bci = bci; - // Derive a unique method_id from the sequential counter so repeated - // bci bytes still produce distinct traces and eventually cross the - // 49152-entry expansion threshold. - frame.method_id = reinterpret_cast( - static_cast(0x10000ULL + (u64)(put_seq & 0xFFFF))); - put_seq++; - - u64 id = g_storage->put(1, &frame, false, 1); - if (id > 0 && id != CallTraceStorage::DROPPED_TRACE_ID) { - pending.insert(id); - just_cleared = false; - } - - } else if (op < 0xC0) { - // processTraces() — verify I1 and I2 - std::unordered_set seen; - g_storage->processTraces([&](const std::unordered_set& traces) { - for (CallTrace* t : traces) { - if (t) seen.insert(t->trace_id); - } - }); - - if (just_cleared) { - // I2: after clear() only the sentinel should be present. - // pending is empty; seen should contain exactly DROPPED_TRACE_ID. - if (seen.size() != 1 || - seen.find(CallTraceStorage::DROPPED_TRACE_ID) == seen.end()) { - __builtin_trap(); - } - } else { - // I1: every successfully put trace must appear in the snapshot. - for (u64 id : pending) { - if (seen.find(id) == seen.end()) { - __builtin_trap(); - } - } - } - pending.clear(); - just_cleared = false; - - } else { - // clear() - g_storage->clear(); - pending.clear(); - put_seq = 0; - just_cleared = true; - } - } - - return 0; -} diff --git a/ddprof-lib/src/test/fuzz/fuzz_dwarf.cpp b/ddprof-lib/src/test/fuzz/fuzz_dwarf.cpp deleted file mode 100644 index fa6dc2c81..000000000 --- a/ddprof-lib/src/test/fuzz/fuzz_dwarf.cpp +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * libFuzzer fuzz target for the DWARF exception frame parser. - * - * The DwarfParser parses .eh_frame_hdr sections from ELF files to build - * unwind tables for stack walking. This is a critical attack surface because: - * - It processes untrusted binary data from loaded libraries - * - Parsing happens in signal handler context (crashes = JVM crashes) - * - Minimal bounds checking in LEB128 decoding and pointer arithmetic - * - * Expected bug classes: - * - Buffer over-reads from malformed LEB128 sequences - * - Integer overflows in offset calculations - * - Infinite loops from cyclic or malformed structures - * - Null pointer dereferences from missing validation - */ - -#include -#include -#include -#include - -// Include the DWARF parser -#include "dwarf.h" -#include "dwarf.h" - -// Provide a minimal stub for Log since dwarf.cpp uses it -#include "log.h" - -// Silence logging during fuzzing to avoid noise -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -namespace { - struct LogSilencer { - LogSilencer() { - // Fuzzing mode - suppress warnings - } - }; - static LogSilencer log_silencer; -} -#endif - -/** - * LLVMFuzzerTestOneInput - The libFuzzer entry point. - * - * This function is called by libFuzzer with mutated input data. - * The fuzzer will try to maximize code coverage by generating - * inputs that explore new paths through the DWARF parser. - * - * @param data Pointer to fuzzer-generated input bytes - * @param size Size of the input data in bytes - * @return 0 (required by libFuzzer interface) - */ -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - // Minimum size for a valid .eh_frame_hdr header: - // - 1 byte version - // - 1 byte eh_frame_ptr encoding - // - 1 byte fde_count encoding - // - 1 byte table encoding - // - 4 bytes eh_frame_ptr - // - 4 bytes fde_count - // Minimum: 12 bytes, but we need at least 16 for the table pointer - if (size < 16) { - return 0; - } - - // Cap size to avoid excessive memory allocation during parsing. - // Real .eh_frame_hdr sections are typically < 1MB - if (size > 1024 * 1024) { - size = 1024 * 1024; - } - - // Create a copy of the input data that's null-terminated for safety - // (some string operations might expect null termination) - char *eh_frame_hdr = new char[size + 1]; - memcpy(eh_frame_hdr, data, size); - eh_frame_hdr[size] = '\0'; - - // The DwarfParser constructor takes: - // - name: library name (for logging) - // - image_base: base address of the loaded image - // - eh_frame_hdr: pointer to the .eh_frame_hdr section - // - // We use the input data as the image base to create valid relative - // pointers within the fuzzed data. - const char *name = "fuzz_target"; - const char *image_base = eh_frame_hdr; // Relative addresses are within fuzzed data - - // The DwarfParser constructor calls parse() internally, which is where - // most of the interesting parsing happens. - // - // DwarfParser has no destructor: it malloc()s its FrameDesc table in - // init() and transfers ownership to the caller via table() (see dwarf.h). - // Production callers hand that pointer to CodeCache::setDwarfTable(), which - // later free()s it. We must do the same here, otherwise the table leaks. - DwarfParser parser(name, image_base, eh_frame_hdr, size, - DwarfParser::EhFrameHdrTag{}, image_base + size); - free(parser.table()); // free(), not delete[], to match the malloc() in init() - - delete[] eh_frame_hdr; - return 0; -} diff --git a/ddprof-lib/src/test/fuzz/fuzz_elf.cpp b/ddprof-lib/src/test/fuzz/fuzz_elf.cpp deleted file mode 100644 index 92f920262..000000000 --- a/ddprof-lib/src/test/fuzz/fuzz_elf.cpp +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * libFuzzer fuzz target for the Linux ELF parsing code in symbols_linux.cpp. - * - * The profiler parses ELF images for every shared library loaded into the JVM - * (see Symbols::parseLibraries). A corrupt or malicious .so therefore feeds - * fully attacker-controlled bytes into two distinct parsers, both exercised - * here from a single ELF-blob corpus: - * - * 1. SymbolsLinux::extractBuildIdFromMemory(base, size, &len) - * An in-memory parser that walks the program-header table and PT_NOTE - * segments to recover the GNU build-id. It guards its reads with manual - * bounds checks such as `e_phoff + e_phnum * sizeof(Phdr) > elf_size` - * and `p_offset + p_filesz > elf_size` — both u64 additions that can - * wrap and defeat the check. Driven against a tight heap buffer so ASan - * catches any over-read precisely. - * - * 2. ElfParser::parseFile (via Symbols::parseElfFileForFuzzing) - * The core symbol/section/relocation loader: validHeader(), findSection() - * (indexes the section header table by attacker-controlled e_shoff / - * e_shentsize / e_shstrndx), loadSymbolTable() (iterates sh_size bytes in - * sh_entsize strides, indexes the string table by st_name) and, with - * use_debug, addRelocationSymbols(). parseFile() mmaps a real file, so the - * harness materialises the input as a temp file to mirror production - * exactly. - * - * Expected bug classes: - * - Integer overflow in the build-id bounds checks -> heap over-read - * - Out-of-bounds section/symbol/string-table reads from bad offsets - * - Infinite loop / DoS from a zero sh_entsize stride - * - Memory corruption in the malloc'd hex build-id string - * - * use_debug is enabled to reach the .plt/.rela.plt relocation path. The - * external debuginfo lookups (build-id / debuglink) it can also trigger are - * neutralised by clearing the relevant environment variables, so the fuzzer - * never fans out to unrelated files on disk. - */ - -#include -#include -#include -#include -#include -#include - -#include "codeCache.h" -#include "symbols_linux.h" // SymbolsLinux::extractBuildIdFromMemory - -// ElfParser is a translation-unit-local class in symbols_linux.cpp; forward- -// declare its public static parseFile() so the harness can drive it directly, -// mirroring how elfparser_ut.cpp reaches it (no production-side hook needed). -class ElfParser { - public: - static bool parseFile(CodeCache* cc, const char* base, const char* file_name, bool use_debug); -}; - -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION -namespace { - -// One-time process setup: neutralise the external debuginfo lookups that -// ElfParser::loadSymbols() performs when use_debug is set and the image has no -// .symtab. With these unset, getDebuginfodCache() yields nothing and the -// /usr/lib/debug build-id path simply fails to open — the fuzzer stays focused -// on in-image parsing instead of walking the host filesystem. -struct ElfFuzzSetup { - int fd = -1; - char path[64]; - - ElfFuzzSetup() { - unsetenv("DEBUGINFOD_CACHE_PATH"); - unsetenv("XDG_CACHE_HOME"); - unsetenv("HOME"); - - // A single reusable temp file backs the parseFile() path; truncated and - // rewritten each iteration to avoid per-iteration mkstemp churn. - strcpy(path, "/tmp/fuzz_elf_XXXXXX"); - fd = mkstemp(path); - } - - ~ElfFuzzSetup() { - if (fd != -1) { - close(fd); - unlink(path); - } - } -}; - -ElfFuzzSetup g_setup; - -} // namespace -#endif - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - if (size == 0) { - return 0; - } - - // Real shared libraries are well under this; cap to bound memory/time. - if (size > 4 * 1024 * 1024) { - size = 4 * 1024 * 1024; - } - - // --- Parser 1: in-memory build-id extraction (ASan-tight heap buffer) --- - // Copy into an exact-sized allocation so any over-read past `size` is an - // immediate ASan heap-buffer-overflow rather than a silent read into slack. - uint8_t *buf = (uint8_t *)malloc(size); - if (buf != nullptr) { - memcpy(buf, data, size); - size_t build_id_len = 0; - char *hex = SymbolsLinux::extractBuildIdFromMemory(buf, size, &build_id_len); - free(hex); // buildIdToHex() returns a malloc'd string (or NULL) - free(buf); - } - -#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION - // --- Parser 2: full symbol/section/relocation loader (production mmap path) --- - if (g_setup.fd != -1) { - if (ftruncate(g_setup.fd, 0) == 0 && - pwrite(g_setup.fd, data, size, 0) == (ssize_t)size) { - CodeCache cc("fuzz_elf"); - // base==NULL keeps symbol addresses as raw st_value (never - // dereferenced) and makes calcVirtualLoadAddress() a no-op, so the - // only reads are file-offset-relative into the mmap'd image — the - // untrusted parsing surface we want to exercise. - ElfParser::parseFile(&cc, /*base=*/NULL, g_setup.path, /*use_debug=*/true); - } - } -#endif - - return 0; -} diff --git a/ddprof-lib/src/test/fuzz/fuzz_stringDictionary.cpp b/ddprof-lib/src/test/fuzz/fuzz_stringDictionary.cpp deleted file mode 100644 index 49df7ac7d..000000000 --- a/ddprof-lib/src/test/fuzz/fuzz_stringDictionary.cpp +++ /dev/null @@ -1,153 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * libFuzzer fuzz target for the triple-buffered StringDictionary. - * - * The fuzzer interprets each input as a sequence of dictionary operations and - * verifies the documented sequential invariants of StringDictionary: - * - * I1. Once a key has an id, every subsequent successful lookup of that key - * returns the same id — across any number of rotate()/clearStandby() cycles. - * - * I2. The signal-safe read-only bounded_lookup(key, len) never returns an id - * for a key that was never inserted into the active buffer. - * - * I3. lookupDuringDump(key) either returns 0 or returns an id that is also - * resolvable from standby()->collect(). - * - * I4. clearAll() resets state — after clearAll(), no previously recorded id - * must be observable via any read path. - * - * Address/UB sanitizer is expected to be enabled by the fuzz build; UAFs and - * heap corruption from the malloc'd key storage will be caught automatically. - */ - -#include -#include -#include - -#include -#include -#include - -#include "stringDictionary.h" - -namespace { - -// Bound the working set so a pathological corpus does not OOM the fuzzer. -constexpr int kMaxUniqueKeys = 4096; -constexpr int kBoundedSizeLimit = 8192; -constexpr size_t kMaxKeyLen = 31; // mask of the length byte - -// Read a length-prefixed key from the input. '\0' bytes are mapped to '_' so -// the key is a valid C string (StringDictionary uses NUL-terminated comparison). -// Advances pos. Returns an empty key when the input is exhausted. -std::string readKey(const uint8_t *data, size_t size, size_t &pos) { - if (pos >= size) return {}; - size_t len = data[pos++] & kMaxKeyLen; - if (pos + len > size) len = size - pos; - std::string out; - out.reserve(len); - for (size_t i = 0; i < len; i++) { - char c = (char)data[pos + i]; - out.push_back(c == '\0' ? '_' : c); - } - pos += len; - return out; -} - -} // namespace - -extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - if (size < 2) return 0; - - StringDictionary dict; - std::unordered_map shadow; // key -> expected id - bool has_dump_buffer = false; // true after first rotate() - - size_t pos = 0; - while (pos < size) { - uint8_t op = data[pos++]; - - if (op < 0x40) { - // lookup (insert into active) - std::string k = readKey(data, size, pos); - if (shadow.size() >= kMaxUniqueKeys && shadow.find(k) == shadow.end()) { - continue; - } - u32 id = dict.lookup(k.c_str(), k.size()); - if (id == 0) continue; - auto it = shadow.find(k); - if (it == shadow.end()) { - shadow.emplace(k, id); - } else if (it->second != id) { - __builtin_trap(); // I1: id changed for known key - } - } else if (op < 0x60) { - // bounded_lookup with insert (high cap) - std::string k = readKey(data, size, pos); - if (shadow.size() >= kMaxUniqueKeys && shadow.find(k) == shadow.end()) { - continue; - } - u32 id = dict.bounded_lookup(k.c_str(), k.size(), kBoundedSizeLimit); - if (id == 0) continue; // at cap is legitimate - auto it = shadow.find(k); - if (it == shadow.end()) { - shadow.emplace(k, id); - } else if (it->second != id) { - __builtin_trap(); // I1 - } - } else if (op < 0x70) { - // bounded_lookup signal-safe (read-only) - std::string k = readKey(data, size, pos); - u32 id = dict.bounded_lookup(k.c_str(), k.size()); - auto it = shadow.find(k); - if (it == shadow.end()) { - if (id != 0) __builtin_trap(); // I2: phantom id - } else if (id != 0 && id != it->second) { - __builtin_trap(); // I1: id changed - } - // Note: id may legitimately be 0 if the key only ever existed in - // standby/dump (e.g. inserted, then clearAll, then nothing); we - // already cleared the shadow in that case, so shadow.find would miss. - } else if (op < 0x80) { - // lookupDuringDump — only legal once we have a dump buffer. - std::string k = readKey(data, size, pos); - if (!has_dump_buffer) continue; - if (shadow.size() >= kMaxUniqueKeys && shadow.find(k) == shadow.end()) { - continue; - } - u32 id = dict.lookupDuringDump(k.c_str(), k.size()); - if (id == 0) continue; - auto it = shadow.find(k); - if (it == shadow.end()) { - shadow.emplace(k, id); - } else if (it->second != id) { - __builtin_trap(); // I1 - } - // I3: lookupDuringDump must leave the key resolvable from standby(). - std::map snap; - dict.standby()->collect(snap); - auto found = snap.find(id); - if (found == snap.end() || k != found->second) { - __builtin_trap(); - } - } else if (op < 0x90) { - // rotate - dict.rotate(); - has_dump_buffer = true; - } else if (op < 0xA0) { - // clearStandby (clears scratch; does not affect active) - dict.clearStandby(); - } else if (op < 0xA8) { - // clearAll — sequential fuzz only; not exercising signal-time semantics. - dict.clearAll(); - shadow.clear(); - has_dump_buffer = false; - } - // 0xA8-0xFF: noop / spacer — keeps the corpus density adjustable. - } - - return 0; -} diff --git a/ddprof-lib/src/test/make/Makefile b/ddprof-lib/src/test/make/Makefile deleted file mode 100644 index c2fe3c9e5..000000000 --- a/ddprof-lib/src/test/make/Makefile +++ /dev/null @@ -1,33 +0,0 @@ -CC := g++ -SRCDIR := ../../main/cpp -OBJDIR := ./../../../build/scanbuild_obj -CFLAGS := -O0 -Wall -std=c++17 -fno-omit-frame-pointer -momit-leaf-frame-pointer -fvisibility=hidden -SRCS := $(shell find ${SRCDIR} -name '*.cpp') -OBJS := $(patsubst ${SRCDIR}/%.cpp,${OBJDIR}/%.o,$(SRCS)) -INCLUDES := -I$(SRCDIR) -I$(JAVA_HOME)/include -I../../../../malloc-shim/src/main/public - -OS := $(shell uname -s) -ifeq ($(OS),Darwin) - CFLAGS += -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE - INCLUDES += -I$(JAVA_HOME)/include/darwin -else - CFLAGS += -Wl,-z,defs -Wl,-z,nodelete - INCLUDES += -I$(JAVA_HOME)/include/linux - ifeq ($(findstring musl,$(shell ldd /bin/ls)),musl) - CFLAGS += -D__musl__ - endif -endif - -.PHONY: all clean - -all: $(OBJDIR) $(OBJS) - -$(OBJDIR): - mkdir -p $(OBJDIR) - -$(OBJDIR)/%.o : ${SRCDIR}/%.cpp | $(OBJDIR) - @mkdir -p $(dir $@) - ${CC} ${CFLAGS} -DDEBUG -DPROFILER_VERSION=\"snapshot\" ${INCLUDES} -c $< -o $@ - -clean : - @rm -rf $(OBJDIR) diff --git a/ddprof-lib/src/test/resources/native-libs/reladyn-lib/Makefile b/ddprof-lib/src/test/resources/native-libs/reladyn-lib/Makefile deleted file mode 100644 index 6f50129b3..000000000 --- a/ddprof-lib/src/test/resources/native-libs/reladyn-lib/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -TARGET_DIR = ../build/test/resources/native-libs/reladyn-lib -all: - g++ -fPIC -shared -o $(TARGET_DIR)/libreladyn.so reladyn.c diff --git a/ddprof-lib/src/test/resources/native-libs/reladyn-lib/reladyn.c b/ddprof-lib/src/test/resources/native-libs/reladyn-lib/reladyn.c deleted file mode 100644 index 4d87f57ed..000000000 --- a/ddprof-lib/src/test/resources/native-libs/reladyn-lib/reladyn.c +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ -#include -#include -// Force pthread_setspecific into .rela.dyn with R_X86_64_GLOB_DAT. -int (*indirect_pthread_setspecific)(pthread_key_t, const void*); -// Force pthread_exit into .rela.dyn with R_X86_64_64. -void (*static_pthread_exit)(void*) = pthread_exit; -void* thread_function(void* arg) { - printf("Thread running\n"); - return NULL; -} -// Not indended to be executed. -int reladyn() { - pthread_t thread; - pthread_key_t key; - pthread_key_create(&key, NULL); - // Direct call, forces into .rela.plt. - pthread_create(&thread, NULL, thread_function, NULL); - // Assign to a function pointer at runtime, forces into .rela.dyn as R_X86_64_GLOB_DAT. - indirect_pthread_setspecific = pthread_setspecific; - indirect_pthread_setspecific(key, "Thread-specific value"); - // Use pthread_exit via the static pointer, forces into .rela.dyn as R_X86_64_64. - static_pthread_exit(NULL); - return 0; -} \ No newline at end of file diff --git a/ddprof-lib/src/test/resources/native-libs/small-lib/Makefile b/ddprof-lib/src/test/resources/native-libs/small-lib/Makefile deleted file mode 100644 index 86b19bd36..000000000 --- a/ddprof-lib/src/test/resources/native-libs/small-lib/Makefile +++ /dev/null @@ -1,3 +0,0 @@ -TARGET_DIR = ../build/test/resources/native-libs/small-lib -all: - g++ -fPIC -shared -o $(TARGET_DIR)/libsmall-lib.so small_lib.cpp diff --git a/ddprof-lib/src/test/resources/native-libs/small-lib/small_lib.cpp b/ddprof-lib/src/test/resources/native-libs/small-lib/small_lib.cpp deleted file mode 100644 index d7de9f5dc..000000000 --- a/ddprof-lib/src/test/resources/native-libs/small-lib/small_lib.cpp +++ /dev/null @@ -1,6 +0,0 @@ -#include -#include "small_lib.h" - -extern "C" void hello() { - std::cout << "Hello, World from shared library!" << std::endl; -} diff --git a/ddprof-lib/src/test/resources/native-libs/small-lib/small_lib.h b/ddprof-lib/src/test/resources/native-libs/small-lib/small_lib.h deleted file mode 100644 index d20c52dc2..000000000 --- a/ddprof-lib/src/test/resources/native-libs/small-lib/small_lib.h +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once - -extern "C" { -void hello(); -} \ No newline at end of file diff --git a/ddprof-lib/src/test/resources/native-libs/unresolved-functions/Makefile b/ddprof-lib/src/test/resources/native-libs/unresolved-functions/Makefile deleted file mode 100644 index 2f3f66f1f..000000000 --- a/ddprof-lib/src/test/resources/native-libs/unresolved-functions/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -TARGET_DIR = ../build/test/resources/unresolved-functions -all: - gcc -c main.c -o $(TARGET_DIR)/main.o - gcc -o $(TARGET_DIR)/main $(TARGET_DIR)/main.o -T linker.ld \ No newline at end of file diff --git a/ddprof-lib/src/test/resources/native-libs/unresolved-functions/linker.ld b/ddprof-lib/src/test/resources/native-libs/unresolved-functions/linker.ld deleted file mode 100644 index 0e930a398..000000000 --- a/ddprof-lib/src/test/resources/native-libs/unresolved-functions/linker.ld +++ /dev/null @@ -1,43 +0,0 @@ -PHDRS -{ - headers PT_PHDR PHDRS ; - interp PT_INTERP ; - text PT_LOAD FILEHDR PHDRS ; - data PT_LOAD ; -} - -SECTIONS -{ - . = 0x10000; - .text : { - *(.text) - } :text - - . = 0x20000; - .data : { - *(.data) - } :data - - .bss : { - *(.bss) - } - - . = 0x30000; - unresolved_symbol = .; - . = 0xffffffffffffffff; - unresolved_function = .; - - /* Add the .init_array section */ - .init_array : { - __init_array_start = .; - KEEP(*(.init_array)) - __init_array_end = .; - } - - /* Add the .fini_array section */ - .fini_array : { - __fini_array_start = .; - KEEP(*(.fini_array)) - __fini_array_end = .; - } -} diff --git a/ddprof-lib/src/test/resources/native-libs/unresolved-functions/main.c b/ddprof-lib/src/test/resources/native-libs/unresolved-functions/main.c deleted file mode 100644 index e55838b2e..000000000 --- a/ddprof-lib/src/test/resources/native-libs/unresolved-functions/main.c +++ /dev/null @@ -1,10 +0,0 @@ -#include - -extern int unresolved_symbol; -extern int unresolved_function(); - -int main() { - printf("Value of unresolved_symbol: %p\n", &unresolved_symbol); - printf("Value of unresolved_function: %p\n", &unresolved_function); - return 0; -} diff --git a/ddprof-lib/src/test/resources/native-libs/unresolved-functions/readme.txt b/ddprof-lib/src/test/resources/native-libs/unresolved-functions/readme.txt deleted file mode 100644 index bbfe23baf..000000000 --- a/ddprof-lib/src/test/resources/native-libs/unresolved-functions/readme.txt +++ /dev/null @@ -1,4 +0,0 @@ -# Description - -This binary tests that we are able to parse symbols even when they point to unresolved functions. -The function is set to point to a 0xffffffffffffffff address. diff --git a/ddprof-stresstest/README.md b/ddprof-stresstest/README.md deleted file mode 100644 index f22caff0e..000000000 --- a/ddprof-stresstest/README.md +++ /dev/null @@ -1,307 +0,0 @@ -# ddprof-stresstest - Performance Benchmarking Suite - -## Overview - -This module contains JMH-based performance benchmarks for the Java Profiler. The benchmarks are organized into two main categories: - -- **Throughput Benchmarks** (`scenarios.throughput`): Measure raw performance and scalability -- **Counter Benchmarks** (`scenarios.counters`): Measure feature-specific behavior with profiler metrics - -## Prerequisites - -### Running Benchmarks - -All benchmarks require the WhiteboxProfiler to be enabled, which starts/stops the profiler between iterations and collects internal metrics. - -### Basic JMH Commands - -**Run all benchmarks:** -```bash -./gradlew :ddprof-stresstest:jmh -``` - -**Run specific benchmark class:** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - CallTraceStorageQuickBenchmark -``` - -**Run specific benchmark method:** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - CallTraceStorageBaselineBenchmark.baseline01Thread -``` - -### Common Options - -- `-Pjmh.fork=N`: Number of JVM forks (default: 3) -- `-Pjmh.wi=N`: Warmup iterations (default: 3-5) -- `-Pjmh.i=N`: Measurement iterations (default: 3-5) -- `-Pjmh.wt=N`: Warmup time in seconds (default: 1-2) -- `-Pjmh.t=N`: Measurement time in seconds (default: 3-5) -- `-Pjmh.resultFormat=json|csv|text`: Output format -- `-Pjmh.resultFile=path`: Output file path -- `-Pjmh.prof='profiler'`: JMH profiler to use - -### Fast Iterations for Development - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - -Pjmh.fork=1 -Pjmh.wi=1 -Pjmh.i=2 \ - YourBenchmark -``` - -## Throughput Benchmarks - -Located in `scenarios.throughput.*` - -### Profiler Throughput Benchmarks - -Measure end-to-end profiling engine performance including signal handlers, stack walking, CallTraceStorage operations, and JFR processing under various thread lifecycle patterns. - -**Quick smoke test (~2 minutes):** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ProfilerThroughputQuickBenchmark -``` - -**Baseline scaling (~12 minutes):** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ProfilerThroughputBaselineBenchmark -``` - -**Thread churn (~20-30 minutes):** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ProfilerThroughputThreadChurnBenchmark -``` - -**Slot exhaustion (~15-20 minutes):** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ProfilerThroughputSlotExhaustionBenchmark -``` - -**Documentation**: See `doc/architecture/CallTraceStorage.md` for detailed CallTraceStorage architecture, benchmark results analysis, and optimization recommendations. - -### ThreadContext Benchmarks - -Compare performance of JNI-based native vs DirectByteBuffer-based Java implementations for thread context storage. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ThreadContextBenchmark -``` - -Tests various thread counts to measure both single-threaded overhead and multi-threaded contention. - -### ThreadFilter Benchmarks - -Measure thread filtering performance and overhead. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - scenarios.throughput.ThreadFilterBenchmark -``` - -## Counter Benchmarks - -Located in `scenarios.counters.*` - -These benchmarks focus on measuring specific profiler features with metric collection enabled. - -### TracedParallelWork - -Measures profiler behavior under parallel work with distributed tracing context propagation. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - TracedParallelWork -``` - -**Parameters:** -- `tagCardinality`: Number of unique tag values (10, 100, 1000) -- `command`: Profiler configuration with attributes - -### DumpRecording - -Measures overhead of JFR recording dump operations. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - DumpRecording -``` - -### GraphMutation / GraphState - -Measures profiler performance with complex object graph mutations. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - GraphMutation -``` - -### NanoTime - -Measures timing overhead and profiler impact on high-frequency time measurements. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - NanoTime -``` - -### CapturingLambdas - -Measures profiler impact on lambda capture and invocation performance. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - CapturingLambdas -``` - -### ThreadFilter (Counters) - -Thread filtering with counter metrics collection. - -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - scenarios.counters.ThreadFilterBenchmark -``` - -## Saving Results - -### JSON Output -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - -Pjmh.resultFormat=json \ - -Pjmh.resultFile=build/benchmark-results.json \ - YourBenchmark -``` - -### CSV Output -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.resultFormat=csv \ - -Pjmh.resultFile=build/benchmark-results.csv \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - YourBenchmark -``` - -## Customizing Parameters - -Override benchmark parameters: -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - -Pjmh.p='command=cpu=50us,wall=50us' \ - YourBenchmark -``` - -## Troubleshooting - -### "No benchmarks matched the filter" -Use simple class names, not regex patterns: -- ❌ Wrong: `-Pjmh.includes='.*CallTrace.*'` -- ✅ Right: `CallTraceStorageQuickBenchmark` - -### Benchmark takes too long -Use reduced iterations: -```bash --Pjmh.fork=1 -Pjmh.wi=1 -Pjmh.i=1 -``` - -### Profiler fails to start -Verify profiler library loads: -```bash -./gradlew :ddprof-test:testDebug -Ptests=JavaProfilerTest.testGetInstance -``` - -**Note**: The `-Ptests` property works uniformly across all platforms with config-specific test tasks. - -### Out of memory errors -- Reduce concurrent thread counts -- Use smaller parameter values -- Increase JVM heap: `-Pjmh.jvmArgs='-Xmx4g'` - -## CI Integration - -For CI environments with reduced iterations: -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - -Pjmh.fork=1 -Pjmh.wi=2 -Pjmh.i=3 \ - -Pjmh.resultFormat=json \ - -Pjmh.resultFile=build/ci-results.json \ - CallTraceStorageQuickBenchmark -``` - -## Project Structure - -``` -ddprof-stresstest/ -├── README.md # This file -├── src/jmh/java/ -│ └── com/datadoghq/profiler/stresstest/ -│ ├── Configuration.java # Base benchmark configuration -│ ├── WhiteboxProfiler.java # Custom JMH profiler -│ └── scenarios/ -│ ├── throughput/ # Raw performance benchmarks -│ │ ├── ProfilerThroughput* # End-to-end profiling engine suite -│ │ ├── ThreadContext* # ThreadContext benchmarks -│ │ └── ThreadFilter* # ThreadFilter benchmarks -│ └── counters/ # Feature-specific benchmarks -│ ├── TracedParallelWork # Distributed tracing overhead -│ ├── DumpRecording # JFR dump overhead -│ ├── GraphMutation # Object graph mutations -│ ├── NanoTime # Timing overhead -│ ├── CapturingLambdas # Lambda performance -│ └── ThreadFilter* # Thread filtering with counters -``` - -## Documentation - -- **CallTraceStorage Architecture**: `doc/architecture/CallTraceStorage.md` - Detailed triple-buffer architecture, benchmark results, and optimization guide -- **Main README**: `README.md` (project root) - General project overview -- **Build Configuration**: `CLAUDE.md` - Build system and development guidelines - -## Contributing - -When adding new benchmarks: - -1. Place in appropriate category (`throughput` or `counters`) -2. Extend `Configuration.java` for common setup -3. Use `WhiteboxProfiler` for profiler metric collection -4. Document in this README with: - - Purpose - - Example command - - Key parameters -5. Add detailed analysis to `doc/` if architectural (like CallTraceStorage) - -## Performance Targets - -Benchmarks help establish and validate: - -- **Scalability**: Linear scaling up to core count -- **Overhead**: <5% impact on application performance -- **Throughput**: Millions of samples per second -- **Latency**: <1μs per profiling operation -- **Memory**: Bounded memory usage under load - -Run benchmarks before and after changes to validate performance regressions. diff --git a/ddprof-stresstest/build.gradle.kts b/ddprof-stresstest/build.gradle.kts deleted file mode 100644 index 8119efb61..000000000 --- a/ddprof-stresstest/build.gradle.kts +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import com.datadoghq.native.util.PlatformUtils - -plugins { - java - id("me.champeau.jmh") version "0.7.3" - id("com.datadoghq.java-conventions") -} - -dependencies { - implementation(project(mapOf("path" to ":ddprof-lib", "configuration" to "debug"))) - implementation(project(":ddprof-test-tracer")) - implementation(libs.bundles.jmh) -} - -sourceSets { - named("jmh") { - java { - runtimeClasspath += project(":ddprof-lib").sourceSets["main"].output - } - } -} - -jmh { - // Set the JVM executable - this is used by the JMH plugin to fork benchmark processes - jvm.set(PlatformUtils.testJavaExecutable()) - - // Explicitly set fork to use external JVM (not Gradle's JVM) - fork.set(3) - - iterations.set(3) - timeOnIteration.set("3s") - warmup.set("1s") - warmupIterations.set(3) - - val jmhInclude: String? by project - if (jmhInclude != null) { - includes.set(listOf(jmhInclude!!)) - } -} - -// Configure all JMH-related JavaExec tasks to use the correct JDK -tasks.withType().matching { it.name.startsWith("jmh") }.configureEach { - setExecutable(PlatformUtils.testJavaExecutable()) -} - -tasks.named("jmhJar") { - manifest { - attributes( - "Main-Class" to "com.datadoghq.profiler.stresstest.Main", - ) - } - archiveFileName.set("stresstests.jar") -} - -// --- chaos harness --------------------------------------------------------- -// Long-running antagonist workload driven by the reliability CI cell. NOT a JMH -// benchmark — runs for a wall-clock budget and exits 0 on clean shutdown; JVM -// crashes propagate as non-zero exit codes. Black-box w.r.t. the profiler: -// runs under a dd-java-agent.jar patched with the locally built ddprof.jar. - -sourceSets { - create("chaos") -} - -dependencies { - "chaosImplementation"(libs.asm) - // dd-trace-api: annotations only at compile time. The patched dd-java-agent - // provides the (relocated) runtime classes and intercepts @Trace. - "chaosCompileOnly"(libs.dd.trace.api) - // ddprof-lib public API: compile-only; the patched dd-java-agent provides the - // classes at runtime for antagonists that call JavaProfiler/ThreadContext directly. - "chaosCompileOnly"(project(mapOf("path" to ":ddprof-lib", "configuration" to "debug"))) -} - -tasks.register("chaosJar") { - group = "build" - description = "Fat jar of the chaos reliability harness" - archiveFileName.set("chaos.jar") - from(sourceSets["chaos"].output) - from({ - configurations["chaosRuntimeClasspath"].map { if (it.isDirectory) it else zipTree(it) } - }) - duplicatesStrategy = DuplicatesStrategy.EXCLUDE - manifest { - attributes("Main-Class" to "com.datadoghq.profiler.chaos.Main") - } -} - -tasks.register("runStressTests") { - dependsOn(tasks.named("jmhJar")) - - group = "application" - description = "Run JMH stresstests" - commandLine( - PlatformUtils.testJavaExecutable(), - "-jar", - "build/libs/stresstests.jar", - "-prof", - "com.datadoghq.profiler.stresstest.WhiteboxProfiler", - "counters.*", - ) -} diff --git a/ddprof-stresstest/scripts/run-threadcontext-benchmark.sh b/ddprof-stresstest/scripts/run-threadcontext-benchmark.sh deleted file mode 100755 index 86608bf7b..000000000 --- a/ddprof-stresstest/scripts/run-threadcontext-benchmark.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -# Run ThreadContext performance benchmark -# Compares JNI-based vs DirectByteBuffer-based implementations - -set -e - -JAVA_TEST_HOME="${JAVA_TEST_HOME:-$JAVA_HOME}" - -echo "Building JMH benchmark JAR..." -./gradlew :ddprof-stresstest:jmhJar - -echo "" -echo "Running ThreadContext benchmark..." -echo "This will take several minutes as it runs multiple configurations with 1, 2, 4, 8, and 16 threads" -echo "" - -# Run the benchmark with specific pattern to match our ThreadContext tests -"${JAVA_TEST_HOME}/bin/java" -jar ddprof-stresstest/build/libs/stresstests.jar \ - "ThreadContextBenchmark.*" \ - -rf json \ - -rff build/threadcontext-benchmark-results.json - -echo "" -echo "Benchmark complete!" -echo "Results saved to: build/threadcontext-benchmark-results.json" - diff --git a/ddprof-stresstest/src/chaos/README.md b/ddprof-stresstest/src/chaos/README.md deleted file mode 100644 index 5dfcabb70..000000000 --- a/ddprof-stresstest/src/chaos/README.md +++ /dev/null @@ -1,33 +0,0 @@ -# Chaos harness - -Long-running antagonist workload driving the reliability CI cell. Runs as a -black-box application under a `dd-java-agent.jar` patched with the locally -built `ddprof.jar` (see `utils/patch-dd-java-agent.sh`). The only failure -signal is a JVM crash — non-zero exit code plus `hs_err_pid*.log` captured by -the runner script. - -## Antagonist roster - -| Name | Targets | -| ----------------- | ------------------------------------------------------------------------ | -| `thread-churn` | signal-vs-teardown races, slot reuse, calltrace storage put/get | -| `vthread-churn` | virtual thread mount/unmount, carrier-thread context, `ProfiledThread` | -| `classloader-churn` | class unload racing stack walk, `CodeCache`/`Symbols` invalidation | -| `alloc-storm` | Java alloc engine + GOT-patched libc malloc/free | -| `trace-context` | `setContext`/`clearContext` racing signals, span ID propagation | - -## Deferred - -- **`dlopen`/`dlclose` churn** — load and unload throwaway native libraries to - exercise the profiler's `CodeCache`/`Symbols` modules during library - teardown. Deferred because it requires shipping a dummy `.so` per supported - architecture; the simplest path is to build those libraries per-arch as a - preparation step in the same reliability CI run. Largely overlaps with - `classloader-churn` for the symbol-resolution race surface, so revisit only - if that antagonist proves insufficient. - -## Multi-engine signal pressure - -Not an antagonist — applied at the runner level via launch flags -(`-Ddd.profiling.ddprof.alloc.enabled=true`, `...wall.enabled=true`, -nativemem, aggressive sampling intervals). diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/AllocStormAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/AllocStormAntagonist.java deleted file mode 100644 index d6ee4c697..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/AllocStormAntagonist.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.reflect.Field; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Drives concurrent Java heap allocation and native ({@code malloc}/{@code free}) - * pressure to exercise the alloc engine and the GOT-patched libc allocator path. - * - *

Two driver threads: - *

    - *
  • {@code chaos-alloc-java}: cycles short-lived {@code byte[]} of varied sizes - *
  • {@code chaos-alloc-native}: tight {@code allocateMemory}/{@code freeMemory} - * loop via reflective {@code sun.misc.Unsafe} access. No-ops gracefully if - * the access is blocked. - *
- */ -public final class AllocStormAntagonist implements Antagonist { - - private static final int[] SIZES = {64, 256, 1024, 4096, 16_384, 65_536}; - private static final long NATIVE_BLOCK_SIZE = 4096L; - - private static final Object UNSAFE; - private static final MethodHandle ALLOCATE_MEMORY; - private static final MethodHandle FREE_MEMORY; - - static { - Object instance = null; - MethodHandle alloc = null; - MethodHandle free = null; - try { - Class unsafeClass = Class.forName("sun.misc.Unsafe"); - Field f = unsafeClass.getDeclaredField("theUnsafe"); - f.setAccessible(true); - instance = f.get(null); - MethodHandles.Lookup lookup = MethodHandles.lookup(); - alloc = lookup.findVirtual(unsafeClass, "allocateMemory", - MethodType.methodType(long.class, long.class)); - free = lookup.findVirtual(unsafeClass, "freeMemory", - MethodType.methodType(void.class, long.class)); - } catch (Throwable t) { - // Fall through; native loop will no-op. - } - UNSAFE = instance; - ALLOCATE_MEMORY = alloc; - FREE_MEMORY = free; - } - - private volatile boolean running; - private Thread javaDriver; - private Thread nativeDriver; - private final AtomicLong sink = new AtomicLong(); - - @Override - public String name() { - return "alloc-storm"; - } - - @Override - public void start() { - running = true; - javaDriver = new Thread(this::javaLoop, "chaos-alloc-java"); - javaDriver.setDaemon(true); - javaDriver.start(); - nativeDriver = new Thread(this::nativeLoop, "chaos-alloc-native"); - nativeDriver.setDaemon(true); - nativeDriver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - long millis = timeout.toMillis(); - try { - javaDriver.join(millis); - nativeDriver.join(millis); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void javaLoop() { - long acc = 0L; - int idx = 0; - while (running) { - byte[] buf = new byte[SIZES[idx]]; - // Touch first/last byte so the JIT cannot elide the allocation. - buf[0] = (byte) idx; - acc += buf[buf.length - 1]; - idx = (idx + 1) % SIZES.length; - } - sink.addAndGet(acc); - } - - private void nativeLoop() { - if (UNSAFE == null || ALLOCATE_MEMORY == null || FREE_MEMORY == null) { - System.out.println("[chaos] alloc-storm: native loop skipped (Unsafe not accessible)"); - return; - } - long acc = 0L; - try { - while (running) { - long addr = (long) ALLOCATE_MEMORY.invoke(UNSAFE, NATIVE_BLOCK_SIZE); - acc += addr; - FREE_MEMORY.invoke(UNSAFE, addr); - } - } catch (Throwable t) { - // reflective failure; JVM crash is the signal we watch for - } - sink.addAndGet(acc); - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Antagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Antagonist.java deleted file mode 100644 index 2fb6d0d58..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Antagonist.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.time.Duration; - -/** - * A long-running workload that deliberately exercises a profiler failure surface - * (thread teardown vs. signal, classloader unloading mid-walk, dlclose during - * symbol resolution, calltrace storage contention, ...). - * - *

Implementations must be safe to start, run for an arbitrary duration, and - * shut down on demand. They are NOT measured — the only failure signal is a JVM - * crash (the harness exits non-zero via SIGSEGV/SIGABRT propagation). - */ -public interface Antagonist { - - String name(); - - /** Spawn driver threads. Must return promptly. */ - void start(); - - /** Signal driver threads to stop and wait up to {@code timeout}. */ - void stopGracefully(Duration timeout); -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/BoundedThreadPoolAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/BoundedThreadPoolAntagonist.java deleted file mode 100644 index 045bf090e..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/BoundedThreadPoolAntagonist.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.time.Duration; -import java.util.concurrent.RejectedExecutionException; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledThreadPoolExecutor; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Maintains 4 scheduled thread pools that cross-submit tasks to simulate - * I/O-callback fan-out from Netty/gRPC event loops. Every 5 s one pool is - * torn down and recreated — a burst of thread-end events while in-flight - * tasks are running on sibling pools. - * - *

Targets: signal-vs-thread-end race during pool shutdown; JVMTI stack - * walk on executor threads being torn down while the scheduler ticks. - */ -public final class BoundedThreadPoolAntagonist implements Antagonist { - - private static final int POOL_COUNT = 4; - private static final int POOL_SIZE = 4; - private static final long TASK_PERIOD_MS = 50L; - private static final long RECYCLE_INTERVAL_MS = 5_000L; - - private final ScheduledExecutorService[] pools = new ScheduledExecutorService[POOL_COUNT]; - private volatile boolean running; - private Thread recycler; - private final AtomicLong sink = new AtomicLong(); - - @Override - public String name() { - return "bounded-pool"; - } - - @Override - public void start() { - running = true; - for (int i = 0; i < POOL_COUNT; i++) { - pools[i] = newPool(i); - } - for (int i = 0; i < POOL_COUNT; i++) { - schedulePoolTasks(i); - } - recycler = new Thread(this::recycleLoop, "chaos-bounded-pool-recycler"); - recycler.setDaemon(true); - recycler.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - recycler.interrupt(); - try { - recycler.join(timeout.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - for (ScheduledExecutorService pool : pools) { - if (pool != null) { - pool.shutdownNow(); - } - } - } - - private ScheduledExecutorService newPool(final int index) { - ScheduledThreadPoolExecutor pool = new ScheduledThreadPoolExecutor( - POOL_SIZE, - new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r, "chaos-bounded-pool-" + index); - t.setDaemon(true); - return t; - } - }); - pool.setRemoveOnCancelPolicy(true); - return pool; - } - - private void schedulePoolTasks(final int poolIdx) { - ScheduledExecutorService pool = pools[poolIdx]; - final int nextIdx = (poolIdx + 1) % POOL_COUNT; - pool.scheduleAtFixedRate( - new Runnable() { - @Override - public void run() { - if (!running) return; - long seed = System.nanoTime(); - sink.addAndGet(burn(seed)); - ScheduledExecutorService sibling = pools[nextIdx]; - if (sibling != null && !sibling.isShutdown()) { - try { - final long s = seed; - sibling.submit(new Runnable() { - @Override - public void run() { - sink.addAndGet(burn(s ^ 0xdeadbeefL)); - } - }); - } catch (RejectedExecutionException ignored) { - // sibling shut down concurrently - } - } - } - }, - TASK_PERIOD_MS, TASK_PERIOD_MS, TimeUnit.MILLISECONDS); - } - - private void recycleLoop() { - int victim = 0; - while (running) { - try { - Thread.sleep(RECYCLE_INTERVAL_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - if (!running) return; - ScheduledExecutorService old = pools[victim]; - pools[victim] = null; - if (old != null) { - old.shutdownNow(); - } - ScheduledExecutorService fresh = newPool(victim); - pools[victim] = fresh; - schedulePoolTasks(victim); - victim = (victim + 1) % POOL_COUNT; - } - } - - private static long burn(long seed) { - long r = seed; - for (int i = 0; i < 500; i++) { - r = r * 6364136223846793005L + 1442695040888963407L; - } - return r; - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ClassLoaderChurnAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ClassLoaderChurnAntagonist.java deleted file mode 100644 index d3f9bd569..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ClassLoaderChurnAntagonist.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import org.objectweb.asm.ClassWriter; -import org.objectweb.asm.MethodVisitor; -import org.objectweb.asm.Opcodes; - -import java.lang.reflect.Method; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Continuously generates throwaway classes inside disposable class loaders so - * that class unload races stack walking and {@code CodeCache}/{@code Symbols} - * cleanup. Each iteration: - * - *

    - *
  1. generates a unique class with a single static {@code compute(long)} method - *
  2. defines it in a fresh {@link ClassLoader} - *
  3. invokes the method (forces JIT exposure, registration in profiler tables) - *
  4. drops every reference so the loader and class become unloadable - *
- */ -public final class ClassLoaderChurnAntagonist implements Antagonist { - - private static final AtomicLong COUNTER = new AtomicLong(); - - private volatile boolean running; - private Thread driver; - private final AtomicLong sink = new AtomicLong(); - - @Override - public String name() { - return "classloader-churn"; - } - - @Override - public void start() { - running = true; - driver = new Thread(this::loop, "chaos-classloader-churn"); - driver.setDaemon(true); - driver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - try { - driver.join(timeout.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void loop() { - while (running) { - long uid = COUNTER.incrementAndGet(); - String simpleName = "Generated_" + uid; - String binaryName = "chaos.gen." + simpleName; - String internalName = "chaos/gen/" + simpleName; - byte[] bytecode = generate(internalName); - ChurnLoader loader = new ChurnLoader(); - try { - Class klass = loader.define(binaryName, bytecode); - Method m = klass.getMethod("compute", long.class); - Object result = m.invoke(null, uid); - sink.addAndGet((long) result); - } catch (Throwable t) { - // transient; JVM crash is the signal we watch for - } - // loader + klass go out of scope here. - // Pace class loading so old-gen GC can reclaim ClassLoader - // instances before they accumulate across a long run. - try { - Thread.sleep(1L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - private static byte[] generate(String internalName) { - ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); - cw.visit(Opcodes.V1_8, - Opcodes.ACC_PUBLIC | Opcodes.ACC_FINAL, - internalName, - null, - "java/lang/Object", - null); - - // public static long compute(long x) { return x * 1103515245L + 12345L; } - MethodVisitor mv = cw.visitMethod( - Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, - "compute", - "(J)J", - null, - null); - mv.visitCode(); - mv.visitVarInsn(Opcodes.LLOAD, 0); - mv.visitLdcInsn(1103515245L); - mv.visitInsn(Opcodes.LMUL); - mv.visitLdcInsn(12345L); - mv.visitInsn(Opcodes.LADD); - mv.visitInsn(Opcodes.LRETURN); - mv.visitMaxs(0, 0); - mv.visitEnd(); - - cw.visitEnd(); - return cw.toByteArray(); - } - - private static final class ChurnLoader extends ClassLoader { - ChurnLoader() { - super(ClassLoaderChurnAntagonist.class.getClassLoader()); - } - - Class define(String name, byte[] bytes) { - return defineClass(name, bytes, 0, bytes.length); - } - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ConsumerGroupAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ConsumerGroupAntagonist.java deleted file mode 100644 index 12e14f3ae..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ConsumerGroupAntagonist.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.time.Duration; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Maintains 16 "consumer" threads doing light spin work. Every 3 s it - * replaces 4 threads simultaneously (all interrupted at once, all - * replacements started before waiting for exits) to simulate a Kafka - * consumer group rebalance burst. - * - *

Targets: ProfiledThread recycling under burst replacement; - * signal-vs-thread-end race window widened by simultaneous stops; - * calltrace storage put() racing thread destruction. - */ -public final class ConsumerGroupAntagonist implements Antagonist { - - private static final int GROUP_SIZE = 16; - private static final int BURST_SIZE = 4; - private static final long REBALANCE_INTERVAL_MS = 3_000L; - - private final Thread[] consumers = new Thread[GROUP_SIZE]; - private volatile boolean running; - private Thread rebalancer; - private final AtomicLong sink = new AtomicLong(); - - @Override - public String name() { - return "consumer-group"; - } - - @Override - public void start() { - running = true; - for (int i = 0; i < GROUP_SIZE; i++) { - consumers[i] = newConsumer(i); - consumers[i].start(); - } - rebalancer = new Thread(new Runnable() { - @Override public void run() { rebalanceLoop(); } - }, "chaos-consumer-rebalancer"); - rebalancer.setDaemon(true); - rebalancer.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - long deadline = System.currentTimeMillis() + timeout.toMillis(); - rebalancer.interrupt(); - try { - rebalancer.join(timeout.toMillis() / 2); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - Thread[] snapshot = consumers.clone(); - for (Thread t : snapshot) { - if (t != null) { - t.interrupt(); - long remaining = Math.max(0L, deadline - System.currentTimeMillis()); - if (remaining == 0L) break; - try { t.join(remaining); } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } - } - - private Thread newConsumer(final int index) { - Thread t = new Thread(new Runnable() { - @Override - public void run() { - long r = ThreadLocalRandom.current().nextLong(); - while (running && !Thread.currentThread().isInterrupted()) { - for (int i = 0; i < 100; i++) { - r = r * 6364136223846793005L + 1442695040888963407L; - } - sink.addAndGet(r); - Thread.yield(); - } - } - }, "chaos-consumer-" + index); - t.setDaemon(true); - return t; - } - - private void rebalanceLoop() { - int offset = 0; - while (running) { - try { - Thread.sleep(REBALANCE_INTERVAL_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - if (!running) return; - - // Interrupt all victims simultaneously (burst) - Thread[] victims = new Thread[BURST_SIZE]; - for (int i = 0; i < BURST_SIZE; i++) { - int idx = offset + i; - victims[i] = consumers[idx]; - consumers[idx] = null; - if (victims[i] != null) { - victims[i].interrupt(); - } - } - // Start replacements before waiting for victims to fully exit - for (int i = 0; i < BURST_SIZE; i++) { - int idx = offset + i; - consumers[idx] = newConsumer(idx); - consumers[idx].start(); - } - // Now wait for victims - for (Thread victim : victims) { - if (victim != null) { - try { victim.join(1_000L); } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - offset = (offset + BURST_SIZE) % GROUP_SIZE; - } - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ContextHopAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ContextHopAntagonist.java deleted file mode 100644 index 439444dac..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ContextHopAntagonist.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.time.Duration; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Chains {@link CompletableFuture} stages across three distinct thread pools - * (A→B→C→A). Each stage sets a {@link ThreadLocal} to a random value, burns - * briefly, then clears it before submitting the next stage. - * - *

Eight self-renewing chains run concurrently. - * - *

Targets: RefCountGuard slot contention racing cross-pool handoff; - * wall-clock signal hitting a thread between ThreadLocal set and remove. - */ -public final class ContextHopAntagonist implements Antagonist { - - private static final int CHAIN_COUNT = 8; - private static final ThreadLocal CONTEXT = new ThreadLocal(); - - private final ExecutorService poolA; - private final ExecutorService poolB; - private final ExecutorService poolC; - private volatile boolean running; - private final AtomicLong sink = new AtomicLong(); - - public ContextHopAntagonist() { - poolA = newPool("A"); - poolB = newPool("B"); - poolC = newPool("C"); - } - - @Override - public String name() { - return "context-hop"; - } - - @Override - public void start() { - running = true; - for (int i = 0; i < CHAIN_COUNT; i++) { - startChain(i); - } - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - poolA.shutdown(); - poolB.shutdown(); - poolC.shutdown(); - long slice = timeout.toMillis() / 3; - try { - poolA.awaitTermination(slice, TimeUnit.MILLISECONDS); - poolB.awaitTermination(slice, TimeUnit.MILLISECONDS); - poolC.awaitTermination(slice, TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void startChain(final int chainId) { - if (!running) return; - final long seed = ThreadLocalRandom.current().nextLong() ^ ((long) chainId << 32); - CompletableFuture - .runAsync(new Runnable() { - @Override public void run() { hop(seed); } - }, poolA) - .thenRunAsync(new Runnable() { - @Override public void run() { hop(seed * 2L); } - }, poolB) - .thenRunAsync(new Runnable() { - @Override public void run() { hop(seed * 3L); } - }, poolC) - .thenRunAsync(new Runnable() { - @Override public void run() { startChain(chainId); } - }, poolA) - .exceptionally(t -> null); - } - - private void hop(long value) { - CONTEXT.set(value); - try { - long r = value; - for (int i = 0; i < 200; i++) { - r = r * 1103515245L + 12345L; - } - sink.addAndGet(r); - } finally { - CONTEXT.remove(); - } - } - - private static ExecutorService newPool(final String label) { - return Executors.newFixedThreadPool(4, new ThreadFactory() { - @Override - public Thread newThread(Runnable r) { - Thread t = new Thread(r, "chaos-context-hop-" + label); - t.setDaemon(true); - return t; - } - }); - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DirectMemoryAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DirectMemoryAntagonist.java deleted file mode 100644 index 40bcba786..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DirectMemoryAntagonist.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.nio.ByteBuffer; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Drives direct (off-heap) memory allocation at high rate via - * {@link ByteBuffer#allocateDirect}. Two concurrent modes: - *

    - *
  • Ring-buffer mode: allocates buffers of {4 KB, 64 KB, 512 KB}, - * keeps 32 live slots, evicts the oldest each cycle. - *
  • Burst mode: allocates and immediately drops 1 KB buffers in a - * tight loop. - *
- * - *

Targets: liveness-table overflow under high off-heap churn; jweak ref - * release racing realloc failure cleanup. - */ -public final class DirectMemoryAntagonist implements Antagonist { - - private static final int RING_SIZE = 32; - private static final int[] RING_SIZES_BYTES = {4_096, 65_536, 524_288}; - private static final int BURST_SIZE_BYTES = 1_024; - - private volatile boolean running; - private Thread ringDriver; - private Thread burstDriver; - private final AtomicLong sink = new AtomicLong(); - - @Override - public String name() { - return "direct-memory"; - } - - @Override - public void start() { - running = true; - ringDriver = new Thread(new Runnable() { - @Override public void run() { ringLoop(); } - }, "chaos-direct-memory-ring"); - ringDriver.setDaemon(true); - ringDriver.start(); - burstDriver = new Thread(new Runnable() { - @Override public void run() { burstLoop(); } - }, "chaos-direct-memory-burst"); - burstDriver.setDaemon(true); - burstDriver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - long slice = timeout.toMillis() / 2; - try { - ringDriver.join(slice); - burstDriver.join(slice); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void ringLoop() { - ByteBuffer[] ring = new ByteBuffer[RING_SIZE]; - int slot = 0; - int sizeIdx = 0; - while (running) { - ring[slot] = null; // evict oldest — GC + Cleaner handles dealloc - try { - ByteBuffer buf = ByteBuffer.allocateDirect(RING_SIZES_BYTES[sizeIdx]); - buf.put(0, (byte) slot); // touch to prevent elision - sink.addAndGet(buf.capacity()); - ring[slot] = buf; - } catch (OutOfMemoryError e) { - // Direct memory exhausted; clear ring to allow recovery - for (int i = 0; i < RING_SIZE; i++) { - ring[i] = null; - } - } - slot = (slot + 1) % RING_SIZE; - sizeIdx = (sizeIdx + 1) % RING_SIZES_BYTES.length; - } - for (int i = 0; i < RING_SIZE; i++) { - ring[i] = null; - } - } - - private void burstLoop() { - long acc = 0L; - while (running) { - try { - ByteBuffer buf = ByteBuffer.allocateDirect(BURST_SIZE_BYTES); - buf.put(0, (byte) 42); - acc += buf.limit(); - } catch (OutOfMemoryError ignored) { - // direct memory exhausted; fall through to sleep so Cleaner can drain - } - // Pace allocations so the Cleaner daemon thread can drain the - // PhantomReference queue before direct memory fills up across a - // long run. Without this, burstLoop saturates MaxDirectMemorySize - // even though each buf is immediately dropped. - try { - Thread.sleep(1L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - sink.addAndGet(acc); - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java deleted file mode 100644 index 26ff07147..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; - -/** - * Spawns short-lived, frequently-renamed threads that each generate distinct - * stack shapes, maximising churn in the profiler's thread-name table and - * call-trace storage right as recording chunks rotate. - * - *

Targets: {@code Recording::switchChunk/writeCpool}, - * {@code updateJavaThreadNames -> ThreadInfo::set}, {@code Dictionary::clear}. - * Pair with a short profiler recording interval so dumps fire continuously. - */ -public final class DumpStormAntagonist implements Antagonist { - - private final int concurrentThreads; - private volatile boolean running; - private Thread driver; - - public DumpStormAntagonist() { - this(96); - } - - public DumpStormAntagonist(int concurrentThreads) { - this.concurrentThreads = concurrentThreads; - } - - @Override - public String name() { - return "dump-storm"; - } - - @Override - public void start() { - running = true; - driver = new Thread(this::loop, "chaos-dump-storm"); - driver.setDaemon(true); - driver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - try { - if (driver != null) driver.join(timeout.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void loop() { - long seq = 0; - while (running) { - List batch = new ArrayList<>(concurrentThreads); - for (int i = 0; i < concurrentThreads && running; i++) { - final long id = seq++; - Thread t = new Thread(() -> distinctStack(id, 0)); - t.setName("dump-storm-" + id); - t.setDaemon(true); - t.start(); - batch.add(t); - } - for (Thread t : batch) { - try { - t.join(500L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - } - - // Recurse to a per-thread depth so each thread yields a unique stack shape, - // forcing new call-trace + symbol entries that the dump path must serialise. - // Depth floor is (id % 32) + 1 so id=0 (and every 32nd id) still recurse at - // least once, avoiding identical single-frame stacks for those threads. - private long distinctStack(long id, int depth) { - if (depth >= (int) (id % 32) + 1) { - long r = id; - for (int i = 0; i < 5000; i++) r = (r * 1103515245L + 12345L) & 0x7fffffffL; - return r; - } - return distinctStack(id, depth + 1) + depth; - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/HiddenClassChurnAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/HiddenClassChurnAntagonist.java deleted file mode 100644 index d496b4060..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/HiddenClassChurnAntagonist.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import org.objectweb.asm.ClassWriter; -import org.objectweb.asm.MethodVisitor; -import org.objectweb.asm.Opcodes; - -import java.lang.invoke.MethodHandles; -import java.lang.reflect.Array; -import java.lang.reflect.Method; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Generates hidden classes via {@code MethodHandles.Lookup.defineHiddenClass} - * (Java 15+) and immediately drops all references to make them GC-eligible. - * - *

Gracefully no-ops on JDKs that do not support hidden classes. - * - *

Targets: StringDictionary concurrent eviction racing hidden-class GC; - * class-map lookup for a class being unloaded while the profiler dumps. - */ -public final class HiddenClassChurnAntagonist implements Antagonist { - - private static final Method DEFINE_HIDDEN_CLASS; - private static final Object EMPTY_OPTIONS; // ClassOption[0] at runtime - - static { - Method m = null; - Object opts = null; - try { - Class optionClass = Class.forName("java.lang.invoke.MethodHandles$Lookup$ClassOption"); - // Build ClassOption[] type without arrayType() (requires Java 12+) - Object emptyArray = Array.newInstance(optionClass, 0); - m = MethodHandles.Lookup.class.getMethod( - "defineHiddenClass", byte[].class, boolean.class, emptyArray.getClass()); - opts = emptyArray; - } catch (Throwable t) { - // Java < 15: defineHiddenClass not available - } - DEFINE_HIDDEN_CLASS = m; - EMPTY_OPTIONS = opts; - } - - private static final AtomicLong COUNTER = new AtomicLong(); - private volatile boolean running; - private Thread driver; - private final AtomicLong sink = new AtomicLong(); - - @Override - public String name() { - return "hidden-class-churn"; - } - - @Override - public void start() { - running = true; - driver = new Thread(new Runnable() { - @Override public void run() { loop(); } - }, "chaos-hidden-class-churn"); - driver.setDaemon(true); - driver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - try { - driver.join(timeout.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void loop() { - if (DEFINE_HIDDEN_CLASS == null) { - System.out.println("[chaos] hidden-class-churn: skipping (defineHiddenClass not available on this JDK)"); - return; - } - MethodHandles.Lookup lookup = MethodHandles.lookup(); - while (running) { - long uid = COUNTER.incrementAndGet(); - byte[] bytecode = generateClass(uid); - try { - // invoke(lookup, byte[], boolean, ClassOption[]) → MethodHandles.Lookup - MethodHandles.Lookup hiddenLookup = (MethodHandles.Lookup) - DEFINE_HIDDEN_CLASS.invoke(lookup, - new Object[]{bytecode, Boolean.FALSE, EMPTY_OPTIONS}); - Class klass = hiddenLookup.lookupClass(); - // Invoke compute() to force JIT registration in profiler tables - Object result = klass.getMethod("compute", long.class).invoke(null, uid); - sink.addAndGet((long) result); - // klass and hiddenLookup go out of scope → hidden class GC-eligible - } catch (Throwable t) { - // transient; JVM crash is the signal we watch for - } - // Pace hidden-class generation so the GC can evict unloaded - // classes before they accumulate across a long run. - try { - Thread.sleep(1L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - private static byte[] generateClass(long uid) { - // Unique internal name per class so each one gets a distinct entry - // in the profiler's class map / StringDictionary. - String internalName = "com/datadoghq/profiler/chaos/Gen" + uid; - ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); - cw.visit(Opcodes.V11, - Opcodes.ACC_PUBLIC | Opcodes.ACC_FINAL, - internalName, null, "java/lang/Object", null); - MethodVisitor mv = cw.visitMethod( - Opcodes.ACC_PUBLIC | Opcodes.ACC_STATIC, "compute", "(J)J", null, null); - mv.visitCode(); - mv.visitVarInsn(Opcodes.LLOAD, 0); - mv.visitLdcInsn(1103515245L); - mv.visitInsn(Opcodes.LMUL); - mv.visitLdcInsn(12345L); - mv.visitInsn(Opcodes.LADD); - mv.visitInsn(Opcodes.LRETURN); - mv.visitMaxs(0, 0); - mv.visitEnd(); - cw.visitEnd(); - return cw.toByteArray(); - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java deleted file mode 100644 index 9b2cccc43..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; - -/** - * Long-running chaos harness driving the reliability CI cell. - * - *

The harness is fully black-box with respect to the profiler: it runs as a - * regular application under a {@code dd-java-agent.jar} that has been patched - * with the locally built {@code ddprof.jar} (see {@code utils/patch-dd-java-agent.sh}). - * That keeps the runtime classloader/relocation path identical to production - * and avoids parallel-classloader conflicts when the agent loads its own - * (relocated) profiler classes. - * - *

The only failure signal is a JVM crash (SIGSEGV/SIGABRT in the profiler - * native code), which propagates as a non-zero exit code and an - * {@code hs_err_pid*.log} captured by the reliability runner script. - * - *

Lifecycle: - *

    - *
  1. parse {@code --duration} (e.g. {@code 30m}, {@code 2h}) and {@code --antagonists} - *
  2. start the requested antagonists, sleep until the deadline - *
  3. stop antagonists, exit 0 on clean shutdown - *
- */ -public final class Main { - - public static void main(String[] args) throws Exception { - Args parsed = Args.parse(args); - log("starting duration=" + parsed.duration + " antagonists=" + parsed.antagonists); - - List running = new ArrayList<>(); - for (String name : parsed.antagonists) { - Antagonist a = create(name); - a.start(); - running.add(a); - log("antagonist started: " + a.name()); - } - - long deadlineNanos = System.nanoTime() + parsed.duration.toNanos(); - try { - while (System.nanoTime() < deadlineNanos) { - Thread.sleep(1_000L); - } - } finally { - for (Antagonist a : running) { - a.stopGracefully(Duration.ofSeconds(10)); - } - } - - log("completed cleanly"); - } - - private static Antagonist create(String name) { - switch (name) { - case "thread-churn": - return new ThreadChurnAntagonist(); - case "alloc-storm": - return new AllocStormAntagonist(); - case "vthread-churn": - return new VirtualThreadChurnAntagonist(); - case "classloader-churn": - return new ClassLoaderChurnAntagonist(); - case "trace-context": - return new TraceContextAntagonist(); - case "bounded-pool": - return new BoundedThreadPoolAntagonist(); - case "context-hop": - return new ContextHopAntagonist(); - case "consumer-group": - return new ConsumerGroupAntagonist(); - case "hidden-class-churn": - return new HiddenClassChurnAntagonist(); - case "direct-memory": - return new DirectMemoryAntagonist(); - case "weakref-wave": - return new WeakRefWaveAntagonist(); - case "dump-storm": - return new DumpStormAntagonist(); - case "reapply-context": - return new ReapplyContextAntagonist(); - // Deferred: dlopen-churn (needs per-arch dummy .so built in CI prep). - default: - throw new IllegalArgumentException("unknown antagonist: " + name); - } - } - - private static void log(String msg) { - System.out.println("[chaos] " + msg); - } - - private static final class Args { - Duration duration = Duration.ofMinutes(5); - List antagonists = new ArrayList<>(Arrays.asList("thread-churn")); - - static Args parse(String[] argv) { - Args a = new Args(); - for (int i = 0; i < argv.length; i++) { - switch (argv[i]) { - case "--duration": - a.duration = parseDuration(argv[++i]); - break; - case "--antagonists": - a.antagonists = Arrays.asList(argv[++i].split(",")); - break; - default: - throw new IllegalArgumentException("unknown arg: " + argv[i]); - } - } - return a; - } - - // Accepts: 90s, 30m, 2h, or ISO-8601 like PT30M. - private static Duration parseDuration(String s) { - String t = s.trim().toLowerCase(Locale.ROOT); - if (t.startsWith("pt")) return Duration.parse(t.toUpperCase(Locale.ROOT)); - char unit = t.charAt(t.length() - 1); - long n = Long.parseLong(t.substring(0, t.length() - 1)); - switch (unit) { - case 's': return Duration.ofSeconds(n); - case 'm': return Duration.ofMinutes(n); - case 'h': return Duration.ofHours(n); - default: throw new IllegalArgumentException("bad duration: " + s); - } - } - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java deleted file mode 100644 index e853fa3c8..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ReapplyContextAntagonist.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import com.datadoghq.profiler.ContextSetter; -import com.datadoghq.profiler.JavaProfiler; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Arrays; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -/** - * Drives the reapply-by-id-and-bytes hot path continuously from multiple threads while the - * profiler's wall-clock signal fires, racing the {@code detach}/{@code attach} window. - * - *

Each worker thread loops: span-activate (wiping slots) → reapply snapshot. This mirrors - * dd-trace-java's {@code reapplyAppContext} pattern and exercises the single-window invariant - * (no partial publish visible to a signal handler) under high thread count and signal pressure. - * - *

The only expected failure signal is a JVM crash. An unexpected {@code false} return from - * reapply (which should never happen since the record is always valid right after - * {@code setContext}) throws {@link IllegalStateException} to surface the bug immediately. - */ -public final class ReapplyContextAntagonist implements Antagonist { - - private static final String[] ATTR_NAMES = { - "http.route", "http.method", "http.status", "db.operation", "rpc.service" - }; - - private static final String[] ROUTES = { - "GET /api/users", - "POST /api/orders", - "GET /api/health", - "PUT /api/users/{id}", - "DELETE /api/sessions" - }; - - private final int workerCount; - private final ExecutorService pool; - private volatile boolean running; - - public ReapplyContextAntagonist() { - this(8); - } - - public ReapplyContextAntagonist(int workerCount) { - this.workerCount = workerCount; - this.pool = - Executors.newFixedThreadPool( - workerCount, - r -> { - Thread t = new Thread(r, "chaos-reapply-context"); - t.setDaemon(true); - return t; - }); - } - - @Override - public String name() { - return "reapply-context"; - } - - @Override - public void start() { - running = true; - for (int i = 0; i < workerCount; i++) { - pool.submit(this::workerLoop); - } - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - pool.shutdown(); - try { - pool.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void workerLoop() { - JavaProfiler profiler; - try { - profiler = JavaProfiler.getInstance(); - } catch (Exception e) { - System.err.println("[chaos] reapply-context: failed to get profiler: " + e); - return; - } - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList(ATTR_NAMES)); - - // Prime the per-thread encoding cache and capture a stable snapshot. - long spanId = Thread.currentThread().getId() + 1; - long localRootSpanId = spanId * 31L; - long traceIdLow = spanId * 6364136223846793005L + 1442695040888963407L; - profiler.setContext(localRootSpanId, spanId, 0, traceIdLow); - for (int i = 0; i < ROUTES.length; i++) { - contextSetter.setContextValue(i, ROUTES[i]); - } - int[] constantIds = contextSetter.snapshotTags(); - byte[][] utf8 = new byte[ROUTES.length][]; - for (int i = 0; i < ROUTES.length; i++) { - utf8[i] = ROUTES[i].getBytes(StandardCharsets.UTF_8); - } - - while (running) { - // Span activation wipes all custom slots. - profiler.setContext(localRootSpanId, spanId, 0, traceIdLow); - // Reapply restores them — this is the hot path under test. - if (!contextSetter.setContextValuesByIdAndBytes(constantIds, utf8)) { - throw new IllegalStateException("reapply failed unexpectedly — record should be valid after setContext"); - } - } - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ThreadChurnAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ThreadChurnAntagonist.java deleted file mode 100644 index 4526d92a2..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/ThreadChurnAntagonist.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Continuously spawns batches of short-lived threads that race signal delivery, - * stack walking, and RefCountGuard slot allocation/release. - * - *

Targets: signal-vs-teardown races in the wall/cpu engines, slot reuse under - * contention, calltrace storage put() racing thread destruction. - */ -public final class ThreadChurnAntagonist implements Antagonist { - - private final int concurrentThreads; - private final int threadLifetimeMillis; - - private volatile boolean running; - private Thread driver; - private final AtomicLong totalWork = new AtomicLong(); - - public ThreadChurnAntagonist() { - this(64, 5); - } - - public ThreadChurnAntagonist(int concurrentThreads, int threadLifetimeMillis) { - this.concurrentThreads = concurrentThreads; - this.threadLifetimeMillis = threadLifetimeMillis; - } - - @Override - public String name() { - return "thread-churn"; - } - - @Override - public void start() { - running = true; - driver = new Thread(this::loop, "chaos-thread-churn"); - driver.setDaemon(true); - driver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - try { - driver.join(timeout.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void loop() { - while (running) { - List batch = new ArrayList<>(concurrentThreads); - for (int i = 0; i < concurrentThreads && running; i++) { - final long seed = System.nanoTime() + i; - Thread t = new Thread(() -> totalWork.addAndGet(burn(seed))); - t.setDaemon(true); - t.start(); - batch.add(t); - } - for (Thread t : batch) { - try { - t.join(1_000L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - } - - private long burn(long seed) { - long endNanos = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(threadLifetimeMillis); - long r = seed; - while (System.nanoTime() < endNanos) { - for (int i = 0; i < 1000; i++) { - r = (r * 1103515245L + 12345L) & 0x7fffffffL; - } - } - return r; - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java deleted file mode 100644 index 8abee751b..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/TraceContextAntagonist.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import datadog.trace.api.Trace; - -import java.time.Duration; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Drives nested {@link Trace}-annotated invocations from a pool of worker - * threads. Under a {@code dd-java-agent}-instrumented JVM the tracer drives - * {@code JavaProfiler.setContext}/{@code clearContext} on every span - * activation, so a high enter/exit rate stresses that path against signal - * delivery and stack walking. - * - *

{@code dd-trace-api} is a {@code compileOnly} dependency — at runtime - * the patched {@code dd-java-agent} provides the (relocated) classes and - * intercepts the annotation. - */ -public final class TraceContextAntagonist implements Antagonist { - - private final int workerCount; - private final ExecutorService pool; - private volatile boolean running; - private final AtomicLong sink = new AtomicLong(); - - public TraceContextAntagonist() { - this(8); - } - - public TraceContextAntagonist(int workerCount) { - this.workerCount = workerCount; - this.pool = Executors.newFixedThreadPool(workerCount, r -> { - Thread t = new Thread(r, "chaos-trace-context"); - t.setDaemon(true); - return t; - }); - } - - @Override - public String name() { - return "trace-context"; - } - - @Override - public void start() { - running = true; - for (int i = 0; i < workerCount; i++) { - pool.submit(this::workerLoop); - } - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - pool.shutdown(); - try { - pool.awaitTermination(timeout.toMillis(), TimeUnit.MILLISECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void workerLoop() { - long r = ThreadLocalRandom.current().nextLong(); - while (running) { - r = outer(r); - } - sink.addAndGet(r); - } - - @Trace(operationName = "chaos.outer", resourceName = "chaos.outer") - private long outer(long seed) { - long r = seed; - for (int i = 0; i < 64; i++) { - r = inner(r); - } - return r; - } - - @Trace(operationName = "chaos.inner", resourceName = "chaos.inner") - private long inner(long seed) { - return seed * 1103515245L + 12345L; - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java deleted file mode 100644 index 3a91f941d..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/VirtualThreadChurnAntagonist.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.lang.reflect.Method; -import java.time.Duration; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Spawns short-lived virtual threads at high rate to exercise mount/unmount on - * carrier threads and {@code ProfiledThread} recycling racing signal delivery. - * - *

Reflectively detects {@code Thread.ofVirtual()} (Java 21+); gracefully - * no-ops on older runtimes. - */ -public final class VirtualThreadChurnAntagonist implements Antagonist { - - private static final Method OF_VIRTUAL = resolveOfVirtual(); - private static final Method BUILDER_START = resolveBuilderStart(); - - private final int batchSize; - - private volatile boolean running; - private Thread driver; - private final AtomicLong sink = new AtomicLong(); - - public VirtualThreadChurnAntagonist() { - this(256); - } - - public VirtualThreadChurnAntagonist(int batchSize) { - this.batchSize = batchSize; - } - - @Override - public String name() { - return "vthread-churn"; - } - - @Override - public void start() { - running = true; - driver = new Thread(this::loop, "chaos-vthread-churn"); - driver.setDaemon(true); - driver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - try { - driver.join(timeout.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void loop() { - if (OF_VIRTUAL == null || BUILDER_START == null) { - System.out.println("[chaos] vthread-churn: skipping (Thread.ofVirtual not available)"); - return; - } - while (running) { - for (int i = 0; i < batchSize && running; i++) { - final long seed = System.nanoTime() ^ i; - try { - Object builder = OF_VIRTUAL.invoke(null); - BUILDER_START.invoke(builder, (Runnable) () -> sink.addAndGet(burn(seed))); - } catch (Throwable t) { - return; - } - } - // Yield briefly so the scheduler can drain mounts before the next batch. - try { - Thread.sleep(1L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - private static long burn(long seed) { - long r = seed; - for (int i = 0; i < 10_000; i++) { - r = r * 6364136223846793005L + 1442695040888963407L; - } - // A yield raises the chance of unmount/remount on a different carrier. - Thread.yield(); - return r; - } - - private static Method resolveOfVirtual() { - try { - return Thread.class.getMethod("ofVirtual"); - } catch (NoSuchMethodException e) { - return null; - } - } - - private static Method resolveBuilderStart() { - try { - Class builder = Class.forName("java.lang.Thread$Builder"); - return builder.getMethod("start", Runnable.class); - } catch (ClassNotFoundException | NoSuchMethodException e) { - return null; - } - } -} diff --git a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/WeakRefWaveAntagonist.java b/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/WeakRefWaveAntagonist.java deleted file mode 100644 index c58aa8915..000000000 --- a/ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/WeakRefWaveAntagonist.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.lang.ref.WeakReference; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.atomic.AtomicLong; - -/** - * Alternates fill and drop phases to drive wave-pattern weak-reference - * allocation and GC. A concurrent reader thread walks the weak-ref list - * during the fill phase. - * - *

Fill: allocates 10 k {@code byte[]} objects with strong + weak refs. - * Drop: releases strong refs (objects become weakly reachable), calls - * {@link System#gc()}, counts survivors. - * - *

Targets: jweak ref leak during liveness table overflow; concurrent - * read of a weakref cleared mid-wave; liveness table clearAll() race. - */ -public final class WeakRefWaveAntagonist implements Antagonist { - - private static final int WAVE_SIZE = 10_000; - private static final int[] OBJECT_SIZES = {64, 256, 1_024, 4_096}; - private static final long INTER_WAVE_MS = 200L; - - private volatile boolean running; - private Thread waveDriver; - private Thread reader; - - // Written by waveDriver, read by reader. Volatile for visibility. - private volatile List> currentWave = new ArrayList>(); - private final AtomicLong sink = new AtomicLong(); - - @Override - public String name() { - return "weakref-wave"; - } - - @Override - public void start() { - running = true; - reader = new Thread(new Runnable() { - @Override public void run() { readerLoop(); } - }, "chaos-weakref-reader"); - reader.setDaemon(true); - reader.start(); - waveDriver = new Thread(new Runnable() { - @Override public void run() { waveLoop(); } - }, "chaos-weakref-wave"); - waveDriver.setDaemon(true); - waveDriver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - long slice = timeout.toMillis() / 2; - try { - waveDriver.join(slice); - reader.join(slice); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void waveLoop() { - while (running) { - // Fill: allocate objects, hold both strong and weak refs - List strongRefs = new ArrayList(WAVE_SIZE); - List> weakRefs = new ArrayList>(WAVE_SIZE); - for (int i = 0; i < WAVE_SIZE && running; i++) { - int size = OBJECT_SIZES[i % OBJECT_SIZES.length]; - byte[] obj = new byte[size]; - obj[0] = (byte) i; - strongRefs.add(obj); - weakRefs.add(new WeakReference(obj)); - } - // Publish filled list to reader - currentWave = weakRefs; - - // Drop: release strong refs — objects now only weakly reachable - strongRefs.clear(); - System.gc(); - - // Count survivors (concurrent with reader on same list) - long alive = 0; - for (WeakReference ref : weakRefs) { - if (ref.get() != null) alive++; - } - sink.addAndGet(alive); - - // Replace shared reference; weakRefs goes out of scope - currentWave = new ArrayList>(); - - // Pause between waves so concurrent GC can reclaim without being - // overwhelmed by back-to-back System.gc() calls. - try { - Thread.sleep(INTER_WAVE_MS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - - private void readerLoop() { - while (running) { - List> wave = currentWave; // snapshot - long alive = 0; - for (WeakReference ref : wave) { - if (ref.get() != null) alive++; - } - sink.addAndGet(alive); - Thread.yield(); - } - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/AbstractFormatter.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/AbstractFormatter.java deleted file mode 100644 index c63189e20..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/AbstractFormatter.java +++ /dev/null @@ -1,22 +0,0 @@ -package com.datadoghq.profiler.stresstest; - -import java.io.IOException; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; - -public abstract class AbstractFormatter implements Formatter { - protected final PrintStream out; - - AbstractFormatter(Path file) throws IOException { - Files.deleteIfExists(file); - Files.createFile(file); - this.out = new PrintStream(file.toFile()); - } - - final public void format() { - print(); - } - - abstract protected void print(); -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/CompositeFormatter.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/CompositeFormatter.java deleted file mode 100644 index cfbca6944..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/CompositeFormatter.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.datadoghq.profiler.stresstest; - -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -public class CompositeFormatter implements Formatter { - private final Set formatters = new HashSet<>(); - - public static Formatter of(Formatter formatter, Formatter ... formatters) { - return new CompositeFormatter(formatter, formatters); - } - - private CompositeFormatter(Formatter formatter, Formatter ... formatters) { - this.formatters.add(formatter); - this.formatters.addAll(Arrays.asList(formatters)); - } - - @Override - public void format() { - formatters.forEach(Formatter::format); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/Configuration.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/Configuration.java deleted file mode 100644 index f4915cc26..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/Configuration.java +++ /dev/null @@ -1,14 +0,0 @@ -package com.datadoghq.profiler.stresstest; - -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.State; - -@State(Scope.Benchmark) -public class Configuration { - - public static final String BASE_COMMAND = "cpu=100us,wall=100us"; - - @Param({BASE_COMMAND}) - public String command; -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/Formatter.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/Formatter.java deleted file mode 100644 index 987389aff..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/Formatter.java +++ /dev/null @@ -1,6 +0,0 @@ -package com.datadoghq.profiler.stresstest; - -@FunctionalInterface -public interface Formatter { - void format(); -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/HtmlCommentFormatter.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/HtmlCommentFormatter.java deleted file mode 100644 index 532301d9e..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/HtmlCommentFormatter.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.datadoghq.profiler.stresstest; - -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.infra.BenchmarkParams; -import org.openjdk.jmh.results.Result; -import org.openjdk.jmh.results.RunResult; - -import java.io.IOException; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collection; -import java.util.SortedSet; -import java.util.TreeSet; - -import static com.datadoghq.profiler.stresstest.Main.SCENARIOS_PACKAGE; - -public class HtmlCommentFormatter extends AbstractFormatter { - - private final Collection results; - - private final String delimiter = "~"; - private final Mode mode; - - public HtmlCommentFormatter(Collection results, Mode mode) throws IOException { - super(Paths.get(System.getProperty("user.dir")).resolve("jmh-comment.html")); - this.results = results; - this.mode = mode; - } - - private void startHtml() { - out.print(""); - } - - private void endHtml() { - out.print(""); - } - - private void printEnvironment() { - out.println(""); - out.println(""); - out.println(""); - out.print(""); - out.print(""); - out.print(""); - out.print(""); - out.print(""); - out.print(""); - out.print(""); - out.println(""); - out.println("
CommitLibCJVM VendorVersionOperating SystemArchitecture
" + System.getenv("TEST_COMMIT") + "" + System.getenv("LIBC") + "" + System.getProperty("java.vm.vendor") + "" + System.getProperty("java.version") + "" + System.getProperty("os.name") +"" + System.getProperty("os.arch") + "
"); - } - - public void print() { - startHtml(); - printEnvironment(); - SortedSet params = new TreeSet<>(); - for (RunResult res : results) { - params.addAll(res.getParams().getParamsKeys()); - } - out.println("

Results

"); - String lastGroup = ""; - for (RunResult result : results) { - BenchmarkParams benchParams = result.getParams(); - String benchmarkName = benchParams.getBenchmark().replace(SCENARIOS_PACKAGE, ""); - String command = benchParams.getParam("command"); - String nodeCount = benchParams.getParam("nodeCount"); - String tagCardinality = benchParams.getParam("tagCardinality"); - String group = benchmarkName + "#" + command + "#" + nodeCount + "#" + tagCardinality; - if (!group.equals(lastGroup)) { - out.println("
"); - out.print("" + benchmarkName + " [command='" + command + "'"); - if (nodeCount != null) { - out.print(", nodeCount=" + nodeCount); - } - if (tagCardinality != null) { - out.print(", tagCardinality=" + tagCardinality); - } - out.println("]🔍"); - out.println(""); - out.println(""); - for (String label : result.getSecondaryResults().keySet()) { - Result metric = result.getSecondaryResults().get(label); - out.println(""); - out.print(""); - out.println(""); - out.println(""); - } - out.println("
MetricScore
" + metric.getLabel() + "" + emit(metric.getScore()) + "
"); - out.println("
"); - lastGroup = group; - } - } - endHtml(); - } - - private String emit(String v) { - return v != null ? v : "" ; - } - - private String emit(int i) { - return emit(String.format("%d", i)); - } - - private String emit(long l) { - return emit(String.format("%d", l)); - } - - private String emit(double d) { - return emit(String.format("%f", d)); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/HtmlFormatter.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/HtmlFormatter.java deleted file mode 100644 index baf023338..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/HtmlFormatter.java +++ /dev/null @@ -1,146 +0,0 @@ -package com.datadoghq.profiler.stresstest; - -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.infra.BenchmarkParams; -import org.openjdk.jmh.results.Result; -import org.openjdk.jmh.results.RunResult; - -import java.io.IOException; -import java.io.PrintStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Collection; -import java.util.SortedSet; -import java.util.TreeSet; - -import static com.datadoghq.profiler.stresstest.Main.SCENARIOS_PACKAGE; - -public class HtmlFormatter extends AbstractFormatter { - - private final Collection results; - private final String delimiter = "~"; - private final Mode mode; - - public HtmlFormatter(Collection results, Mode mode) throws IOException { - super(Paths.get(System.getProperty("user.dir")).resolve("jmh-result.html")); - this.results = results; - this.mode = mode; - } - - private void printHeader(SortedSet params) { - out.print(""); - out.print("Scenario"); - out.print("Metric"); - out.print("Score"); - for (String k : params) { - out.print(""); - out.print(k); - out.print(""); - } - out.print(""); - } - - private void startHtml() { - out.print(""); - } - - private void endHtml() { - out.print(""); - } - - private void startTable() { - out.print(""); - } - - private void endTable() { - out.print("
"); - } - - private void startTableBody() { - out.print(""); - } - - private void endTableBody() { - out.print(""); - } - - private void printEnvironment() { - out.print("

Setup

"); - startTable(); - startTableBody(); - out.print("Commit" + System.getenv("TEST_COMMIT") + ""); - out.print("LibC" + System.getenv("LIBC") + ""); - out.print("JVM Vendor" + System.getProperty("java.vm.vendor") + ""); - out.print("Version" + System.getProperty("java.version") + ""); - out.print("Operating System" + System.getProperty("os.name") +""); - out.print("Architecture" + System.getProperty("os.arch") + ""); - endTableBody(); - endTable(); - } - - public void print() { - startHtml(); - out.print("

Stress Tests

"); - printEnvironment(); - SortedSet params = new TreeSet<>(); - for (RunResult res : results) { - params.addAll(res.getParams().getParamsKeys()); - } - out.print("

Results

"); - startTable(); - printHeader(params); - startTableBody(); - for (RunResult result : results) { - BenchmarkParams benchParams = result.getParams(); - Result timing = result.getPrimaryResult(); - String benchmarkName = benchParams.getBenchmark().replace(SCENARIOS_PACKAGE, ""); - printRow(benchmarkName, mode.shortLabel(), benchParams, params, timing); - for (String label : result.getSecondaryResults().keySet()) { - Result metric = result.getSecondaryResults().get(label); - printRow(benchmarkName, metric.getLabel(), benchParams, params, metric); - } - } - endTableBody(); - endTable(); - endHtml(); - } - - private void printRow(String scenario, String metric, BenchmarkParams benchmarkParams, SortedSet params, Result result) { - out.print(""); - out.print(""); - out.print(scenario); - out.print(""); - out.print(""); - out.print(metric); - out.print(""); - out.print(""); - out.print(emit(result.getScore())); - out.print(""); - for (String p : params) { - out.print(""); - String v = benchmarkParams.getParam(p); - if (v != null) { - out.print(emit(v)); - } - out.print(""); - } - out.print(""); - } - - private String emit(String v) { - return v != null ? v : " "; - } - - private String emit(int i) { - return emit(String.format("%d", i)); - } - - private String emit(long l) { - return emit(String.format("%d", l)); - } - - private String emit(double d) { - return emit(String.format("%f", d)); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/Main.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/Main.java deleted file mode 100644 index c8d5994f1..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/Main.java +++ /dev/null @@ -1,33 +0,0 @@ -package com.datadoghq.profiler.stresstest; - -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.results.RunResult; -import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.options.CommandLineOptions; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.runner.options.OptionsBuilder; -import org.openjdk.jmh.runner.options.TimeValue; - -import java.util.Collection; -import java.util.concurrent.TimeUnit; - -public class Main { - - public static final String SCENARIOS_PACKAGE = "com.datadoghq.profiler.stresstest.scenarios."; - - public static void main(String... args) throws Exception { - CommandLineOptions commandLineOptions = new CommandLineOptions(args); - Mode mode = Mode.AverageTime; - Options options = new OptionsBuilder() - .parent(new CommandLineOptions(args)) - .forks(commandLineOptions.getForkCount().orElse(1)) - .warmupIterations(commandLineOptions.getWarmupIterations().orElse(0)) - .measurementIterations(commandLineOptions.getMeasurementIterations().orElse(1)) - .measurementTime(commandLineOptions.getMeasurementTime().orElse(TimeValue.seconds(5))) - .timeUnit(commandLineOptions.getTimeUnit().orElse(TimeUnit.MICROSECONDS)) - .mode(mode) - .build(); - Collection results = new Runner(options).run(); - CompositeFormatter.of(new HtmlCommentFormatter(results, mode), new HtmlFormatter(results, mode)).format(); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/WhiteboxProfiler.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/WhiteboxProfiler.java deleted file mode 100644 index 99899eb45..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/WhiteboxProfiler.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.datadoghq.profiler.stresstest; - -import com.datadoghq.profiler.JavaProfiler; -import org.openjdk.jmh.infra.BenchmarkParams; -import org.openjdk.jmh.infra.IterationParams; -import org.openjdk.jmh.profile.InternalProfiler; -import org.openjdk.jmh.results.AggregationPolicy; -import org.openjdk.jmh.results.IterationResult; -import org.openjdk.jmh.results.Result; -import org.openjdk.jmh.results.ScalarResult; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; - -public class WhiteboxProfiler implements InternalProfiler { - - private Path jfr; - - @Override - public String getDescription() { - return "ddprof-whitebox"; - } - - @Override - public void beforeIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams) { - try { - jfr = Files.createTempFile(benchmarkParams.getBenchmark() + System.currentTimeMillis(), ".jfr"); - String command = "start," + benchmarkParams.getParam("command") - + ",jfr,file=" + jfr.toAbsolutePath(); - JavaProfiler.getInstance().execute(command); - } catch (IOException e) { - e.printStackTrace(); - } - } - - @Override - public Collection afterIteration(BenchmarkParams benchmarkParams, IterationParams iterationParams, IterationResult result) { - // TODO unit encoded in counter name for now, so results are effectively dimensionless - try { - JavaProfiler.getInstance().stop(); - long fileSize = Files.size(jfr); - Files.deleteIfExists(jfr); - if (!Boolean.parseBoolean(benchmarkParams.getParam("skipResults"))) { - List results = new ArrayList<>(); - results.add(new ScalarResult("jfr_filesize_bytes", fileSize, "", AggregationPolicy.MAX)); - for (Map.Entry counter : JavaProfiler.getInstance().getDebugCounters().entrySet()) { - results.add(new ScalarResult(counter.getKey(), counter.getValue(), "", AggregationPolicy.MAX)); - } - return results; - } else { - return Collections.emptyList(); - } - } catch (IOException e) { - e.printStackTrace(); - return Collections.emptyList(); - } - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/CapturingLambdas.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/CapturingLambdas.java deleted file mode 100644 index a6d2e06e3..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/CapturingLambdas.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.datadoghq.profiler.stresstest.scenarios.counters; - -import com.datadoghq.profiler.stresstest.Configuration; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.CompilerControl; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.State; - -import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; -import java.util.function.Supplier; - -@State(Scope.Benchmark) -public class CapturingLambdas extends Configuration { - - @Param(BASE_COMMAND + ",memory=1048576:a") - public String command; - - @Benchmark - public Object capturingLambda() { - return lambda(UUID.randomUUID()).get(); - } - - - @CompilerControl(CompilerControl.Mode.DONT_INLINE) - public Supplier lambda(UUID state) { - return () -> state.getLeastSignificantBits() * ThreadLocalRandom.current().nextLong(); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/DumpRecording.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/DumpRecording.java deleted file mode 100644 index 83c4cc29c..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/DumpRecording.java +++ /dev/null @@ -1,35 +0,0 @@ -package com.datadoghq.profiler.stresstest.scenarios.counters; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.stresstest.Configuration; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Threads; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.UUID; -import java.util.concurrent.ThreadLocalRandom; - -@State(Scope.Benchmark) -public class DumpRecording extends Configuration { - - @Param(BASE_COMMAND + ",memory=1048576:a") - public String command; - - @Benchmark - @Threads(2) - public Object dumpRecording(GraphState graph) throws IOException { - for (int i = 0; i < 100; i++) { - int object = ThreadLocalRandom.current().nextInt(graph.nodeCount); - int subject = ThreadLocalRandom.current().nextInt(graph.nodeCount); - graph.nodes[subject].link(graph.nodes[object]); - } - Path tmpRecording = Files.createTempFile(UUID.randomUUID().toString(), ".jfr"); - JavaProfiler.getInstance().dump(tmpRecording); - return Files.deleteIfExists(tmpRecording); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/GraphMutation.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/GraphMutation.java deleted file mode 100644 index f986c8ad3..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/GraphMutation.java +++ /dev/null @@ -1,34 +0,0 @@ -package com.datadoghq.profiler.stresstest.scenarios.counters; - -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Threads; - -import java.util.concurrent.ConcurrentLinkedQueue; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicLong; - -public class GraphMutation { - - public static class GraphNode { - private volatile ConcurrentLinkedQueue nodes = new ConcurrentLinkedQueue<>(); - private final AtomicLong linkCount = new AtomicLong(0); - - public void link(GraphNode node) { - nodes.add(node); - - // Switch to new data structure every 1,000 operations - // Other threads can finish with the old one - if (linkCount.incrementAndGet() % 1000 == 0) { - nodes = new ConcurrentLinkedQueue<>(); - } - } - } - - @Benchmark - @Threads(8) - public void mutateGraph(GraphState graph) { - int object = ThreadLocalRandom.current().nextInt(graph.nodeCount); - int subject = ThreadLocalRandom.current().nextInt(graph.nodeCount); - graph.nodes[subject].link(graph.nodes[object]); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/GraphState.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/GraphState.java deleted file mode 100644 index cd612832b..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/GraphState.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.datadoghq.profiler.stresstest.scenarios.counters; - -import com.datadoghq.profiler.stresstest.Configuration; -import org.openjdk.jmh.annotations.*; - -import java.util.Arrays; - -@State(Scope.Benchmark) -public class GraphState extends Configuration { - @Param("1024") - int nodeCount; - - public GraphMutation.GraphNode[] nodes; - - @Setup(Level.Iteration) - public void setup() { - nodes = new GraphMutation.GraphNode[nodeCount]; - Arrays.setAll(nodes, i -> new GraphMutation.GraphNode()); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/NanoTime.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/NanoTime.java deleted file mode 100644 index 8e4b7dd12..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/NanoTime.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.datadoghq.profiler.stresstest.scenarios.counters; - -import com.datadoghq.profiler.stresstest.Configuration; -import org.openjdk.jmh.annotations.Benchmark; - -public class NanoTime { - - @Benchmark - public long nanoTime(Configuration config) { - return System.nanoTime(); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/ThreadFilterBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/ThreadFilterBenchmark.java deleted file mode 100644 index 0d280bdf2..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/ThreadFilterBenchmark.java +++ /dev/null @@ -1,88 +0,0 @@ -package com.datadoghq.profiler.stresstest.scenarios.counters; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.stresstest.Configuration; -import org.openjdk.jmh.annotations.*; - -import java.io.FileWriter; -import java.io.IOException; -import java.io.PrintWriter; -import java.util.concurrent.*; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLongArray; - -@State(Scope.Benchmark) -public class ThreadFilterBenchmark extends Configuration { - - @Param({"true", "false"}) // Parameterize the filter usage - public boolean useThreadFilters; - - private JavaProfiler profiler; - private static final int ARRAY_SIZE = 1024; // Larger array to stress memory - private static final long[] sharedArray = new long[ARRAY_SIZE]; - private static final AtomicLongArray atomicArray = new AtomicLongArray(ARRAY_SIZE); - private static final int CACHE_LINE_SIZE = 64; // Typical cache line size - private static final int STRIDE = CACHE_LINE_SIZE / Integer.BYTES; // Elements per cache line - - @Setup(Level.Trial) - public void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 1, warmups = 1) - @Warmup(iterations = 1, time = 1) - @Measurement(iterations = 1, time = 2) - @Threads(15) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void threadFilterStress() throws InterruptedException { - long threadId = Thread.currentThread().getId(); - // Memory-intensive operations that would be sensitive to false sharing - for (int j = 0; j < ARRAY_SIZE; j += STRIDE) { - if (useThreadFilters) { - // Register thread at the start of each cache line operation - profiler.addThread(); - } - - // Each thread writes to its own cache line - long baseIndex = (threadId * STRIDE) % ARRAY_SIZE; - for (int k = 0; k < STRIDE; k++) { - int index = (int)(baseIndex + k) % ARRAY_SIZE; - // Write to shared array - sharedArray[index] = threadId; - // Read and modify - long value = sharedArray[index] + 1; - // Atomic operation - atomicArray.set(index, value); - } - - if (useThreadFilters) { - // Remove thread after cache line operation - profiler.removeThread(); - } - } - - // More memory operations with thread registration - for (int j = 0; j < ARRAY_SIZE; j += STRIDE) { - if (useThreadFilters) { - // Register thread at the start of each cache line operation - profiler.addThread(); - } - - long baseIndex = (threadId * STRIDE) % ARRAY_SIZE; - for (int k = 0; k < STRIDE; k++) { - int index = (int)(baseIndex + k) % ARRAY_SIZE; - long value = atomicArray.get(index); - sharedArray[index] = value * 2; - } - - if (useThreadFilters) { - // Remove thread after cache line operation - profiler.removeThread(); - } - } - } -} \ No newline at end of file diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java deleted file mode 100644 index 9b538e3db..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/counters/TracedParallelWork.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.datadoghq.profiler.stresstest.scenarios.counters; - -import com.datadoghq.profiler.ContextSetter; -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.context.ContextExecutor; -import com.datadoghq.profiler.context.Tracing; -import com.datadoghq.profiler.stresstest.Configuration; -import org.openjdk.jmh.annotations.*; -import org.openjdk.jmh.infra.Blackhole; - -import java.io.IOException; -import java.util.Arrays; -import java.util.UUID; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadLocalRandom; -import java.util.stream.IntStream; - -public class TracedParallelWork { - - @State(Scope.Benchmark) - public static class BenchmarkState extends Configuration { - - public static final String COMMAND = BASE_COMMAND + ",attributes=tag0;tag1"; - - @Param(COMMAND) - String command; - @Param({"10", "100", "1000"}) - int tagCardinality; - ContextExecutor executor; - JavaProfiler profiler; - ContextSetter contextSetter; - - int tag0; - int tag1; - - String[] tagValues; - - public long newTraceId() { - return ThreadLocalRandom.current().nextLong(); - } - - public String getTag(long id) { - int offset = (int) (Math.abs(id) % tagCardinality); - return tagValues[offset]; - } - - @Setup(Level.Trial) - public void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - executor = new ContextExecutor(200, profiler); - contextSetter = new ContextSetter(profiler, Arrays.asList("tag0", "tag1")); - tag0 = contextSetter.offsetOf("tag0"); - tag1 = contextSetter.offsetOf("tag1"); - tagValues = IntStream.range(0, tagCardinality).mapToObj(i -> UUID.randomUUID().toString()) - .toArray(String[]::new); - } - } - - @Benchmark - @Threads(8) - @SuppressWarnings("deprecation") - public Object work(BenchmarkState state, Blackhole bh) throws ExecutionException, InterruptedException { - try (Tracing.Context context = Tracing.newContext(state::newTraceId, state.profiler)) { - state.profiler.setContext(context.getSpanId(), context.getRootSpanId()); - state.contextSetter.setContextValue(state.tag0, state.getTag(context.getSpanId())); - state.contextSetter.setContextValue(state.tag1, state.getTag(context.getSpanId() + 1)); - Future f = state.executor.submit(() -> compute(state)); - bh.consume(compute(state)); - return f.get(); - } - } - - public long compute(BenchmarkState state) { - long x = ThreadLocalRandom.current().nextLong(); - for (int i = 0; i < 10_000; i++) { - state.contextSetter.setContextValue(state.tag0, state.getTag(x)); - x ^= ThreadLocalRandom.current().nextLong(); - state.contextSetter.setContextValue(state.tag1, state.getTag(x)); - } - return x; - } - -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/NativeSocketOverheadBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/NativeSocketOverheadBenchmark.java deleted file mode 100644 index 82becf128..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/NativeSocketOverheadBenchmark.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.stresstest.scenarios.throughput; - -import com.datadoghq.profiler.JavaProfiler; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; -import java.net.ServerSocket; -import java.net.Socket; -import java.nio.ByteBuffer; -import java.nio.channels.SocketChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.TimeUnit; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; - -/** - * Measures overhead of the nativesocket PLT write/read hooks on: - *
    - *
  • fileWrite — write() to a regular file. After the first call the - * fd-type cache classifies it as non-socket (one atomic load per subsequent - * call). This is the worst-case overhead scenario for code that does heavy - * file I/O with nativesocket enabled. - *
  • socketWrite — write() to a TCP socket (blocking, PlainSocket). - *
  • nioSocketWrite — write() via NIO SocketChannel (JDK11+ path). - *
- * - *

Compare {@code profilerActive=false} vs {@code profilerActive=true} to - * quantify the hook overhead. Revert if fileWrite throughput degrades > 5%. - * - *

The profiler uses time-weighted (duration-based) inverse-transform - * sampling: {@code P(sample) = 1 - exp(-duration_ticks / interval_ticks)}. - * Slow I/O calls are sampled more often; fast calls are down-sampled. - * - *

- *   ./gradlew :ddprof-stresstest:jmh -PjmhInclude="NativeSocketOverheadBenchmark"
- * 
- */ -@BenchmarkMode(Mode.Throughput) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -@Fork(value = 1, warmups = 0) -@Warmup(iterations = 3, time = 2) -@Measurement(iterations = 5, time = 3) -@State(Scope.Thread) -public class NativeSocketOverheadBenchmark { - - private static final int CHUNK = 4096; - - @Param({"false", "true"}) - public boolean profilerActive; - - private Path tmpFile; - private OutputStream fileOut; - - private ServerSocket server; - private Socket client; - private Socket serverConn; - private OutputStream sockOut; - private Thread serverAcceptor; - - private ServerSocket nioServer; - private SocketChannel nioClient; - private ByteBuffer nioBuf; - - private final byte[] buf = new byte[CHUNK]; - - @Setup(Level.Trial) - public void setup() throws Exception { - if (profilerActive) { - JavaProfiler profiler = JavaProfiler.getInstance(); - Path jfr = Files.createTempFile("nativesocket-bench", ".jfr"); - profiler.execute("start,natsock,jfr,file=" + jfr.toAbsolutePath()); - } - - // File I/O: regular file, will be classified non-socket after first write - tmpFile = Files.createTempFile("nativesocket-bench-file", ".bin"); - fileOut = Files.newOutputStream(tmpFile); - - // Blocking TCP socket (PlainSocket / JDK 8 send path) - server = new ServerSocket(0); - serverAcceptor = new Thread(() -> { - try { - serverConn = server.accept(); - // drain so the client write buffer never fills - InputStream drain = serverConn.getInputStream(); - byte[] dbuf = new byte[CHUNK]; - while (drain.read(dbuf) != -1) { /* drain */ } - } catch (IOException ignored) {} - }); - serverAcceptor.setDaemon(true); - serverAcceptor.start(); - client = new Socket("127.0.0.1", server.getLocalPort()); - sockOut = client.getOutputStream(); - - // NIO SocketChannel (write(2) path used by JDK 11+) - nioServer = new ServerSocket(0); - Thread nioAcceptor = new Thread(() -> { - try { - Socket ac = nioServer.accept(); - // drain - InputStream drain = ac.getInputStream(); - byte[] dbuf = new byte[CHUNK]; - while (drain.read(dbuf) != -1) { /* drain */ } - } catch (IOException ignored) {} - }); - nioAcceptor.setDaemon(true); - nioAcceptor.start(); - nioClient = SocketChannel.open(new InetSocketAddress("127.0.0.1", nioServer.getLocalPort())); - nioBuf = ByteBuffer.allocate(CHUNK); - } - - @TearDown(Level.Trial) - public void teardown() throws Exception { - if (profilerActive) { - JavaProfiler.getInstance().execute("stop"); - } - fileOut.close(); - Files.deleteIfExists(tmpFile); - client.close(); - if (serverConn != null) serverConn.close(); - server.close(); - nioClient.close(); - nioServer.close(); - } - - /** write() to a regular file — measures fd-type-cache overhead on non-socket fds. */ - @Benchmark - public void fileWrite() throws IOException { - fileOut.write(buf); - } - - /** write() to a blocking TCP socket — the socket sampling path. */ - @Benchmark - public void socketWrite() throws IOException { - sockOut.write(buf); - } - - /** SocketChannel.write() — the NIO path used by JDK 11+ java.net.Socket. */ - @Benchmark - public long nioSocketWrite() throws IOException { - nioBuf.clear(); - nioBuf.put(buf); - nioBuf.flip(); - return nioClient.write(nioBuf); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputFullBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputFullBenchmark.java deleted file mode 100644 index 6c19b9714..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputFullBenchmark.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.stresstest.scenarios.throughput; - -import com.datadoghq.profiler.JavaProfiler; -import java.io.IOException; -import java.util.concurrent.TimeUnit; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; - -/** - * Full benchmark for profiler throughput with comprehensive thread counts and statistical rigor. - * - *

This benchmark measures end-to-end profiling overhead including: - * - Signal handler interrupts (cpu=100us, wall=100us) - * - Stack walking via ASGCT - * - CallTraceStorage::put() operations with RefCountGuard memory reclamation - * - Atomic operations in the refcount guard lifecycle - * - JFR background processing - * - *

Tests the complete profiling pipeline across 7 thread counts (1,2,4,8,16,32,64) with - * high statistical confidence (3 forks, 5 measurement iterations). Runtime: ~45-60 minutes. - * - *

Use {@link ProfilerThroughputQuickBenchmark} for rapid development iteration (~2 minutes). - */ -@State(Scope.Benchmark) -public class ProfilerThroughputFullBenchmark { - - @Param({"cpu=100us,wall=100us"}) - public String command; - - @Param({"false"}) - public String skipResults; - - private JavaProfiler profiler; - - @Setup(Level.Trial) - public void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - } - - /** - * Performs computational work to generate stack samples. - * The profiler will interrupt these threads and call CallTraceStorage::put(). - */ - private long doWork(long seed) { - long result = seed; - // Enough work to generate multiple samples - for (int i = 0; i < 100_000; i++) { - result = (result * 1103515245L + 12345L) & 0x7fffffffL; - } - return result; - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 3, time = 2) - @Measurement(iterations = 5, time = 5) - @Threads(1) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline01Thread(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 3, time = 2) - @Measurement(iterations = 5, time = 5) - @Threads(2) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline02Threads(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 3, time = 2) - @Measurement(iterations = 5, time = 5) - @Threads(4) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline04Threads(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 3, time = 2) - @Measurement(iterations = 5, time = 5) - @Threads(8) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline08Threads(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 3, time = 2) - @Measurement(iterations = 5, time = 5) - @Threads(16) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline16Threads(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 3, time = 2) - @Measurement(iterations = 5, time = 5) - @Threads(32) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline32Threads(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 3, time = 2) - @Measurement(iterations = 5, time = 5) - @Threads(64) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline64Threads(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputQuickBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputQuickBenchmark.java deleted file mode 100644 index 44ff75e30..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputQuickBenchmark.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.stresstest.scenarios.throughput; - -import com.datadoghq.profiler.JavaProfiler; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; - -/** - * Quick smoke test for CallTraceStorage short-lived thread performance. - * - *

This benchmark provides fast feedback (~2 minutes) on key performance metrics: - * - Baseline stable thread performance (1, 8, 32 threads) - * - Short-lived thread churn overhead - * - Moderate concurrency slot allocation - * - *

Use this for rapid iteration during development. For comprehensive analysis, - * use the full benchmark suite (CallTraceStorageBaselineBenchmark, etc.). - */ -@State(Scope.Benchmark) -public class ProfilerThroughputQuickBenchmark { - - @Param({"cpu=100us,wall=100us"}) - public String command; - - @Param({"false"}) - public String skipResults; - - private JavaProfiler profiler; - private final AtomicLong totalWork = new AtomicLong(0); - - @Setup(Level.Trial) - public void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - totalWork.set(0); - } - - private long doWork(long seed) { - long result = seed; - for (int i = 0; i < 100_000; i++) { - result = (result * 1103515245L + 12345L) & 0x7fffffffL; - } - return result; - } - - private long doShortWork(long seed, long durationMillis) { - long result = seed; - long endTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationMillis); - while (System.nanoTime() < endTime) { - for (int i = 0; i < 1000; i++) { - result = (result * 1103515245L + 12345L) & 0x7fffffffL; - } - } - return result; - } - - // === Baseline Tests === - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 1, warmups = 0) - @Warmup(iterations = 1, time = 1) - @Measurement(iterations = 2, time = 2) - @Threads(1) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline01Thread(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 1, warmups = 0) - @Warmup(iterations = 1, time = 1) - @Measurement(iterations = 2, time = 2) - @Threads(8) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline08Threads(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 1, warmups = 0) - @Warmup(iterations = 1, time = 1) - @Measurement(iterations = 2, time = 2) - @Threads(32) - @OutputTimeUnit(TimeUnit.SECONDS) - public void baseline32Threads(Blackhole bh) { - bh.consume(doWork(System.nanoTime())); - } - - // === Thread Churn Tests === - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 1, warmups = 0) - @Warmup(iterations = 1, time = 1) - @Measurement(iterations = 2, time = 3) - @OutputTimeUnit(TimeUnit.SECONDS) - public void churn10ThreadsShort(Blackhole bh) throws InterruptedException { - List threads = new ArrayList<>(10); - for (int i = 0; i < 10; i++) { - final long seed = System.nanoTime() + i; - Thread thread = new Thread(() -> { - long result = doShortWork(seed, 5); - totalWork.addAndGet(result); - }); - thread.start(); - threads.add(thread); - } - for (Thread thread : threads) { - thread.join(); - } - bh.consume(totalWork.get()); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 1, warmups = 0) - @Warmup(iterations = 1, time = 1) - @Measurement(iterations = 2, time = 3) - @OutputTimeUnit(TimeUnit.SECONDS) - public void churn50ThreadsShort(Blackhole bh) throws InterruptedException { - List threads = new ArrayList<>(50); - for (int i = 0; i < 50; i++) { - final long seed = System.nanoTime() + i; - Thread thread = new Thread(() -> { - long result = doShortWork(seed, 5); - totalWork.addAndGet(result); - }); - thread.start(); - threads.add(thread); - } - for (Thread thread : threads) { - thread.join(); - } - bh.consume(totalWork.get()); - } - - // === Slot Allocation Test === - - @Benchmark - @BenchmarkMode(Mode.SingleShotTime) - @Fork(value = 1, warmups = 0) - @Warmup(iterations = 1) - @Measurement(iterations = 3) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void burst500Threads(Blackhole bh) throws InterruptedException { - List threads = new ArrayList<>(500); - for (int i = 0; i < 500; i++) { - final long seed = System.nanoTime() + i; - Thread thread = new Thread(() -> { - long result = doShortWork(seed, 100); - totalWork.addAndGet(result); - }); - thread.start(); - threads.add(thread); - } - for (Thread thread : threads) { - thread.join(); - } - bh.consume(totalWork.get()); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputSlotExhaustionBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputSlotExhaustionBenchmark.java deleted file mode 100644 index b69825252..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputSlotExhaustionBenchmark.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.stresstest.scenarios.throughput; - -import com.datadoghq.profiler.JavaProfiler; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; - -/** - * Benchmark testing RefCountGuard slot allocation under extreme concurrency. - * - *

This benchmark stress-tests the slot allocation mechanism by creating - * large numbers of concurrent threads, approaching or exceeding typical usage - * patterns. Key metrics: - * - *

    - *
  • Slot allocation latency as occupancy increases - *
  • Prime probing efficiency under high contention - *
  • Risk of slot exhaustion (MAX_THREADS=8192 limit) - *
  • Cache line contention effects - *
- * - *

The RefCountGuard::getThreadRefCountSlot() function uses prime probing with - * MAX_PROBE_DISTANCE=32 attempts. Under high concurrency, probing distance - * increases and may reach the limit, causing allocation failures. - * - *

Test scenarios: - * - Burst thread creation: Spawn many threads simultaneously - * - Wave pattern: Create threads in successive waves - * - Sustained high concurrency: Maintain high thread count - */ -@State(Scope.Benchmark) -public class ProfilerThroughputSlotExhaustionBenchmark { - - @Param({"cpu=100us,wall=100us"}) - public String command; - - @Param({"false"}) - public String skipResults; - - @Param({"500", "1000", "2000", "4000"}) - private int burstThreadCount; - - @Param({"50", "100", "200"}) - private int threadLifetimeMillis; - - private JavaProfiler profiler; - private final AtomicLong totalWork = new AtomicLong(0); - private final AtomicLong successfulSlots = new AtomicLong(0); - - @Setup(Level.Trial) - public void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - totalWork.set(0); - successfulSlots.set(0); - } - - /** - * Work performed by each thread during slot exhaustion test. - * Simulates real application work that would trigger profiling samples. - */ - private long doWork(long seed, long durationMillis) { - long result = seed; - long endTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(durationMillis); - - while (System.nanoTime() < endTime) { - for (int i = 0; i < 5000; i++) { - result = (result * 1103515245L + 12345L) & 0x7fffffffL; - } - } - return result; - } - - /** - * Burst test: Creates many threads simultaneously to stress slot allocation. - * All threads start together and run for the specified lifetime. - * - *

This tests the worst-case scenario where slot allocation happens - * under maximum contention. High probe distances and potential allocation - * failures may occur if burstThreadCount approaches MAX_THREADS. - */ - @Benchmark - @BenchmarkMode(Mode.SingleShotTime) - @Fork(value = 3, warmups = 0) - @Warmup(iterations = 2) - @Measurement(iterations = 5) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void burstThreadCreation(Blackhole bh) throws InterruptedException { - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch completionLatch = new CountDownLatch(burstThreadCount); - List threads = new ArrayList<>(burstThreadCount); - - // Create all threads first - for (int i = 0; i < burstThreadCount; i++) { - final long seed = System.nanoTime() + i; - Thread thread = new Thread(() -> { - try { - // Wait for signal to start all threads simultaneously - startLatch.await(); - - long result = doWork(seed, threadLifetimeMillis); - totalWork.addAndGet(result); - successfulSlots.incrementAndGet(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - completionLatch.countDown(); - } - }); - threads.add(thread); - thread.start(); - } - - // Start all threads simultaneously - long startTime = System.nanoTime(); - startLatch.countDown(); - - // Wait for all threads to complete - completionLatch.await(); - long duration = System.nanoTime() - startTime; - - bh.consume(totalWork.get()); - bh.consume(successfulSlots.get()); - bh.consume(duration); - } - - /** - * Wave test: Creates threads in successive waves to test slot reuse. - * Tests whether slots from completed threads are efficiently reclaimed - * and reused by subsequent threads. - */ - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 2, time = 2) - @Measurement(iterations = 5, time = 5) - @OutputTimeUnit(TimeUnit.SECONDS) - public void waveThreadCreation(Blackhole bh) throws InterruptedException { - int threadsPerWave = burstThreadCount / 4; - int waves = 4; - - for (int wave = 0; wave < waves; wave++) { - List threads = new ArrayList<>(threadsPerWave); - - for (int i = 0; i < threadsPerWave; i++) { - final long seed = System.nanoTime() + i; - Thread thread = new Thread(() -> { - long result = doWork(seed, threadLifetimeMillis / 4); - totalWork.addAndGet(result); - }); - thread.start(); - threads.add(thread); - } - - // Wait for this wave to complete before starting next - for (Thread thread : threads) { - thread.join(); - } - } - - bh.consume(totalWork.get()); - } - - /** - * Sustained concurrency test: Maintains high thread count continuously. - * Tests long-term stability under sustained high slot occupancy. - */ - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 2, time = 2) - @Measurement(iterations = 5, time = 5) - @OutputTimeUnit(TimeUnit.SECONDS) - public void sustainedHighConcurrency(Blackhole bh) throws InterruptedException { - // Maintain constant high thread count by spawning replacement threads - int activeThreads = burstThreadCount / 2; - List threads = new ArrayList<>(activeThreads); - - // Initial thread pool - for (int i = 0; i < activeThreads; i++) { - Thread thread = createWorkThread(System.nanoTime() + i); - thread.start(); - threads.add(thread); - } - - // Replace completed threads to maintain high occupancy - for (int i = 0; i < activeThreads; i++) { - threads.get(i).join(); - Thread replacement = createWorkThread(System.nanoTime() + activeThreads + i); - replacement.start(); - threads.set(i, replacement); - } - - // Wait for final batch - for (Thread thread : threads) { - thread.join(); - } - - bh.consume(totalWork.get()); - } - - private Thread createWorkThread(long seed) { - return new Thread(() -> { - long result = doWork(seed, threadLifetimeMillis); - totalWork.addAndGet(result); - }); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputThreadChurnBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputThreadChurnBenchmark.java deleted file mode 100644 index ef05ba739..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ProfilerThroughputThreadChurnBenchmark.java +++ /dev/null @@ -1,179 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.stresstest.scenarios.throughput; - -import com.datadoghq.profiler.JavaProfiler; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; - -/** - * Benchmark for profiler throughput under high thread churn. - * - *

This benchmark measures end-to-end profiling overhead with frequent - * thread creation and destruction. Key components tested: - * - *

    - *
  • Signal handler interrupts (cpu/wall profiling) - *
  • Stack walking via ASGCT - *
  • RefCountGuard slot allocation (prime probing with up to 32 attempts) - *
  • CallTraceStorage::put() operations (5 atomic ops per call) - *
  • JFR background processing - *
  • Slot cleanup and reuse efficiency - *
- * - *

Each short-lived thread: - * 1. Acquires a RefCountGuard slot via getThreadRefCountSlot() - * 2. Performs work that may be sampled by the profiler (full stack walking) - * 3. Releases the slot in the destructor - * - *

High thread churn tests whether: - * - Slots are efficiently reused - * - Prime probing remains effective under contention - * - The system avoids slot exhaustion (MAX_THREADS=8192) - */ -@State(Scope.Benchmark) -public class ProfilerThroughputThreadChurnBenchmark { - - @Param({"cpu=100us,wall=100us"}) - public String command; - - @Param({"false"}) - public String skipResults; - - @Param({"1", "5", "10"}) - private int threadLifetimeMillis; - - @Param({"10", "50", "100"}) - private int concurrentThreads; - - private JavaProfiler profiler; - private final AtomicLong totalWork = new AtomicLong(0); - - @Setup(Level.Trial) - public void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - totalWork.set(0); - } - - /** - * Work performed by each short-lived thread. - * Designed to generate stack samples while avoiding excessive CPU time. - */ - private long doShortLivedWork(long seed) { - long result = seed; - long endTime = System.nanoTime() + TimeUnit.MILLISECONDS.toNanos(threadLifetimeMillis); - - while (System.nanoTime() < endTime) { - // Light work to keep thread alive and generate samples - for (int i = 0; i < 1000; i++) { - result = (result * 1103515245L + 12345L) & 0x7fffffffL; - } - } - return result; - } - - /** - * Spawns a batch of short-lived threads and waits for them to complete. - * Measures throughput in terms of thread batches processed per second. - */ - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 3, time = 3) - @Measurement(iterations = 5, time = 10) - @OutputTimeUnit(TimeUnit.SECONDS) - public void threadChurn(Blackhole bh) throws InterruptedException { - List threads = new ArrayList<>(concurrentThreads); - - // Spawn concurrent short-lived threads - for (int i = 0; i < concurrentThreads; i++) { - final long seed = System.nanoTime() + i; - Thread thread = new Thread(() -> { - long result = doShortLivedWork(seed); - totalWork.addAndGet(result); - }); - thread.start(); - threads.add(thread); - } - - // Wait for all threads to complete - for (Thread thread : threads) { - thread.join(); - } - - bh.consume(totalWork.get()); - } - - /** - * Mixed workload: background long-lived threads plus continuous short-lived thread spawning. - * Tests realistic scenario where stable workers coexist with transient threads. - */ - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 1) - @Warmup(iterations = 3, time = 3) - @Measurement(iterations = 5, time = 10) - @OutputTimeUnit(TimeUnit.SECONDS) - public void mixedWorkload(Blackhole bh) throws InterruptedException { - // Background work simulating long-lived threads - long backgroundWork = 0; - for (int i = 0; i < 10_000; i++) { - backgroundWork = (backgroundWork * 1103515245L + 12345L) & 0x7fffffffL; - } - - // Spawn short-lived threads concurrently with background work - List threads = new ArrayList<>(concurrentThreads / 2); - for (int i = 0; i < concurrentThreads / 2; i++) { - final long seed = System.nanoTime() + i; - Thread thread = new Thread(() -> { - long result = doShortLivedWork(seed); - totalWork.addAndGet(result); - }); - thread.start(); - threads.add(thread); - } - - // More background work while short-lived threads execute - for (int i = 0; i < 10_000; i++) { - backgroundWork = (backgroundWork * 1103515245L + 12345L) & 0x7fffffffL; - } - - // Wait for short-lived threads - for (Thread thread : threads) { - thread.join(); - } - - bh.consume(backgroundWork); - bh.consume(totalWork.get()); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java deleted file mode 100644 index dbd4eec4c..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.stresstest.scenarios.throughput; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.ThreadContext; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; - -/** - * Benchmarks for ThreadContext operations — measures the per-call cost of - * context set/get/attribute operations on the span start/end hot path. - * - *

Run: ./gradlew :ddprof-stresstest:jmh -PjmhInclude="ThreadContextBenchmark" - * - *

Run with different JAVA_HOME to compare JNI (17+) vs ByteBuffer (<17) paths. - */ -@BenchmarkMode(Mode.AverageTime) -@OutputTimeUnit(TimeUnit.NANOSECONDS) -@Fork(value = 2, warmups = 0) -@Warmup(iterations = 3, time = 1) -@Measurement(iterations = 5, time = 2) -public class ThreadContextBenchmark { - - private static final String[] ROUTES = { - "GET /api/users", "POST /api/orders", "GET /api/health", - "PUT /api/users/{id}", "DELETE /api/sessions" - }; - - @State(Scope.Benchmark) - public static class ProfilerState { - JavaProfiler profiler; - - @Setup(Level.Trial) - public void setup() throws Exception { - profiler = JavaProfiler.getInstance(); - Path jfr = Files.createTempFile("bench", ".jfr"); - profiler.execute("start,cpu=10ms,attributes=http.route,jfr,file=" + jfr.toAbsolutePath()); - } - } - - @State(Scope.Thread) - public static class ThreadState { - ThreadContext ctx; - long spanId; - long localRootSpanId; - long traceIdLow; - int counter; - - @Setup(Level.Trial) - public void setup(ProfilerState ps) { - ctx = ps.profiler.getThreadContext(); - spanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - localRootSpanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - traceIdLow = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - } - } - - @State(Scope.Thread) - public static class ReapplyState { - ThreadContext ctx; - long spanId; - long localRootSpanId; - long traceIdLow; - int[] constantIds; - byte[][] utf8; - - @Setup(Level.Trial) - public void setup(ProfilerState ps) { - ctx = ps.profiler.getThreadContext(); - spanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - localRootSpanId = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - traceIdLow = ThreadLocalRandom.current().nextLong(1, Long.MAX_VALUE); - - // Prime the normal path to obtain constant IDs, then snapshot for reapply. - ctx.put(localRootSpanId, spanId, 0, traceIdLow); - for (int i = 0; i < ROUTES.length; i++) { - ctx.setContextAttribute(i, ROUTES[i]); - } - constantIds = new int[10]; - ctx.copyCustoms(constantIds); - utf8 = new byte[10][]; - for (int i = 0; i < ROUTES.length; i++) { - utf8[i] = ROUTES[i].getBytes(StandardCharsets.UTF_8); - } - } - - @Setup(Level.Iteration) - public void resetToSteadyState() { - // Re-establish a live span (valid=1) and pre-populate attrs_data with all slots - // before each measurement window. Without this, reapplyByIdAndBytes sees a - // different attrs_data state on the first invocation of each iteration (empty - // after put() in the previous iteration or after the trial setup), causing a - // bimodal distribution across forks due to JIT profile divergence. - ctx.put(localRootSpanId, spanId, 0, traceIdLow); - if (!ctx.setContextAttributesByIdAndBytes(constantIds, utf8)) { - throw new IllegalStateException( - "resetToSteadyState: setContextAttributesByIdAndBytes failed; benchmark state invalid"); - } - } - } - - @Benchmark - public void setContextFull(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - } - - @Benchmark - public boolean setAttrCacheHit(ThreadState ts) { - return ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - public void spanLifecycle(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - @Threads(2) - public void setContextFull_2t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - } - - @Benchmark - @Threads(4) - public void setContextFull_4t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - } - - @Benchmark - @Threads(2) - public void spanLifecycle_2t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - @Threads(4) - public void spanLifecycle_4t(ThreadState ts) { - ts.ctx.put(ts.localRootSpanId, ts.spanId, 0, ts.traceIdLow); - ts.ctx.setContextAttribute(0, ROUTES[ts.counter++ % ROUTES.length]); - } - - @Benchmark - public long getSpanId(ThreadState ts) { - return ts.ctx.getSpanId(); - } - - @Benchmark - public void clearContext(ThreadState ts) { - ts.ctx.put(0, 0, 0, 0); - } - - /** Bare reapply cost with constant IDs and bytes already in hand — no Dictionary lookup. */ - @Benchmark - public boolean reapplyByIdAndBytes(ReapplyState rs) { - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } - - /** Full reapply cycle: span activation wipes slots, then reapply restores them. */ - @Benchmark - public boolean reapplyCycle(ReapplyState rs) { - rs.ctx.put(rs.localRootSpanId, rs.spanId, 0, rs.spanId); - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } - - @Benchmark - @Threads(2) - public boolean reapplyCycle_2t(ReapplyState rs) { - rs.ctx.put(rs.localRootSpanId, rs.spanId, 0, rs.spanId); - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } - - @Benchmark - @Threads(4) - public boolean reapplyCycle_4t(ReapplyState rs) { - rs.ctx.put(rs.localRootSpanId, rs.spanId, 0, rs.spanId); - return rs.ctx.setContextAttributesByIdAndBytes(rs.constantIds, rs.utf8); - } -} diff --git a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadFilterBenchmark.java b/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadFilterBenchmark.java deleted file mode 100644 index f56bd258b..000000000 --- a/ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadFilterBenchmark.java +++ /dev/null @@ -1,125 +0,0 @@ -package com.datadoghq.profiler.stresstest.scenarios.throughput; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.stresstest.Configuration; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Level; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Param; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.TearDown; -import org.openjdk.jmh.annotations.Threads; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.Blackhole; - -import java.io.IOException; -import java.util.concurrent.TimeUnit; - -@State(Scope.Benchmark) -public class ThreadFilterBenchmark extends Configuration { - private JavaProfiler profiler; - - @Param(BASE_COMMAND + ",filter=1") - public String command; - - @Param("true") - public String skipResults; - - @Param({"0", "7", "70000"}) - public String workload; - - private long workloadNum = 0; - - @Setup(Level.Trial) - public void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - workloadNum = Long.parseLong(workload); - // Start profiling to enable JVMTI ThreadStart callbacks - // Add JFR file parameter to satisfy profiler requirements - profiler.execute("start," + command + ",jfr,file=/tmp/thread-filter-benchmark.jfr"); - } - - @TearDown(Level.Trial) - public void tearDown() throws IOException { - // Stop profiling and clean up - if (profiler != null) { - profiler.execute("stop"); - } - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 3) - @Warmup(iterations = 5) - @Measurement(iterations = 8) - @Threads(1) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void threadFilterStress01() throws InterruptedException { - profiler.addThread(); - // Simulate per-thread work - Blackhole.consumeCPU(workloadNum); - profiler.removeThread(); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 3) - @Warmup(iterations = 5) - @Measurement(iterations = 8) - @Threads(2) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void threadFilterStress02() throws InterruptedException { - profiler.addThread(); - // Simulate per-thread work - Blackhole.consumeCPU(workloadNum); - profiler.removeThread(); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 3) - @Warmup(iterations = 5) - @Measurement(iterations = 8) - @Threads(4) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void threadFilterStress04() throws InterruptedException { - profiler.addThread(); - // Simulate per-thread work - Blackhole.consumeCPU(workloadNum); - profiler.removeThread(); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 3) - @Warmup(iterations = 5) - @Measurement(iterations = 8) - @Threads(8) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void threadFilterStress08() throws InterruptedException { - profiler.addThread(); - // Simulate per-thread work - Blackhole.consumeCPU(workloadNum); - profiler.removeThread(); - } - - @Benchmark - @BenchmarkMode(Mode.Throughput) - @Fork(value = 3, warmups = 3) - @Warmup(iterations = 5) - @Measurement(iterations = 8) - @Threads(16) - @OutputTimeUnit(TimeUnit.MILLISECONDS) - public void threadFilterStress16() throws InterruptedException { - profiler.addThread(); - // Simulate per-thread work - Blackhole.consumeCPU(workloadNum); - profiler.removeThread(); - } -} diff --git a/ddprof-test-native/build.gradle.kts b/ddprof-test-native/build.gradle.kts deleted file mode 100644 index 969a92629..000000000 --- a/ddprof-test-native/build.gradle.kts +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Native test helper libraries (JNI helpers for Java tests). - */ - -import com.datadoghq.native.model.Platform -import com.datadoghq.native.util.PlatformUtils - -plugins { - id("com.datadoghq.simple-native-lib") -} - -description = "Native test helper libraries (JNI helpers for Java tests)" - -simpleNativeLib { - libraryName.set("ddproftest") - - // Use C compiler (not C++) for .c files - compiler.set(if (PlatformUtils.currentPlatform == Platform.MACOS) "clang" else "gcc") - linker.set(if (PlatformUtils.currentPlatform == Platform.MACOS) "clang" else "gcc") - - includeJni.set(true) - - // Note: No optimization (-O0) to prevent inlining of static functions like do_primes() - // which need to be visible in stack traces for profiler testing - compilerArgs.set( - when (PlatformUtils.currentPlatform) { - Platform.LINUX -> listOf("-fPIC") - Platform.MACOS -> emptyList() - }, - ) - - linkerArgs.set( - when (PlatformUtils.currentPlatform) { - Platform.LINUX -> listOf("-shared", "-Wl,--build-id") - Platform.MACOS -> listOf("-dynamiclib") - }, - ) - - // Create consumable configurations for other projects to depend on - createConfigurations.set(true) -} diff --git a/ddprof-test-native/src/main/cpp/nativealloc.c b/ddprof-test-native/src/main/cpp/nativealloc.c deleted file mode 100644 index 2fb080523..000000000 --- a/ddprof-test-native/src/main/cpp/nativealloc.c +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -/* Drive malloc through libddproftest.so's PLT, which is patched by MallocTracer. - * ByteBuffer.allocateDirect() routes through libjvm.so which may use -Bsymbolic-functions, - * binding malloc internally and bypassing GOT patching entirely. */ -JNIEXPORT void JNICALL -Java_com_datadoghq_profiler_nativemem_NativeAllocHelper_nativeMalloc( - JNIEnv *env, jclass clazz, jlong size, jint count) { - for (jint i = 0; i < count; i++) { - void *p = malloc((size_t)size); - if (p != NULL) { - free(p); - } - } -} diff --git a/ddprof-test-native/src/main/cpp/nativethread.c b/ddprof-test-native/src/main/cpp/nativethread.c deleted file mode 100644 index 18d711d3f..000000000 --- a/ddprof-test-native/src/main/cpp/nativethread.c +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -#include -#include - -#include - -#define MAX_PRIME 100000 - -// Burn CPU -static void do_primes() { - unsigned long i, num, primes = 0; - for (num = 1; num <= MAX_PRIME; ++num) { - for (i = 2; (i <= num) && (num % i != 0); ++i); - if (i == num) - ++primes; - } -} - -// Function to be executed by the new thread -void* thread_function(void* arg) { - do_primes(); - pthread_exit(NULL); // Terminate the thread, optionally returning a value -} - -jlong JNICALL Java_com_datadoghq_profiler_nativethread_NativeThreadCreator_createNativeThread - (JNIEnv * env, jclass clz) { - - // Create a new thread - // Arguments: - // 1. &thread_id: Pointer to the pthread_t variable where the new thread's ID will be stored. - // 2. NULL: Pointer to thread attributes (using default attributes here). - // 3. thread_function: Pointer to the function the new thread will execute. - // 4. NULL: Pointer to the argument to pass to the thread_function. - pthread_t thread_id; - int result = pthread_create(&thread_id, NULL, thread_function, NULL); - - if (result != 0) { - perror("Error creating thread"); - return -1L; - } - - return (jlong) thread_id; -} - -void JNICALL Java_com_datadoghq_profiler_nativethread_NativeThreadCreator_waitNativeThread - (JNIEnv * env, jclass clz, jlong threadId) { - pthread_t thread_id = (pthread_t)threadId; - // Wait for the created thread to finish - // Arguments: - // 1. thread_id: The ID of the thread to wait for. - // 2. NULL: Pointer to store the return value of the joined thread (not used here). - pthread_join(thread_id, NULL); -} \ No newline at end of file diff --git a/ddprof-test-native/src/main/cpp/remotesym.c b/ddprof-test-native/src/main/cpp/remotesym.c deleted file mode 100644 index 7f4c647bf..000000000 --- a/ddprof-test-native/src/main/cpp/remotesym.c +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -/** - * Recursive CPU-burning function that creates a distinct call stack. - * This ensures native frames from this library appear in profiling samples. - */ -static uint64_t burn_cpu_recursive(uint64_t n, uint64_t depth) { - if (depth == 0) { - // Base case: perform actual computation - uint64_t sum = 0; - for (uint64_t i = 0; i < n; i++) { - sum += i * i; - } - return sum; - } - - // Recursive case: go deeper - return burn_cpu_recursive(n, depth - 1) + depth; -} - -/** - * Entry point for CPU-burning work. - * Called from Java to generate CPU samples with this library on the stack. - */ -JNIEXPORT jlong JNICALL -Java_com_datadoghq_profiler_RemoteSymHelper_burnCpu(JNIEnv *env, jclass clazz, jlong iterations, jint depth) { - return (jlong)burn_cpu_recursive((uint64_t)iterations, (uint32_t)depth); -} - -/** - * Additional function to create more stack depth. - */ -static uint64_t compute_fibonacci(uint32_t n) { - if (n <= 1) return n; - - uint64_t a = 0, b = 1; - for (uint32_t i = 2; i <= n; i++) { - uint64_t temp = a + b; - a = b; - b = temp; - } - return b; -} - -/** - * Another entry point with different stack signature. - */ -JNIEXPORT jlong JNICALL -Java_com_datadoghq_profiler_RemoteSymHelper_computeFibonacci(JNIEnv *env, jclass clazz, jint n) { - // Call multiple times to increase likelihood of sampling - uint64_t result = 0; - for (int i = 0; i < 1000; i++) { - result += compute_fibonacci((uint32_t)n); - } - return (jlong)result; -} diff --git a/ddprof-test-tracer/build.gradle.kts b/ddprof-test-tracer/build.gradle.kts deleted file mode 100644 index ed13dcd45..000000000 --- a/ddprof-test-tracer/build.gradle.kts +++ /dev/null @@ -1,8 +0,0 @@ -plugins { - java - id("com.datadoghq.java-conventions") -} - -dependencies { - implementation(project(mapOf("path" to ":ddprof-lib", "configuration" to "release"))) -} diff --git a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java deleted file mode 100644 index edded2ea9..000000000 --- a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextExecutor.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.datadoghq.profiler.context; - -import com.datadoghq.profiler.JavaProfiler; - -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.RunnableFuture; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -public class ContextExecutor extends ThreadPoolExecutor { - - private final JavaProfiler profiler; - public ContextExecutor(int corePoolSize, JavaProfiler profiler) { - super(corePoolSize, corePoolSize, 30, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(128), new RegisteringThreadFactory(profiler)); - this.profiler = profiler; - } - - @Override - protected RunnableFuture newTaskFor(Runnable runnable, T value) { - return ContextTask.wrap(runnable, value); - } - - @Override - protected void beforeExecute(Thread t, Runnable r) { - super.beforeExecute(t, r); - profiler.addThread(); - // Prime OTEL context TLS to avoid race condition with wall clock signals. - // TLS is lazily initialized on first setContext() call, which happens in - // ContextTask.run() after this method returns. If a wall clock signal - // arrives between now and then, the context would be uninitialized. - profiler.setContext(0, 0); - } - - @Override - protected void afterExecute(Runnable r, Throwable t) { - profiler.removeThread(); - super.afterExecute(r, t); - } -} diff --git a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextTask.java b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextTask.java deleted file mode 100644 index 62c99876e..000000000 --- a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/ContextTask.java +++ /dev/null @@ -1,24 +0,0 @@ -package com.datadoghq.profiler.context; - -import java.util.concurrent.FutureTask; - -public class ContextTask extends FutureTask { - - public static FutureTask wrap(Runnable task, T value) { - return new ContextTask<>(Tracing.capture(), task, value); - } - - public ContextTask(Tracing.MigratingContext context, Runnable task, T value) { - super(task, value); - this.context = context; - } - - @Override - public void run() { - try (Tracing.Context activation = this.context.activate()) { - super.run(); - } - } - - private final Tracing.MigratingContext context; -} diff --git a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/RegisteringThreadFactory.java b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/RegisteringThreadFactory.java deleted file mode 100644 index 5c52686ff..000000000 --- a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/RegisteringThreadFactory.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.datadoghq.profiler.context; - -import com.datadoghq.profiler.JavaProfiler; - -import java.util.concurrent.ThreadFactory; - -final class RegisteringThreadFactory implements ThreadFactory { - private final JavaProfiler profiler; - - RegisteringThreadFactory(JavaProfiler profiler) { - this.profiler = profiler; - } - - @Override - public Thread newThread(Runnable task) { - Thread thread = new Thread(() -> { - profiler.addThread(); - task.run(); - }); - thread.setDaemon(true); - return thread; - } -} diff --git a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java b/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java deleted file mode 100644 index 18a15a471..000000000 --- a/ddprof-test-tracer/src/main/java/com/datadoghq/profiler/context/Tracing.java +++ /dev/null @@ -1,114 +0,0 @@ -package com.datadoghq.profiler.context; - -import com.datadoghq.profiler.JavaProfiler; - -import java.util.ArrayDeque; -import java.util.Deque; -import java.util.Optional; -import java.util.function.LongSupplier; - -public class Tracing { - - private static final ThreadLocal STACK = ThreadLocal.withInitial(Tracing::new); - - public static Context newContext(LongSupplier idSupplier, JavaProfiler profiler) { - return STACK.get().create(idSupplier, profiler); - } - - public static MigratingContext capture() { - return STACK.get().startMigration(); - } - - private final Deque stack = new ArrayDeque<>(); - - private Context create(LongSupplier supplier, JavaProfiler profiler) { - long id = supplier.getAsLong(); - long rootSpanId = Optional.ofNullable(stack.peekLast()).map(Context::getRootSpanId).orElse(id); - Context context = new Context(profiler, stack, rootSpanId, id); - stack.addFirst(context); - return context; - } - - public MigratingContext startMigration() { - return Optional.ofNullable(stack.peekFirst()).map(Context::snapshot).orElse(null); - } - - Context endMigration(MigratingContext context) { - Context activated = new Context(this.stack, context); - stack.addFirst(activated); - return activated; - } - - public static class MigratingContext { - - private final JavaProfiler profiler; - private final long rootSpanId; - private final long spanId; - - public MigratingContext(JavaProfiler profiler, long rootSpanId, long spanId) { - this.profiler = profiler; - this.rootSpanId = rootSpanId; - this.spanId = spanId; - } - - public Context activate() { - Context context = STACK.get().endMigration(this); - context.notifyProfiler(); - return context; - } - } - - public static class Context implements AutoCloseable { - - private final JavaProfiler profiler; - private final Deque stack; - private final long rootSpanId; - private final long spanId; - - public Context(Deque stack, MigratingContext context) { - this(context.profiler, stack, context.rootSpanId, context.spanId); - } - - public Context(JavaProfiler profiler, Deque stack, long rootSpanId, long spanId) { - this.stack = stack; - this.rootSpanId = rootSpanId; - this.spanId = spanId; - this.profiler = profiler; - notifyProfiler(); - } - - public MigratingContext snapshot() { - return new MigratingContext(profiler, rootSpanId, spanId); - } - - private void notifyProfiler() { - profiler.setContext(spanId, rootSpanId); - } - - public long getRootSpanId() { - return rootSpanId; - } - - public long getSpanId() { - return spanId; - } - - @Override - public void close() { - if (stack != null) { - Context top = stack.removeFirst(); - assert top == this : "expected " + this + " on stack but have " + top; - Optional.ofNullable(stack.peekFirst()).ifPresent(Context::notifyProfiler); - } - } - - - @Override - public String toString() { - return "Context{" + - "rootSpanId=" + rootSpanId + - ", spanId=" + spanId + - '}'; - } - } -} diff --git a/ddprof-test/build.gradle.kts b/ddprof-test/build.gradle.kts deleted file mode 100644 index 035e13037..000000000 --- a/ddprof-test/build.gradle.kts +++ /dev/null @@ -1,76 +0,0 @@ -import com.datadoghq.profiler.ProfilerTestExtension - -plugins { - java - `java-library` - application - id("com.datadoghq.profiler-test") - id("com.datadoghq.java-conventions") -} - -// Reference to native test helpers library directory -val testNativeLibDir = project(":ddprof-test-native").layout.buildDirectory.dir("lib") - -// Configure profiler test plugin - this generates all multi-config tasks automatically -configure { - // Native library path for JNI test helpers - nativeLibDir.set(testNativeLibDir) - - // Enable multi-config task generation - profilerLibProject.set(":ddprof-lib") - - // Extra JVM args specific to this project's tests - extraJvmArgs.addAll( - "-Dddprof.disable_unsafe=true", - "-XX:OnError=/tmp/do_stuff.sh", - ) -} - -// Generate JNI headers using javac -val jniHeadersDir = layout.buildDirectory.dir("generated/jni-headers") -tasks.named("compileJava") { - options.compilerArgs.addAll(listOf("-h", jniHeadersDir.get().asFile.absolutePath)) -} - -// Application configuration (for the run task) -application { - mainClass.set("com.datadoghq.profiler.unwinding.UnwindingValidator") -} - -// Add common dependencies to test and main configurations -// The plugin creates testCommon and mainCommon configurations eagerly -dependencies { - // Test dependencies - "testCommon"(libs.bundles.testing) - "testCommon"(libs.bundles.profiler.runtime) - "testCommon"(libs.asm) - - // Main/application dependencies - "mainCommon"(libs.slf4j.simple) - "mainCommon"(libs.bundles.profiler.runtime) -} - -// Additional test task configuration beyond what the plugin provides -// The plugin creates Test tasks on glibc/macOS and Exec tasks on musl -// Both need the native test library to be built first -tasks.matching { it.name.startsWith("test") && it.name != "test" }.configureEach { - // Ensure native test library is built before running tests - dependsOn(":ddprof-test-native:linkLib") -} - -// Disable the default 'test' task - we use config-specific tasks instead -tasks.named("test") { - onlyIf { false } -} - -// Java compilation settings handled by java-conventions plugin (--release 8) - -// Ensure compileTestJava has access to the test dependencies for compilation -// (must be set after project evaluation when the configuration is created) -gradle.projectsEvaluated { - configurations.findByName("testReleaseImplementation")?.let { testReleaseCfg -> - tasks.withType().configureEach { - classpath += testReleaseCfg - } - } -} diff --git a/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/TestResult.java b/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/TestResult.java deleted file mode 100644 index 3f9902825..000000000 --- a/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/TestResult.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.unwinding; - -/** - * Standardized result object for unwinding validation tests. - * Provides consistent structure for reporting test outcomes across different scenarios. - */ -public class TestResult { - - public enum Status { - EXCELLENT("🟢", "Excellent unwinding quality"), - GOOD("🟢", "Good unwinding quality"), - MODERATE("🟡", "Moderate unwinding quality - improvement recommended"), - NEEDS_WORK("🔴", "Poor unwinding quality - requires attention"); - - private final String indicator; - private final String description; - - Status(String indicator, String description) { - this.indicator = indicator; - this.description = description; - } - - public String getIndicator() { return indicator; } - public String getDescription() { return description; } - } - - private final String testName; - private final String scenarioDescription; - private final UnwindingMetrics.UnwindingResult metrics; - private final Status status; - private final String statusMessage; - private final long executionTimeMs; - - public TestResult(String testName, String scenarioDescription, - UnwindingMetrics.UnwindingResult metrics, - Status status, String statusMessage, long executionTimeMs) { - this.testName = testName; - this.scenarioDescription = scenarioDescription; - this.metrics = metrics; - this.status = status; - this.statusMessage = statusMessage; - this.executionTimeMs = executionTimeMs; - } - - public String getTestName() { return testName; } - public String getScenarioDescription() { return scenarioDescription; } - public UnwindingMetrics.UnwindingResult getMetrics() { return metrics; } - public Status getStatus() { return status; } - public String getStatusMessage() { return statusMessage; } - public long getExecutionTimeMs() { return executionTimeMs; } - - /** - * Determine test status based on error rate and other quality metrics. - */ - public static Status determineStatus(UnwindingMetrics.UnwindingResult result) { - double errorRate = result.getErrorRate(); - - if (errorRate < 0.1) { - return Status.EXCELLENT; - } else if (errorRate < 1.0) { - return Status.GOOD; - } else if (errorRate < 5.0) { - return Status.MODERATE; - } else { - return Status.NEEDS_WORK; - } - } - - /** - * Generate appropriate status message based on metrics. - */ - public static String generateStatusMessage(UnwindingMetrics.UnwindingResult result, Status status) { - StringBuilder sb = new StringBuilder(); - - switch (status) { - case EXCELLENT: - sb.append("Error rate < 0.1% - exceptional unwinding quality"); - break; - case GOOD: - sb.append("Error rate < 1.0% - good unwinding performance"); - break; - case MODERATE: - sb.append("Error rate ").append(String.format("%.2f%%", result.getErrorRate())) - .append(" - moderate, consider optimization"); - break; - case NEEDS_WORK: - sb.append("Error rate ").append(String.format("%.2f%%", result.getErrorRate())) - .append(" - requires investigation"); - break; - } - - // Add specific issue highlights for problematic cases - if (result.errorSamples > 0 && (status == Status.MODERATE || status == Status.NEEDS_WORK)) { - if (!result.errorTypeBreakdown.isEmpty()) { - sb.append(" (").append(result.errorTypeBreakdown.keySet().iterator().next()).append(")"); - } - } - - return sb.toString(); - } - - /** - * Create a TestResult from metrics with automatic status determination. - */ - public static TestResult create(String testName, String scenarioDescription, - UnwindingMetrics.UnwindingResult metrics, - long executionTimeMs) { - Status status = determineStatus(metrics); - String statusMessage = generateStatusMessage(metrics, status); - return new TestResult(testName, scenarioDescription, metrics, status, statusMessage, executionTimeMs); - } - - @Override - public String toString() { - return String.format("TestResult{name='%s', status=%s, errorRate=%.2f%%, samples=%d}", - testName, status, metrics.getErrorRate(), metrics.totalSamples); - } -} \ No newline at end of file diff --git a/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/UnwindingDashboard.java b/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/UnwindingDashboard.java deleted file mode 100644 index 3bf8e0dae..000000000 --- a/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/UnwindingDashboard.java +++ /dev/null @@ -1,375 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.unwinding; - -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Unified dashboard for displaying unwinding test results in a consistent, - * easy-to-scan format. Replaces scattered console output with structured reporting. - */ -public class UnwindingDashboard { - - /** - * Generate a comprehensive dashboard report for all test results. - */ - public static String generateReport(List results) { - if (results.isEmpty()) { - return "=== No Test Results Available ===\n"; - } - - StringBuilder sb = new StringBuilder(); - - // Header - sb.append("=== Unwinding Quality Dashboard ===\n"); - generateSummaryTable(sb, results); - - // Overall assessment - generateOverallAssessment(sb, results); - - // Detailed breakdowns for problematic tests - generateDetailedBreakdowns(sb, results); - - // Performance summary - generatePerformanceSummary(sb, results); - - return sb.toString(); - } - - private static void generateSummaryTable(StringBuilder sb, List results) { - sb.append("\n"); - sb.append(String.format("%-35s | %6s | %8s | %10s | %12s | %s\n", - "Test Scenario", "Status", "Error%", "Samples", "Native%", "Execution")); - sb.append(String.format("%-35s-|-%6s-|-%8s-|-%10s-|-%12s-|-%s\n", - "-----------------------------------", "------", "--------", "----------", "------------", "----------")); - - for (TestResult result : results) { - UnwindingMetrics.UnwindingResult metrics = result.getMetrics(); - - sb.append(String.format("%-35s | %4s | %7.2f%% | %10d | %12.1f%% | %7dms\n", - truncateTestName(result.getTestName()), - result.getStatus().getIndicator(), - metrics.getErrorRate(), - metrics.totalSamples, - metrics.getNativeRate(), - result.getExecutionTimeMs())); - } - } - - private static void generateOverallAssessment(StringBuilder sb, List results) { - sb.append("\n=== Overall Assessment ===\n"); - - long excellentCount = results.stream().mapToLong(r -> r.getStatus() == TestResult.Status.EXCELLENT ? 1 : 0).sum(); - long goodCount = results.stream().mapToLong(r -> r.getStatus() == TestResult.Status.GOOD ? 1 : 0).sum(); - long moderateCount = results.stream().mapToLong(r -> r.getStatus() == TestResult.Status.MODERATE ? 1 : 0).sum(); - long needsWorkCount = results.stream().mapToLong(r -> r.getStatus() == TestResult.Status.NEEDS_WORK ? 1 : 0).sum(); - - double avgErrorRate = results.stream() - .mapToDouble(r -> r.getMetrics().getErrorRate()) - .average() - .orElse(0.0); - - int totalSamples = results.stream() - .mapToInt(r -> r.getMetrics().totalSamples) - .sum(); - - int totalErrors = results.stream() - .mapToInt(r -> r.getMetrics().errorSamples) - .sum(); - - sb.append(String.format("Tests: %d excellent, %d good, %d moderate, %d needs work\n", - excellentCount, goodCount, moderateCount, needsWorkCount)); - sb.append(String.format("Overall: %.3f%% average error rate (%d errors / %d samples)\n", - avgErrorRate, totalErrors, totalSamples)); - - // Overall quality assessment - if (needsWorkCount > 0) { - sb.append("🔴 ATTENTION: Some scenarios require investigation\n"); - } else if (moderateCount > 0) { - sb.append("🟡 MODERATE: Good overall quality, some optimization opportunities\n"); - } else { - sb.append("🟢 EXCELLENT: All unwinding scenarios performing well\n"); - } - } - - private static void generateDetailedBreakdowns(StringBuilder sb, List results) { - List problematicResults = results.stream() - .filter(r -> r.getStatus() == TestResult.Status.MODERATE || - r.getStatus() == TestResult.Status.NEEDS_WORK) - .collect(Collectors.toList()); - - if (problematicResults.isEmpty()) { - return; - } - - sb.append("\n=== Issue Details ===\n"); - - for (TestResult result : problematicResults) { - sb.append(String.format("\n%s %s:\n", result.getStatus().getIndicator(), result.getTestName())); - sb.append(String.format(" %s\n", result.getStatusMessage())); - - UnwindingMetrics.UnwindingResult metrics = result.getMetrics(); - - // Show error breakdown if available - if (!metrics.errorTypeBreakdown.isEmpty()) { - sb.append(" Error types: "); - metrics.errorTypeBreakdown.forEach((type, count) -> - sb.append(String.format("%s:%d ", type, count))); - sb.append("\n"); - } - - // Show stub coverage if relevant - if (metrics.stubSamples > 0 && !metrics.stubTypeBreakdown.isEmpty()) { - sb.append(" Stub types: "); - metrics.stubTypeBreakdown.forEach((type, count) -> - sb.append(String.format("%s:%d ", type, count))); - sb.append("\n"); - } - - // Key metrics - if (metrics.nativeSamples > 0) { - sb.append(String.format(" Native coverage: %d/%d samples (%.1f%%)\n", - metrics.nativeSamples, metrics.totalSamples, metrics.getNativeRate())); - } - } - } - - private static void generatePerformanceSummary(StringBuilder sb, List results) { - sb.append("\n=== Test Execution Summary ===\n"); - - long totalExecutionTime = results.stream().mapToLong(TestResult::getExecutionTimeMs).sum(); - long maxExecutionTime = results.stream().mapToLong(TestResult::getExecutionTimeMs).max().orElse(0); - String slowestTest = results.stream() - .filter(r -> r.getExecutionTimeMs() == maxExecutionTime) - .map(TestResult::getTestName) - .findFirst() - .orElse("unknown"); - - sb.append(String.format("Total execution: %d seconds\n", totalExecutionTime / 1000)); - sb.append(String.format("Slowest test: %s (%d seconds)\n", truncateTestName(slowestTest), maxExecutionTime / 1000)); - - // Test coverage summary - int totalSamples = results.stream().mapToInt(r -> r.getMetrics().totalSamples).sum(); - int totalNativeSamples = results.stream().mapToInt(r -> r.getMetrics().nativeSamples).sum(); - int totalStubSamples = results.stream().mapToInt(r -> r.getMetrics().stubSamples).sum(); - - sb.append(String.format("Sample coverage: %d total, %d native (%.1f%%), %d stub (%.1f%%)\n", - totalSamples, - totalNativeSamples, totalSamples > 0 ? (double) totalNativeSamples / totalSamples * 100 : 0.0, - totalStubSamples, totalSamples > 0 ? (double) totalStubSamples / totalSamples * 100 : 0.0)); - } - - private static String truncateTestName(String testName) { - if (testName.length() <= 35) { - return testName; - } - return testName.substring(0, 32) + "..."; - } - - /** - * Generate a compact single-line summary suitable for CI logs. - */ - public static String generateCompactSummary(List results) { - if (results.isEmpty()) { - return "UNWINDING: No tests executed"; - } - - long problemCount = results.stream() - .mapToLong(r -> (r.getStatus() == TestResult.Status.MODERATE || - r.getStatus() == TestResult.Status.NEEDS_WORK) ? 1 : 0) - .sum(); - - double avgErrorRate = results.stream() - .mapToDouble(r -> r.getMetrics().getErrorRate()) - .average() - .orElse(0.0); - - int totalSamples = results.stream() - .mapToInt(r -> r.getMetrics().totalSamples) - .sum(); - - String status = problemCount == 0 ? "PASS" : "ISSUES"; - - return String.format("UNWINDING: %s - %d tests, %.3f%% avg error rate, %d samples, %d issues", - status, results.size(), avgErrorRate, totalSamples, problemCount); - } - - /** - * Generate a GitHub Actions Job Summary compatible markdown report. - */ - public static String generateMarkdownReport(List results) { - if (results.isEmpty()) { - return "## 🔍 Unwinding Quality Report\n\n❌ No test results available\n"; - } - - StringBuilder md = new StringBuilder(); - - // Header with timestamp and platform info - md.append("## 🔍 Unwinding Quality Report\n\n"); - md.append("**Generated**: ").append(java.time.Instant.now()).append(" \n"); - md.append("**Platform**: ").append(System.getProperty("os.name")) - .append(" ").append(System.getProperty("os.arch")).append(" \n"); - md.append("**Java**: ").append(System.getProperty("java.version")).append("\n\n"); - - // Overall status summary - generateMarkdownSummary(md, results); - - // Detailed results table - generateMarkdownResultsTable(md, results); - - // Issue details if any - generateMarkdownIssueDetails(md, results); - - // Performance footer - generateMarkdownPerformanceFooter(md, results); - - return md.toString(); - } - - private static void generateMarkdownSummary(StringBuilder md, List results) { - long excellentCount = results.stream().mapToLong(r -> r.getStatus() == TestResult.Status.EXCELLENT ? 1 : 0).sum(); - long goodCount = results.stream().mapToLong(r -> r.getStatus() == TestResult.Status.GOOD ? 1 : 0).sum(); - long moderateCount = results.stream().mapToLong(r -> r.getStatus() == TestResult.Status.MODERATE ? 1 : 0).sum(); - long needsWorkCount = results.stream().mapToLong(r -> r.getStatus() == TestResult.Status.NEEDS_WORK ? 1 : 0).sum(); - - double avgErrorRate = results.stream() - .mapToDouble(r -> r.getMetrics().getErrorRate()) - .average() - .orElse(0.0); - - int totalSamples = results.stream() - .mapToInt(r -> r.getMetrics().totalSamples) - .sum(); - - int totalErrors = results.stream() - .mapToInt(r -> r.getMetrics().errorSamples) - .sum(); - - // Summary section with badges - md.append("### 📊 Summary\n\n"); - - if (needsWorkCount > 0) { - md.append("🔴 **ATTENTION**: Some scenarios require investigation \n"); - } else if (moderateCount > 0) { - md.append("🟡 **MODERATE**: Good overall quality, optimization opportunities available \n"); - } else { - md.append("🟢 **EXCELLENT**: All unwinding scenarios performing well \n"); - } - - md.append("**Results**: "); - if (excellentCount > 0) md.append("🟢 ").append(excellentCount).append(" excellent "); - if (goodCount > 0) md.append("🟢 ").append(goodCount).append(" good "); - if (moderateCount > 0) md.append("🟡 ").append(moderateCount).append(" moderate "); - if (needsWorkCount > 0) md.append("🔴 ").append(needsWorkCount).append(" needs work "); - md.append(" \n"); - - md.append("**Error Rate**: ").append(String.format("%.3f%%", avgErrorRate)) - .append(" (").append(totalErrors).append(" errors / ").append(totalSamples).append(" samples) \n\n"); - } - - private static void generateMarkdownResultsTable(StringBuilder md, List results) { - md.append("### 🎯 Scenario Results\n\n"); - - md.append("| Scenario | Status | Error Rate | Samples | Native % | Duration |\n"); - md.append("|----------|--------|------------|---------|----------|---------|\n"); - - for (TestResult result : results) { - UnwindingMetrics.UnwindingResult metrics = result.getMetrics(); - - md.append("| ").append(truncateForTable(result.getTestName(), 25)) - .append(" | ").append(result.getStatus().getIndicator()) - .append(" | ").append(String.format("%.2f%%", metrics.getErrorRate())) - .append(" | ").append(String.format("%,d", metrics.totalSamples)) - .append(" | ").append(String.format("%.1f%%", metrics.getNativeRate())) - .append(" | ").append(String.format("%.1fs", result.getExecutionTimeMs() / 1000.0)) - .append(" |\n"); - } - - md.append("\n"); - } - - private static void generateMarkdownIssueDetails(StringBuilder md, List results) { - List problematicResults = results.stream() - .filter(r -> r.getStatus() == TestResult.Status.MODERATE || - r.getStatus() == TestResult.Status.NEEDS_WORK) - .collect(Collectors.toList()); - - if (problematicResults.isEmpty()) { - return; - } - - md.append("### ⚠️ Issues Requiring Attention\n\n"); - - for (TestResult result : problematicResults) { - UnwindingMetrics.UnwindingResult metrics = result.getMetrics(); - - md.append("#### ").append(result.getStatus().getIndicator()).append(" ") - .append(result.getTestName()).append("\n\n"); - md.append("**Issue**: ").append(result.getStatusMessage()).append(" \n"); - - if (!metrics.errorTypeBreakdown.isEmpty()) { - md.append("**Error types**: "); - metrics.errorTypeBreakdown.forEach((type, count) -> - md.append("`").append(truncateForTable(type, 30)).append("`:") - .append(count).append(" ")); - md.append(" \n"); - } - - if (metrics.nativeSamples > 0) { - md.append("**Native coverage**: ").append(metrics.nativeSamples) - .append("/").append(metrics.totalSamples) - .append(" (").append(String.format("%.1f%%", metrics.getNativeRate())).append(") \n"); - } - - md.append("\n"); - } - } - - private static void generateMarkdownPerformanceFooter(StringBuilder md, List results) { - long totalExecutionTime = results.stream().mapToLong(TestResult::getExecutionTimeMs).sum(); - long maxExecutionTime = results.stream().mapToLong(TestResult::getExecutionTimeMs).max().orElse(0); - String slowestTest = results.stream() - .filter(r -> r.getExecutionTimeMs() == maxExecutionTime) - .map(TestResult::getTestName) - .findFirst() - .orElse("unknown"); - - int totalSamples = results.stream().mapToInt(r -> r.getMetrics().totalSamples).sum(); - int totalNativeSamples = results.stream().mapToInt(r -> r.getMetrics().nativeSamples).sum(); - int totalStubSamples = results.stream().mapToInt(r -> r.getMetrics().stubSamples).sum(); - - md.append("---\n\n"); - md.append("**⚡ Performance**: ").append(String.format("%.1fs", totalExecutionTime / 1000.0)) - .append(" total execution time \n"); - md.append("**🐌 Slowest test**: ").append(truncateForTable(slowestTest, 20)) - .append(" (").append(String.format("%.1fs", maxExecutionTime / 1000.0)).append(") \n"); - md.append("**📈 Coverage**: ").append(String.format("%,d", totalSamples)).append(" total samples, ") - .append(String.format("%,d", totalNativeSamples)).append(" native (") - .append(String.format("%.1f%%", totalSamples > 0 ? (double) totalNativeSamples / totalSamples * 100 : 0.0)) - .append("), ").append(String.format("%,d", totalStubSamples)).append(" stub (") - .append(String.format("%.1f%%", totalSamples > 0 ? (double) totalStubSamples / totalSamples * 100 : 0.0)) - .append(") \n"); - } - - private static String truncateForTable(String text, int maxLength) { - if (text.length() <= maxLength) { - return text; - } - return text.substring(0, maxLength - 3) + "..."; - } -} \ No newline at end of file diff --git a/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/UnwindingMetrics.java b/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/UnwindingMetrics.java deleted file mode 100644 index 748eb34b9..000000000 --- a/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/UnwindingMetrics.java +++ /dev/null @@ -1,213 +0,0 @@ -package com.datadoghq.profiler.unwinding; - -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -/** - * Utility class for collecting and analyzing stub unwinding metrics from JFR data. - * Provides standardized measurement and comparison of stackwalking performance - * across different tests and configurations. - */ -public class UnwindingMetrics { - - public static class UnwindingResult { - public final int totalSamples; - public final int nativeSamples; - public final int errorSamples; - public final int stubSamples; - public final int pltSamples; - public final int jniSamples; - public final int reflectionSamples; - public final int jitSamples; - public final int methodHandleSamples; - public final Map errorTypeBreakdown; - public final Map stubTypeBreakdown; - - public UnwindingResult(int totalSamples, int nativeSamples, int errorSamples, - int stubSamples, int pltSamples, int jniSamples, - int reflectionSamples, int jitSamples, int methodHandleSamples, - Map errorTypeBreakdown, - Map stubTypeBreakdown) { - this.totalSamples = totalSamples; - this.nativeSamples = nativeSamples; - this.errorSamples = errorSamples; - this.stubSamples = stubSamples; - this.pltSamples = pltSamples; - this.jniSamples = jniSamples; - this.reflectionSamples = reflectionSamples; - this.jitSamples = jitSamples; - this.methodHandleSamples = methodHandleSamples; - this.errorTypeBreakdown = errorTypeBreakdown; - this.stubTypeBreakdown = stubTypeBreakdown; - } - - public double getErrorRate() { - return totalSamples > 0 ? (double) errorSamples / totalSamples * 100 : 0.0; - } - - public double getNativeRate() { - return totalSamples > 0 ? (double) nativeSamples / totalSamples * 100 : 0.0; - } - - public double getStubRate() { - return totalSamples > 0 ? (double) stubSamples / totalSamples * 100 : 0.0; - } - - public double getPLTRate() { - return totalSamples > 0 ? (double) pltSamples / totalSamples * 100 : 0.0; - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - sb.append("UnwindingResult{\n"); - sb.append(" totalSamples=").append(totalSamples).append("\n"); - sb.append(" errorSamples=").append(errorSamples).append(" (").append(String.format("%.2f%%", getErrorRate())).append(")\n"); - sb.append(" nativeSamples=").append(nativeSamples).append(" (").append(String.format("%.2f%%", getNativeRate())).append(")\n"); - sb.append(" stubSamples=").append(stubSamples).append(" (").append(String.format("%.2f%%", getStubRate())).append(")\n"); - sb.append(" pltSamples=").append(pltSamples).append(" (").append(String.format("%.2f%%", getPLTRate())).append(")\n"); - sb.append(" jniSamples=").append(jniSamples).append("\n"); - sb.append(" reflectionSamples=").append(reflectionSamples).append("\n"); - sb.append(" jitSamples=").append(jitSamples).append("\n"); - sb.append(" methodHandleSamples=").append(methodHandleSamples).append("\n"); - - if (!errorTypeBreakdown.isEmpty()) { - sb.append(" errorTypes=").append(errorTypeBreakdown).append("\n"); - } - if (!stubTypeBreakdown.isEmpty()) { - sb.append(" stubTypes=").append(stubTypeBreakdown).append("\n"); - } - sb.append("}"); - return sb.toString(); - } - } - - /** - * Analyze JFR execution samples and extract comprehensive unwinding metrics. - */ - public static UnwindingResult analyzeUnwindingData(Iterable cpuSamples, - IMemberAccessor modeAccessor) { - AtomicInteger totalSamples = new AtomicInteger(0); - AtomicInteger nativeSamples = new AtomicInteger(0); - AtomicInteger errorSamples = new AtomicInteger(0); - AtomicInteger stubSamples = new AtomicInteger(0); - AtomicInteger pltSamples = new AtomicInteger(0); - AtomicInteger jniSamples = new AtomicInteger(0); - AtomicInteger reflectionSamples = new AtomicInteger(0); - AtomicInteger jitSamples = new AtomicInteger(0); - AtomicInteger methodHandleSamples = new AtomicInteger(0); - - Map errorTypes = new HashMap<>(); - Map stubTypes = new HashMap<>(); - - for (IItemIterable samples : cpuSamples) { - IMemberAccessor stacktraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(samples.getType()); - - for (IItem item : samples) { - totalSamples.incrementAndGet(); - String stackTrace = stacktraceAccessor.getMember(item); - String mode = modeAccessor.getMember(item); - - if ("NATIVE".equals(mode)) { - nativeSamples.incrementAndGet(); - } - - if (containsJNIMethod(stackTrace)) { - jniSamples.incrementAndGet(); - } - - if (containsStubMethod(stackTrace)) { - stubSamples.incrementAndGet(); - categorizeStubType(stackTrace, stubTypes); - } - - if (containsPLTReference(stackTrace)) { - pltSamples.incrementAndGet(); - } - - if (containsReflectionMethod(stackTrace)) { - reflectionSamples.incrementAndGet(); - } - - if (containsJITReference(stackTrace)) { - jitSamples.incrementAndGet(); - } - - if (containsMethodHandleReference(stackTrace)) { - methodHandleSamples.incrementAndGet(); - } - - if (containsError(stackTrace)) { - errorSamples.incrementAndGet(); - categorizeErrorType(stackTrace, errorTypes); - } - } - } - - // Convert AtomicInteger maps to regular Integer maps - Map errorTypeBreakdown = new HashMap<>(); - errorTypes.forEach((k, v) -> errorTypeBreakdown.put(k, v.get())); - - Map stubTypeBreakdown = new HashMap<>(); - stubTypes.forEach((k, v) -> stubTypeBreakdown.put(k, v.get())); - - return new UnwindingResult( - totalSamples.get(), nativeSamples.get(), errorSamples.get(), - stubSamples.get(), pltSamples.get(), jniSamples.get(), - reflectionSamples.get(), jitSamples.get(), methodHandleSamples.get(), - errorTypeBreakdown, stubTypeBreakdown - ); - } - - private static void categorizeErrorType(String stackTrace, Map errorTypes) { - Set observedErrors = Arrays.stream(stackTrace.split(System.lineSeparator())).filter(UnwindingMetrics::containsError).collect(Collectors.toSet()); - observedErrors.forEach(f -> errorTypes.computeIfAbsent(f, k -> new AtomicInteger()).incrementAndGet()); - } - - private static void categorizeStubType(String stackTrace, Map stubTypes) { - Set observedStubs = Arrays.stream(stackTrace.split(System.lineSeparator())).filter(UnwindingMetrics::containsStubMethod).collect(Collectors.toSet()); - observedStubs.forEach(f -> stubTypes.computeIfAbsent(f, k -> new AtomicInteger()).incrementAndGet()); - } - - private static boolean containsAny(String target, String ... values) { - return Arrays.stream(values).anyMatch(target::contains); - } - - // Pattern detection methods (reused from individual tests) - private static boolean containsJNIMethod(String stackTrace) { - return containsAny(stackTrace, "DirectByteBuffer", "Unsafe", "System.arraycopy", "ByteBuffer.get", "ByteBuffer.put", "ByteBuffer.allocateDirect"); - } - - private static boolean containsStubMethod(String value) { - return containsAny(value, "stub", "Stub", "jni_", "_stub", "call_stub", "adapter"); - } - - private static boolean containsPLTReference(String stackTrace) { - return containsAny(stackTrace, "@plt", ".plt", "PLT", "_plt", "plt_", "dl_runtime", "_dl_fixup"); - } - - private static boolean containsReflectionMethod(String stackTrace) { - return containsAny(stackTrace, "Method.invoke", "reflect", "NativeMethodAccessor"); - } - - private static boolean containsJITReference(String stackTrace) { - return containsAny(stackTrace, "Compile", "C1", "C2", "OSR", "Tier", "I2C", "C2I", "I2OSR"); - } - - private static boolean containsMethodHandleReference(String stackTrace) { - return containsAny(stackTrace, "MethodHandle", "java.lang.invoke", "LambdaForm", "DirectMethodHandle", "BoundMethodHandle"); - } - - private static boolean containsError(String value) { - return containsAny(value, ".break_", "BCI_ERROR", ".invalid_", ".unknown()"); - } -} \ No newline at end of file diff --git a/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/UnwindingValidator.java b/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/UnwindingValidator.java deleted file mode 100644 index 968db5e53..000000000 --- a/ddprof-test/src/main/java/com/datadoghq/profiler/unwinding/UnwindingValidator.java +++ /dev/null @@ -1,1918 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.unwinding; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.Platform; -import com.github.luben.zstd.Zstd; -import net.jpountz.lz4.LZ4Compressor; -import net.jpountz.lz4.LZ4Factory; -import net.jpountz.lz4.LZ4FastDecompressor; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.item.ItemFilters; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.unit.UnitLookup; -import org.openjdk.jmc.common.IMCStackTrace; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.reflect.Method; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.LongAdder; -import java.util.concurrent.locks.LockSupport; -import java.io.FileWriter; -import java.io.IOException; - -import static org.openjdk.jmc.common.item.Attribute.attr; -import static org.openjdk.jmc.common.unit.UnitLookup.*; - -/** - * Comprehensive JIT unwinding validation tool that focuses on C2 compilation scenarios - * and related stub unwinding challenges. - * - * The tool executes intensive computational workloads designed to trigger various JIT - * compilation scenarios including: - * - Heavy arithmetic operations that force C2 compilation - * - On-Stack Replacement (OSR) through long-running loops and deep recursion - * - Array processing with complex mathematical transformations and sorting - * - Mixed native/Java transitions using ByteBuffer operations and reflection calls - * - Matrix multiplication operations for sustained computational load - * - String processing with intensive building, splitting, and pattern matching - * - Polymorphic call sites that trigger deoptimization scenarios - * - Exception handling patterns that cause uncommon traps - * - Dynamic class loading during active profiling - * - Null check and array bounds deoptimization triggers - * - Cross-library native method calls (LZ4, ZSTD compression) - * - Concurrent compilation stress with multiple threads - * - PLT resolution scenarios for dynamic symbol lookup - * - Stack boundary stress testing with deep recursion - * - * Each scenario runs with active profiling to capture unwinding behavior during - * JIT compilation transitions, deoptimization events, and complex call chains. - * The tool analyzes JFR recordings to measure unwinding success rates and identify - * problematic scenarios that produce '.unknown()', 'invalid_*' or 'broken_*' frames. - * - * Usage: - * java UnwindingValidator [options] - * - * Options: - * --scenario= Run specific scenario (default: all) - * --output-format= Output format: text, json, markdown (default: text) - * --output-file= Output file path (default: stdout) - * --help Show this help message - */ -public class UnwindingValidator { - - public enum OutputFormat { - TEXT, JSON, MARKDOWN - } - public enum Scenario { - C2_COMPILATION_TRIGGERS("C2CompilationTriggers"), - OSR_SCENARIOS("OSRScenarios"), - CONCURRENT_C2_COMPILATION("ConcurrentC2Compilation"), - C2_DEOPT_SCENARIOS("C2DeoptScenarios"), - EXTENDED_JNI_SCENARIOS("ExtendedJNIScenarios"), - MULTIPLE_STRESS_ROUNDS("MultipleStressRounds"), - EXTENDED_PLT_SCENARIOS("ExtendedPLTScenarios"), - ACTIVE_PLT_RESOLUTION("ActivePLTResolution"), - CONCURRENT_COMPILATION_STRESS("ConcurrentCompilationStress"), - VENEER_HEAVY_SCENARIOS("VeneerHeavyScenarios"), - RAPID_TIER_TRANSITIONS("RapidTierTransitions"), - DYNAMIC_LIBRARY_OPS("DynamicLibraryOps"), - STACK_BOUNDARY_STRESS("StackBoundaryStress"); - - public final String name; - Scenario(String name) { - this.name = name; - } - - public static Scenario of(String name) { - for (Scenario scenario : Scenario.values()) { - if (scenario.name.equals(name)) { - return scenario; - } - } - return null; - } - } - @FunctionalInterface - public interface TestScenario { - long execute() throws Exception; - } - - // Profiler management - private JavaProfiler profiler; - private Path jfrDump; - private boolean profilerStarted = false; - - // Configuration - private String targetScenario = "all"; - private OutputFormat outputFormat = OutputFormat.TEXT; - private String outputFile = null; - - // Attributes for JFR analysis - public static final IAttribute THREAD_EXECUTION_MODE = - attr("mode", "mode", "Execution Mode", PLAIN_TEXT); - public static final IAttribute STACK_TRACE = - attr("stackTrace", "stackTrace", "", UnitLookup.STACKTRACE); - - public static void main(String[] args) { - UnwindingValidator validator = new UnwindingValidator(); - - try { - validator.parseArguments(args); - validator.run(); - } catch (Exception e) { - System.err.println("Error: " + e.getMessage()); - System.exit(1); - } - } - - private void parseArguments(String[] args) { - for (String arg : args) { - if (arg.equals("--help")) { - showHelp(); - System.exit(0); - } else if (arg.startsWith("--scenario=")) { - targetScenario = arg.substring("--scenario=".length()); - } else if (arg.startsWith("--output-format=")) { - String format = arg.substring("--output-format=".length()).toUpperCase(); - try { - outputFormat = OutputFormat.valueOf(format); - } catch (IllegalArgumentException e) { - throw new RuntimeException("Invalid output format: " + format + - ". Valid options: text, json, markdown"); - } - } else if (arg.startsWith("--output-file=")) { - outputFile = arg.substring("--output-file=".length()); - } else if (!arg.isEmpty()) { - throw new RuntimeException("Unknown argument: " + arg); - } - } - } - - private void showHelp() { - System.out.println("UnwindingValidator - Comprehensive JIT unwinding validation tool"); - System.out.println(); - System.out.println("Usage: java UnwindingValidator [options]"); - System.out.println(); - System.out.println("Options:"); - System.out.println(" --scenario= Run specific scenario"); - System.out.println(" Available: C2CompilationTriggers, OSRScenarios,"); - System.out.println(" ConcurrentC2Compilation, C2DeoptScenarios,"); - System.out.println(" ExtendedJNIScenarios, MultipleStressRounds,"); - System.out.println(" ExtendedPLTScenarios, ActivePLTResolution,"); - System.out.println(" ConcurrentCompilationStress, VeneerHeavyScenarios,"); - System.out.println(" RapidTierTransitions, DynamicLibraryOps,"); - System.out.println(" StackBoundaryStress"); - System.out.println(" Default: all"); - System.out.println(" --output-format= Output format: text, json, markdown"); - System.out.println(" Default: text"); - System.out.println(" --output-file= Output file path (default: stdout)"); - System.out.println(" --help Show this help message"); - System.out.println(); - System.out.println("Examples:"); - System.out.println(" java UnwindingValidator"); - System.out.println(" java UnwindingValidator --scenario=C2CompilationTriggers"); - System.out.println(" java UnwindingValidator --output-format=markdown --output-file=report.md"); - } - - private void run() throws Exception { - if (Platform.isZing() || Platform.isJ9()) { - System.err.println("Skipping unwinding validation on unsupported JVM: " + - (Platform.isZing() ? "Zing" : "OpenJ9")); - return; - } - - System.err.println("=== Comprehensive Unwinding Validation Tool ==="); - System.err.println("Platform: " + System.getProperty("os.name") + " " + System.getProperty("os.arch")); - System.err.println("Java Version: " + System.getProperty("java.version")); - System.err.println("Is musl: " + Platform.isMusl()); - System.err.println("Scenario: " + targetScenario); - System.err.println("Output format: " + outputFormat.name().toLowerCase()); - if (outputFile != null) { - System.err.println("Output file: " + outputFile); - } - System.err.println(); - - List results = new ArrayList<>(); - - // Execute scenarios based on target - if ("all".equals(targetScenario)) { - results.addAll(executeAllScenarios()); - } else { - TestResult result = executeScenario(Scenario.of(targetScenario)); - if (result != null) { - results.add(result); - } else { - throw new RuntimeException("Unknown scenario: " + targetScenario); - } - } - - // Generate and output report - String report = generateReport(results); - outputReport(report); - - // Print summary to stderr for visibility - System.err.println("\n=== VALIDATION SUMMARY ==="); - System.err.println(UnwindingDashboard.generateCompactSummary(results)); - - // Check for CI environment to avoid failing builds - use same pattern as build.gradle - boolean isCI = System.getenv("CI") != null; - - // Exit with non-zero if there are critical issues (unless in CI mode) - boolean hasCriticalIssues = results.stream() - .anyMatch(r -> r.getStatus() == TestResult.Status.NEEDS_WORK); - if (hasCriticalIssues && !isCI) { - System.err.println("WARNING: Critical unwinding issues detected!"); - System.exit(1); - } else if (hasCriticalIssues && isCI) { - System.err.println("INFO: Critical unwinding issues detected, but continuing in CI mode"); - } - } - - private List executeAllScenarios() throws Exception { - List results = new ArrayList<>(); - - for (Scenario s : Scenario.values()) { - results.add(executeScenario(s)); - }; - - return results; - } - - private TestResult executeScenario(Scenario scenario) throws Exception { - if (scenario == null) { - return null; - } - switch (scenario) { - case C2_COMPILATION_TRIGGERS: - return executeIndividualScenario(scenario.name, "C2 compilation triggers with computational workloads", () -> { - long work = 0; - for (int round = 0; round < 10; round++) { - work += performC2CompilationTriggers(); - if (round % 3 == 0) { - LockSupport.parkNanos(5_000_000); - } - } - return work; - }); - - case OSR_SCENARIOS: - return executeIndividualScenario(scenario.name, "On-Stack Replacement compilation scenarios", () -> { - long work = 0; - for (int round = 0; round < 5; round++) { - work += performOSRScenarios(); - LockSupport.parkNanos(10_000_000); - } - return work; - }); - - case CONCURRENT_C2_COMPILATION: - return executeIndividualScenario(scenario.name, "Concurrent C2 compilation stress", - this::performConcurrentC2Compilation); - - case C2_DEOPT_SCENARIOS: - return executeIndividualScenario(scenario.name, "C2 deoptimization and transition edge cases", () -> { - long work = 0; - for (int round = 0; round < 200; round++) { - work += performC2DeoptScenarios(); - LockSupport.parkNanos(1_000_000); - } - return work; - }); - - case EXTENDED_JNI_SCENARIOS: - return executeIndividualScenario(scenario.name, "Extended basic JNI scenarios", () -> { - long work = 0; - for (int i = 0; i < 200; i++) { - work += performBasicJNIScenarios(); - if (i % 50 == 0) { - LockSupport.parkNanos(5_000_000); - } - } - return work; - }); - - case MULTIPLE_STRESS_ROUNDS: - return executeIndividualScenario(scenario.name, "Multiple concurrent stress rounds", () -> { - long work = 0; - for (int round = 0; round < 3; round++) { - work += executeStressScenarios(); - LockSupport.parkNanos(10_000_000); - } - return work; - }); - - case EXTENDED_PLT_SCENARIOS: - return executeIndividualScenario(scenario.name, "Extended PLT/veneer scenarios", () -> { - long work = 0; - for (int i = 0; i < 500; i++) { - work += performPLTScenarios(); - if (i % 100 == 0) { - LockSupport.parkNanos(2_000_000); - } - } - return work; - }); - - case ACTIVE_PLT_RESOLUTION: - return executeIndividualScenario(scenario.name, "Intensive PLT resolution during profiling", - this::performActivePLTResolution); - - case CONCURRENT_COMPILATION_STRESS: - return executeIndividualScenario(scenario.name, "Heavy JIT compilation + native activity", - this::performConcurrentCompilationStress); - - case VENEER_HEAVY_SCENARIOS: - return executeIndividualScenario(scenario.name, "ARM64 veneer/trampoline intensive workloads", - this::performVeneerHeavyScenarios); - - case RAPID_TIER_TRANSITIONS: - return executeIndividualScenario(scenario.name, "Rapid compilation tier transitions", - this::performRapidTierTransitions); - - case DYNAMIC_LIBRARY_OPS: - return executeIndividualScenario(scenario.name, "Dynamic library operations during profiling", - this::performDynamicLibraryOperations); - - case STACK_BOUNDARY_STRESS: - return executeIndividualScenario(scenario.name, "Stack boundary stress scenarios", - this::performStackBoundaryStress); - - default: - return null; - } - } - - private String generateReport(List results) { - switch (outputFormat) { - case JSON: - return generateJsonReport(results); - case MARKDOWN: - return UnwindingDashboard.generateMarkdownReport(results); - case TEXT: - default: - return UnwindingDashboard.generateReport(results); - } - } - - private String generateJsonReport(List results) { - StringBuilder json = new StringBuilder(); - json.append("{\n"); - json.append(" \"timestamp\": \"").append(java.time.Instant.now()).append("\",\n"); - json.append(" \"platform\": {\n"); - json.append(" \"os\": \"").append(System.getProperty("os.name")).append("\",\n"); - json.append(" \"arch\": \"").append(System.getProperty("os.arch")).append("\",\n"); - json.append(" \"java_version\": \"").append(System.getProperty("java.version")).append("\"\n"); - json.append(" },\n"); - json.append(" \"results\": [\n"); - - for (int i = 0; i < results.size(); i++) { - TestResult result = results.get(i); - UnwindingMetrics.UnwindingResult metrics = result.getMetrics(); - - json.append(" {\n"); - json.append(" \"testName\": \"").append(result.getTestName()).append("\",\n"); - json.append(" \"description\": \"").append(result.getScenarioDescription()).append("\",\n"); - json.append(" \"status\": \"").append(result.getStatus()).append("\",\n"); - json.append(" \"statusMessage\": \"").append(result.getStatusMessage()).append("\",\n"); - json.append(" \"executionTimeMs\": ").append(result.getExecutionTimeMs()).append(",\n"); - json.append(" \"metrics\": {\n"); - json.append(" \"totalSamples\": ").append(metrics.totalSamples).append(",\n"); - json.append(" \"errorSamples\": ").append(metrics.errorSamples).append(",\n"); - json.append(" \"errorRate\": ").append(String.format("%.3f", metrics.getErrorRate())).append(",\n"); - json.append(" \"nativeSamples\": ").append(metrics.nativeSamples).append(",\n"); - json.append(" \"nativeRate\": ").append(String.format("%.3f", metrics.getNativeRate())).append(",\n"); - json.append(" \"stubSamples\": ").append(metrics.stubSamples).append(",\n"); - json.append(" \"pltSamples\": ").append(metrics.pltSamples).append("\n"); - json.append(" }\n"); - json.append(" }"); - if (i < results.size() - 1) { - json.append(","); - } - json.append("\n"); - } - - json.append(" ],\n"); - json.append(" \"summary\": {\n"); - json.append(" \"totalTests\": ").append(results.size()).append(",\n"); - - double avgErrorRate = results.stream() - .mapToDouble(r -> r.getMetrics().getErrorRate()) - .average() - .orElse(0.0); - json.append(" \"averageErrorRate\": ").append(String.format("%.3f", avgErrorRate)).append(",\n"); - - int totalSamples = results.stream() - .mapToInt(r -> r.getMetrics().totalSamples) - .sum(); - json.append(" \"totalSamples\": ").append(totalSamples).append("\n"); - json.append(" }\n"); - json.append("}\n"); - - return json.toString(); - } - - private void outputReport(String report) throws IOException { - if (outputFile != null) { - Path outputPath = Paths.get(outputFile); - Files.createDirectories(outputPath.getParent()); - try (FileWriter writer = new FileWriter(outputFile)) { - writer.write(report); - } - } else { - System.out.println(report); - } - } - - /** - * Start profiler with aggressive settings for unwinding validation. - */ - private void startProfiler(String testName) throws Exception { - if (profilerStarted) { - throw new IllegalStateException("Profiler already started"); - } - - // Create JFR recording file - use current working directory in case /tmp has issues - Path rootDir; - try { - rootDir = Paths.get("/tmp/unwinding-recordings"); - Files.createDirectories(rootDir); - } catch (Exception e) { - // Fallback to current directory if /tmp is not writable - rootDir = Paths.get("./unwinding-recordings"); - Files.createDirectories(rootDir); - } - jfrDump = Files.createTempFile(rootDir, testName + "-", ".jfr"); - - // Use less aggressive profiling for musl environments which may be more sensitive - profiler = JavaProfiler.getInstance(); - String interval = Platform.isMusl() ? "100us" : "10us"; - String command = "start,cpu=" + interval + ",cstack=vm,jfr,file=" + jfrDump.toAbsolutePath(); - - try { - profiler.execute(command); - profilerStarted = true; - } catch (Exception e) { - System.err.println("Failed to start profiler: " + e.getMessage()); - // Try with even less aggressive settings as fallback - try { - command = "start,cpu=1ms,jfr,file=" + jfrDump.toAbsolutePath(); - profiler.execute(command); - profilerStarted = true; - System.err.println("Started profiler with fallback settings"); - } catch (Exception fallbackE) { - throw new RuntimeException("Failed to start profiler with both standard and fallback settings", fallbackE); - } - } - - // Give profiler more time to initialize on potentially slower environments - Thread.sleep(Platform.isMusl() ? 500 : 100); - } - - /** - * Stop profiler and return path to JFR recording. - */ - private Path stopProfiler() throws Exception { - if (!profilerStarted) { - throw new IllegalStateException("Profiler not started"); - } - - profiler.stop(); - profilerStarted = false; - - // Wait a bit for profiler to flush data - Thread.sleep(200); - - return jfrDump; - } - - /** - * Verify events from JFR recording and return samples. - */ - private Iterable verifyEvents(String eventType) throws Exception { - if (jfrDump == null || !Files.exists(jfrDump)) { - throw new RuntimeException("No JFR dump available"); - } - - IItemCollection events = JfrLoaderToolkit.loadEvents(jfrDump.toFile()); - return events.apply(ItemFilters.type(eventType)); - } - - /** - * Execute a single scenario with its own profiler session and JFR recording. - */ - private TestResult executeIndividualScenario(String testName, String description, - TestScenario scenario) throws Exception { - System.err.println("Executing scenario: " + testName); - long startTime = System.currentTimeMillis(); - - // Start profiler for this specific scenario - startProfiler(testName); - - try { - // Execute the scenario - long workCompleted = scenario.execute(); - - // Stop profiler for this scenario - stopProfiler(); - - // Analyze results for this specific scenario - Iterable cpuSamples = verifyEvents("datadog.ExecutionSample"); - IMemberAccessor modeAccessor = null; - - for (IItemIterable samples : cpuSamples) { - modeAccessor = THREAD_EXECUTION_MODE.getAccessor(samples.getType()); - break; - } - - if (modeAccessor == null) { - throw new RuntimeException("Could not get mode accessor for scenario: " + testName); - } - - UnwindingMetrics.UnwindingResult metrics = - UnwindingMetrics.analyzeUnwindingData(cpuSamples, modeAccessor); - - // Check if we got meaningful data - if (metrics.totalSamples == 0) { - System.err.println("WARNING: " + testName + " captured 0 samples - profiler may not be working properly"); - - // In CI, try to give a bit more time for sample collection - boolean isCI = System.getenv("CI") != null; - if (isCI) { - System.err.println("CI mode: Extending scenario execution time..."); - // Re-run scenario with longer execution - startProfiler(testName); - Thread.sleep(1000); // Wait 1 second before scenario - scenario.execute(); - Thread.sleep(1000); // Wait 1 second after scenario - stopProfiler(); - - // Re-analyze - cpuSamples = verifyEvents("datadog.ExecutionSample"); - modeAccessor = null; - for (IItemIterable samples : cpuSamples) { - modeAccessor = THREAD_EXECUTION_MODE.getAccessor(samples.getType()); - break; - } - if (modeAccessor != null) { - metrics = UnwindingMetrics.analyzeUnwindingData(cpuSamples, modeAccessor); - } - } - } - - long executionTime = System.currentTimeMillis() - startTime; - - TestResult result = TestResult.create(testName, description, metrics, executionTime); - - System.err.println("Completed: " + testName + " (" + executionTime + "ms, " + - metrics.totalSamples + " samples, " + - String.format("%.2f%%", metrics.getErrorRate()) + " error rate)"); - - return result; - - } catch (Exception e) { - // Ensure profiler is stopped even on failure - if (profilerStarted) { - try { - stopProfiler(); - } catch (Exception stopException) { - System.err.println("Warning: Failed to stop profiler: " + stopException.getMessage()); - } - } - - // Create a failed result - UnwindingMetrics.UnwindingResult emptyResult = new UnwindingMetrics.UnwindingResult( - 0, 0, 0, 0, 0, 0, 0, 0, 0, - java.util.Collections.emptyMap(), java.util.Collections.emptyMap()); - - long executionTime = System.currentTimeMillis() - startTime; - TestResult failedResult = new TestResult(testName, description, emptyResult, - TestResult.Status.NEEDS_WORK, "Scenario execution failed: " + e.getMessage(), - executionTime); - - System.err.println("Failed: " + testName + " (" + executionTime + "ms) - " + e.getMessage()); - return failedResult; - } - } - - // =============== SCENARIO IMPLEMENTATION METHODS =============== - // All the performance scenario methods from the original test are included here - // (Note: Including abbreviated versions for brevity - full implementations would be copied) - - private long performC2CompilationTriggers() { - long work = 0; - - // Computational intensive methods that trigger C2 - for (int round = 0; round < 20; round++) { - work += heavyArithmeticMethod(round * 1000); - work += complexArrayOperations(round); - work += mathIntensiveLoop(round); - work += nestedLoopOptimizations(round); - - // Mix with native calls to create transition points - if (round % 5 == 0) { - work += performMixedNativeCallsDuringCompilation(); - } - } - - return work; - } - - private long performOSRScenarios() { - long work = 0; - - // Very long-running loops that will trigger OSR - work += longRunningLoopWithOSR(5000); - work += recursiveMethodWithOSR(10); - work += arrayProcessingWithOSR(); - - return work; - } - - private long performConcurrentC2Compilation() throws Exception { - int threads = 6; - int iterationsPerThread = 15; - ExecutorService executor = Executors.newFixedThreadPool(threads); - CountDownLatch latch = new CountDownLatch(threads); - List results = new ArrayList<>(); - - for (int i = 0; i < threads; i++) { - final int threadId = i; - executor.submit(() -> { - try { - long work = 0; - for (int j = 0; j < iterationsPerThread; j++) { - // Each thread performs different C2-triggering patterns - work += heavyArithmeticMethod(threadId * 1000 + j); - work += complexMatrixOperations(threadId); - work += stringProcessingWithJIT(threadId); - - // Mix with native operations - work += performNativeMixDuringC2(threadId); - - if (j % 3 == 0) { - LockSupport.parkNanos(2_000_000); - } - } - synchronized (results) { - results.add(work); - } - } finally { - latch.countDown(); - } - }); - } - - if (!latch.await(90, TimeUnit.SECONDS)) { - throw new RuntimeException("Concurrent C2 compilation test timeout"); - } - executor.shutdown(); - - return results.stream().mapToLong(Long::longValue).sum(); - } - - private long performC2DeoptScenarios() { - long work = 0; - - try { - // Scenarios that commonly trigger deoptimization - work += polymorphicCallSites(); - work += exceptionHandlingDeopt(); - work += classLoadingDuringExecution(); - work += nullCheckDeoptimization(); - work += arrayBoundsDeoptimization(); - - } catch (Exception e) { - work += e.hashCode() % 1000; - } - - return work; - } - - // Include abbreviated versions of other key scenario methods - // (Full implementations would be copied from the original test file) - - private long performBasicJNIScenarios() { - long work = 0; - - try { - // Direct ByteBuffer operations - ByteBuffer direct = ByteBuffer.allocateDirect(2048); - for (int i = 0; i < 512; i++) { - direct.putInt(ThreadLocalRandom.current().nextInt()); - } - work += direct.position(); - - // Reflection operations - Method method = String.class.getMethod("length"); - String testStr = "validation" + ThreadLocalRandom.current().nextInt(); - work += (Integer) method.invoke(testStr); - - // Array operations - int[] array = new int[500]; - int[] copy = new int[500]; - for (int i = 0; i < array.length; i++) { - array[i] = ThreadLocalRandom.current().nextInt(); - } - System.arraycopy(array, 0, copy, 0, array.length); - work += copy[copy.length - 1]; - - } catch (Exception e) { - work += e.hashCode() % 1000; - } - - return work; - } - - private long executeStressScenarios() throws Exception { - int threads = 5; - int iterationsPerThread = 25; - ExecutorService executor = Executors.newFixedThreadPool(threads); - CountDownLatch latch = new CountDownLatch(threads); - List threadResults = new ArrayList<>(); - - // Concurrent JNI operations - for (int i = 0; i < threads; i++) { - final int threadId = i; - executor.submit(() -> { - try { - long work = 0; - for (int j = 0; j < iterationsPerThread; j++) { - work += performDeepJNIChain(5); - work += performLargeBufferOps(); - work += performComplexReflection(); - if (j % 5 == 0) LockSupport.parkNanos(2_000_000); - } - synchronized (threadResults) { - threadResults.add(work); - } - } finally { - latch.countDown(); - } - }); - } - - if (!latch.await(60, TimeUnit.SECONDS)) { - throw new RuntimeException("Stress scenarios timeout"); - } - executor.shutdown(); - - return threadResults.stream().mapToLong(Long::longValue).sum(); - } - - // Additional abbreviated helper methods (full implementations would be included) - - private long performPLTScenarios() { - long work = 0; - - try { - // Multiple native library calls (PLT entries) - LZ4FastDecompressor decompressor = LZ4Factory.nativeInstance().fastDecompressor(); - LZ4Compressor compressor = LZ4Factory.nativeInstance().fastCompressor(); - - ByteBuffer source = ByteBuffer.allocateDirect(512); - byte[] data = new byte[256]; - ThreadLocalRandom.current().nextBytes(data); - source.put(data); - source.flip(); - - ByteBuffer compressed = ByteBuffer.allocateDirect(compressor.maxCompressedLength(source.remaining())); - compressor.compress(source, compressed); - compressed.flip(); - - ByteBuffer decompressed = ByteBuffer.allocateDirect(256); - decompressor.decompress(compressed, decompressed); - work += decompressed.position(); - - // Method handle operations (veneers) - MethodHandles.Lookup lookup = MethodHandles.lookup(); - MethodType mt = MethodType.methodType(long.class); - MethodHandle nanoHandle = lookup.findStatic(System.class, "nanoTime", mt); - work += (Long) nanoHandle.invoke(); - - } catch (Throwable e) { - work += e.hashCode() % 1000; - } - - return work; - } - - // Enhanced implementations for CI reliability - - private long performActivePLTResolution() { - // Create conditions where PLT stubs are actively resolving during profiling - // This maximizes the chance of catching signals during incomplete stack setup - - System.err.println(" Creating intensive PLT resolution activity..."); - long work = 0; - - // Use multiple threads to force PLT resolution under concurrent load - ExecutorService executor = Executors.newFixedThreadPool(4); - CountDownLatch latch = new CountDownLatch(4); - - for (int thread = 0; thread < 4; thread++) { - executor.submit(() -> { - try { - // Force many different native library calls to trigger PLT resolution - for (int i = 0; i < 1000; i++) { - // Mix of different libraries to force PLT entries - performIntensiveLZ4Operations(); - performIntensiveZSTDOperations(); - performIntensiveReflectionCalls(); - performIntensiveSystemCalls(); - - // No sleep - maximum PLT activity - if (i % 100 == 0 && Thread.currentThread().isInterrupted()) break; - } - } finally { - latch.countDown(); - } - }); - } - - try { - latch.await(30, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - executor.shutdown(); - return work + 1000; - } - - private long performConcurrentCompilationStress() { - // Start JIT compilation and immediately begin profiling during active compilation - System.err.println(" Starting concurrent compilation + profiling..."); - long work = 0; - - // Create multiple compilation contexts simultaneously - ExecutorService compilationExecutor = Executors.newFixedThreadPool(6); - CountDownLatch compilationLatch = new CountDownLatch(6); - - final LongAdder summer = new LongAdder(); - for (int thread = 0; thread < 6; thread++) { - final int threadId = thread; - compilationExecutor.submit(() -> { - try { - // Each thread triggers different compilation patterns - switch (threadId % 3) { - case 0: - // Heavy C2 compilation triggers - for (int i = 0; i < 500; i++) { - summer.add(performIntensiveArithmetic(i * 1000)); - summer.add(performIntensiveBranching(i)); - } - break; - case 1: - // OSR compilation scenarios - performLongRunningLoops(1000); - break; - case 2: - // Mixed native/Java transitions - for (int i = 0; i < 300; i++) { - performMixedNativeJavaTransitions(); - } - break; - } - } finally { - compilationLatch.countDown(); - } - }); - } - - try { - compilationLatch.await(45, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - System.out.println("=== blackhole: " + summer.sumThenReset()); - - compilationExecutor.shutdown(); - return work + 2000; - } - - private long performVeneerHeavyScenarios() { - if (!Platform.isAarch64()) { - // no veneers on non-aarch64 - return 0; - } - // ARM64-specific: create conditions requiring veneers/trampolines - System.err.println(" Creating veneer-heavy call patterns..."); - long work = 0; - - // Create call patterns that require long jumps (potential veneers on ARM64) - for (int round = 0; round < 200; round++) { - // Cross-library calls that may require veneers - work += performCrossLibraryCalls(); - - // Deep recursion that spans different code sections - work += performDeepCrossModuleRecursion(20); - - // Rapid library switching - work += performRapidLibrarySwitching(); - - // No delays - keep veneer activity high - } - - return work; - } - - private long performRapidTierTransitions() { - // Force rapid interpreter -> C1 -> C2 transitions during active profiling - System.err.println(" Forcing rapid compilation tier transitions..."); - long work = 0; - - // Use multiple patterns to trigger different tier transitions - ExecutorService tierExecutor = Executors.newFixedThreadPool(3); - CountDownLatch tierLatch = new CountDownLatch(3); - - for (int thread = 0; thread < 50; thread++) { - final int threadId = thread; - tierExecutor.submit(() -> { - try { - for (int cycle = 0; cycle < 200; cycle++) { - // Force decompilation -> recompilation cycles - switch (threadId) { - case 0: - forceDeoptimizationCycle(cycle); - break; - case 1: - forceOSRCompilationCycle(cycle); - break; - case 2: - forceUncommonTrapCycle(cycle); - break; - } - - // Brief pause to allow tier transitions - if (cycle % 50 == 0) { - LockSupport.parkNanos(1_000_000); // 1ms - } - } - } finally { - tierLatch.countDown(); - } - }); - } - - try { - tierLatch.await(60, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - tierExecutor.shutdown(); - return work + 3000; - } - - private long performDynamicLibraryOperations() { - // Force dynamic library operations during profiling to stress symbol resolution - long work = 0; - - ExecutorService libraryExecutor = Executors.newFixedThreadPool(2); - CountDownLatch libraryLatch = new CountDownLatch(2); - - for (int thread = 0; thread < 2; thread++) { - libraryExecutor.submit(() -> { - try { - // Force class loading and native method resolution during profiling - for (int i = 0; i < 100; i++) { - // Force dynamic loading of native methods by class loading - forceClassLoading(i); - - // Force JNI method resolution - forceJNIMethodResolution(); - - // Force reflection method caching - forceReflectionMethodCaching(i); - - // Brief yield to maximize chance of signal during resolution - Thread.yield(); - } - } finally { - libraryLatch.countDown(); - } - }); - } - - try { - libraryLatch.await(30, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - libraryExecutor.shutdown(); - return work + 1000; - } - - private long performStackBoundaryStress() { - // Create scenarios that stress stack walking at boundaries - long work = 0; - - ExecutorService boundaryExecutor = Executors.newFixedThreadPool(3); - CountDownLatch boundaryLatch = new CountDownLatch(3); - - for (int thread = 0; thread < 3; thread++) { - final int threadId = thread; - boundaryExecutor.submit(() -> { - try { - switch (threadId) { - case 0: - // Deep recursion to stress stack boundaries - for (int i = 0; i < 50; i++) { - performDeepRecursionWithNativeCalls(30); - } - break; - case 1: - // Rapid stack growth/shrinkage - for (int i = 0; i < 200; i++) { - performRapidStackChanges(i); - } - break; - case 2: - // Exception-based stack unwinding stress - for (int i = 0; i < 100; i++) { - performExceptionBasedUnwindingStress(); - } - break; - } - } finally { - boundaryLatch.countDown(); - } - }); - } - - try { - boundaryLatch.await(45, TimeUnit.SECONDS); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - - boundaryExecutor.shutdown(); - return work + 2000; - } - - // Computational helper methods (abbreviated - full versions would be copied) - - private long heavyArithmeticMethod(int seed) { - long result = seed; - - for (int i = 0; i < 500; i++) { - result = result * 31 + i; - result = Long.rotateLeft(result, 5); - result ^= (result >>> 21); - result *= 0x9e3779b97f4a7c15L; - - if (result % 17 == 0) { - result += Math.abs(result % 1000); - } - } - - return result; - } - - private long complexArrayOperations(int size) { - int arraySize = 1000 + (size % 500); - long[] array1 = new long[arraySize]; - long[] array2 = new long[arraySize]; - long result = 0; - - for (int i = 0; i < arraySize; i++) { - array1[i] = i * 13 + size; - array2[i] = (i * 17) ^ size; - } - - for (int pass = 0; pass < 5; pass++) { - for (int i = 0; i < arraySize - 1; i++) { - array1[i] = array1[i] + array2[i + 1] * pass; - array2[i] = array2[i] ^ (array1[i] >>> 3); - result += array1[i] + array2[i]; - } - } - - return result; - } - - private long mathIntensiveLoop(int iterations) { - double result = 1.0 + iterations; - - for (int i = 0; i < 200; i++) { - result = Math.sin(result) * Math.cos(i); - result = Math.sqrt(Math.abs(result)) + Math.log(Math.abs(result) + 1); - result = Math.pow(result, 1.1); - - if (i % 10 == 0) { - long intResult = (long) result; - intResult = Long.rotateLeft(intResult, 7); - result = intResult + Math.PI; - } - } - - return (long) result; - } - - private long nestedLoopOptimizations(int depth) { - long result = 0; - - for (int i = 0; i < 50; i++) { - for (int j = 0; j < 30; j++) { - for (int k = 0; k < 10; k++) { - result += i * j + k * depth; - result ^= (i << j) | (k << depth); - } - } - } - - return result; - } - - // Additional helper methods would be included... - // (For brevity, showing abbreviated implementations) - - private long longRunningLoopWithOSR(int iterations) { - long result = 0; - for (int i = 0; i < iterations; i++) { - result += i * 31L; - result ^= (result << 13); - result ^= (result >>> 17); - result ^= (result << 5); - - if (i % 1000 == 0) { - result += Math.abs(result % 1000); - String.valueOf(result).hashCode(); - } - - if (i % 10000 == 0) { - LockSupport.parkNanos(100_000); - } - } - return result; - } - private long recursiveMethodWithOSR(int depth) { - if (depth <= 0) return 0; - - long result = depth * 13L; - - for (int i = 0; i < 100; i++) { - result ^= (result << 7); - result += i * depth; - result = Long.rotateRight(result, 3); - } - - if (depth > 1) { - result += recursiveMethodWithOSR(depth - 1); - result += recursiveMethodWithOSR(depth - 1); - } - - return result; - } - private long arrayProcessingWithOSR() { - int arraySize = 5000; - long[] data = new long[arraySize]; - long[] temp = new long[arraySize]; - long result = 0; - - for (int i = 0; i < arraySize; i++) { - data[i] = ThreadLocalRandom.current().nextLong(); - } - - for (int pass = 0; pass < 50; pass++) { - for (int i = 0; i < arraySize - 1; i++) { - temp[i] = data[i] + data[i + 1] * pass; - data[i] = temp[i] ^ (data[i] >>> 7); - result += data[i]; - } - - if (pass % 10 == 0) { - Arrays.sort(data); - result += data[data.length - 1]; - } - - System.arraycopy(data, 0, temp, 0, arraySize); - } - - return result; - } - private long performMixedNativeCallsDuringCompilation() { - long result = 0; - - try { - ByteBuffer direct = ByteBuffer.allocateDirect(1024); - for (int i = 0; i < 256; i++) { - direct.putInt(ThreadLocalRandom.current().nextInt()); - } - result += direct.position(); - - result += performIntensiveArithmetic(100); - - int[] array = new int[500]; - System.arraycopy(array, 0, new int[500], 0, array.length); - result += array.hashCode(); - - String test = String.valueOf(result); - result += test.hashCode(); - - } catch (Exception e) { - result += e.hashCode() % 1000; - } - - return result; - } - private long complexMatrixOperations(int threadId) { - int size = 50 + (threadId % 20); - double[][] matrix1 = new double[size][size]; - double[][] matrix2 = new double[size][size]; - double[][] result = new double[size][size]; - - for (int i = 0; i < size; i++) { - for (int j = 0; j < size; j++) { - matrix1[i][j] = ThreadLocalRandom.current().nextDouble(); - matrix2[i][j] = ThreadLocalRandom.current().nextDouble(); - } - } - - for (int i = 0; i < size; i++) { - for (int j = 0; j < size; j++) { - result[i][j] = 0; - for (int k = 0; k < size; k++) { - result[i][j] += matrix1[i][k] * matrix2[k][j]; - } - } - } - - return (long) (result[size-1][size-1] * 1000); - } - private long stringProcessingWithJIT(int threadId) { - StringBuilder sb = new StringBuilder(); - long result = 0; - - for (int i = 0; i < 1000; i++) { - sb.append("thread").append(threadId).append("_").append(i); - if (i % 100 == 0) { - String str = sb.toString(); - result += str.hashCode(); - sb.setLength(0); - } - } - - String finalStr = sb.toString(); - String[] parts = finalStr.split("_"); - for (String part : parts) { - if (part.contains(String.valueOf(threadId))) { - result += part.length(); - } - } - - return result; - } - private long performNativeMixDuringC2(int threadId) { - long result = 0; - - try { - result += performIntensiveArithmetic(threadId * 100); - - performIntensiveLZ4Operations(); - result += threadId * 10; - - Method method = String.class.getMethod("valueOf", int.class); - String value = (String) method.invoke(null, threadId); - result += value.hashCode(); - - performIntensiveSystemCalls(); - result += threadId * 5; - - result += performIntensiveBranching(threadId * 50); - - } catch (Exception e) { - result += e.hashCode() % 1000; - } - - return result; - } - private long polymorphicCallSites() { - long result = 0; - Object[] objects = {"string", Integer.valueOf(42), Double.valueOf(3.14), new StringBuilder(), new ArrayList<>()}; - - for (int round = 0; round < 200; round++) { - for (Object obj : objects) { - if (obj instanceof String) { - result += ((String) obj).length(); - } else if (obj instanceof Integer) { - result += ((Integer) obj).intValue(); - } else if (obj instanceof Double) { - result += ((Double) obj).longValue(); - } else if (obj instanceof StringBuilder) { - ((StringBuilder) obj).append(round); - result += obj.hashCode(); - } else if (obj instanceof ArrayList) { - ((ArrayList) obj).add(round); - result += ((ArrayList) obj).size(); - } - } - } - - return result; - } - private long exceptionHandlingDeopt() { - long result = 0; - - for (int i = 0; i < 1000; i++) { - try { - if (i % 3 == 0) { - throw new RuntimeException("Deopt trigger " + i); - } else if (i % 5 == 0) { - throw new IllegalArgumentException("Another deopt " + i); - } else { - result += performIntensiveArithmetic(i); - } - } catch (IllegalArgumentException e) { - result += e.getMessage().length(); - } catch (RuntimeException e) { - result += e.getMessage().hashCode(); - } catch (Exception e) { - result += e.hashCode() % 100; - } - } - - return result; - } - private long classLoadingDuringExecution() { - long result = 0; - - try { - String[] classNames = { - "java.util.concurrent.ConcurrentHashMap", - "java.util.concurrent.ThreadPoolExecutor", - "java.security.SecureRandom", - "java.util.zip.CRC32", - "java.nio.channels.FileChannel" - }; - - for (int round = 0; round < 20; round++) { - for (String className : classNames) { - Class clazz = Class.forName(className); - result += clazz.hashCode(); - - Method[] methods = clazz.getDeclaredMethods(); - for (int i = 0; i < Math.min(methods.length, 5); i++) { - result += methods[i].hashCode(); - } - } - - result += performIntensiveArithmetic(round * 10); - } - - } catch (ClassNotFoundException e) { - result += e.hashCode() % 1000; - } - - return result; - } - private long nullCheckDeoptimization() { - long result = 0; - String[] strings = new String[1000]; - - for (int i = 0; i < strings.length; i++) { - if (i % 3 == 0) { - strings[i] = "value_" + i; - } - } - - for (int round = 0; round < 50; round++) { - for (int i = 0; i < strings.length; i++) { - String s = strings[i]; - if (s != null) { - result += s.hashCode(); - } else { - result += i; - } - - if (round % 10 == 0 && i % 100 == 0) { - strings[i] = null; - } - } - } - - return result; - } - private long arrayBoundsDeoptimization() { - long result = 0; - int[] array = new int[1000]; - - for (int i = 0; i < array.length; i++) { - array[i] = ThreadLocalRandom.current().nextInt(); - } - - for (int round = 0; round < 100; round++) { - for (int i = 0; i < 1200; i++) { - try { - if (i < array.length) { - result += array[i]; - } else { - result += array[array.length - 1]; - } - } catch (ArrayIndexOutOfBoundsException e) { - result += e.hashCode() % 100; - } - } - - try { - int randomIndex = ThreadLocalRandom.current().nextInt(-10, array.length + 10); - result += array[randomIndex]; - } catch (ArrayIndexOutOfBoundsException e) { - result += 42; - } - } - - return result; - } - private long performIntensiveArithmetic(int cycles) { - // Heavy arithmetic computation to trigger C2 compilation - long result = 0; - for (int i = 0; i < cycles; i++) { - result = result * 31 + i; - result = Long.rotateLeft(result, 5); - result ^= (result >>> 21); - result *= 0x9e3779b97f4a7c15L; - } - return result; - } - - private long performIntensiveBranching(int cycles) { - // Heavy branching patterns to trigger compilation - long result = 0; - for (int i = 0; i < cycles; i++) { - if (i % 2 == 0) { - result += i * 3L; - } else if (i % 3 == 0) { - result += i * 7L; - } else if (i % 5 == 0) { - result += i * 11L; - } else { - result += i; - } - } - return result; - } - - private void performLongRunningLoops(int iterations) { - // Long-running loops that trigger OSR compilation - long sum = 0; - for (int i = 0; i < iterations; i++) { - sum += (long) i * ThreadLocalRandom.current().nextInt(100); - if (i % 100 == 0) { - // Force memory access to prevent optimization - String.valueOf(sum).hashCode(); - } - } - System.out.println("=== blackhole: " + sum); - } - - private void performIntensiveLZ4Operations() { - if (Platform.isMusl()) { - // lz4 native lib not available on musl - simulate equivalent work - performAlternativeNativeWork(); - return; - } - try { - LZ4Compressor compressor = LZ4Factory.nativeInstance().fastCompressor(); - LZ4FastDecompressor decompressor = LZ4Factory.nativeInstance().fastDecompressor(); - - ByteBuffer source = ByteBuffer.allocateDirect(1024); - source.putInt(ThreadLocalRandom.current().nextInt()); - source.flip(); - - ByteBuffer compressed = ByteBuffer.allocateDirect(compressor.maxCompressedLength(source.limit())); - compressor.compress(source, compressed); - - compressed.flip(); - ByteBuffer decompressed = ByteBuffer.allocateDirect(source.limit()); - decompressor.decompress(compressed, decompressed); - } catch (Exception e) { - // Expected during rapid PLT resolution - } - } - - private void performIntensiveZSTDOperations() { - try { - ByteBuffer source = ByteBuffer.allocateDirect(1024); - source.putLong(ThreadLocalRandom.current().nextLong()); - source.flip(); - - ByteBuffer compressed = ByteBuffer.allocateDirect(Math.toIntExact(Zstd.compressBound(source.limit()))); - Zstd.compress(compressed, source); - } catch (Exception e) { - // Expected during rapid PLT resolution - } - } - - private void performIntensiveReflectionCalls() { - try { - Method method = String.class.getMethod("valueOf", int.class); - for (int i = 0; i < 10; i++) { - method.invoke(null, i); - } - } catch (Exception e) { - // Expected during rapid reflection - } - } - - private void performIntensiveSystemCalls() { - // System calls that go through different stubs - int[] array1 = new int[100]; - int[] array2 = new int[100]; - System.arraycopy(array1, 0, array2, 0, array1.length); - - // String operations that may use native methods - String.valueOf(ThreadLocalRandom.current().nextInt()).hashCode(); - } - - private void performAlternativeNativeWork() { - // Alternative native work for musl where LZ4 is not available - // Focus on JNI calls that are available on musl - try { - // Array operations that go through native code - int[] source = new int[256]; - int[] dest = new int[256]; - for (int i = 0; i < source.length; i++) { - source[i] = ThreadLocalRandom.current().nextInt(); - } - System.arraycopy(source, 0, dest, 0, source.length); - - // String interning and native operations - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < 10; i++) { - sb.append("test").append(i); - } - String result = sb.toString(); - result.hashCode(); - - // Reflection calls that exercise native method resolution - Method method = String.class.getMethod("length"); - method.invoke(result); - - // Math operations that may use native implementations - for (int i = 0; i < 50; i++) { - Math.sin(i * Math.PI / 180); - Math.cos(i * Math.PI / 180); - } - - } catch (Exception e) { - // Expected during alternative native work - } - } - - private long performMixedNativeJavaTransitions() { - long work = 0; - - // Rapid Java -> Native -> Java transitions - work += performIntensiveArithmetic(100); - performIntensiveLZ4Operations(); - work += performIntensiveBranching(50); - performIntensiveSystemCalls(); - work += performIntensiveArithmetic(75); - - return work; - } - - private long performDeepJNIChain(int depth) { - if (depth <= 0) return ThreadLocalRandom.current().nextInt(100); - - try { - // JNI -> Java -> JNI chain - ByteBuffer buffer = ByteBuffer.allocateDirect(1024); - buffer.putLong(System.nanoTime()); - - // Reflection in the middle - Method method = buffer.getClass().getMethod("position"); - Integer pos = (Integer) method.invoke(buffer); - - // More JNI - use platform-appropriate operations - long workResult; - if (Platform.isMusl()) { - // Alternative native operations for musl - performAlternativeNativeWork(); - workResult = buffer.position(); - } else { - // LZ4 operations for non-musl platforms - LZ4Compressor compressor = LZ4Factory.nativeInstance().fastCompressor(); - ByteBuffer source = ByteBuffer.allocateDirect(256); - ByteBuffer compressed = ByteBuffer.allocateDirect(compressor.maxCompressedLength(256)); - - byte[] data = new byte[256]; - ThreadLocalRandom.current().nextBytes(data); - source.put(data); - source.flip(); - - compressor.compress(source, compressed); - workResult = compressed.position(); - } - - return pos + workResult + performDeepJNIChain(depth - 1); - - } catch (Exception e) { - return e.hashCode() % 1000 + performDeepJNIChain(depth - 1); - } - } - - private long performLargeBufferOps() { - long work = 0; - - try { - ByteBuffer large = ByteBuffer.allocateDirect(16384); - byte[] data = new byte[8192]; - ThreadLocalRandom.current().nextBytes(data); - large.put(data); - large.flip(); - - // ZSTD compression - ByteBuffer compressed = ByteBuffer.allocateDirect(Math.toIntExact(Zstd.compressBound(large.remaining()))); - work += Zstd.compress(compressed, large); - - // ZSTD decompression - compressed.flip(); - ByteBuffer decompressed = ByteBuffer.allocateDirect(8192); - work += Zstd.decompress(decompressed, compressed); - - } catch (Exception e) { - work += e.hashCode() % 1000; - } - - return work; - } - - private long performComplexReflection() { - long work = 0; - try { - // Complex reflection patterns that stress unwinder - Class clazz = ByteBuffer.class; - Method[] methods = clazz.getDeclaredMethods(); - for (Method method : methods) { - if (method.getName().startsWith("put") && method.getParameterCount() == 1) { - work += method.hashCode(); - // Create method handle for more complex unwinding - MethodHandles.Lookup lookup = MethodHandles.lookup(); - MethodHandle handle = lookup.unreflect(method); - work += handle.hashCode(); - break; - } - } - - // Nested reflection calls - Method lengthMethod = String.class.getMethod("length"); - for (int i = 0; i < 10; i++) { - String testStr = "test" + i; - work += (Integer) lengthMethod.invoke(testStr); - } - - } catch (Throwable e) { - work += e.hashCode() % 1000; - } - return work; - } - - // Supporting methods for cross-library and tier transition scenarios - - private long performCrossLibraryCalls() { - long work = 0; - - // Mix calls across different native libraries - try { - // LZ4 -> ZSTD -> System -> Reflection - performIntensiveLZ4Operations(); - performIntensiveZSTDOperations(); - performIntensiveSystemCalls(); - performIntensiveReflectionCalls(); - work += 10; - } catch (Exception e) { - // Expected during cross-library transitions - } - - return work; - } - - private long performDeepCrossModuleRecursion(int depth) { - if (depth <= 0) return 1; - - // Mix native and Java calls in recursion - performIntensiveLZ4Operations(); - long result = performDeepCrossModuleRecursion(depth - 1); - performIntensiveSystemCalls(); - - return result + depth; - } - - private long performRapidLibrarySwitching() { - long work = 0; - - // Rapid switching between different native libraries - for (int i = 0; i < 20; i++) { - switch (i % 4) { - case 0: performIntensiveLZ4Operations(); break; - case 1: performIntensiveZSTDOperations(); break; - case 2: performIntensiveSystemCalls(); break; - case 3: performIntensiveReflectionCalls(); break; - } - work++; - } - - return work; - } - - private void forceDeoptimizationCycle(int cycle) { - // Pattern that forces deoptimization - Object obj = (cycle % 2 == 0) ? "string" : Integer.valueOf(cycle); - - // This will cause uncommon trap and deoptimization - if (obj instanceof String) { - performIntensiveArithmetic(cycle); - } else { - performIntensiveBranching(cycle); - } - } - - private void forceOSRCompilationCycle(int cycle) { - // Long-running loop that triggers OSR - long sum = 0; - for (int i = 0; i < 1000; i++) { - sum += (long) i * cycle; - if (i % 100 == 0) { - // Force native call during OSR - performIntensiveSystemCalls(); - } - } - } - - private void forceUncommonTrapCycle(int cycle) { - // Pattern that creates uncommon traps - try { - Class clazz = (cycle % 3 == 0) ? String.class : Integer.class; - Method method = clazz.getMethod("toString"); - method.invoke((cycle % 2 == 0) ? "test" : Integer.valueOf(cycle)); - } catch (Exception e) { - // Creates uncommon trap scenarios - } - } - - // Additional supporting methods for dynamic library operations - - private void forceClassLoading(int iteration) { - try { - // Force loading of classes with native methods - String className = (iteration % 3 == 0) ? "java.util.zip.CRC32" : - (iteration % 3 == 1) ? "java.security.SecureRandom" : - "java.util.concurrent.ThreadLocalRandom"; - - Class clazz = Class.forName(className); - // Force static initialization which may involve native method resolution - clazz.getDeclaredMethods(); - } catch (Exception e) { - // Expected during dynamic loading - } - } - - private void forceJNIMethodResolution() { - // Operations that force JNI method resolution - try { - // These operations force native method lookup - System.identityHashCode(new Object()); - Runtime.getRuntime().availableProcessors(); - System.nanoTime(); - - // Force string native operations - "test".intern(); - - } catch (Exception e) { - // Expected during method resolution - } - } - - private void forceReflectionMethodCaching(int iteration) { - try { - // Force method handle caching and native method resolution - Class clazz = String.class; - Method method = clazz.getMethod("valueOf", int.class); - - // This forces method handle creation and caching - for (int i = 0; i < 5; i++) { - method.invoke(null, iteration + i); - } - } catch (Exception e) { - // Expected during reflection operations - } - } - - // Stack boundary stress supporting methods - - private void performDeepRecursionWithNativeCalls(int depth) { - if (depth <= 0) return; - - // Mix native calls in recursion - performIntensiveLZ4Operations(); - System.arraycopy(new int[10], 0, new int[10], 0, 10); - - performDeepRecursionWithNativeCalls(depth - 1); - - // More native calls on return path - String.valueOf(depth).hashCode(); - } - - private void performRapidStackChanges(int iteration) { - // Create rapid stack growth and shrinkage patterns - try { - switch (iteration % 4) { - case 0: - rapidStackGrowth1(iteration); - break; - case 1: - rapidStackGrowth2(iteration); - break; - case 2: - rapidStackGrowth3(iteration); - break; - case 3: - rapidStackGrowth4(iteration); - break; - } - } catch (StackOverflowError e) { - // Expected - this stresses stack boundaries - } - } - - private void rapidStackGrowth1(int depth) { - if (depth > 50) return; - performIntensiveSystemCalls(); - rapidStackGrowth1(depth + 1); - } - - private void rapidStackGrowth2(int depth) { - if (depth > 50) return; - performIntensiveLZ4Operations(); - rapidStackGrowth2(depth + 1); - } - - private void rapidStackGrowth3(int depth) { - if (depth > 50) return; - performIntensiveReflectionCalls(); - rapidStackGrowth3(depth + 1); - } - - private void rapidStackGrowth4(int depth) { - if (depth > 50) return; - performIntensiveZSTDOperations(); - rapidStackGrowth4(depth + 1); - } - - private void performExceptionBasedUnwindingStress() { - // Use exceptions to force stack unwinding during native operations - try { - try { - try { - performIntensiveLZ4Operations(); - throw new RuntimeException("Force unwinding"); - } catch (RuntimeException e1) { - performIntensiveSystemCalls(); - throw new IllegalArgumentException("Force unwinding 2"); - } - } catch (IllegalArgumentException e2) { - performIntensiveReflectionCalls(); - throw new UnsupportedOperationException("Force unwinding 3"); - } - } catch (UnsupportedOperationException e3) { - // Final catch - forces multiple stack unwind operations - performIntensiveZSTDOperations(); - } - } -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java deleted file mode 100644 index 6c0bc0685..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProcessProfilerTest.java +++ /dev/null @@ -1,140 +0,0 @@ -package com.datadoghq.profiler; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.LockSupport; -import java.util.function.Function; - -import static org.junit.jupiter.api.Assertions.*; - - -public abstract class AbstractProcessProfilerTest { - public static final class LaunchResult { - public final boolean inTime; - public final int exitCode; - - public LaunchResult(boolean inTime, int exitCode) { - this.inTime = inTime; - this.exitCode = exitCode; - } - } - - public enum LineConsumerResult { - CONTINUE, - STOP, - IGNORE - } - - protected final LaunchResult launch(String target, List jvmArgs, String commands, Function onStdoutLine, Function onStderrLine) throws Exception { - return launch(target, jvmArgs, commands, Collections.emptyMap(), onStdoutLine, onStderrLine); - } - - protected final LaunchResult launch(String target, List jvmArgs, String commands, Map env, Function onStdoutLine, Function onStderrLine) throws Exception { - String javaHome = System.getenv("JAVA_TEST_HOME"); - if (javaHome == null) { - javaHome = System.getenv("JAVA_HOME"); - } - if (javaHome == null) { - javaHome = System.getProperty("java.home"); - } - assertNotNull(javaHome); - - List args = new ArrayList<>(); - args.add(javaHome + "/bin/java"); - args.addAll(jvmArgs); - args.add("-cp"); - args.add(System.getProperty("java.class.path")); - args.add(ExternalLauncher.class.getName()); - args.add(target); - if (commands != null && !commands.isEmpty()) { - args.add(commands); - } - - ProcessBuilder pb = new ProcessBuilder(args); - pb.environment().putAll(env); - Process p = pb.start(); - Thread stdoutReader = new Thread(() -> { - Function lineProcessor = onStdoutLine != null ? onStdoutLine : l -> LineConsumerResult.CONTINUE; - try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream()))) { - String line; - while ((line = br.readLine()) != null) { - System.out.println("[out] " + line); - LineConsumerResult lResult = lineProcessor.apply(line); - switch (lResult) { - case STOP: { - try { - p.getOutputStream().write(1); - p.getOutputStream().flush(); - } catch (IOException ignored) { - } - break; - } - case CONTINUE: { - if (line.contains("[ready]")) { - p.getOutputStream().write(1); - p.getOutputStream().flush(); - } - break; - } - case IGNORE: { - // ignore - break; - } - } - } - } catch (IOException ignored) { - } catch (Exception e) { - throw new RuntimeException(e); - } - }, "stdout-reader"); - Thread stderrReader = new Thread(() -> { - Function lineProcessor = onStderrLine != null ? onStderrLine : l -> LineConsumerResult.CONTINUE; - try (BufferedReader br = new BufferedReader(new InputStreamReader(p.getErrorStream()))) { - String line; - while ((line = br.readLine()) != null) { - System.out.println("[err] " + line); - LineConsumerResult lResult = lineProcessor.apply(line); - switch (lResult) { - case STOP: { - try { - p.getOutputStream().write(1); - p.getOutputStream().flush(); - } catch (IOException ignored) { - } - break; - } - case CONTINUE: { - break; - } - case IGNORE: { - // ignore - break; - } - } - } - } catch (IOException ignored) { - } catch (Exception e) { - throw new RuntimeException(e); - } - }, "stderr-reader"); - - stdoutReader.setDaemon(true); - stderrReader.setDaemon(true); - - stdoutReader.start(); - stderrReader.start(); - - boolean val = p.waitFor(10, TimeUnit.SECONDS); - if (!val) { - p.destroyForcibly(); - p.waitFor(5, TimeUnit.SECONDS); - } - return new LaunchResult(val, p.exitValue()); - } -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java deleted file mode 100644 index de75c2f06..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/AbstractProfilerTest.java +++ /dev/null @@ -1,507 +0,0 @@ -package com.datadoghq.profiler; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.time.Duration; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInfo; -import org.openjdk.jmc.common.IMCStackTrace; -import org.openjdk.jmc.common.item.Attribute; - -import static org.junit.jupiter.api.Assertions.*; -import static org.openjdk.jmc.common.item.Attribute.attr; -import static org.openjdk.jmc.common.unit.UnitLookup.*; - -import org.openjdk.jmc.common.IMCType; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.item.IItemFilter; -import org.openjdk.jmc.common.item.ItemFilters; -import org.openjdk.jmc.common.item.IType; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.common.unit.QuantityConversionException; -import org.openjdk.jmc.common.unit.UnitLookup; -import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -public abstract class AbstractProfilerTest { - private static final boolean ALLOW_NATIVE_CSTACKS = true; - - private boolean stopped = true; - private Map sanitizerLogSizesBefore = new HashMap<>(); - - public static final String LAMBDA_QUALIFIER = Platform.isJavaVersionAtLeast(21) ? "$$Lambda." : "$$Lambda$"; - public static final IQuantity ZERO_BYTES = BYTE.quantity(0); - public static final IAttribute SIZE = attr("size", "size", "", BYTE.getContentType()); - public static final IAttribute WEIGHT = attr("weight", "weight", "weight", NUMBER); - - public static final IAttribute SCALED_SIZE = new Attribute("scaledSize", "scaled size", "", BYTE.getContentType()) { - @Override - public IMemberAccessor customAccessor(IType type) { - IMemberAccessor sizeAccessor = SIZE.getAccessor(type); - IMemberAccessor weightAccessor = WEIGHT.getAccessor(type); - if (sizeAccessor == null || weightAccessor == null) { - return i -> ZERO_BYTES; - } - return i -> sizeAccessor.getMember(i).multiply(weightAccessor.getMember(i).doubleValue()); - } - }; - - public static final IAttribute LOCAL_ROOT_SPAN_ID = attr("localRootSpanId", "localRootSpanId", - "localRootSpanId", NUMBER); - public static final IAttribute SPAN_ID = attr("spanId", "spanId", - "spanId", NUMBER); - - public static final IAttribute OPERATION = attr("operation", "operation", - "operation", PLAIN_TEXT); - - - - public static final IAttribute THREAD_STATE = - attr("state", "state", "Thread State", PLAIN_TEXT); - - public static final IAttribute THREAD_EXECUTION_MODE = - attr("mode", "mode", "Execution Mode", PLAIN_TEXT); - - public static final IAttribute TAG_1 = attr("tag1", "", "", PLAIN_TEXT); - public static final IAttribute TAG_2 = attr("tag2", "", "", PLAIN_TEXT); - public static final IAttribute TAG_3 = attr("tag3", "", "", PLAIN_TEXT); - - public static final IAttribute STACK_TRACE = attr("stackTrace", "stackTrace", "", UnitLookup.STACKTRACE); - - public static final IAttribute CPU_INTERVAL = attr("cpuInterval", "cpuInterval", "", TIMESPAN); - public static final IAttribute CPU_ENGINE = attr("cpuEngine", "", "", PLAIN_TEXT); - - public static final IAttribute WALL_INTERVAL = attr("wallInterval", "wallInterval", "", TIMESPAN); - - public static final IAttribute NAME = attr("name", "", "", PLAIN_TEXT); - - public static final IAttribute COUNT = attr("count", "", "", NUMBER); - - protected JavaProfiler profiler; - private Path jfrDump; - - private Duration cpuInterval; - private Duration wallInterval; - - private Map testParams; - - protected static Map mapOf(Object ... vals) { - Map map = new HashMap<>(); - for (int i = 0; i < vals.length; i += 2) { - map.put(vals[i].toString(), vals[i + 1]); - } - return map; - } - - protected AbstractProfilerTest(Map testParams) { - this.testParams = testParams != null ? new HashMap<>(testParams) : Collections.emptyMap(); - } - - protected AbstractProfilerTest() { - this(null); - } - - private static Duration parseInterval(String command, String part) { - String prefix = part + "="; - int start = command.indexOf(prefix); - if (start >= 0) { - start += prefix.length(); - int end = command.indexOf(",", start); - if (end < 0) { - end = command.length(); - } - String interval = command.substring(start, end); - int unitFirstChar = 0; - int durationFirstChar = interval.charAt(0) == '~' ? 1 : 0; - for (int i = 0; i < interval.length(); i++) { - if (Character.isAlphabetic(interval.charAt(i))) { - unitFirstChar = i; - break; - } - } - long duration = Long.parseLong(interval.substring(durationFirstChar, unitFirstChar)); - String unit = interval.substring(unitFirstChar).toLowerCase(); - switch (unit) { - case "s": - return Duration.ofSeconds(duration); - case "ms": - return Duration.ofMillis(duration); - // backend assumes we report duration in millis, - // so we can't express these more accurately than 0 - case "us": - case "ns": - default: - } - } - return Duration.ofMillis(0); - } - - protected final boolean isAsan() { - return System.getenv("ASAN_OPTIONS") != null; - } - - private static long getPid() { - try { - String name = java.lang.management.ManagementFactory.getRuntimeMXBean().getName(); - return Long.parseLong(name.split("@")[0]); - } catch (NumberFormatException e) { - return 0L; - } - } - - private static List getSanitizerLogPaths() { - List paths = new ArrayList<>(); - String pid = String.valueOf(getPid()); - for (String envVar : new String[]{"ASAN_OPTIONS", "UBSAN_OPTIONS"}) { - String options = System.getenv(envVar); - if (options == null) continue; - for (String opt : options.split(":")) { - if (opt.startsWith("log_path=")) { - String template = opt.substring("log_path=".length()); - String path = template.replace("%p", pid); - paths.add(Paths.get(path)); - } - } - } - return paths; - } - - private void dumpSanitizerLogs() { - for (Path logPath : getSanitizerLogPaths()) { - try { - if (!Files.exists(logPath)) continue; - long sizeBefore = sanitizerLogSizesBefore.getOrDefault(logPath, 0L); - long currentSize = Files.size(logPath); - if (currentSize <= sizeBefore) continue; - byte[] bytes = Files.readAllBytes(logPath); - if (bytes.length > (int) sizeBefore) { - String newContent = new String(bytes, (int) sizeBefore, bytes.length - (int) sizeBefore); - String label = logPath.getFileName().toString().toUpperCase(); - System.err.println("=== " + label + " errors detected during test ==="); - System.err.println(newContent); - System.err.println("=== End " + label + " errors ==="); - } - } catch (Exception e) { - // best effort - } - } - } - - protected final boolean isTsan() { - return System.getenv("TSAN_OPTIONS") != null; - } - - protected boolean isPlatformSupported() { - return true; - } - - protected void withTestAssumptions() {} - - @BeforeEach - public void setupProfiler(TestInfo testInfo) throws Exception { - Assumptions.assumeTrue(isPlatformSupported()); - withTestAssumptions(); - - String testConfig = System.getenv("TEST_CONFIGURATION"); - testConfig = testConfig == null ? "" : testConfig; - Path rootDir = Paths.get("/tmp/recordings"); - Files.createDirectories(rootDir); - - String cstack = (String)testParams.get("cstack"); - - if (cstack != null) { - rootDir = rootDir.resolve(cstack); - Files.createDirectories(rootDir); - } - - jfrDump = Files.createTempFile(rootDir, testInfo.getTestMethod().map(m -> m.getDeclaringClass().getSimpleName() + "_" + m.getName()).orElse("unknown") + (testConfig.isEmpty() ? "" : "-" + testConfig.replace('/', '_')), ".jfr"); - profiler = JavaProfiler.getInstance(); - String command = "start," + getAmendedProfilerCommand() + ",jfr,file=" + jfrDump.toAbsolutePath(); - cpuInterval = command.contains("cpu") ? parseInterval(command, "cpu") : (command.contains("interval") ? parseInterval(command, "interval") : Duration.ZERO); - wallInterval = parseInterval(command, "wall"); - // Record sanitizer log sizes before test so we can dump new errors after - sanitizerLogSizesBefore.clear(); - for (Path logPath : getSanitizerLogPaths()) { - try { - if (Files.exists(logPath)) { - sanitizerLogSizesBefore.put(logPath, Files.size(logPath)); - } - } catch (Exception e) { - // best effort - } - } - - System.out.println("===> command: " + command); - profiler.execute(command); - stopped = false; - before(); - } - - @AfterEach - public void cleanup() throws Exception { - after(); - stopProfiler(); - dumpSanitizerLogs(); - System.out.println("===> keep_jfrs: " + Boolean.getBoolean("ddprof_test.keep_jfrs")); - if (jfrDump != null && !Boolean.getBoolean("ddprof_test.keep_jfrs")) { - Files.deleteIfExists(jfrDump); - } - } - - protected void before() throws Exception { - } - - protected void after() throws Exception { - } - - public static final boolean isInCI() { - return Boolean.getBoolean("ddprof_test.ci"); - } - - private void checkConfig() { - try { - IItemCollection profilerConfig = verifyEvents("datadog.DatadogProfilerConfig"); - for (IItemIterable items : profilerConfig) { - IMemberAccessor cpuIntervalAccessor = CPU_INTERVAL.getAccessor(items.getType()); - IMemberAccessor wallIntervalAccessor = WALL_INTERVAL.getAccessor(items.getType()); - for (IItem item : items) { - long cpuIntervalMillis = cpuIntervalAccessor.getMember(item).longValueIn(MILLISECOND); - long wallIntervalMillis = wallIntervalAccessor.getMember(item).longValueIn(MILLISECOND); - if (!Platform.isJ9() && Platform.isJavaVersionAtLeast(11)) { - // fixme J9 engine have weird defaults and need fixing - // Only assert intervals that were explicitly requested in the profiler - // command; engines not requested carry default intervals that do not - // match the (absent) command value. - if (cpuInterval.toMillis() > 0) { - assertEquals(cpuInterval.toMillis(), cpuIntervalMillis); - } - if (wallInterval.toMillis() > 0) { - assertEquals(wallInterval.toMillis(), wallIntervalMillis); - } - } - } - } - } catch (QuantityConversionException e) { - Assertions.fail(e.getMessage()); - } - } - - protected static IItemFilter allocatedTypeFilter(String className) { - return type -> { - IMemberAccessor accessor = JdkAttributes.OBJECT_CLASS.getAccessor(type); - return iItem -> { - return accessor != null && accessor.getMember(iItem).getFullName().equals(className); - }; - }; - } - - protected void runTests(Runnable... runnables) throws InterruptedException { - Thread[] threads = new Thread[runnables.length]; - for (int i = 0; i < runnables.length; i++) { - threads[i] = new Thread(runnables[i]); - } - for (Thread thread : threads) { - thread.start(); - } - for (Thread thread : threads) { - thread.join(); - } - stopProfiler(); - } - - - public final void stopProfiler() { - if (!stopped) { - profiler.stop(); - profiler.resetThreadContext(); - stopped = true; - checkConfig(); - } - } - - protected void dump(Path recording) { - if (!stopped) { - profiler.dump(recording); - } - } - - /** - * Waits for the profiler to reach RUNNING state by polling getStatus(). - * This ensures all engines are initialized and ready to collect samples - * before test workload begins. - * - * @param timeoutMs Maximum time to wait in milliseconds - * @throws IllegalStateException if profiler doesn't reach RUNNING state within timeout - * @throws InterruptedException if interrupted while waiting - */ - protected void waitForProfilerReady(long timeoutMs) throws InterruptedException { - long deadline = System.currentTimeMillis() + timeoutMs; - long waitTime = 0; - - while (System.currentTimeMillis() < deadline) { - String status = profiler.getStatus(); - if (status.contains("Running : true")) { - System.out.println("[Profiler Ready] Took " + waitTime + "ms to initialize"); - return; - } - Thread.sleep(10); - waitTime += 10; - } - - // Timeout reached - throw with diagnostic info - String finalStatus = profiler.getStatus(); - throw new IllegalStateException( - "Profiler failed to reach RUNNING state within " + timeoutMs + "ms\n" + - "Final status:\n" + finalStatus); - } - - public final void registerCurrentThreadForWallClockProfiling() { - profiler.addThread(); - } - - private String getAmendedProfilerCommand() { - String profilerCommand = getProfilerCommand(); - String testCstack = (String)testParams.get("cstack"); - if (testCstack != null) { - profilerCommand += ",cstack=" + testCstack; - } else if(!(ALLOW_NATIVE_CSTACKS || profilerCommand.contains("cstack="))) { - profilerCommand += ",cstack=fp"; - } - // FIXME - test framework doesn't seem to be forking each test, so need to sync - // these across test cases for now - // Only add attributes if not already specified - if (!profilerCommand.contains("attributes=")) { - profilerCommand += ",attributes=tag1;tag2;tag3"; - } - return profilerCommand; - } - - protected abstract String getProfilerCommand(); - - - protected void verifyEventsPresent(String... expectedEventTypes) { - verifyEventsPresent(jfrDump, expectedEventTypes); - } - - protected void verifyEventsPresent(Path recording, String... expectedEventTypes) { - try { - IItemCollection events = JfrLoaderToolkit.loadEvents(Files.newInputStream(recording)); - assertTrue(events.hasItems()); - for (String expectedEventType : expectedEventTypes) { - IItemCollection filtered = events.apply(ItemFilters.type(expectedEventType)); - assertTrue(filtered.hasItems(), - expectedEventType + " was empty for " + getAmendedProfilerCommand()); - System.out.println(expectedEventType + " count: " + filtered.stream().count()); - } - } catch (Throwable t) { - fail(getProfilerCommand() + " " + t.getMessage()); - } - } - - public final IItemCollection verifyEvents(String eventType) { - return verifyEvents(eventType, true); - } - - protected IItemCollection verifyEvents(String eventType, boolean failOnEmpty) { - return verifyEvents(jfrDump, eventType, failOnEmpty); - } - - protected IItemCollection verifyEvents(Path recording, String eventType, boolean failOnEmpty) { - try { - IItemCollection events = JfrLoaderToolkit.loadEvents(Files.newInputStream(recording)); - assertTrue(events.hasItems()); - IItemCollection collection = events.apply(ItemFilters.type(eventType)); - System.out.println(eventType + " count: " + collection.stream().flatMap(IItemIterable::stream).count()); - if (failOnEmpty) { - assertTrue(collection.hasItems(), - eventType + " was empty for " + getAmendedProfilerCommand()); - } - return collection; - } catch (Throwable t) { - fail(getProfilerCommand() + " " + t); - return null; - } - } - - protected final void verifyCStackSettings() { - String cstack = (String)testParams.get("cstack"); - if (cstack == null) { - // not a forced cstack mode - return; - } - IItemCollection settings = verifyEvents("jdk.ActiveSetting"); - for (IItemIterable settingEvents : settings) { - IMemberAccessor nameAccessor = JdkAttributes.REC_SETTING_NAME.getAccessor(settingEvents.getType()); - IMemberAccessor valueAccessor = JdkAttributes.REC_SETTING_VALUE.getAccessor(settingEvents.getType()); - for (IItem item : settingEvents) { - String name = nameAccessor.getMember(item); - if (name.equals("cstack")) { - assertEquals(cstack, valueAccessor.getMember(item)); - } - } - } - } - - protected void verifyStackTraces(String eventType, String... patterns) { - verifyStackTraces(jfrDump, eventType, patterns); - } - - protected void verifyStackTraces(Path recording, String eventType, String... patterns) { - Set unmatched = new HashSet<>(Arrays.asList(patterns)); - long cumulatedEvents = 0; - outer: for (IItemIterable sample : verifyEvents(recording, eventType, false)) { - cumulatedEvents += sample.getItemCount(); - IMemberAccessor stackTraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(sample.getType()); - for (IItem item : sample) { - String stackTrace = stackTraceAccessor.getMember(item); - if (stackTrace != null) { - unmatched.removeIf(stackTrace::contains); - if (unmatched.isEmpty()) { - break outer; - } - } - } - } - assertNotEquals(0, cumulatedEvents, "no events found for " + eventType); - assertTrue(unmatched.isEmpty(), "couldn't find " + eventType + " with " + unmatched); - } - - /** - * Returns the value of a named counter from {@code datadog.ProfilerCounter} events in the JFR - * recording. These events are written before the final cleanup ({@code processTraces}), so they - * capture the pre-cleanup state. - * - * @return the counter value, or -1 if no matching event is found - */ - public long getRecordedCounterValue(String counterName) { - IItemCollection events = verifyEvents("datadog.ProfilerCounter", false); - for (IItemIterable iterable : events) { - IMemberAccessor nameAccessor = NAME.getAccessor(iterable.getType()); - IMemberAccessor countAccessor = COUNT.getAccessor(iterable.getType()); - if (nameAccessor == null || countAccessor == null) continue; - for (IItem item : iterable) { - if (counterName.equals(nameAccessor.getMember(item))) { - return countAccessor.getMember(item).longValue(); - } - } - } - return -1; - } -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java deleted file mode 100644 index 853b06431..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/BufferWriterTest.java +++ /dev/null @@ -1,373 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler; - -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Comprehensive tests for {@link BufferWriter} class. - * - *

This test validates: - *

    - *
  • Correct implementation selection based on Java version
  • - *
  • Basic write and read operations with ordered (release) semantics
  • - *
  • Release (ordered) semantics for single-threaded signal handler safety
  • - *
  • Int and long write operations
  • - *
  • Various offsets and buffer positions
  • - *
  • Edge cases and boundary conditions
  • - *
- */ -public class BufferWriterTest { - - private BufferWriter bufferWriter; - - @BeforeEach - public void setUp() { - bufferWriter = new BufferWriter(); - } - - /** - * Helper method to create a direct ByteBuffer with native byte order. - * BufferWriter implementations use native byte order, so tests must read in the same order. - */ - private ByteBuffer createBuffer(int capacity) { - return ByteBuffer.allocateDirect(capacity).order(ByteOrder.nativeOrder()); - } - - /** - * Tests that the correct implementation is loaded based on Java version. - */ - @Test - public void testCorrectImplementationLoaded() { - assertNotNull(bufferWriter, "BufferWriter instance should not be null"); - - // Verify implementation can be used without throwing exceptions - ByteBuffer buffer = createBuffer(16); - assertDoesNotThrow(() -> bufferWriter.writeOrderedLong(buffer, 0, 42L)); - assertDoesNotThrow(() -> bufferWriter.writeOrderedLong(buffer, 8, 99L)); - } - - /** - * Tests basic ordered long write and read functionality at offset 0. - */ - @Test - public void testWriteOrderedLongAtOffsetZero() { - ByteBuffer buffer = createBuffer(16); - long expectedValue = 0x123456789ABCDEF0L; - - bufferWriter.writeOrderedLong(buffer, 0, expectedValue); - long actualValue = buffer.getLong(0); - - assertEquals(expectedValue, actualValue, "Buffer value should match written value"); - } - - /** - * Tests basic ordered int write and read functionality. - */ - @Test - public void testWriteOrderedInt() { - ByteBuffer buffer = createBuffer(16); - int expectedValue = 0x12345678; - - bufferWriter.writeOrderedInt(buffer, 0, expectedValue); - int actualValue = buffer.getInt(0); - - assertEquals(expectedValue, actualValue, "Buffer value should match written int value"); - } - - /** - * Tests ordered long write operations at various offsets within the buffer. - */ - @Test - public void testWriteOrderedLongAtVariousOffsets() { - ByteBuffer buffer = createBuffer(64); - - // Test at different 8-byte aligned offsets - int[] offsets = {0, 8, 16, 24, 32, 40, 48, 56}; - long[] expectedValues = { - 0x1111111111111111L, - 0x2222222222222222L, - 0x3333333333333333L, - 0x4444444444444444L, - 0x5555555555555555L, - 0x6666666666666666L, - 0x7777777777777777L, - 0x8888888888888888L - }; - - for (int i = 0; i < offsets.length; i++) { - bufferWriter.writeOrderedLong(buffer, offsets[i], expectedValues[i]); - } - - for (int i = 0; i < offsets.length; i++) { - long actualValue = buffer.getLong(offsets[i]); - assertEquals(expectedValues[i], actualValue, - String.format("Buffer value at offset %d should match", offsets[i])); - } - } - - /** - * Tests int write operations at various offsets within the buffer. - */ - @Test - public void testWriteIntAtVariousOffsets() { - ByteBuffer buffer = createBuffer(64); - - // Test at different 4-byte aligned offsets - int[] offsets = {0, 4, 8, 12, 16, 20, 24, 28}; - int[] expectedValues = { - 0x11111111, - 0x22222222, - 0x33333333, - 0x44444444, - 0x55555555, - 0x66666666, - 0x77777777, - 0x88888888 - }; - - for (int i = 0; i < offsets.length; i++) { - bufferWriter.writeOrderedInt(buffer, offsets[i], expectedValues[i]); - } - - for (int i = 0; i < offsets.length; i++) { - int actualValue = buffer.getInt(offsets[i]); - assertEquals(expectedValues[i], actualValue, - String.format("Int value at offset %d should match", offsets[i])); - } - } - - /** - * Tests writing special long values (min, max, zero, negative). - */ - @Test - public void testSpecialLongValues() { - ByteBuffer buffer = createBuffer(40); - long[] specialValues = { - 0L, // Zero - Long.MIN_VALUE, // Minimum long - Long.MAX_VALUE, // Maximum long - -1L, // All bits set - 0xDEADBEEFCAFEBABEL // Arbitrary pattern - }; - - for (int i = 0; i < specialValues.length; i++) { - int offset = i * 8; - bufferWriter.writeOrderedLong(buffer, offset, specialValues[i]); - long actualValue = buffer.getLong(offset); - assertEquals(specialValues[i], actualValue, - String.format("Special value 0x%X should be written correctly", specialValues[i])); - } - } - - /** - * Tests writing special int values (min, max, zero, negative). - */ - @Test - public void testSpecialIntValues() { - ByteBuffer buffer = createBuffer(24); - int[] specialValues = { - 0, // Zero - Integer.MIN_VALUE, // Minimum int - Integer.MAX_VALUE, // Maximum int - -1, // All bits set - 0xDEADBEEF, // Arbitrary pattern - 0x12345678 // Another pattern - }; - - for (int i = 0; i < specialValues.length; i++) { - int offset = i * 4; - bufferWriter.writeOrderedInt(buffer, offset, specialValues[i]); - int actualValue = buffer.getInt(offset); - assertEquals(specialValues[i], actualValue, - String.format("Special int value 0x%X should be written correctly", specialValues[i])); - } - } - - /** - * Tests that multiple consecutive writes work correctly without interference. - */ - @Test - public void testConsecutiveWrites() { - ByteBuffer buffer = createBuffer(16); - int offset = 0; - - // Write multiple times to the same offset - bufferWriter.writeOrderedLong(buffer, offset, 100L); - assertEquals(100L, buffer.getLong(offset)); - - bufferWriter.writeOrderedLong(buffer, offset, 200L); - assertEquals(200L, buffer.getLong(offset)); - - bufferWriter.writeOrderedLong(buffer, offset, 300L); - assertEquals(300L, buffer.getLong(offset)); - } - - /** - * Tests that writes to adjacent locations don't interfere with each other. - */ - @Test - public void testNonInterference() { - ByteBuffer buffer = createBuffer(32); - - long value1 = 0x1111111111111111L; - long value2 = 0x2222222222222222L; - long value3 = 0x3333333333333333L; - - bufferWriter.writeOrderedLong(buffer, 0, value1); - bufferWriter.writeOrderedLong(buffer, 8, value2); - bufferWriter.writeOrderedLong(buffer, 16, value3); - - assertEquals(value1, buffer.getLong(0), "First value should not be affected"); - assertEquals(value2, buffer.getLong(8), "Second value should not be affected"); - assertEquals(value3, buffer.getLong(16), "Third value should not be affected"); - } - - /** - * Tests writing to a buffer at maximum valid offset for longs. - */ - @Test - public void testMaximumValidOffsetLong() { - int bufferSize = 1024; - ByteBuffer buffer = createBuffer(bufferSize); - int maxValidOffset = bufferSize - 8; // 8 bytes for a long - - long expectedValue = 0xFEDCBA9876543210L; - bufferWriter.writeOrderedLong(buffer, maxValidOffset, expectedValue); - - long actualValue = buffer.getLong(maxValidOffset); - assertEquals(expectedValue, actualValue, - "Value at maximum offset should be written correctly"); - } - - /** - * Tests writing to a buffer at maximum valid offset for ints. - */ - @Test - public void testMaximumValidOffsetInt() { - int bufferSize = 1024; - ByteBuffer buffer = createBuffer(bufferSize); - int maxValidOffset = bufferSize - 4; // 4 bytes for an int - - int expectedValue = 0xFEDCBA98; - bufferWriter.writeOrderedInt(buffer, maxValidOffset, expectedValue); - - int actualValue = buffer.getInt(maxValidOffset); - assertEquals(expectedValue, actualValue, - "Int value at maximum offset should be written correctly"); - } - - /** - * Tests that overwriting values works correctly. - */ - @Test - public void testOverwrite() { - ByteBuffer buffer = createBuffer(16); - int offset = 0; - - // Write initial pattern - bufferWriter.writeOrderedLong(buffer, offset, 0xAAAAAAAAAAAAAAAAL); - assertEquals(0xAAAAAAAAAAAAAAAAL, buffer.getLong(offset)); - - // Overwrite with different pattern - bufferWriter.writeOrderedLong(buffer, offset, 0x5555555555555555L); - assertEquals(0x5555555555555555L, buffer.getLong(offset)); - - // Overwrite with zeros - bufferWriter.writeOrderedLong(buffer, offset, 0L); - assertEquals(0L, buffer.getLong(offset)); - } - - /** - * Tests parallel writes to different offsets from multiple threads. - */ - @Test - public void testParallelWrites() throws InterruptedException { - ByteBuffer buffer = createBuffer(128); - int numThreads = 8; - Thread[] threads = new Thread[numThreads]; - CountDownLatch startLatch = new CountDownLatch(1); - CountDownLatch doneLatch = new CountDownLatch(numThreads); - - for (int i = 0; i < numThreads; i++) { - final int threadIndex = i; - final int offset = threadIndex * 16; - final long expectedValue = (long) threadIndex * 0x1111111111111111L; - - threads[i] = new Thread(() -> { - try { - startLatch.await(5, TimeUnit.SECONDS); - bufferWriter.writeOrderedLong(buffer, offset, expectedValue); - doneLatch.countDown(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }); - threads[i].start(); - } - - // Start all threads simultaneously - startLatch.countDown(); - - // Wait for all threads to complete - assertTrue(doneLatch.await(5, TimeUnit.SECONDS), - "All threads should complete within timeout"); - - // Verify all values were written correctly - for (int i = 0; i < numThreads; i++) { - int offset = i * 16; - long expectedValue = (long) i * 0x1111111111111111L; - long actualValue = buffer.getLong(offset); - assertEquals(expectedValue, actualValue, - String.format("Thread %d's value at offset %d should be correct", i, offset)); - } - - for (Thread thread : threads) { - thread.join(1000); - } - } - - /** - * Tests that the buffer's original position and limit are not affected by writes. - */ - @Test - public void testBufferStatePreservation() { - ByteBuffer buffer = createBuffer(64); - buffer.position(10); - buffer.limit(50); - - int originalPosition = buffer.position(); - int originalLimit = buffer.limit(); - - bufferWriter.writeOrderedLong(buffer, 16, 0x123456789ABCDEF0L); - bufferWriter.writeOrderedLong(buffer, 24, 0xFEDCBA9876543210L); - bufferWriter.writeOrderedInt(buffer, 32, 0x12345678); - bufferWriter.writeOrderedInt(buffer, 36, 0x9ABCDEF0); - - assertEquals(originalPosition, buffer.position(), - "Buffer position should not be affected by writes"); - assertEquals(originalLimit, buffer.limit(), - "Buffer limit should not be affected by writes"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/CStackAwareAbstractProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/CStackAwareAbstractProfilerTest.java deleted file mode 100644 index 3cf797abf..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/CStackAwareAbstractProfilerTest.java +++ /dev/null @@ -1,13 +0,0 @@ -package com.datadoghq.profiler; - - -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.CStackInjector; -import org.junit.jupiter.api.extension.ExtendWith; - -@ExtendWith(CStackInjector.class) -public abstract class CStackAwareAbstractProfilerTest extends AbstractProfilerTest { - public CStackAwareAbstractProfilerTest(@CStack String cstack) { - super(mapOf("cstack", cstack)); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ContendedCallTraceStorageTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ContendedCallTraceStorageTest.java deleted file mode 100644 index b95b58d60..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ContendedCallTraceStorageTest.java +++ /dev/null @@ -1,257 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.datadoghq.profiler; - -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.IMCStackTrace; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.ItemFilters; -import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; -import org.openjdk.jmc.flightrecorder.CouldNotLoadRecordingException; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.CyclicBarrier; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Test to validate that CallTraceStorage::put() contention is low - * when exclusive operations (processTraces) are running concurrently. - * - * This test exercises contention between: - * - Multiple threads calling put() operations (shared lock) - * - JFR dump operations calling processTraces() (exclusive lock) - */ -public class ContendedCallTraceStorageTest extends AbstractProfilerTest { - - @Override - protected String getProfilerCommand() { - // Generate a lot of CPU samples - return "cpu=1ms"; - } - - @Override - protected boolean isPlatformSupported() { - // CTimer::unregisterThread races with concurrent thread teardown on musl-aarch64 debug; - // tracked separately as a pre-existing native bug: - // https://github.com/DataDog/java-profiler/issues/534 - return !Platform.isJ9() && !(Platform.isMusl() && Platform.isAarch64()); - } - - @Test - public void shouldShowImprovedContentionWithRetries() throws Exception { - List currentResults = measureContention(); - - // The test validates that the measurement infrastructure works - // In practice, you would modify CallTraceStorage::put to accept retry count - // and test with higher values like tryLockShared(100) - - for (ContentionResult currentResult : currentResults) { - // For this test, we verify that contention measurement works - assertTrue(currentResult.totalAttempts > 0, "Should measure total attempts"); - assertTrue( - currentResult.totalAttempts > 0 && - currentResult.droppedSamples / (double) currentResult.totalAttempts < 0.1f, - "Should measure total attempts and not drop more than 10% of samples" - ); - } - - // The key insight: this test framework can be used to validate - // that increasing retry counts reduces dropped samples - } - - private List measureContention() throws Exception { - Path jfrFile = Paths.get("contention-test.jfr"); - List recordings = new ArrayList<>(); - recordings.add(jfrFile); - - try { - // Create high contention scenario - int numThreads = Runtime.getRuntime().availableProcessors() * 2; - CyclicBarrier startBarrier = new CyclicBarrier(numThreads + 1); - CountDownLatch finishLatch = new CountDownLatch(numThreads); - - // Start concurrent allocation threads - for (int i = 0; i < numThreads; i++) { - final int threadId = i; - Thread worker = new Thread(() -> { - try { - startBarrier.await(); // Synchronize start - - // Generate CPU load for 5 seconds to ensure samples - long endTime = System.currentTimeMillis() + 5000; - while (System.currentTimeMillis() < endTime) { - performCpuIntensiveWork(threadId); - } - } catch (Exception e) { - throw new RuntimeException(e); - } finally { - finishLatch.countDown(); - } - }); - worker.start(); - } - - // Wait for all threads to be ready - startBarrier.await(); - - // Let allocation threads run for a bit, then trigger contention with dumps - Thread.sleep(500); - - // Trigger contention by calling dump during heavy allocation - // This forces processTraces() to acquire exclusive lock while put() operations are active - for (int i = 0; i < 3; i++) { - Path tempDump = Paths.get("temp-contention-" + i + ".jfr"); - dump(tempDump); // This will cause contention in CallTraceStorage - recordings.add(tempDump); - Thread.sleep(500); - } - - // Wait for all allocation threads to finish - finishLatch.await(); - - // Final dump to get all data - dump(jfrFile); - - // Analyze contention from JFR data - return analyzeContentionFromJFR(recordings); - - } finally { - recordings.forEach(f -> { - try { - Files.deleteIfExists(f); - } catch (IOException e) { - // ignore - } - }); - } - } - - private List analyzeContentionFromJFR(List recordings) throws IOException, CouldNotLoadRecordingException { - List results = new ArrayList<>(); - for (Path jfrFile : recordings) { - IItemCollection events = JfrLoaderToolkit.loadEvents(Files.newInputStream(jfrFile)); - - // Count profiling events - represents successful put() operations - IItemCollection cpuEvents = events.apply(ItemFilters.type("datadog.ExecutionSample")); - IItemCollection allocationEvents = events.apply(ItemFilters.type("jdk.ObjectAllocationInNewTLAB")); - - // Count events with regular stack traces vs dropped traces - long cpuWithRegularStack = countEventsWithRegularStackTrace(cpuEvents); - long cpuWithDroppedStack = countEventsWithDroppedStackTrace(cpuEvents); - long allocWithRegularStack = countEventsWithRegularStackTrace(allocationEvents); - long allocWithDroppedStack = countEventsWithDroppedStackTrace(allocationEvents); - - // Events with dropped stack traces indicate contention - CallTraceStorage::put() returned DROPPED_TRACE_ID - long contentionDrops = cpuWithDroppedStack + allocWithDroppedStack; - long totalEvents = cpuWithRegularStack + cpuWithDroppedStack + allocWithRegularStack + allocWithDroppedStack; - - System.out.printf("JFR Contention Analysis:%n"); - System.out.printf(" CPU: %d with regular stack, %d with dropped stack%n", cpuWithRegularStack, cpuWithDroppedStack); - System.out.printf(" Alloc: %d with regular stack, %d with dropped stack%n", allocWithRegularStack, allocWithDroppedStack); - System.out.printf(" Contention drops: %d/%d (%.2f%%)%n", - contentionDrops, totalEvents, - totalEvents > 0 ? (double) contentionDrops / totalEvents * 100 : 0); - results.add(new ContentionResult(contentionDrops, totalEvents)); - } - - return results; - } - - private long countEventsWithRegularStackTrace(IItemCollection events) { - if (!events.hasItems()) return 0; - - long count = 0; - for (IItemIterable iterable : events) { - for (IItem item : iterable) { - IMCStackTrace stackTrace = STACK_TRACE.getAccessor(iterable.getType()).getMember(item); - if (stackTrace != null && !stackTrace.getFrames().isEmpty()) { - // Check if this is NOT the dropped trace (contains method with "dropped") - String topMethodName = stackTrace.getFrames().get(0).getMethod().getMethodName(); - if (!topMethodName.contains("dropped")) { - count++; - } - } - } - } - return count; - } - - private long countEventsWithDroppedStackTrace(IItemCollection events) { - if (!events.hasItems()) return 0; - - long count = 0; - for (IItemIterable iterable : events) { - for (IItem item : iterable) { - IMCStackTrace stackTrace = STACK_TRACE.getAccessor(iterable.getType()).getMember(item); - if (stackTrace != null && !stackTrace.getFrames().isEmpty()) { - // Check if this is the special dropped trace (single frame with "dropped" method) - if (stackTrace.getFrames().size() == 1) { - String methodName = stackTrace.getFrames().get(0).getMethod().getMethodName(); - if (methodName.contains("dropped")) { - count++; - } - } - } - } - } - return count; - } - - private void performCpuIntensiveWork(int threadId) { - // Simple CPU-intensive loop similar to ProfiledCode.burnCycles() - burnCycles(threadId); - } - - private void burnCycles(int threadId) { - // CPU burning pattern that ensures we get profiling samples - long sink = 0; - for (int i = 0; i < 100000; i++) { - sink += i * threadId; - sink ^= threadId; - if (i % 1000 == 0) { - // Add some method calls to create interesting stack traces - sink += computeHash(sink, threadId); - } - } - // Store in volatile to prevent optimization - volatileResult = sink; - } - - private long computeHash(long value, int threadId) { - // Another method in the stack trace - long result = value; - for (int i = 0; i < 100; i++) { - result = Long.rotateLeft(result, 1); - result ^= (threadId + i); - } - return result; - } - - private volatile long volatileResult; // Prevent optimization - - private static class ContentionResult { - final long droppedSamples; - final long totalAttempts; - - ContentionResult(long droppedSamples, long totalAttempts) { - this.droppedSamples = droppedSamples; - this.totalAttempts = totalAttempts; - } - - double getDropRate() { - return totalAttempts > 0 ? (double) droppedSamples / totalAttempts : 0.0; - } - } -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java deleted file mode 100644 index cee538aea..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ExternalLauncher.java +++ /dev/null @@ -1,79 +0,0 @@ -package com.datadoghq.profiler; - -import java.lang.management.ManagementFactory; -import java.lang.management.ThreadMXBean; -import java.util.Random; -import java.util.concurrent.atomic.LongAdder; - -/** - * External launcher for the profiler under test. - *

- * This class is used to launch the profiler in a separate process for testing purposes. - *

- * The main method takes the following arguments: - *
    - *
  • library - loads the profiler library
  • - *
  • profiler [comma delimited profiler command list] - starts the profiler
  • - *
  • profiler-work: [comma delimited profiler command list] - starts the profiler and runs a CPU-intensive task
  • - *
- */ -public class ExternalLauncher { - public static void main(String[] args) throws Exception { - Thread worker = null; - try { - if (args.length < 1) { - throw new RuntimeException(); - } - if (args[0].equals("library")) { - JVMAccess.getInstance(); - } else if (args[0].equals("profiler")) { - JavaProfiler instance = JavaProfiler.getInstance(); - if (args.length == 2) { - String commands = args[1]; - if (!commands.isEmpty()) { - instance.execute(commands); - } - } - } else if (args[0].startsWith("profiler-work:")) { - long expectedCpuTime = Long.parseLong(args[0].substring("profiler-work:".length())); - ThreadMXBean thrdBean = ManagementFactory.getThreadMXBean(); - JavaProfiler instance = JavaProfiler.getInstance(); - if (args.length == 2) { - String commands = args[1]; - if (!commands.isEmpty()) { - instance.execute(commands); - worker = new Thread(() -> { - Random rnd = new Random(); - LongAdder adder = new LongAdder(); - long counter = 0; - long cpuTime = thrdBean.getThreadCpuTime(Thread.currentThread().getId()); - while (!Thread.currentThread().isInterrupted()) { - adder.add(rnd.nextLong()); - // make sure we caused some CPU load and print the progress - if (++counter % 1000000 == 0) { - if (thrdBean.getThreadCpuTime(Thread.currentThread().getId()) - cpuTime > expectedCpuTime * 1_000_000L) { - cpuTime = thrdBean.getThreadCpuTime(Thread.currentThread().getId()); - System.out.println("[working]"); - System.out.flush(); - } - } - } - System.out.println("[async] " + adder.sum()); - }); - worker.start(); - } - } - } - } finally { - System.out.println("[ready]"); - System.out.flush(); - System.err.flush(); - } - // wait for signal to exit - System.in.read(); - if (worker != null) { - worker.interrupt(); - worker.join(); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/JVMAccessTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/JVMAccessTest.java deleted file mode 100644 index bdd17feeb..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/JVMAccessTest.java +++ /dev/null @@ -1,109 +0,0 @@ -package com.datadoghq.profiler; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.util.Collections; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicReference; - -import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assumptions.assumeFalse; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class JVMAccessTest extends AbstractProcessProfilerTest { - @BeforeAll - static void setUp() { - assumeFalse(Platform.isJ9() || Platform.isZing()); // J9 and Zing do not support vmstructs - } - - @Test - void sanityInitailizationTest() throws Exception { - String config = System.getProperty("ddprof_test.config"); - assumeTrue("debug".equals(config)); - - AtomicBoolean initLibraryFound = new AtomicBoolean(false); - AtomicBoolean initProfilerFound = new AtomicBoolean(false); - - boolean rslt = launch("library", Collections.emptyList(), null, - l -> { - initLibraryFound.set(initLibraryFound.get() | l.contains("[TEST::INFO] VM::initLibrary")); - initProfilerFound.set(initProfilerFound.get() | l.contains("[TEST::INFO] VM::initProfilerBridge")); - return LineConsumerResult.CONTINUE; - }, - null - ).inTime; - - assertTrue(rslt); - - assertTrue(initLibraryFound.get(), "initLibrary not found"); - assertFalse(initProfilerFound.get(), "initProfilerBridge found"); - } - - @Test - void jvmVersionTest() throws Exception { - String config = System.getProperty("ddprof_test.config"); - assumeTrue("debug".equals(config)); - - String javaVersion = System.getenv("JAVA_VERSION"); - assumeTrue(javaVersion != null); - if (javaVersion.startsWith("8u")) { - // convert 8u432 to nomralized 8.0.432 format which is expected - javaVersion = "8.0." + javaVersion.split("u")[1]; - } - - AtomicReference foundVersion = new AtomicReference<>(null); - - boolean rslt = launch("library", Collections.emptyList(), null, l -> { - if (l.contains("[TEST::INFO] jvm_version#")) { - foundVersion.set(l.split("#")[1]); - return LineConsumerResult.STOP; - } - return LineConsumerResult.CONTINUE; - }, null).inTime; - - assertTrue(rslt); - - assertNotNull(foundVersion.get(), "java version not found in logs"); - assertEquals(javaVersion, foundVersion.get(), "invalid java version"); - } - - @Test - void testGetFlag() { - JVMAccess.Flags flags = JVMAccess.getInstance().flags(); - // non-existent flag - assertNull(flags.getStringFlag("test")); - - // The test relies on the gradle test task setting the JVM flags to expected values - assertEquals("build/hs_err_pid%p.log", flags.getStringFlag("ErrorFile")); // set to 'build/hs_err_pid%p.log' in the test task - assertTrue(flags.getBooleanFlag("ResizeTLAB")); // set to 'true' in the test task - assertEquals(512 * 1024 * 1024, flags.getIntFlag("MaxHeapSize")); // set to 512m in the test task - assertNotNull(flags.getStringFlag("OnError")); - } - - @Test - void testGetFlagMismatch() { - JVMAccess.Flags flags = JVMAccess.getInstance().flags(); - - assertNull(flags.getStringFlag("ResizeTLAB")); // default is 'null' - assertFalse(flags.getBooleanFlag("ErrorFile")); // default is 'false' - assertEquals(0, flags.getFloatFlag("MaxHeapSize")); // default is '0' - } - - @Test - void testMutableFlags() { - JVMAccess.Flags flags = JVMAccess.getInstance().flags(); - String errorFile = "/tmp/hs_err_pid%p.log"; - flags.setStringFlag("ErrorFile", errorFile); - assertEquals(errorFile, flags.getStringFlag("ErrorFile")); - } - - @Test - void testMutableFlagsMismatch() { - JVMAccess.Flags flags = JVMAccess.getInstance().flags(); - String val = flags.getStringFlag("ErrorFile"); - flags.setBooleanFlag("ErrorFile", true); - // make sure the flag value is not changed and overwritten with rubbish - assertEquals(val, flags.getStringFlag("ErrorFile")); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java deleted file mode 100644 index b3052f3a2..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerApiSurfaceTest.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.datadoghq.profiler; - -import org.junit.jupiter.api.Test; - -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; - -import static org.junit.jupiter.api.Assertions.assertFalse; - -public class JavaProfilerApiSurfaceTest { - @Test - public void ownedBlockHooksAreNotPublicApiBeforeTaskBlockInstrumentation() throws Exception { - assertNotPublic(JavaProfiler.class.getDeclaredMethod("parkEnter")); - assertNotPublic(JavaProfiler.class.getDeclaredMethod( - "parkExit", long.class, long.class)); - assertNotPublic(JavaProfiler.class.getDeclaredMethod("blockEnter", int.class)); - assertNotPublic(JavaProfiler.class.getDeclaredMethod("blockExit", long.class)); - } - - private static void assertNotPublic(Method method) { - assertFalse(Modifier.isPublic(method.getModifiers()), - method.getName() + " must remain non-public until PR2 wires TaskBlock instrumentation"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerTest.java deleted file mode 100644 index 5648b29f2..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/JavaProfilerTest.java +++ /dev/null @@ -1,136 +0,0 @@ -package com.datadoghq.profiler; - -import org.junit.jupiter.api.Test; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.LockSupport; - -import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assumptions.assumeFalse; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class JavaProfilerTest extends AbstractProcessProfilerTest { - @Test - void sanityInitailizationTest() throws Exception { - String config = System.getProperty("ddprof_test.config"); - assumeTrue("debug".equals(config)); - - AtomicInteger initFlag = new AtomicInteger(0); - boolean val = launch("profiler", Collections.emptyList(), "", l -> { - if (l.contains("[TEST::INFO] VM::initLibrary")) { - initFlag.set(initFlag.get() | 1); - } else if (l.contains("[TEST::INFO] VM::initProfilerBridge")) { - initFlag.set(initFlag.get() | 2); - } - // found both expected sections; can terminate the test now - return initFlag.get() != 3 ? LineConsumerResult.CONTINUE : LineConsumerResult.STOP; - }, null).inTime; - - assertTrue(val); - } - - @Test - void testJ9DefaultSanity() throws Exception { - String config = System.getProperty("ddprof_test.config"); - assumeTrue("debug".equals(config)); - assumeFalse(Platform.isMac()); // crashy on mac - assumeTrue(Platform.isJ9()); - - Path jfr = Files.createTempFile("j9", ".jfr"); - jfr.toFile().deleteOnExit(); - - // ASGCT re-enabled for versions containing the OpenJ9 0.51 fix (eclipse-openj9/openj9#20577) - String sampler = "jvmti"; - if (Platform.isJavaVersion(8) && Platform.isJavaVersionAtLeast(8, 0, 451)) { - sampler = "asgct"; - } else if (Platform.isJavaVersion(11) && Platform.isJavaVersionAtLeast(11, 0, 27)) { - sampler = "asgct"; - } else if (Platform.isJavaVersion(17) && Platform.isJavaVersionAtLeast(17, 0, 15)) { - sampler = "asgct"; - } else if (Platform.isJavaVersion(21) && Platform.isJavaVersionAtLeast(21, 0, 7)) { - sampler = "asgct"; - } - - AtomicReference usedSampler = new AtomicReference<>(""); - AtomicBoolean hasWall = new AtomicBoolean(false); - boolean val = launch("profiler", Collections.emptyList(), "start,cpu,file=" + jfr, l -> { - if (l.contains("J9[cpu]")) { - usedSampler.set(l.split("=")[1]); - return LineConsumerResult.STOP; - } else if (l.contains("J9[wall]")) { - hasWall.set(true); - return LineConsumerResult.STOP; - } - return LineConsumerResult.CONTINUE; - }, null).inTime; - assertTrue(val); - assertEquals(sampler, usedSampler.get()); - assertFalse(hasWall.get()); - } - - @Test - void testJ9ForceJvmtiSanity() throws Exception { - String config = System.getProperty("ddprof_test.config"); - assumeTrue("debug".equals(config)); - assumeFalse(Platform.isMac()); // crashy on mac - assumeTrue(Platform.isJ9()); - - Path jfr = Files.createTempFile("j9", ".jfr"); - jfr.toFile().deleteOnExit(); - - String sampler = "jvmti"; - - AtomicReference usedSampler = new AtomicReference<>(""); - AtomicBoolean hasWall = new AtomicBoolean(false); - List args = new ArrayList<>(); - args.add("-XX:+KeepJNIIDs"); - args.add("-Ddd.profiling.ddprof.j9.sampler=jvmti"); - boolean val = launch("profiler", args, "start,cpu,file=" + jfr, l -> { - if (l.contains("J9[cpu]")) { - usedSampler.set(l.split("=")[1]); - return LineConsumerResult.STOP; - } else if (l.contains("J9[wall]")) { - hasWall.set(true); - return LineConsumerResult.STOP; - } - return LineConsumerResult.CONTINUE; - }, null).inTime; - assertTrue(val); - assertEquals(sampler, usedSampler.get()); - assertFalse(hasWall.get()); - } - - @Test - void vmStackwalkerCrashRecoveryTest() throws Exception { - assumeFalse(Platform.isJ9() || Platform.isZing()); // J9 and Zing do not support vmstructs - String config = System.getProperty("ddprof_test.config"); - assumeTrue("debug".equals(config)); - - Path jfr = Files.createTempFile("work", ".jfr"); - jfr.toFile().deleteOnExit(); - - Map env = Collections.singletonMap("DDPROF_FORCE_STACKWALK_CRASH", "1"); - // run the profiled process and generate at least 50ms of cpu activity - LaunchResult rslt = launch("profiler-work:50", Collections.emptyList(), "start,cpu=1ms,cstack=vm,file=" + jfr, env, l -> { - if (l.contains("[ready]")) { - return LineConsumerResult.IGNORE; - } - if (l.contains("[working]")) { - return LineConsumerResult.STOP; - } - return LineConsumerResult.CONTINUE; - }, null); - - assertTrue(rslt.inTime); - assertEquals(0, rslt.exitCode, "exit code should be 0"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/MoreAssertions.java b/ddprof-test/src/test/java/com/datadoghq/profiler/MoreAssertions.java deleted file mode 100644 index 0f80d5fbb..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/MoreAssertions.java +++ /dev/null @@ -1,18 +0,0 @@ -package com.datadoghq.profiler; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class MoreAssertions { - - public static final int DICTIONARY_PAGE_SIZE = (128 * (3 * 8 + 8) + 4); - - public static void assertBoundedBy(long value, long maximum, String error) { - if (value >= maximum) { - throw new AssertionError(error + ". Too large: " + value + " > " + maximum); - } - } - - public static void assertInRange(double value, double min, double max) { - assertTrue(value >= min && value <= max, value + " not in (" + min + "," + max + ")"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/MuslDetectionTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/MuslDetectionTest.java deleted file mode 100644 index 5f93868da..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/MuslDetectionTest.java +++ /dev/null @@ -1,23 +0,0 @@ -package com.datadoghq.profiler; - -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; - -import java.io.IOException; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class MuslDetectionTest { - - @Test - public void testIsMusl() throws IOException { - Assumptions.assumeTrue(Platform.isLinux(), "not running on linux"); - String libc = System.getenv("LIBC"); - Assumptions.assumeTrue(libc != null, "not running in CI, so LIBC envvar not set"); - boolean isMusl = "musl".equalsIgnoreCase(libc); - OperatingSystem os = OperatingSystem.current(); - assertEquals(isMusl, os.isMuslProcSelfMaps()); - assertEquals(isMusl, os.isMuslJavaExecutable()); - assertEquals(isMusl, os.isMusl()); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/PlatformTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/PlatformTest.java deleted file mode 100644 index 77e912202..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/PlatformTest.java +++ /dev/null @@ -1,133 +0,0 @@ -package com.datadoghq.profiler; - -import java.util.stream.Stream; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.Arguments; -import org.junit.jupiter.params.provider.MethodSource; - - -import static org.junit.jupiter.api.Assertions.*; -import static org.junit.jupiter.api.Assumptions.*; - -class PlatformTest { - @Test - void isAtLeastJava7() { - assertTrue(Platform.isJavaVersionAtLeast(7)); - } - - @Test - void isAtLeastJava8() { - assumeTrue(!System.getProperty("java.version").startsWith("1.") - || System.getProperty("java.version").startsWith("1.8.")); - assertTrue(Platform.isJavaVersionAtLeast(8) && Platform.isJavaVersionAtLeast(7)); - } - - @Test - void isAtLeastJava11() { - assumeTrue(!System.getProperty("java.version").startsWith("1.") - && !(System.getProperty("java.version").startsWith("9.") - || System.getProperty("java.version").startsWith("10."))); - assertTrue(Platform.isJavaVersionAtLeast(11) && Platform.isJavaVersionAtLeast(8)); - } - - @ParameterizedTest - @MethodSource("parseArgsExact") - void testParse(String version, int major, int minor, int update) { - Platform.Version javaVersion = Platform.parseJavaVersion(version); - - assertEquals(major, javaVersion.major); - assertEquals(minor, javaVersion.minor); - assertEquals(update, javaVersion.update); - assertTrue(javaVersion.is(major)); - assertTrue(javaVersion.is(major, minor)); - assertTrue(javaVersion.is(major, minor, update)); - assertTrue(javaVersion.isAtLeast(major, minor, update)); - assertTrue(javaVersion.isBetween(major, minor, update, Integer.MAX_VALUE, Integer.MAX_VALUE, Integer.MAX_VALUE)); - assertFalse(javaVersion.isBetween(major, minor, update, major, minor, update)); - assertFalse(javaVersion.isBetween(major, minor, update, major - 1, 0, 0)); - assertFalse(javaVersion.isBetween(major, minor, update, major, minor -1, 0)); - assertFalse(javaVersion.isBetween(major, minor, update, major, minor, update - 1)); - assertTrue(javaVersion.isBetween(major, minor, update, major + 1, 0, 0)); - assertTrue(javaVersion.isBetween(major, minor, update, major, minor + 1, 0)); - assertTrue(javaVersion.isBetween(major, minor, update, major, minor, update + 1)); - } - - @ParameterizedTest - @MethodSource("parseArgsAtLeast") - void testParseAtLeast(String version, int major, int minor, int update) { - Platform.Version javaVersion = Platform.parseJavaVersion(version); - assertTrue(javaVersion.isAtLeast(major, minor, update)); - } - - @ParameterizedTest - @MethodSource("parseArgsWeird") - void tetParseWeird(String propVersion, String rtVersion, String propName, String propVendor, String version, String patch, String name, String vendor) { - Platform.JvmRuntime runtime = new Platform.JvmRuntime(propVersion, rtVersion, propName, propVendor); - - assertEquals(version, runtime.version); - assertEquals(patch, runtime.patches); - assertEquals(name, runtime.name); - assertEquals(vendor, runtime.vendor); - } - - private static Stream parseArgsExact() { - return Stream.of( - Arguments.of("" , 0 , 0 , 0), - Arguments.of("a.0.0" , 0 , 0 , 0), - Arguments.of("0.a.0" , 0 , 0 , 0), - Arguments.of("0.0.a" , 0 , 0 , 0), - Arguments.of("1.a.0_0" , 0 , 0 , 0), - Arguments.of("1.8.a_0" , 0 , 0 , 0), - Arguments.of("1.8.0_a" , 0 , 0 , 0), - Arguments.of("1.7" , 7 , 0 , 0), - Arguments.of("1.7.0" , 7 , 0 , 0), - Arguments.of("1.7.0_221" , 7 , 0 , 221), - Arguments.of("1.8" , 8 , 0 , 0), - Arguments.of("1.8.0" , 8 , 0 , 0), - Arguments.of("1.8.0_212" , 8 , 0 , 212), - Arguments.of("1.8.0_292" , 8 , 0 , 292), - Arguments.of("9-ea" , 9 , 0 , 0), - Arguments.of("9.0.4" , 9 , 0 , 4), - Arguments.of("9.1.2" , 9 , 1 , 2), - Arguments.of("10.0.2" , 10 , 0 , 2), - Arguments.of("11" , 11 , 0 , 0), - Arguments.of("11.0.6" , 11 , 0 , 6), - Arguments.of("11.0.11" , 11 , 0 , 11), - Arguments.of("12.0.2" , 12 , 0 , 2), - Arguments.of("13.0.2" , 13 , 0 , 2), - Arguments.of("14" , 14 , 0 , 0), - Arguments.of("14.0.2" , 14 , 0 , 2), - Arguments.of("15" , 15 , 0 , 0), - Arguments.of("15.0.2" , 15 , 0 , 2), - Arguments.of("16.0.1" , 16 , 0 , 1), - Arguments.of("11.0.9.1+1", 11 , 0 , 9), - Arguments.of("11.0.6+10" , 11 , 0 , 6), - Arguments.of("17.0.4-x" , 17 , 0 , 4) - ); - } - - private static Stream parseArgsAtLeast() { - return Stream.of( - Arguments.of("17.0.5+8" , 17 , 0 , 5), - Arguments.of("17.0.5" , 17 , 0 , 5), - Arguments.of("17.0.6+8" , 17 , 0 , 5), - Arguments.of("11.0.17+8" , 11 , 0 , 17), - Arguments.of("11.0.18+8" , 11 , 0 , 17), - Arguments.of("11.0.17" , 11 , 0 , 17), - Arguments.of("1.8.0_352" , 8 , 0 , 352), - Arguments.of("1.8.0_362" , 8 , 0 , 352) - ); - } - - private static Stream parseArgsWeird() { - return Stream.of( - Arguments.of("1.8.0_265" , "1.8.0_265-b01" , "OpenJDK" , "AdoptOpenJDK" , "1.8.0_265" , "b01" , "OpenJDK" , "AdoptOpenJDK"), - Arguments.of("1.8.0_265" , "1.8-b01" , "OpenJDK" , "AdoptOpenJDK" , "1.8.0_265" , "" , "OpenJDK" , "AdoptOpenJDK"), - Arguments.of("19" , "19" , "OpenJDK 64-Bit" , "Homebrew" , "19" , "" , "OpenJDK 64-Bit" , "Homebrew"), - Arguments.of("17" , null , null , null , "17" , "" , "" , ""), - Arguments.of(null , "17" , null , null , "" , "" , "" , "") - ); - } -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ProfilerOwnedBlockHooks.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ProfilerOwnedBlockHooks.java deleted file mode 100644 index f58837f5d..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ProfilerOwnedBlockHooks.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.datadoghq.profiler; - -/** Test bridge for package-scoped owned blocking hooks. */ -public final class ProfilerOwnedBlockHooks { - private ProfilerOwnedBlockHooks() {} - - public static void parkEnter(JavaProfiler profiler) { - profiler.parkEnter(); - } - - public static void parkExit(JavaProfiler profiler, long blocker, long unblockingSpanId) { - profiler.parkExit(blocker, unblockingSpanId); - } - - public static long blockEnter(JavaProfiler profiler, int state) { - return profiler.blockEnter(state); - } - - public static void blockExit(JavaProfiler profiler, long token) { - profiler.blockExit(token); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/RemoteSymHelper.java b/ddprof-test/src/test/java/com/datadoghq/profiler/RemoteSymHelper.java deleted file mode 100644 index af6cf86bd..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/RemoteSymHelper.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.datadoghq.profiler; - -/** - * Helper class for remote symbolication testing. - * Provides JNI methods that burn CPU to ensure native frames appear in profiling samples. - * The native library is built with GNU build-id on Linux for remote symbolication testing. - */ -public class RemoteSymHelper { - static { - System.loadLibrary("ddproftest"); - } - - /** - * Burns CPU cycles by performing recursive computation. - * This creates a distinctive call stack that should appear in profiling samples. - * - * @param iterations Number of iterations for computation - * @param depth Recursion depth - * @return Computed result (to prevent optimization) - */ - public static native long burnCpu(long iterations, int depth); - - /** - * Computes Fibonacci numbers repeatedly to burn CPU. - * - * @param n Fibonacci number to compute - * @return Computed result (to prevent optimization) - */ - public static native long computeFibonacci(int n); -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java deleted file mode 100644 index 5078181df..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/ScopeStackTest.java +++ /dev/null @@ -1,164 +0,0 @@ -package com.datadoghq.profiler; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; - -import java.nio.ByteBuffer; -import java.nio.ByteOrder; - -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; - -/** - * Pure-Java unit test for {@link ScopeStack}. Uses heap-backed {@link ByteBuffer}s so - * no native library is required. Exercises depth accounting, underflow, and round-trip - * preservation of trace/span IDs across fast-path and chunked-path depths. - */ -public class ScopeStackTest { - - // Offsets mirror OtelThreadContextRecord in otel_context.h and the sidecar layout - // built by initializeContextTLS0 in javaApi.cpp. These are spec-fixed; guarded by - // static_asserts in native code. All are absolute within the unified buffer. - private static final int TRACE_ID_OFFSET = 0; - private static final int SPAN_ID_OFFSET = 16; - private static final int VALID_OFFSET = 24; - private static final int ATTRS_DATA_SIZE_OFFSET = 26; - private static final int ATTRS_DATA_OFFSET = 28; - private static final int LRS_OFFSET = 640 + 40; // after 640-byte record + 10 * sizeof(u32) - - private static ThreadContext newContext() { - ByteBuffer buf = ByteBuffer.allocate(ThreadContext.SNAPSHOT_SIZE).order(ByteOrder.nativeOrder()); - long[] metadata = { - VALID_OFFSET, TRACE_ID_OFFSET, SPAN_ID_OFFSET, - ATTRS_DATA_SIZE_OFFSET, ATTRS_DATA_OFFSET, LRS_OFFSET - }; - return new ThreadContext(buf, metadata); - } - - private static void assumeLittleEndian() { - Assumptions.assumeTrue( - ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN, - "ThreadContext only supports little-endian platforms"); - } - - @Test - public void depthBalance() { - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - assertEquals(0, stack.depth()); - stack.enter(ctx); - assertEquals(1, stack.depth()); - stack.enter(ctx); - assertEquals(2, stack.depth()); - stack.exit(ctx); - assertEquals(1, stack.depth()); - stack.exit(ctx); - assertEquals(0, stack.depth()); - } - - @Test - public void exitUnderflowThrows() { - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - assertThrows(IllegalStateException.class, () -> stack.exit(ctx)); - } - - @Test - public void fastPathRoundTrip() { - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - - ctx.put(/*lrs*/ 100L, /*span*/ 200L, /*trHi*/ 0L, /*trLo*/ 300L); - assertEquals(200L, ctx.getSpanId()); - assertEquals(100L, ctx.getRootSpanId()); - - stack.enter(ctx); - ctx.put(500L, 600L, 0L, 700L); - assertEquals(600L, ctx.getSpanId()); - assertEquals(500L, ctx.getRootSpanId()); - - stack.exit(ctx); - assertEquals(200L, ctx.getSpanId(), "span must be restored"); - assertEquals(100L, ctx.getRootSpanId(), "root span must be restored"); - } - - @Test - public void chunkedPathRoundTrip() { - // Push past FAST_DEPTH (6) to exercise the lazy-chunk path and Arrays.copyOf growth. - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - - final int depth = 20; // FAST_DEPTH + one full 12-slot chunk + 2 into the next - for (int i = 0; i < depth; i++) { - ctx.put(1000L + i, 2000L + i, 0L, 3000L + i); - stack.enter(ctx); - } - assertEquals(depth, stack.depth()); - - // Scramble state so restore has something to correct. - ctx.put(99L, 99L, 0L, 99L); - - for (int i = depth - 1; i >= 0; i--) { - stack.exit(ctx); - assertEquals(2000L + i, ctx.getSpanId(), "span mismatch at depth " + i); - assertEquals(1000L + i, ctx.getRootSpanId(), "root mismatch at depth " + i); - } - assertEquals(0, stack.depth()); - } - - @Test - public void reusesStackAfterFullUnwind() { - // After the stack returns to depth 0, re-entering must not leak state from the prior run. - assumeLittleEndian(); - ThreadContext ctx = newContext(); - ScopeStack stack = new ScopeStack(); - - ctx.put(1L, 2L, 0L, 3L); - stack.enter(ctx); - ctx.put(10L, 20L, 0L, 30L); - stack.exit(ctx); - assertEquals(2L, ctx.getSpanId()); - - ctx.put(4L, 5L, 0L, 6L); - stack.enter(ctx); - ctx.put(40L, 50L, 0L, 60L); - stack.exit(ctx); - assertEquals(5L, ctx.getSpanId()); - } - - @Test - public void snapshotOverClearedContextDoesNotRepublish() { - // Regression: snapshot() used to unconditionally re-attach, flipping valid back to 1 - // after a zero-put clear. The clear path leaves attrs_data_size / attrs_data stale and - // relies on valid=0 to keep external readers from seeing the stale bytes. Here we verify - // the valid byte directly since setContextAttribute is a native path unavailable to - // pure-Java tests. - assumeLittleEndian(); - ByteBuffer buf = ByteBuffer.allocate(ThreadContext.SNAPSHOT_SIZE).order(ByteOrder.nativeOrder()); - long[] metadata = { - VALID_OFFSET, TRACE_ID_OFFSET, SPAN_ID_OFFSET, - ATTRS_DATA_SIZE_OFFSET, ATTRS_DATA_OFFSET, LRS_OFFSET - }; - ThreadContext ctx = new ThreadContext(buf, metadata); - ScopeStack stack = new ScopeStack(); - - ctx.put(1L, 2L, 0L, 3L); - assertEquals(1, buf.get(VALID_OFFSET), "record must be published after non-zero put"); - - // Zero-put clear: leaves valid=0 (the all-zero early-return in setContextDirect). - ctx.put(0L, 0L, 0L, 0L); - assertEquals(0, buf.get(VALID_OFFSET), "record must be invalid after zero-put clear"); - - stack.enter(ctx); - assertEquals(0, buf.get(VALID_OFFSET), - "snapshot must preserve valid=0 — not republish a cleared record"); - - stack.exit(ctx); - assertEquals(0, buf.get(VALID_OFFSET), - "restore must replay valid=0 — not republish a cleared record"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/alloc/AllocationProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/alloc/AllocationProfilerTest.java deleted file mode 100644 index d995d065f..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/alloc/AllocationProfilerTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.datadoghq.profiler.alloc; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.Aggregators; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jol.info.GraphLayout; - -import java.util.Random; -import java.util.concurrent.atomic.AtomicLong; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class AllocationProfilerTest extends AbstractProfilerTest { - - @Override - protected boolean isPlatformSupported() { - return !(Platform.isJ9() || Platform.isZing()) && Platform.isJavaVersionAtLeast(11); - } - - @RetryingTest(5) - public void shouldGetObjectAllocationSamples() throws InterruptedException { - - // We seem to hit issues on j9: - // OSR (On stack replacement) creates crashes with the profiler. - // ----------- Stack Backtrace ----------- - // prepareForOSR+0xbf (0x00007F51062A4DDF [libj9jit29.so+0x4a4ddf]) - if (Platform.isJ9() && !Platform.isJavaVersionAtLeast(8)) { - return; - } - Assumptions.assumeFalse(isAsan() || isTsan()); - - AllocatingTarget target1 = new AllocatingTarget(); - AllocatingTarget target2 = new AllocatingTarget(); - runTests(target1, target2); - IItemCollection allocations = verifyEvents("datadog.ObjectSample"); - // FIXME when more tests are ported to this structure - if (!Platform.isMusl()) { - // JOL on musl seems to be locking up randomly - assertAllocations(allocations, int[].class, target1, target2); - assertAllocations(allocations, Integer[].class, target1, target2); - } - } - - private static void assertAllocations(IItemCollection allocations, Class clazz, AllocatingTarget... targets) { - long allocated = 0; - for (AllocatingTarget target : targets) { - allocated += target.getAllocated(clazz); - } - IItemCollection allocationsByType = allocations.apply(allocatedTypeFilter(clazz.getCanonicalName())); - assertTrue(allocationsByType.hasItems()); - long recorded = allocationsByType.getAggregate(Aggregators.sum(SCALED_SIZE)).longValue(); - double error = Math.abs(recorded - allocated) / (double)allocated; - assertTrue(error <= 0.50, - String.format("allocation samples should be within 10pct tolerance of allocated memory (recorded %d, allocated %d :: %4.2f)", - recorded, allocated, error * 100)); - } - - @Override - protected String getProfilerCommand() { - return "memory=" + (256 * 1024) + ":a"; - } - - - public static class AllocatingTarget extends ClassValue implements Runnable { - public static volatile Object sink; - - @Override - public void run() { - Random random = new Random(0); - for (int i = 0; i < 1_000_000; i++) { - allocate(random); - } - } - - public long getAllocated(Class clazz) { - return get(clazz).get(); - } - - private void allocate(Random random) { - Object object; - if (random.nextBoolean()) { - object = new int[128 * 1000]; - } else { - object = new Integer[128 * 1000]; - } - if (!Platform.isMusl()) { - // JOL does not work that well with musl - get(object.getClass()).addAndGet(GraphLayout.parseInstance(object).totalSize()); - } - sink = object; - } - - @Override - protected AtomicLong computeValue(Class type) { - return new AtomicLong(); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/classgc/ClassGCTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/classgc/ClassGCTest.java deleted file mode 100644 index 7f73cbe72..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/classgc/ClassGCTest.java +++ /dev/null @@ -1,153 +0,0 @@ -package com.datadoghq.profiler.classgc; - -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import javax.tools.FileObject; -import javax.tools.ForwardingJavaFileManager; -import javax.tools.JavaCompiler; -import javax.tools.JavaFileManager; -import javax.tools.SimpleJavaFileObject; -import javax.tools.StandardJavaFileManager; -import javax.tools.ToolProvider; -import java.io.ByteArrayOutputStream; -import java.io.OutputStream; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.net.URI; -import java.util.Collections; - -public class ClassGCTest extends AbstractProfilerTest { - @Override - protected String getProfilerCommand() { - return "cpu=1ms,wall=1ms,filter=0,memory=524288:L"; - } - - private static final String CLASS_NAME = "code.Worker"; - private static final String JAVA_CODE = "package code;\n" + - "import java.util.concurrent.ThreadLocalRandom;\n" + - "\n" + - "public class Worker {\n" + - " public static void consumeCpu() {\n" + - " long blackhole = ThreadLocalRandom.current().nextLong();\n" + - " for (int i = 0; i < 1000; i++) {\n" + - " blackhole ^= ThreadLocalRandom.current().nextLong();\n" + - " }\n" + - " }\n" + - "}"; - - @Test - @Timeout(30) - public void profileWithManyShortLivedClasses() throws Throwable { - // TODO temporarily skip this for J9 as it is crashing due to missing safe fetch impl - Assumptions.assumeFalse(Platform.isJ9()); - - registerCurrentThreadForWallClockProfiling(); - // compiles code and loads it with many different classloaders, in the hope that - // the classes will get GC'd by the time the JFR is dumped - byte[] compiledClass = compile(CLASS_NAME, JAVA_CODE); - MethodHandles.Lookup lookup = MethodHandles.publicLookup(); - ClassValue cv = new ClassValue() { - @Override - protected MethodHandle computeValue(Class type) { - try { - return lookup.findStatic(type, "consumeCpu", MethodType.methodType(void.class)); - } catch (Throwable t) { - throw new RuntimeException(t); - } - } - }; - for (int i = 0; i < 10_000; i++) { - Class clazz = Class.forName(CLASS_NAME, true, new TestClassLoader(CLASS_NAME, compiledClass)); - MethodHandle consumeCpu = cv.get(clazz); - consumeCpu.invokeExact(); - if (i % 100 == 0) { - System.gc(); - } - } - } - - - static class TestClassLoader extends ClassLoader { - private final String className; - private final byte[] compiledClass; - - public TestClassLoader(String className, byte[] compiledClass) { - this.className = className; - this.compiledClass = compiledClass; - } - - @Override - public Class loadClass(String name) throws ClassNotFoundException { - if (name.equals(className)) { - try { - return defineClass(compiledClass, 0, compiledClass.length); - } catch (Throwable t) { - throw new ClassNotFoundException(name); - } - } - return ClassLoader.getSystemClassLoader().loadClass(name); - } - } - - // adapted from https://blog.jooq.org/how-to-compile-a-class-at-runtime-with-java-8-and-9/ - private static byte[] compile(String className, String code) { - JavaCompiler compiler = ToolProvider.getSystemJavaCompiler(); - ClassFileManager manager = new ClassFileManager(compiler.getStandardFileManager(null, null, null)); - compiler.getTask(null, manager, null, null, null, Collections.singletonList(new CharSequenceJavaFileObject(className, code))).call(); - return manager.javaFileObject.getBytes(); - } - - private static URI uri(String name, JavaFileObject.Kind kind) { - return URI.create("string:///" + name.replace('.', '/') + kind.extension); - } - - static final class JavaFileObject - extends SimpleJavaFileObject { - final ByteArrayOutputStream bos = new ByteArrayOutputStream(); - - JavaFileObject(String name, JavaFileObject.Kind kind) { - super(uri(name, kind), kind); - } - - byte[] getBytes() { - return bos.toByteArray(); - } - - @Override - public OutputStream openOutputStream() { - return bos; - } - } - - static final class ClassFileManager extends ForwardingJavaFileManager { - JavaFileObject javaFileObject; - - ClassFileManager(StandardJavaFileManager m) { - super(m); - } - - @Override - public JavaFileObject getJavaFileForOutput(JavaFileManager.Location location, String className, JavaFileObject.Kind kind, FileObject sibling) { - return javaFileObject = new JavaFileObject(className, kind); - } - } - - static final class CharSequenceJavaFileObject extends SimpleJavaFileObject { - final CharSequence content; - - public CharSequenceJavaFileObject(String className, CharSequence content) { - super(uri(className, JavaFileObject.Kind.SOURCE), JavaFileObject.Kind.SOURCE); - this.content = content; - } - - @Override - public CharSequence getCharContent(boolean ignoreEncodingErrors) { - return content; - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java deleted file mode 100644 index 8e5f4f6ac..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java +++ /dev/null @@ -1,296 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.context; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.ThreadContext; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Tests for OTEL-compatible context storage (OTEP #4947). - */ -public class OtelContextStorageModeTest { - - private static JavaProfiler profiler; - private boolean profilerStarted = false; - - @BeforeAll - public static void setup() throws IOException { - profiler = JavaProfiler.getInstance(); - } - - @AfterEach - public void cleanup() { - if (profilerStarted) { - profiler.stop(); - profiler.resetThreadContext(); - profilerStarted = false; - } - } - - /** - * Tests that context round-trips correctly. - */ - @Test - public void testOtelStorageModeContext() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-otel", ".jfr"); - - profiler.execute(String.format("start,cpu=1ms,attributes=tag1;tag2;tag3,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - long localRootSpanId = 0x1111222233334444L; - long spanId = 0xAAAABBBBCCCCDDDDL; - long traceIdHigh = 0x5555666677778888L; - long traceIdLow = 0x9999AAAABBBBCCCCL; - profiler.setContext(localRootSpanId, spanId, traceIdHigh, traceIdLow); - - ThreadContext ctx = profiler.getThreadContext(); - assertEquals(spanId, ctx.getSpanId(), "SpanId should match"); - assertEquals(localRootSpanId, ctx.getRootSpanId(), "LocalRootSpanId should match"); - // Verify the 128-bit trace ID round-trips through the OTEP record (big-endian) - assertEquals("55556666777788889999aaaabbbbcccc", ctx.readTraceId(), "TraceId should match"); - } - - - /** - * Tests that custom attributes are correctly written to and read back from - * the OTEP record's attrs_data (via DirectByteBuffer). Verifies the - * sidecar encoding is set and the UTF-8 value appears in attrs_data. - */ - @Test - public void testOtelModeCustomAttributes() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-attrs", ".jfr"); - - profiler.execute(String.format("start,cpu=1ms,attributes=http.route;db.system,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - long localRootSpanId = 0x1111222233334444L; - long spanId = 0xAAAABBBBCCCCDDDDL; - profiler.setContext(localRootSpanId, spanId, 0L, 0x9999L); - - ThreadContext ctx = profiler.getThreadContext(); - boolean result = ctx.setContextAttribute(0, "GET /api/users"); - assertTrue(result, "setContextAttribute should succeed"); - - result = ctx.setContextAttribute(1, "postgresql"); - assertTrue(result, "setContextAttribute for second key should succeed"); - - // Verify attribute values round-trip correctly through attrs_data - assertEquals("GET /api/users", ctx.readContextAttribute(0), "http.route should round-trip"); - assertEquals("postgresql", ctx.readContextAttribute(1), "db.system should round-trip"); - - // Verify trace context is still intact after attribute writes - assertEquals(spanId, ctx.getSpanId(), "SpanId should match after setAttribute"); - assertEquals(localRootSpanId, ctx.getRootSpanId(), "LocalRootSpanId should match after setAttribute"); - } - - /** - * Tests that attrs_data overflow is handled gracefully (returns false, no crash). - */ - @Test - public void testOtelModeAttributeOverflow() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-overflow", ".jfr"); - - profiler.execute(String.format("start,cpu=1ms,attributes=k0;k1;k2;k3;k4,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - profiler.setContext(0x2L, 0x1L, 0L, 0x3L); - - ThreadContext ctx = profiler.getThreadContext(); - - // LRS is a fixed 18-byte entry (key=0, len=16, 16 hex value bytes). - // Available for custom attrs: 612 - 18 = 594 bytes. - // Each 255-char attr = 257 bytes. Two fit (514 ≤ 594); third overflows (771 > 594). - StringBuilder sb = new StringBuilder(255); - for (int i = 0; i < 255; i++) sb.append('x'); - String longValue = sb.toString(); - assertTrue(ctx.setContextAttribute(0, longValue), "First long attr should fit"); - assertTrue(ctx.setContextAttribute(1, longValue), "Second long attr should fit"); - assertFalse(ctx.setContextAttribute(2, longValue), "Third long attr should overflow"); - - // Short values should still work for remaining slots - assertTrue(ctx.setContextAttribute(3, "short"), "Short attr after overflow should work"); - } - - /** - * Tests sequential context updates including boundary values and clearing. - * Verifies that each setContext overwrites the previous values, that MAX_VALUE - * round-trips correctly, and that clearContext resets both IDs to zero. - */ - @Test - public void testSequentialContextUpdates() { - profiler.setContext(2L, 1L, 0, 1L); - assertEquals(1L, profiler.getThreadContext().getSpanId()); - assertEquals(2L, profiler.getThreadContext().getRootSpanId()); - - profiler.setContext(20L, 10L, 0, 10L); - assertEquals(10L, profiler.getThreadContext().getSpanId()); - assertEquals(20L, profiler.getThreadContext().getRootSpanId()); - - profiler.setContext(200L, 100L, 0, 100L); - assertEquals(100L, profiler.getThreadContext().getSpanId()); - assertEquals(200L, profiler.getThreadContext().getRootSpanId()); - - long maxValue = Long.MAX_VALUE; - profiler.setContext(maxValue, maxValue, 0, maxValue); - assertEquals(maxValue, profiler.getThreadContext().getSpanId(), "SpanId should be MAX_VALUE"); - assertEquals(maxValue, profiler.getThreadContext().getRootSpanId(), "RootSpanId should be MAX_VALUE"); - - profiler.clearContext(); - assertEquals(0, profiler.getThreadContext().getSpanId(), "SpanId should be zero after clear"); - assertEquals(0, profiler.getThreadContext().getRootSpanId(), "RootSpanId should be zero after clear"); - } - - @Test - public void testThreadIsolation() throws InterruptedException { - long threadASpanId = 1000L; - long threadARootSpanId = 1001L; - profiler.setContext(threadARootSpanId, threadASpanId, 0, threadASpanId); - assertEquals(threadASpanId, profiler.getThreadContext().getSpanId()); - assertEquals(threadARootSpanId, profiler.getThreadContext().getRootSpanId()); - - final long threadBSpanId = 2000L; - final long threadBRootSpanId = 2001L; - final AssertionError[] threadBError = {null}; - - Thread threadB = new Thread(() -> { - try { - profiler.setContext(threadBRootSpanId, threadBSpanId, 0, threadBSpanId); - assertEquals(threadBSpanId, profiler.getThreadContext().getSpanId()); - assertEquals(threadBRootSpanId, profiler.getThreadContext().getRootSpanId()); - } catch (AssertionError e) { - threadBError[0] = e; - } - }, "TestThread-B"); - - threadB.start(); - threadB.join(); - - if (threadBError[0] != null) throw threadBError[0]; - - // Thread A's context must be unaffected - assertEquals(threadASpanId, profiler.getThreadContext().getSpanId()); - assertEquals(threadARootSpanId, profiler.getThreadContext().getRootSpanId()); - } - - /** - * Tests that a direct span-to-span transition (no clearContext in between) - * does not leak custom attributes from the previous span. - */ - @Test - public void testSpanTransitionClearsAttributes() throws Exception { - Path jfrFile = Files.createTempFile("otel-ctx-transition", ".jfr"); - profiler.execute(String.format("start,cpu=1ms,attributes=http.route,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - // Span A: set a custom attribute - profiler.setContext(0x1L, 0x1L, 0L, 0x1L); - ThreadContext ctx = profiler.getThreadContext(); - ctx.setContextAttribute(0, "/api/spanA"); - - // Transition directly to span B without clearing - profiler.setContext(0x2L, 0x2L, 0L, 0x2L); - - // Span A's attribute must not be visible in span B's context - assertNull(ctx.readContextAttribute(0), "Custom attribute must be cleared on span transition"); - } - - /** - * Stress-tests the OTEP context path with many sequential writes to catch - * buffer corruption or stale-value leaks over repeated updates. - */ - @Test - public void testRepeatedContextWrites() { - for (int i = 1; i <= 1000; i++) { - long spanId = (long) i; - long rootSpanId = (long) (i + 10000); - profiler.setContext(rootSpanId, spanId, 0L, spanId); - assertEquals(spanId, profiler.getThreadContext().getSpanId(), - "spanId mismatch at iteration " + i); - assertEquals(rootSpanId, profiler.getThreadContext().getRootSpanId(), - "rootSpanId mismatch at iteration " + i); - } - profiler.clearContext(); - assertEquals(0, profiler.getThreadContext().getSpanId(), "spanId should be 0 after clear"); - assertEquals(0, profiler.getThreadContext().getRootSpanId(), "rootSpanId should be 0 after clear"); - } - - /** - * Tests that the per-thread attribute cache isolates threads correctly. - * "FB" and "Ea" have equal hashCode() (both 2236), so they map to the same - * cache slot. With per-thread caches each thread owns its slot independently. - */ - @Test - public void testAttributeCacheIsolation() throws Exception { - Path jfrFile = Files.createTempFile("otel-attr-cache-iso", ".jfr"); - profiler.execute(String.format("start,cpu=1ms,attributes=attr0,jfr,file=%s", jfrFile.toAbsolutePath())); - profilerStarted = true; - - final String valueA = "FB"; // hashCode = 2236, slot 188 - final String valueB = "Ea"; // hashCode = 2236, same slot - final AssertionError[] errors = {null, null}; - final CountDownLatch bothWritten = new CountDownLatch(2); - - Thread threadA = new Thread(() -> { - try { - profiler.setContext(1L, 1L, 0L, 1L); - ThreadContext ctx = profiler.getThreadContext(); - assertTrue(ctx.setContextAttribute(0, valueA)); - bothWritten.countDown(); - bothWritten.await(5, TimeUnit.SECONDS); - // After thread B has written "Ea" to its own slot, A must still read "FB" - assertEquals(valueA, ctx.readContextAttribute(0)); - } catch (AssertionError e) { - errors[0] = e; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }, "TestThread-CacheA"); - - Thread threadB = new Thread(() -> { - try { - profiler.setContext(2L, 2L, 0L, 2L); - ThreadContext ctx = profiler.getThreadContext(); - assertTrue(ctx.setContextAttribute(0, valueB)); - bothWritten.countDown(); - bothWritten.await(5, TimeUnit.SECONDS); - assertEquals(valueB, ctx.readContextAttribute(0)); - } catch (AssertionError e) { - errors[1] = e; - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - }, "TestThread-CacheB"); - - threadA.start(); - threadB.start(); - threadA.join(10_000); - threadB.join(10_000); - - if (errors[0] != null) throw errors[0]; - if (errors[1] != null) throw errors[1]; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/ProcessContextTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/ProcessContextTest.java deleted file mode 100644 index 423d325d9..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/ProcessContextTest.java +++ /dev/null @@ -1,180 +0,0 @@ -package com.datadoghq.profiler.context; - -import com.datadoghq.profiler.OTelContext; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; - -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static org.junit.jupiter.api.Assertions.*; - -public class ProcessContextTest { - - @Test - public void testProcessContextMappingCreation() throws IOException { - Assumptions.assumeTrue(Platform.isLinux()); - - String env = "test-env"; - String hostname = "test-hostname"; - String runtimeId = "test-instance-123"; - String service = "test-service"; - String version = "1.0.0"; - String tracerVersion = "3.5.0"; - - OTelContext.getInstance().initializeAllContext(env, hostname, runtimeId, service, version, tracerVersion, new String[0]); - - OtelMappingInfo mapping = findOtelMapping(); - assertNotNull(mapping, "OTEL mapping should exist after initializeAllContext"); - - verifyMappingPermissions(mapping); - - // With no user keys, the published map is exactly the reserved slot. - OTelContext.ProcessContext readContext = OTelContext.getInstance().readProcessContext(); - assertNotNull(readContext); - assertArrayEquals( - new String[] {"datadog.local_root_span_id"}, - readContext.attributeKeyMap); - } - - private static class OtelMappingInfo { - final String startAddress; - final String endAddress; - final String permissions; - - OtelMappingInfo(String startAddress, String endAddress, String permissions) { - this.startAddress = startAddress; - this.endAddress = endAddress; - this.permissions = permissions; - } - } - - private OtelMappingInfo findOtelMapping() throws IOException { - Path mapsFile = Paths.get("/proc/self/maps"); - if (!Files.exists(mapsFile)) { - return null; - } - - // Match any mapping containing OTEL_CTX (memfd, anon, anon_shmem variants) - Pattern otelPattern = Pattern.compile("^([0-9a-f]+)-([0-9a-f]+)\\s+(\\S+)\\s+\\S+\\s+\\S+\\s+\\S+\\s*.*OTEL_CTX.*$"); - - try (BufferedReader reader = Files.newBufferedReader(mapsFile)) { - String line; - while ((line = reader.readLine()) != null) { - Matcher matcher = otelPattern.matcher(line); - if (matcher.matches()) { - return new OtelMappingInfo( - matcher.group(1), - matcher.group(2), - matcher.group(3) - ); - } - } - } - return null; - } - - private void verifyMappingPermissions(OtelMappingInfo mapping) { - assertTrue(mapping.permissions.contains("r"), - "OTEL mapping should have read permission, got: " + mapping.permissions); - assertFalse(mapping.permissions.contains("x"), - "OTEL mapping should not have execute permission, got: " + mapping.permissions); - } - - @Test - public void testProcessContextRoundTrip() { - Assumptions.assumeTrue(Platform.isLinux()); - - String env = "test-env"; - String hostname = "test-hostname"; - String runtimeId = "test-instance-123"; - String service = "test-service"; - String version = "1.0.0"; - String tracerVersion = "3.5.0"; - - OTelContext context = OTelContext.getInstance(); - context.initializeAllContext(env, hostname, runtimeId, service, version, tracerVersion, - new String[] {"http.route", "db.system"}); - - OTelContext.ProcessContext readContext = context.readProcessContext(); - - assertNotNull(readContext); - assertEquals(env, readContext.deploymentEnvironmentName); - assertEquals(hostname, readContext.hostName); - assertEquals(runtimeId, readContext.serviceInstanceId); - assertEquals(service, readContext.serviceName); - assertEquals(version, readContext.serviceVersion); - assertEquals("java", readContext.telemetrySdkLanguage); - assertEquals(tracerVersion, readContext.telemetrySdkVersion); - assertEquals("dd-trace-java", readContext.telemetrySdkName); - // The reserved local_root_span_id slot precedes the caller-provided keys. - assertArrayEquals( - new String[] {"datadog.local_root_span_id", "http.route", "db.system"}, - readContext.attributeKeyMap); - } - - @Test - public void testNullAttributeKeyElementAbortsPublish() { - Assumptions.assumeTrue(Platform.isLinux()); - - OTelContext context = OTelContext.getInstance(); - - // Publish a known-good context first. - context.initializeAllContext("env-a", "host-a", "rt-a", "svc-a", "1.0.0", "3.5.0", - new String[] {"http.route"}); - - OTelContext.ProcessContext before = context.readProcessContext(); - assertNotNull(before); - assertEquals("svc-a", before.serviceName); - - // A null element in the keys array aborts the entire publish: the previously - // published context must remain untouched, not just have the bad key dropped. - context.initializeAllContext("env-b", "host-b", "rt-b", "svc-b", "2.0.0", "4.0.0", - new String[] {"http.route", null, "db.system"}); - - OTelContext.ProcessContext after = context.readProcessContext(); - assertNotNull(after); - assertEquals("svc-a", after.serviceName, - "null element must abort the publish, leaving the previous context intact"); - assertArrayEquals( - new String[] {"datadog.local_root_span_id", "http.route"}, - after.attributeKeyMap); - } - - @Test - public void testAttributeKeysClippedToCapacity() { - Assumptions.assumeTrue(Platform.isLinux()); - - // Native DD_TAGS_CAPACITY: at most this many user keys are published, preceded - // by the reserved datadog.local_root_span_id slot; any extra keys are clipped. - final int capacity = 10; - - String[] keys = new String[capacity + 5]; - for (int i = 0; i < keys.length; i++) { - keys[i] = "key" + i; - } - - OTelContext context = OTelContext.getInstance(); - context.initializeAllContext("test-env", "test-hostname", "test-instance-123", - "test-service", "1.0.0", "3.5.0", keys); - - OTelContext.ProcessContext readContext = context.readProcessContext(); - assertNotNull(readContext); - - // Expect the reserved slot followed by exactly the first `capacity` user keys. - String[] expected = new String[capacity + 1]; - expected[0] = "datadog.local_root_span_id"; - for (int i = 0; i < capacity; i++) { - expected[i + 1] = "key" + i; - } - assertArrayEquals(expected, readContext.attributeKeyMap, - "keys beyond DD_TAGS_CAPACITY must be clipped, keeping the first " + capacity); - } - -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java deleted file mode 100644 index 95c8fecd5..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java +++ /dev/null @@ -1,641 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.context; - -import java.nio.charset.StandardCharsets; -import java.util.Arrays; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.atomic.AtomicLong; -import java.util.stream.IntStream; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.ContextSetter; -import static com.datadoghq.profiler.MoreAssertions.DICTIONARY_PAGE_SIZE; -import static com.datadoghq.profiler.MoreAssertions.assertBoundedBy; -import com.datadoghq.profiler.Platform; - -public class TagContextTest extends AbstractProfilerTest { - - @BeforeEach - void assumeNotJ9() { - // On J9, ProfiledThread (and thus the OTEP TLS buffer) is not allocated until the thread - // is registered for wall-clock profiling, so initializeContextTLS0() returns null and - // ThreadContext creation throws. These tests require a live ThreadContext from the start. - Assumptions.assumeTrue(!Platform.isJ9()); - } - - @RetryingTest(10) - public void test() throws InterruptedException { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2", "tag1")); - - // Use session-unique prefix so each @RetryingTest attempt registers fresh values in the - // native Dictionary. Without this, on musl (no JVM fork) the per-thread attrCacheKeys - // persists across retries: cache hits skip registerConstant0(), leaving - // dictionary_context_keys=0 on every retry after the first. - String pfx = Long.toHexString(System.nanoTime()) + "_"; - String[] strings = IntStream.range(0, 10).mapToObj(i -> pfx + i).toArray(String[]::new); - for (int i = 0; i < strings.length * 10; i++) { - work(contextSetter, "tag1", strings[i % strings.length]); - } - stopProfiler(); - IItemCollection events = verifyEvents("datadog.MethodSample"); - Map weightsByTagValue = new HashMap<>(); - long droppedSamplesCount = 0; - long droppedSamplesWeight = 0; - long totalSamplesCount = 0; - long totalSamplesWeight = 0; - try { - for (IItemIterable wallclockSamples : events) { - IMemberAccessor weightAccessor = WEIGHT.getAccessor(wallclockSamples.getType()); - // this will become more generic in the future - IMemberAccessor tag1Accessor = TAG_1.getAccessor(wallclockSamples.getType()); - assertNotNull(tag1Accessor); - IMemberAccessor tag2Accessor = TAG_2.getAccessor(wallclockSamples.getType()); - assertNotNull(tag2Accessor); - IMemberAccessor stacktraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(wallclockSamples.getType()); - for (IItem sample : wallclockSamples) { - String stacktrace = stacktraceAccessor.getMember(sample); - if (!stacktrace.contains("sleep")) { - // we don't know the context has been set for sure until the sleep has started - continue; - } - - long weight = weightAccessor.getMember(sample).longValue(); - totalSamplesCount++; - totalSamplesWeight += weight; - - if (stacktrace.contains("")) { - // track dropped samples statistics but skip for weight distribution calculation - droppedSamplesCount++; - droppedSamplesWeight += weight; - continue; - } - - String tag = tag1Accessor.getMember(sample); - weightsByTagValue.computeIfAbsent(tag, v -> new AtomicLong()) - .addAndGet(weight); - assertNull(tag2Accessor.getMember(sample)); - } - } - long sum = 0; - long[] weights = new long[strings.length]; - System.out.println("Found tag values: " + weightsByTagValue.keySet()); - for (int i = 0; i < strings.length; i++) { - AtomicLong weight = weightsByTagValue.get(strings[i]); - assertNotNull(weight, "Weight for " + strings[i] + " not found. Found: " + weightsByTagValue.keySet()); - weights[i] = weightsByTagValue.get(strings[i]).get(); - sum += weights[i]; - } - double avg = (double) sum / weights.length; - for (int i = 0; i < weights.length; i++) { - assertTrue(Math.abs(weights[i] - avg) < 0.15 * weights[i], strings[i] - + " more than 15% from mean"); - } - - // now check we have settings to unbundle the dynamic columns - IItemCollection activeSettings = verifyEvents("jdk.ActiveSetting"); - Set recordedContextAttributes = new HashSet<>(); - for (IItemIterable activeSetting : activeSettings) { - IMemberAccessor nameAccessor = JdkAttributes.REC_SETTING_NAME.getAccessor(activeSetting.getType()); - IMemberAccessor valueAccessor = JdkAttributes.REC_SETTING_VALUE.getAccessor(activeSetting.getType()); - for (IItem item : activeSetting) { - String name = nameAccessor.getMember(item); - if ("contextattribute".equals(name)) { - recordedContextAttributes.add(valueAccessor.getMember(item)); - } - } - } - assertEquals(3, recordedContextAttributes.size()); - assertTrue(recordedContextAttributes.contains("tag1")); - assertTrue(recordedContextAttributes.contains("tag2")); - assertTrue(recordedContextAttributes.contains("tag3")); - - // Verify counters from JFR serialized data (not live process counters which are reset) - Map jfrCounters = new HashMap<>(); - for (IItemIterable counterEvent : verifyEvents("datadog.ProfilerCounter")) { - IMemberAccessor nameAccessor = NAME.getAccessor(counterEvent.getType()); - IMemberAccessor countAccessor = COUNT.getAccessor(counterEvent.getType()); - for (IItem item : counterEvent) { - String name = nameAccessor.getMember(item); - jfrCounters.put(name, countAccessor.getMember(item).longValue()); - } - } - - assertFalse(jfrCounters.isEmpty()); - assertEquals(strings.length, jfrCounters.get("dictionary_context_keys")); - } finally { - // Print statistics about dropped samples for debugging - double dropRate = totalSamplesCount > 0 ? (100.0 * droppedSamplesCount / totalSamplesCount) : 0.0; - double dropWeightRate = totalSamplesWeight > 0 ? (100.0 * droppedSamplesWeight / totalSamplesWeight) : 0.0; - System.out.printf("Sample statistics: %d total (%d dropped, %.2f%%), weight %d total (%d dropped, %.2f%%)%n", - totalSamplesCount, droppedSamplesCount, dropRate, - totalSamplesWeight, droppedSamplesWeight, dropWeightRate); - } - } - - /** - * Reads the current value of {@code tag} via {@link ThreadContext#readContextAttribute} - * — the only readback path retained on the Java side (test-only). - */ - private String readTag(ContextSetter contextSetter, String tag) { - return profiler.getThreadContext().readContextAttribute(contextSetter.offsetOf(tag)); - } - - @Test - public void testSnapshotRestore() throws Exception { - // J9 does not initialize ThreadContext for non-profiled threads; skip. - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - // Initially both slots are empty - assertNull(readTag(contextSetter, "tag1")); - assertNull(readTag(contextSetter, "tag2")); - - // Set a value and read it back - assertTrue(contextSetter.setContextValue("tag1", "before")); - assertEquals("before", readTag(contextSetter, "tag1")); - - // Snapshot the string, overwrite, then restore - String saved = readTag(contextSetter, "tag1"); - assertTrue(contextSetter.setContextValue("tag1", "inside")); - assertEquals("inside", readTag(contextSetter, "tag1")); - - // Restore via setContextValue - assertTrue(contextSetter.setContextValue("tag1", saved)); - assertEquals("before", readTag(contextSetter, "tag1")); - - // put/clear/put cycle: verify offset stability across state transitions - assertTrue(contextSetter.clearContextValue("tag1")); - assertNull(readTag(contextSetter, "tag1")); - assertTrue(contextSetter.setContextValue("tag1", "after")); - assertEquals("after", readTag(contextSetter, "tag1")); - - // tag2 was never set; readContextAttribute returns null - assertNull(readTag(contextSetter, "tag2")); - } - - @Test - public void testAttrsDataOverflow() throws Exception { - registerCurrentThreadForWallClockProfiling(); - List attrs = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - attrs.add("tag" + i); - } - ContextSetter contextSetter = new ContextSetter(profiler, attrs); - char[] chars = new char[255]; - java.util.Arrays.fill(chars, 'x'); - String bigValue = new String(chars); - int overflowIndex = -1; - for (int i = 1; i <= 10; i++) { - if (!contextSetter.setContextValue("tag" + i, bigValue)) { - overflowIndex = i; - break; - } - } - assertTrue(overflowIndex >= 0, "Expected at least one write to overflow attrs_data"); - assertNull(readTag(contextSetter, "tag" + overflowIndex), - "Overflowed slot must read null — the entry never landed in attrs_data"); - } - - @Test - public void testPutClearsCustomSlots() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "before-put")); - assertEquals("before-put", readTag(contextSetter, "tag1")); - - // setContext() triggers setContextDirect which resets attrs_data_size to the LRS entry only, - // dropping all user attribute entries — so scanning attrs_data for tag1 returns null. - profiler.setContext(1L, 42L, 0L, 43L); - assertNull(readTag(contextSetter, "tag1"), "tag1 must be null after setContext resets attrs_data"); - } - - @Test - public void testCrossSlotIsolation() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "v1")); - assertTrue(contextSetter.setContextValue("tag2", "v2")); - assertTrue(contextSetter.clearContextValue("tag2")); - assertEquals("v1", readTag(contextSetter, "tag1")); - assertNull(readTag(contextSetter, "tag2")); - } - - @Test - public void testReapplyByIdAndBytes() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - String value = "app-managed"; - - // Set the attribute the normal way, then capture both the constant ID (sidecar) and the - // UTF-8 bytes — exactly what dd-trace-java retains for the reapply hot path. - assertTrue(contextSetter.setContextValue("tag1", value)); - int[] ids = contextSetter.snapshotTags(); - int savedId = ids[slot]; - assertNotEquals(0, savedId); - byte[][] bytes = new byte[ids.length][]; - bytes[slot] = value.getBytes(StandardCharsets.UTF_8); - - // setContext (span activation) wipes both views. - profiler.setContext(1L, 42L, 0L, 43L); - assertNull(readTag(contextSetter, "tag1"), "attrs_data must be wiped by setContext"); - assertEquals(0, contextSetter.snapshotTags()[slot], "sidecar must be wiped by setContext"); - - // Reapply by ID + bytes restores BOTH views. - assertTrue(contextSetter.setContextValuesByIdAndBytes(ids, bytes)); - assertEquals(value, readTag(contextSetter, "tag1"), "attrs_data must be restored"); - assertEquals(savedId, contextSetter.snapshotTags()[slot], "sidecar must be restored"); - } - - @Test - public void testReapplyByIdAndBytesRejectsBadArgs() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - assertTrue(contextSetter.setContextValue("tag1", "v")); - int id = contextSetter.snapshotTags()[slot]; - assertNotEquals(0, id); - - // Null arrays and length mismatch must throw. - assertThrows(NullPointerException.class, - () -> contextSetter.setContextValuesByIdAndBytes(null, new byte[1][])); - assertThrows(NullPointerException.class, - () -> contextSetter.setContextValuesByIdAndBytes(new int[1], null)); - assertThrows(IllegalArgumentException.class, - () -> contextSetter.setContextValuesByIdAndBytes(new int[2], new byte[3][])); - - // A slot with constantId > 0 requires non-null bytes within the size limit. - assertThrows(NullPointerException.class, - () -> contextSetter.setContextValuesByIdAndBytes( - new int[] {id, 0}, new byte[][] {null, null})); - // Record must remain attached (valid=1) after the exception — no detach leak. - assertEquals(id, contextSetter.snapshotTags()[slot], - "sidecar must be unchanged after NPE on active utf8[i]"); - - assertThrows(IllegalArgumentException.class, - () -> contextSetter.setContextValuesByIdAndBytes( - new int[] {id, 0}, new byte[][] {new byte[256], null})); - // Record must remain attached (valid=1) after the exception — no detach leak. - assertEquals(id, contextSetter.snapshotTags()[slot], - "sidecar must be unchanged after IAE on oversized utf8[i]"); - - // 255 bytes is the boundary and must be accepted. - // Register the 255-byte value so its constant ID matches the bytes we pass. - byte[] ok255 = new byte[255]; - Arrays.fill(ok255, (byte) 'x'); - assertTrue(contextSetter.setContextValue("tag1", new String(ok255, StandardCharsets.UTF_8))); - int id255 = contextSetter.snapshotTags()[slot]; - assertNotEquals(0, id255); - assertTrue(contextSetter.setContextValuesByIdAndBytes( - new int[] {id255, 0}, new byte[][] {ok255, null})); - } - - @Test - public void testReapplyByIdAndBytesReplacesExistingValue() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - // Capture the ID + bytes for "first". - assertTrue(contextSetter.setContextValue("tag1", "first")); - int[] idsFirst = contextSetter.snapshotTags(); - byte[][] bytesFirst = new byte[idsFirst.length][]; - bytesFirst[slot] = "first".getBytes(StandardCharsets.UTF_8); - - // Overwrite the live slot with a different value. - assertTrue(contextSetter.setContextValue("tag1", "second")); - assertEquals("second", readTag(contextSetter, "tag1")); - - // Reapply "first" by ID + bytes over the live "second" — exercises the - // compact-then-insert path in replaceOtepAttribute. - assertTrue(contextSetter.setContextValuesByIdAndBytes(idsFirst, bytesFirst)); - assertEquals("first", readTag(contextSetter, "tag1")); - assertEquals(idsFirst[slot], contextSetter.snapshotTags()[slot]); - } - - @Test - public void testReapplyByIdAndBytesAfterClear() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - assertTrue(contextSetter.setContextValue("tag1", "live")); - int[] ids = contextSetter.snapshotTags(); - byte[][] bytes = new byte[ids.length][]; - bytes[slot] = "live".getBytes(StandardCharsets.UTF_8); - - assertTrue(contextSetter.clearContextValue("tag1")); - assertNull(readTag(contextSetter, "tag1")); - assertEquals(0, contextSetter.snapshotTags()[slot]); - - assertTrue(contextSetter.setContextValuesByIdAndBytes(ids, bytes)); - assertEquals("live", readTag(contextSetter, "tag1")); - assertEquals(ids[slot], contextSetter.snapshotTags()[slot]); - } - - @Test - public void testReapplyByIdAndBytesClearedRecord() throws Exception { - // Verifies that setContextValuesByIdAndBytes never resurrects a cleared (span-less) record. - // A cleared record has valid=0 and no trace/span context; re-publishing it would expose - // attribute values with no associated trace, which is meaningless to the signal handler. - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - // Establish a live record with tag1 set. - profiler.setContext(1L, 42L, 0L, 43L); - assertTrue(contextSetter.setContextValue("tag1", "will-be-cleared")); - int[] ids = contextSetter.snapshotTags(); - byte[][] bytes = new byte[ids.length][]; - bytes[slot] = "will-be-cleared".getBytes(StandardCharsets.UTF_8); - - // Drive valid=0 via the all-zero clear path (clearContext → put(0,0,0,0) → no attach()). - profiler.clearContext(); - // readContextAttribute respects valid=0 and returns null, confirming the record is dark. - assertNull(readTag(contextSetter, "tag1")); - - // Reapply must return false and must not resurrect the cleared record. - assertFalse(contextSetter.setContextValuesByIdAndBytes(ids, bytes), - "setContextValuesByIdAndBytes must return false when the record is cleared (valid=0)"); - assertNull(readTag(contextSetter, "tag1"), - "cleared record must not be resurrected by setContextValuesByIdAndBytes"); - } - - @Test - public void testReapplyByIdAndBytesOverflowRollback() throws Exception { - registerCurrentThreadForWallClockProfiling(); - List attrs = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - attrs.add("tag" + i); - } - ContextSetter contextSetter = new ContextSetter(profiler, attrs); - - // Register one 255-byte value to obtain a valid constant ID. - char[] chars = new char[255]; - Arrays.fill(chars, 'x'); - String bigValue = new String(chars); - assertTrue(contextSetter.setContextValue("tag1", bigValue)); - int bigId = contextSetter.snapshotTags()[contextSetter.offsetOf("tag1")]; - assertNotEquals(0, bigId); - byte[] bigBytes = bigValue.getBytes(StandardCharsets.UTF_8); - - // Reapply the same 255-byte value to all 10 slots — attrs_data cannot hold them all. - int[] ids = new int[10]; - byte[][] bytes = new byte[10][]; - Arrays.fill(ids, bigId); - Arrays.fill(bytes, bigBytes); - assertFalse(contextSetter.setContextValuesByIdAndBytes(ids, bytes), - "10 x 255-byte values must overflow attrs_data"); - - // The last slot certainly overflowed: its sidecar must be zeroed and attrs_data empty. - int lastSlot = contextSetter.offsetOf("tag10"); - assertEquals(0, contextSetter.snapshotTags()[lastSlot], - "overflowed slot's sidecar must be zeroed"); - assertNull(readTag(contextSetter, "tag10"), - "overflowed slot must read null — the entry never landed in attrs_data"); - - // Slots processed before the overflow are durably written — false does not mean - // the record is unchanged. At least tag1 (slot 0) must retain the new value. - int firstSlot = contextSetter.offsetOf("tag1"); - assertEquals(bigId, contextSetter.snapshotTags()[firstSlot], - "slot 0 processed before overflow must have its sidecar durably written"); - assertEquals(bigValue, readTag(contextSetter, "tag1"), - "slot 0 processed before overflow must be readable via attrs_data"); - } - - // ----------------------------------------------------------------------- - // Acceptance tests for the MAX_CUSTOM_SLOTS guard fixes - // ----------------------------------------------------------------------- - - /** - * Test 1: setContextValuesByIdAndBytes must throw IllegalArgumentException immediately when - * the arrays are longer than MAX_CUSTOM_SLOTS (10), and must not perform - * any partial write before the rejection. - */ - @Test - public void testSetContextValuesByIdAndBytesRejectsArraysLongerThanMaxSlots() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - int slot = contextSetter.offsetOf("tag1"); - - // Establish a known value and capture its constant ID. - assertTrue(contextSetter.setContextValue("tag1", "original")); - int savedId = contextSetter.snapshotTags()[slot]; - assertNotEquals(0, savedId); - - // Build arrays of length 11 (> MAX_CUSTOM_SLOTS = 10). - int[] ids = new int[11]; - byte[][] utf8 = new byte[11][]; - ids[0] = savedId; - utf8[0] = "original".getBytes(StandardCharsets.UTF_8); - // All other entries remain 0 / null. - - // The call must be rejected with an exception before any write. - assertThrows(IllegalArgumentException.class, - () -> contextSetter.setContextValuesByIdAndBytes(ids, utf8), - "setContextValuesByIdAndBytes must throw when array length > MAX_CUSTOM_SLOTS"); - - // No partial write: the sidecar for slot 0 must be unchanged. - assertEquals(savedId, contextSetter.snapshotTags()[slot], - "sidecar must not be modified before the length guard fires"); - } - - /** - * Test 2: setContextValuesByIdAndBytes must accept arrays of exactly - * MAX_CUSTOM_SLOTS (10) and return true, restoring all sidecar values. - */ - @Test - public void testSetContextValuesByIdAndBytesAcceptsExactlyMaxSlots() throws Exception { - registerCurrentThreadForWallClockProfiling(); - List attrs = new ArrayList<>(); - for (int i = 1; i <= 10; i++) { - attrs.add("tag" + i); - } - ContextSetter contextSetter = new ContextSetter(profiler, attrs); - - // Set all 10 attributes to distinct values and capture constant IDs + bytes. - int[] savedIds = new int[10]; - byte[][] savedBytes = new byte[10][]; - for (int i = 0; i < 10; i++) { - String value = "val" + i; - assertTrue(contextSetter.setContextValue("tag" + (i + 1), value)); - savedBytes[i] = value.getBytes(StandardCharsets.UTF_8); - } - int[] snapshot = contextSetter.snapshotTags(); - for (int i = 0; i < 10; i++) { - savedIds[i] = snapshot[i]; - assertNotEquals(0, savedIds[i], "tag" + (i + 1) + " must have a non-zero sidecar ID"); - } - - // Wipe all slots via setContext (span activation). - profiler.setContext(1L, 42L, 0L, 43L); - - // Reapply with exactly-10-element arrays — must succeed. - assertTrue(contextSetter.setContextValuesByIdAndBytes(savedIds, savedBytes), - "setContextValuesByIdAndBytes must return true for arrays of length == MAX_CUSTOM_SLOTS"); - - // All 10 sidecar IDs must be restored. - int[] restored = contextSetter.snapshotTags(); - for (int i = 0; i < 10; i++) { - assertEquals(savedIds[i], restored[i], - "sidecar for tag" + (i + 1) + " must be restored after reapply"); - } - } - - /** - * Test 3: snapshotTags(int[]) with an oversized buffer (length > attributes.size()) - * must write the managed indices [0, attributes.size()) with the current sidecar values, - * and zero out the extra indices [attributes.size(), snapshot.length). - */ - @Test - public void testSnapshotTagsOversizedBufferCopiesAndZerosExtras() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "v1")); - assertTrue(contextSetter.setContextValue("tag2", "v2")); - - // Verify no-arg overload returns valid IDs. - int[] canonical = contextSetter.snapshotTags(); - assertNotEquals(0, canonical[0]); - assertNotEquals(0, canonical[1]); - - // Oversized buffer: length 5 > attributes.size() == 2. - int[] oversized = new int[5]; - Arrays.fill(oversized, -1); - contextSetter.snapshotTags(oversized); - - // Managed indices [0, attributes.size()) must contain the current sidecar values. - assertEquals(canonical[0], oversized[0], - "oversized buffer[0] must match no-arg snapshotTags()[0]"); - assertEquals(canonical[1], oversized[1], - "oversized buffer[1] must match no-arg snapshotTags()[1]"); - - // Extra indices [attributes.size(), snapshot.length) must be zeroed. - for (int i = 2; i < oversized.length; i++) { - assertEquals(0, oversized[i], - "oversized buffer element [" + i + "] must be zeroed by snapshotTags"); - } - - // No-arg overload must still work correctly. - int[] check = contextSetter.snapshotTags(); - assertEquals(canonical[0], check[0]); - assertEquals(canonical[1], check[1]); - } - - /** - * Test 4: snapshotTags(int[]) with an undersized buffer (length < attributes.size()) - * must be a no-op — existing no-op semantics must be preserved. - */ - @Test - public void testSnapshotTagsUndersizedBufferIsNoOp() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2", "tag3")); - - assertTrue(contextSetter.setContextValue("tag1", "a")); - assertTrue(contextSetter.setContextValue("tag2", "b")); - assertTrue(contextSetter.setContextValue("tag3", "c")); - - // Undersized buffer: length 1 < attributes.size() == 3. - int[] undersized = new int[1]; - undersized[0] = -1; - contextSetter.snapshotTags(undersized); - - assertEquals(-1, undersized[0], - "undersized buffer must not be written by snapshotTags"); - } - - /** - * Test 5: snapshotTags(int[]) with an exact-size buffer (length == attributes.size()) - * must copy the current sidecar values correctly. - */ - @Test - public void testSnapshotTagsExactSizeBufferCopiesCorrectly() throws Exception { - registerCurrentThreadForWallClockProfiling(); - ContextSetter contextSetter = new ContextSetter(profiler, Arrays.asList("tag1", "tag2")); - - assertTrue(contextSetter.setContextValue("tag1", "x")); - assertTrue(contextSetter.setContextValue("tag2", "y")); - - // No-arg overload to obtain expected values. - int[] canonical = contextSetter.snapshotTags(); - assertNotEquals(0, canonical[0]); - assertNotEquals(0, canonical[1]); - - // Exact-size buffer: length 2 == attributes.size() == 2. - int[] exact = new int[2]; - contextSetter.snapshotTags(exact); - - assertEquals(canonical[0], exact[0], - "exact-size buffer[0] must match no-arg snapshotTags()[0]"); - assertEquals(canonical[1], exact[1], - "exact-size buffer[1] must match no-arg snapshotTags()[1]"); - } - - private void work(ContextSetter contextSetter, String contextAttribute, String contextValue) - throws InterruptedException { - assertTrue(contextSetter.setContextValue(contextAttribute, contextValue)); - checkTagValues(contextSetter, contextAttribute); - Thread.sleep(10); - assertTrue(contextSetter.clearContextValue(contextAttribute)); - } - - private void checkTagValues(ContextSetter contextSetter, String contextAttribute) { - int[] tags = contextSetter.snapshotTags(); - // expects tag1/tag2/tag3 - change this if the tested tags change - int offset = Integer.parseInt(contextAttribute.substring(3)) - 1; - for (int i = 0; i < tags.length; i++) { - if (i == offset) { - assertNotEquals(0, tags[i]); - } else { - assertEquals(0, tags[i]); - } - } - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms,filter=0,attributes=tag1;tag2;tag3"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/CTimerSamplerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/CTimerSamplerTest.java deleted file mode 100644 index 812bedc7f..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/CTimerSamplerTest.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.datadoghq.profiler.cpu; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.CStackInjector; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assumptions.assumeFalse; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class CTimerSamplerTest extends CStackAwareAbstractProfilerTest { - - private ProfiledCode profiledCode; - - public CTimerSamplerTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected void before() { - profiledCode = new ProfiledCode(profiler); - } - - @RetryTest(10) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void test(@CStack String cstack) throws ExecutionException, InterruptedException { - // timer_create is available on Linux only - assumeTrue(Platform.isLinux()); - for (int i = 0, id = 1; i < 100; i++, id += 3) { - profiledCode.method1(id); - } - stopProfiler(); - - verifyCStackSettings(); - - IItemCollection events = verifyEvents("datadog.ExecutionSample"); - - for (IItemIterable cpuSamples : events) { - IMemberAccessor frameAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - for (IItem sample : cpuSamples) { - String stackTrace = frameAccessor.getMember(sample); - assertFalse(stackTrace.contains("jvmtiError")); - } - } - } - - @Override - protected void after() throws Exception { - profiledCode.close(); - } - - @Override - protected String getProfilerCommand() { - return "cpu=100us,event=ctimer"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/ContextCpuTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/ContextCpuTest.java deleted file mode 100644 index 3afd598cb..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/ContextCpuTest.java +++ /dev/null @@ -1,132 +0,0 @@ -package com.datadoghq.profiler.cpu; - -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ExecutionException; - -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.CStackInjector; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.api.extension.ExtendWith; -import org.junit.jupiter.params.provider.ValueSource; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import com.datadoghq.profiler.AbstractProfilerTest; -import static com.datadoghq.profiler.MoreAssertions.assertInRange; -import com.datadoghq.profiler.Platform; - -public class ContextCpuTest extends CStackAwareAbstractProfilerTest { - - private ProfiledCode profiledCode; - - public ContextCpuTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected void before() { - profiledCode = new ProfiledCode(profiler); - } - - @RetryTest(10) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void test(@CStack String cstack) throws ExecutionException, InterruptedException { - Assumptions.assumeTrue(!Platform.isJ9()); - for (int i = 0, id = 1; i < 100; i++, id += 3) { - profiledCode.method1(id); - } - stopProfiler(); - - verifyCStackSettings(); - - Set method1SpanIds = profiledCode.spanIdsForMethod("method1Impl"); - Set method2SpanIds = profiledCode.spanIdsForMethod("method2Impl"); - Set method3SpanIds = profiledCode.spanIdsForMethod("method3Impl"); - IItemCollection events = verifyEvents("datadog.ExecutionSample"); - - // on mac the usage of itimer to drive the sampling provides very unreliable outputs - if (!Platform.isMac()) { - - // we have 100 method1, method2, and method3 calls, but can't guarantee we sampled them all - long method1Weight = 0; - long method2Weight = 0; - long method3Weight = 0; - long totalWeight = 0; - for (IItemIterable cpuSamples : events) { - IMemberAccessor frameAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - IMemberAccessor spanIdAccessor = SPAN_ID.getAccessor(cpuSamples.getType()); - IMemberAccessor rootSpanIdAccessor = LOCAL_ROOT_SPAN_ID.getAccessor(cpuSamples.getType()); - IMemberAccessor stateAccessor = THREAD_STATE.getAccessor(cpuSamples.getType()); - for (IItem sample : cpuSamples) { - String stackTrace = frameAccessor.getMember(sample); - long spanId = spanIdAccessor.getMember(sample).longValue(); - long rootSpanId = rootSpanIdAccessor.getMember(sample).longValue(); - String state = stateAccessor.getMember(sample); - assertDoesNotThrow(() -> Thread.State.valueOf(state)); - assertEquals(Thread.State.RUNNABLE, Thread.State.valueOf(state)); - if (stackTrace.contains("method3Impl")) { - // method3 is scheduled after method2, and method1 blocks on it, so spanId == rootSpanId + 2 - if (spanId > 0) { - assertEquals(rootSpanId + 2, spanId, stackTrace); - assertTrue(method3SpanIds.contains(spanId), stackTrace); - method3Weight += 1; - } - } else if (stackTrace.contains("method2Impl")) { - // method2 is called next, so spanId == rootSpanId + 1 - if (spanId > 0) { - assertEquals(rootSpanId + 1, spanId, stackTrace); - assertTrue(method2SpanIds.contains(spanId), stackTrace); - method2Weight += 1; - } - } else if (stackTrace.contains("method1Impl") - && !stackTrace.contains("method2") && !stackTrace.contains("method3")) { - // need to check this after method2 because method1 calls method2 - // it's the root so spanId == rootSpanId - assertEquals(rootSpanId, spanId, stackTrace); - assertTrue(spanId == 0 || method1SpanIds.contains(spanId), stackTrace); - method1Weight += 1; - } - totalWeight++; - } - } - assertInRange(method1Weight / (double) totalWeight, 0.1, 0.6); - assertInRange(method2Weight / (double) totalWeight, 0.1, 0.6); - assertInRange(method3Weight / (double) totalWeight, 0.05, 0.6); - } - // The recording captures counter values before the final cleanup (before processTraces - // runs and frees all traces). Verify the recording contains meaningful data. - assertInRange(getRecordedCounterValue("calltrace_storage_traces"), 1, 100); - assertInRange(getRecordedCounterValue("calltrace_storage_bytes"), 1024, 8 * 1024 * 1024); - // live counters are 0 after stop (all traces freed - correct, non-leaking behaviour) - Map debugCounters = profiler.getDebugCounters(); - assertEquals(0, debugCounters.get("calltrace_storage_traces")); - assertEquals(0, debugCounters.get("calltrace_storage_bytes")); - assertEquals(0, debugCounters.get("linear_allocator_bytes")); - assertEquals(0, debugCounters.get("linear_allocator_chunks")); - } - - - - @Override - protected void after() throws Exception { - profiledCode.close(); - } - - @Override - protected String getProfilerCommand() { - return "cpu=10ms"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/IOBoundCode.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/IOBoundCode.java deleted file mode 100644 index 2314036cb..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/IOBoundCode.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.datadoghq.profiler.cpu; - -import java.io.InputStream; -import java.io.OutputStream; -import java.net.ServerSocket; -import java.net.Socket; -import java.util.concurrent.ThreadLocalRandom; - -public class IOBoundCode { - private static final class IdleClient extends Thread { - private final String host; - private final int port; - - public IdleClient(String host, int port) { - this.host = host; - this.port = port; - setDaemon(true); - } - - @Override - public void run() { - try { - byte[] buf = new byte[4096]; - - try (Socket s = new Socket(host, port)) { - InputStream in = s.getInputStream(); - while (in.read(buf) >= 0) { - // keep reading - } - System.out.println(Thread.currentThread().getName() + " stopped"); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - private static final class BusyClient extends Thread { - private final String host; - private final int port; - - public BusyClient(String host, int port) { - this.host = host; - this.port = port; - setDaemon(true); - } - - @Override - public void run() { - try { - byte[] buf = new byte[4096]; - - try (Socket s = new Socket(host, port)) { - - InputStream in = s.getInputStream(); - while (in.read(buf) >= 0) { - // keep reading - } - System.out.println(Thread.currentThread().getName() + " stopped"); - } - } catch (Exception e) { - e.printStackTrace(); - } - } - } - - void run() throws Exception { - try (ServerSocket s = new ServerSocket(0)) { - String host = "localhost"; - int port = s.getLocalPort(); - Thread t1 = new IdleClient(host, port); - t1.start(); - OutputStream idleClient = s.accept().getOutputStream(); - - Thread t2 = new BusyClient(host, port); - t2.start(); - OutputStream busyClient = s.accept().getOutputStream(); - - byte[] buf = new byte[4096]; - ThreadLocalRandom.current().nextBytes(buf); - - long target = System.nanoTime() + 3_000_000_000L; - for (int i = 0; ; i++) { - if ((i % 10_000_000) == 0) { - idleClient.write(buf, 0, 1); - } else { - busyClient.write(buf); - } - if (System.nanoTime() >= target) { - t1.interrupt(); - t2.interrupt(); - break; - } - } - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/LightweightContextCpuTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/LightweightContextCpuTest.java deleted file mode 100644 index 8d9296728..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/LightweightContextCpuTest.java +++ /dev/null @@ -1,84 +0,0 @@ -package com.datadoghq.profiler.cpu; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.context.ContextExecutor; -import com.datadoghq.profiler.context.Tracing; -import org.junit.jupiter.api.Assumptions; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.*; - -import static com.datadoghq.profiler.MoreAssertions.assertInRange; -import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class LightweightContextCpuTest extends AbstractProfilerTest { - - private ProfiledCode profiledCode; - - @Override - protected void before() { - profiledCode = new ProfiledCode(profiler); - } - - public void test() throws ExecutionException, InterruptedException { - Assumptions.assumeTrue(!Platform.isJ9()); - for (int i = 0, id = 1; i < 100; i++, id += 3) { - profiledCode.method1(id); - } - stopProfiler(); - Set sampledSpanIds = profiledCode.allSampledSpanIds(); - IItemCollection events = verifyEvents("datadog.ExecutionSample"); - - int numNonZeroContexts = 0; - for (IItemIterable cpuSamples : events) { - IMemberAccessor frameAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - IMemberAccessor spanIdAccessor = SPAN_ID.getAccessor(cpuSamples.getType()); - IMemberAccessor rootSpanIdAccessor = LOCAL_ROOT_SPAN_ID.getAccessor(cpuSamples.getType()); - IMemberAccessor stateAccessor = THREAD_STATE.getAccessor(cpuSamples.getType()); - for (IItem sample : cpuSamples) { - String stackTrace = frameAccessor.getMember(sample); - assertNull(stackTrace); - long spanId = spanIdAccessor.getMember(sample).longValue(); - long rootSpanId = rootSpanIdAccessor.getMember(sample).longValue(); - numNonZeroContexts += (spanId != 0 && rootSpanId != 0 ? 1 : 0); - if (spanId > 0) { - assertTrue(sampledSpanIds.contains(spanId)); - } - String state = stateAccessor.getMember(sample); - assertDoesNotThrow(() -> Thread.State.valueOf(state)); - assertEquals(Thread.State.RUNNABLE, Thread.State.valueOf(state)); - } - } - assertTrue(numNonZeroContexts > 0, "no context"); - Map debugCounters = profiler.getDebugCounters(); - // these are here to verify these counters produce reasonable values so they can be used for memory leak detection - assertInRange(debugCounters.get("calltrace_storage_traces"), 10, 10000); - assertInRange(debugCounters.get("calltrace_storage_bytes"), 1024, 8 * 1024 * 1024); - // this allocator is only used for calltrace storage and eagerly allocates chunks of 8MiB - assertEquals(8 * 1024 * 1024, debugCounters.get("linear_allocator_bytes")); - assertEquals(1, debugCounters.get("linear_allocator_chunks")); - } - - @Override - protected void after() throws Exception { - profiledCode.close(); - } - - @Override - protected String getProfilerCommand() { - return "cpu=100us,lightweight=yes"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/ProfiledCode.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/ProfiledCode.java deleted file mode 100644 index e9c4d6bdf..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/ProfiledCode.java +++ /dev/null @@ -1,96 +0,0 @@ -package com.datadoghq.profiler.cpu; - -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.context.ContextExecutor; -import com.datadoghq.profiler.context.Tracing; - -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.TimeUnit; - -public class ProfiledCode implements AutoCloseable { - - private static volatile long sink; - private final ContextExecutor executor; - private final JavaProfiler profiler; - - private final Map> methodsToSpanIds = new ConcurrentHashMap<>(); - - public ProfiledCode(JavaProfiler profiler) { - this.profiler = profiler; - this.executor = new ContextExecutor(1, profiler); - } - - public void method1(int id) throws ExecutionException, InterruptedException { - try (Tracing.Context context = Tracing.newContext(() -> id, profiler)) { - method1Impl(id, context); - } - } - - public void method1Impl(int id, Tracing.Context context) throws ExecutionException, InterruptedException { - burnCycles(); - Future wait = executor.submit(() -> method3(id)); - method2(id); - wait.get(); - record("method1Impl", context); - } - - public void method2(long id) { - try (Tracing.Context context = Tracing.newContext(() -> id + 1, profiler)) { - method2Impl(context); - } - } - - public void method2Impl(Tracing.Context context) { - burnCycles(); - record("method2Impl", context); - } - - public void method3(long id) { - try (Tracing.Context context = Tracing.newContext(() -> id + 2, profiler)) { - method3Impl(context); - } - } - - public void method3Impl(Tracing.Context context) { - burnCycles(); - record("method3Impl", context); - } - - public Set spanIdsForMethod(String methodName) { - return new HashSet<>(methodsToSpanIds.get(methodName)); - } - - public Set allSampledSpanIds() { - Set all = new HashSet<>(); - methodsToSpanIds.values().forEach(all::addAll); - return all; - } - - - private void record(String methodName, Tracing.Context context) { - methodsToSpanIds.computeIfAbsent(methodName, k -> new CopyOnWriteArrayList<>()) - .add(context.getSpanId()); - } - - private void burnCycles() { - long blackhole = sink; - for (int i = 0; i < 1_000_000; i++) { - blackhole ^= ThreadLocalRandom.current().nextLong(); - } - sink = blackhole; - } - - @Override - public void close() throws Exception { - executor.shutdownNow(); - executor.awaitTermination(30, TimeUnit.SECONDS); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/RemoteSymbolicationTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/RemoteSymbolicationTest.java deleted file mode 100644 index eb3e852bd..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/RemoteSymbolicationTest.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.datadoghq.profiler.cpu; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.RemoteSymHelper; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.params.provider.ValueSource; -import org.openjdk.jmc.common.IMCFrame; -import org.openjdk.jmc.common.IMCMethod; -import org.openjdk.jmc.common.IMCStackTrace; -import org.openjdk.jmc.common.IMCType; -import org.openjdk.jmc.common.item.Attribute; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.List; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; -import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT; - -/** - * Integration test for remote symbolication feature. - * - *

Tests that when remotesym=true is enabled: - *

    - *
  • Native frames contain build-id instead of symbol names
  • - *
  • PC offsets are stored instead of symbol addresses
  • - *
  • Build-ids are valid hex strings
  • - *
- */ -public class RemoteSymbolicationTest extends CStackAwareAbstractProfilerTest { - public RemoteSymbolicationTest(@CStack String cstack) { - super(cstack); - } - - @BeforeEach - public void checkPlatform() { - // Remote symbolication with build-id extraction is Linux-only - Assumptions.assumeTrue(Platform.isLinux(), "Remote symbolication test requires Linux"); - // Zing JVM forces cstack=no which disables native stack walking - Assumptions.assumeFalse(Platform.isZing(), "Remote symbolication test requires native stack walking (incompatible with Zing)"); - // OpenJ9 uses JVMTI-based CPU profiling which only captures Java frames, not native library frames - // This test requires native frames from libddproftest.so to validate remote symbolication - Assumptions.assumeFalse(Platform.isJ9(), "Remote symbolication requires native frames; OpenJ9 JVMTI engine only captures Java frames"); - } - - @RetryTest(10) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void testRemoteSymbolicationEnabled(@CStack String cstack) throws Exception { - try (ProfiledCode profiledCode = new ProfiledCode(profiler)) { - for (int i = 0, id = 1; i < 100; i++, id += 3) { - profiledCode.method1(id); - // Call native functions from our test library to ensure - // native frames with build-id appear in the samples - // Increased iterations to ensure profiler captures these frames - RemoteSymHelper.burnCpu(1000000, 10); - RemoteSymHelper.computeFibonacci(35); - } - stopProfiler(); - - verifyCStackSettings(); - - // First verify that our test library (libddproftest) has a build-id - // We use the extended jdk.NativeLibrary event which now includes buildId and loadBias fields - IItemCollection libraryEvents = verifyEvents("jdk.NativeLibrary"); - String testLibBuildId = null; - boolean foundTestLib = false; - - // Create attributes for the custom fields we added to jdk.NativeLibrary - IAttribute buildIdAttr = Attribute.attr("buildId", "buildId", "GNU Build ID", PLAIN_TEXT); - IAttribute nameAttr = Attribute.attr("name", "name", "Name", PLAIN_TEXT); - - for (IItemIterable libItems : libraryEvents) { - IMemberAccessor buildIdAccessor = buildIdAttr.getAccessor(libItems.getType()); - IMemberAccessor nameAccessor = nameAttr.getAccessor(libItems.getType()); - - for (IItem libItem : libItems) { - String name = nameAccessor.getMember(libItem); - String buildId = buildIdAccessor.getMember(libItem); - - System.out.println("Library: " + name + " -> build-id: " + - (buildId != null && !buildId.isEmpty() ? buildId : "")); - - // Check if this is our test library - if (name != null && name.contains("libddproftest")) { - foundTestLib = true; - testLibBuildId = buildId; - System.out.println("Found test library: " + name + " with build-id: " + buildId); - } - } - } - - // Our test library MUST be present and have a build-id - Assumptions.assumeTrue(foundTestLib, - "Test library libddproftest not found in jdk.NativeLibrary events. " - + "The test needs this library to verify remote symbolication."); - Assumptions.assumeTrue(testLibBuildId != null && !testLibBuildId.isEmpty(), - "Test library libddproftest found but has no build-id. " - + "Cannot test remote symbolication without build-id."); - - IItemCollection events = verifyEvents("datadog.ExecutionSample"); - - boolean foundTestLibFrame = false; - boolean foundTestLibRemoteFrame = false; - int sampleCount = 0; - int printCount = 0; - int testLibFrameCount = 0; - - for (IItemIterable cpuSamples : events) { - IMemberAccessor stackTraceAccessor = - STACK_TRACE.getAccessor(cpuSamples.getType()); - - for (IItem sample : cpuSamples) { - IMCStackTrace stackTrace = stackTraceAccessor.getMember(sample); - if (stackTrace == null) { - continue; - } - - sampleCount++; - - // Iterate through frames to check for test library frames - List frames = stackTrace.getFrames(); - - for (IMCFrame frame : frames) { - IMCMethod method = frame.getMethod(); - if (method == null) { - continue; - } - - // Check for jvmtiError in method name - String methodName = method.getMethodName(); - if (methodName != null && methodName.contains("jvmtiError")) { - fail("Found jvmtiError in frame method name: " + methodName); - } - - // Get class name (contains build-id for remote symbolication frames) - IMCType type = method.getType(); - String className = type != null ? type.getFullName() : null; - - // Check if this is a remote symbolication frame from our test library - // Format in JFR: type.name = build-ID (bare, no suffix), method.name = "" - if (methodName != null && methodName.equals("") && - className != null && className.equals(testLibBuildId)) { - foundTestLibRemoteFrame = true; - testLibFrameCount++; - foundTestLibFrame = true; - - // Print first remote frame for debugging - if (printCount == 0) { - System.out.println("=== First remote symbolication frame ==="); - System.out.println("Class: " + className); - System.out.println("Method: " + methodName); - System.out.println("Signature: " + (method.getFormalDescriptor() != null ? method.getFormalDescriptor() : "null")); - printCount++; - } - } - - // With remote symbolication, we should see method names, not resolved symbols - // Log a warning if we find resolved symbols (indicates remote symbolication didn't work for this frame) - if (methodName != null && (methodName.equals("burn_cpu") || methodName.equals("compute_fibonacci"))) { - System.out.println("WARNING: Found resolved symbol instead of remote frame: " + methodName + " (class: " + className + ")"); - } - - // Also count frames with resolved symbols from libddproftest - // (for fallback case or if library name appears in class name) - if ((methodName != null && (methodName.contains("burn_cpu") || methodName.contains("compute_fibonacci"))) || - (className != null && className.contains("libddproftest"))) { - foundTestLibFrame = true; - // Don't increment testLibFrameCount here to avoid double-counting - } - } - } - } - - System.out.println("Total samples analyzed: " + sampleCount); - System.out.println("Samples with test lib frames: " + testLibFrameCount); - System.out.println("Found test lib frames: " + foundTestLibFrame); - System.out.println("Found test lib remote frames: " + foundTestLibRemoteFrame); - System.out.println("Test library build-id: " + testLibBuildId); - - // We call the test library functions extensively, so we MUST see frames from it - assertTrue(foundTestLibFrame, - "No frames from libddproftest found in any samples. " - + "The test called RemoteSymHelper.burnCpu() and computeFibonacci() extensively. " - + "Analyzed " + sampleCount + " samples."); - - // Those frames MUST be in remote symbolication format (not resolved) - assertTrue(foundTestLibRemoteFrame, - "Found frames from libddproftest but they are not in remote symbolication format. " - + "Expected format: " + testLibBuildId + ".(0x). " - + "Analyzed " + testLibFrameCount + " samples with test lib frames."); - } - } - - @Override - protected String getProfilerCommand() { - return "cpu=10ms,remotesym=true"; - } -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/SmokeCpuTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/SmokeCpuTest.java deleted file mode 100644 index fd065bd14..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/SmokeCpuTest.java +++ /dev/null @@ -1,93 +0,0 @@ -package com.datadoghq.profiler.cpu; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.params.provider.ValueSource; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.concurrent.ExecutionException; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.datadoghq.profiler.Platform; - -public class SmokeCpuTest extends CStackAwareAbstractProfilerTest { - public SmokeCpuTest(@CStack String cstack) { - super(cstack); - } - - @RetryTest(10) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void testComputations(@CStack String cstack) throws Exception { - try (ProfiledCode profiledCode = new ProfiledCode(profiler)) { - for (int i = 0, id = 1; i < 100; i++, id += 3) { - profiledCode.method1(id); - } - stopProfiler(); - - verifyCStackSettings(); - - IItemCollection events = verifyEvents("datadog.ExecutionSample"); - - // on mac the usage of itimer to drive the sampling provides very unreliable outputs - for (IItemIterable cpuSamples : events) { - IMemberAccessor frameAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - for (IItem sample : cpuSamples) { - String stackTrace = frameAccessor.getMember(sample); - assertFalse(stackTrace.contains("jvmtiError")); - if ("vmx".equals(stackTrace)) { - // extra checks to make sure we see the mixed stacktraces - assertTrue(stackTrace.contains("JavaCalls::call_virtual()"), - "JavaCalls::call_virtual() (above call_stub) found in stack trace"); - assertTrue(stackTrace.contains("call_stub()"), - "call_stub() (sentinel value used to halt unwinding) found in stack trace"); - } - } - } - } - } - - @RetryTest(10) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void testIOBound(@CStack String cstack) throws Exception { - new IOBoundCode().run(); - - stopProfiler(); - - verifyCStackSettings(); - - IItemCollection events = verifyEvents("datadog.ExecutionSample"); - - // on mac the usage of itimer to drive the sampling provides very unreliable outputs - for (IItemIterable cpuSamples : events) { - IMemberAccessor frameAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - for (IItem sample : cpuSamples) { - String stackTrace = frameAccessor.getMember(sample); - assertFalse(stackTrace.contains("jvmtiError")); - if ("vmx".equals(stackTrace)) { - // extra checks to make sure we see the mixed stacktraces - assertTrue(stackTrace.contains("JavaCalls::call_virtual()"), - "JavaCalls::call_virtual() (above call_stub) found in stack trace"); - assertTrue(stackTrace.contains("call_stub()"), - "call_stub() (sentinel value used to halt unwinding) found in stack trace"); - } - } - } - } - - @Override - protected String getProfilerCommand() { - return "cpu=10ms"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java deleted file mode 100644 index e3c4dfd1f..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/cpu/VtableReceiverFrameTest.java +++ /dev/null @@ -1,97 +0,0 @@ -package com.datadoghq.profiler.cpu; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.concurrent.ThreadLocalRandom; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class VtableReceiverFrameTest extends AbstractProfilerTest { - - @Override - protected String getProfilerCommand() { - return "cpu=1ms"; - } - - abstract static class Shape { - abstract int area(); - } - - // Three implementations force megamorphic vtable dispatch (JIT won't inline). - // ThreadLocalRandom bodies ensure each variant is non-trivial and CPU-bound. - static class Circle extends Shape { - @Override public int area() { return ThreadLocalRandom.current().nextInt() | 1; } - } - - static class Square extends Shape { - @Override public int area() { return ThreadLocalRandom.current().nextInt() | 2; } - } - - static class Triangle extends Shape { - @Override public int area() { return ThreadLocalRandom.current().nextInt() | 4; } - } - - private int profiledWork(Shape... shapes) { - int result = 0; - for (int i = 0; i < 10_000_000; i++) { - for (Shape shape : shapes) { - result += shape.area(); - } - } - return result; - } - - // The vtable_target feature inserts a synthetic frame immediately - // below a vtable stub frame in the call stack. The receiver class (Circle/Square/Triangle) - // is captured as a VMSymbol* in the signal handler and resolved to a class name at - // dump time via SafeAccess-protected reads. If resolution fails or the synthetic frame - // is dropped, the receiver class name will not appear next to a vtable stub in JFR. - @RetryingTest(5) - public void testVtableReceiverFrameInCpuSamples() throws Exception { - Assumptions.assumeFalse(Platform.isZing() || Platform.isJ9() || Platform.isGraal()); - waitForProfilerReady(2000); - int result = profiledWork(new Circle(), new Square(), new Triangle()); - System.err.println(result); - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.ExecutionSample"); - boolean foundVtableWithReceiver = false; - for (IItemIterable cpuSamples : events) { - IMemberAccessor frameAccessor = - JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - if (frameAccessor == null) continue; - for (IItem sample : cpuSamples) { - String stackTrace = frameAccessor.getMember(sample); - if (stackTrace != null && stackTrace.contains(".vtable stub()")) { - System.err.println("=VTABLE STUB TRACE=\n" + stackTrace + "\n=END="); - } - // JMC's STACK_TRACE_STRING HTML-escapes angle brackets in method - // names (it does the same for /), so the synthetic - // method appears as "<vtable_receiver>" in the rendered string. - // Match on the bare token so the test is robust to either form. - if (stackTrace != null - && stackTrace.contains(".vtable stub()") - && stackTrace.contains("vtable_receiver") - && (stackTrace.contains("Circle") - || stackTrace.contains("Square") - || stackTrace.contains("Triangle"))) { - foundVtableWithReceiver = true; - break; - } - } - if (foundVtableWithReceiver) break; - } - assertTrue(foundVtableWithReceiver, - "No CPU sample contained a vtable stub frame, a vtable_receiver synthetic frame, " + - "and a receiver class (Circle/Square/Triangle); signal-handler VMSymbol* capture or " + - "dump-time SafeAccess resolution in Lookup::resolveVTableReceiver is broken"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/endpoints/EndpointTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/endpoints/EndpointTest.java deleted file mode 100644 index 97baba6e2..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/endpoints/EndpointTest.java +++ /dev/null @@ -1,113 +0,0 @@ -package com.datadoghq.profiler.endpoints; - -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; - -import java.util.Arrays; -import java.util.BitSet; -import java.util.Map; -import java.util.UUID; -import java.util.stream.IntStream; - -import static com.datadoghq.profiler.MoreAssertions.assertBoundedBy; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.openjdk.jmc.common.item.Attribute.attr; -import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT; - -public class EndpointTest extends AbstractProfilerTest { - - @Test - public void testEndpoints() { - Endpoint[] endpoints = IntStream.range(0, 1000) - .mapToObj(i -> new Endpoint(i, i + "", i % 2 == 0 ? "op" + i : null)) - .toArray(Endpoint[]::new); - int sizeLimit = endpoints.length; - // insert up to limit - for (Endpoint endpoint : endpoints) { - record(endpoint, true, sizeLimit); - } - // idempotency - for (Endpoint endpoint : endpoints) { - record(endpoint, true, sizeLimit); - } - // reject above size limit - record(new Endpoint(0, UUID.randomUUID().toString(), UUID.randomUUID().toString()), false, sizeLimit); - - Map debugCounters = profiler.getDebugCounters(); - assertEquals(endpoints.length, debugCounters.get("dictionary_endpoints_keys")); - stopProfiler(); - IItemCollection events = verifyEvents("datadog.Endpoint"); - IAttribute endpointAttribute = attr("endpoint", "endpoint", "endpoint", - PLAIN_TEXT); - BitSet recovered = new BitSet(); - for (IItemIterable it : events) { - IMemberAccessor endpointAccessor = endpointAttribute.getAccessor(it.getType()); - IMemberAccessor rootSpanIdAccessor = LOCAL_ROOT_SPAN_ID.getAccessor(it.getType()); - IMemberAccessor operationAccessor = OPERATION.getAccessor(it.getType()); - for (IItem event : it) { - long rootSpanId = rootSpanIdAccessor.getMember(event).longValue(); - String operation = operationAccessor.getMember(event); - Endpoint endpoint = endpoints[(int) rootSpanId]; - recovered.set((int) rootSpanId); - String message = endpoint.toString(); - String recordedEndpoint = endpointAccessor.getMember(event); - assertEquals(endpoint.endpoint, recordedEndpoint, message); - assertEquals(endpoint.rootSpanId, rootSpanId, message); - assertEquals(endpoint.operation, operation, message); - } - } - for (int i = 0; i < endpoints.length; i++) { - assertTrue(recovered.get(i), i + " not tested"); - } - assertEquals(Arrays.stream(endpoints).mapToInt(ep -> ep.endpoint.length() + 1).sum(), debugCounters.get("dictionary_endpoints_keys_bytes")); - // SBTable geometry (post-StringDictionary refactor): ROWS=128, CELLS=3. - // 1000 keys distribute ~8 entries per row, so nearly every row in the - // active buffer overflows, allocating one new SBTable per row → up to - // ~129 pages per buffer (1 root + 128 overflow). With 3 buffers and - // potential dump-cycle rotation, 512 is a safe upper bound. - assertBoundedBy(debugCounters.get("dictionary_endpoints_pages"), 512, - "endpoint storage too many SBTable pages"); - // dictionary_endpoints_bytes covers SBTable nodes plus arena Chunk(s). - // Worst case for 1000 short keys: 3 buffers × (sizeof(SBTable) + - // sizeof(Chunk)) ≈ 2.3 MB; bound at 4 MB for safety. - assertBoundedBy(debugCounters.get("dictionary_endpoints_bytes"), 4L * 1024 * 1024, - "endpoint storage too many bytes"); - } - - private void record(Endpoint endpoint, boolean shouldAccept, int sizeLimit) { - assertEquals(shouldAccept, profiler.recordTraceRoot(endpoint.rootSpanId, endpoint.endpoint, endpoint.operation, sizeLimit)); - } - - @Override - protected String getProfilerCommand() { - return "wall=~1ms"; - } - - static class Endpoint { - private final long rootSpanId; - private final String endpoint; - private final String operation; - - Endpoint(long rootSpanId, String endpoint, String operation) { - this.rootSpanId = rootSpanId; - this.endpoint = endpoint; - this.operation = operation; - } - - @Override - public String toString() { - return "Endpoint{" + - "rootSpanId=" + rootSpanId + - ", endpoint='" + endpoint + '\'' + - ", operation='" + operation + '\'' + - '}'; - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java deleted file mode 100644 index 2595ed0c6..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/filter/ThreadFilterSmokeTest.java +++ /dev/null @@ -1,55 +0,0 @@ -package com.datadoghq.profiler.filter; - -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; - -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -public class ThreadFilterSmokeTest extends AbstractProfilerTest { - private ExecutorService executorService; - - @BeforeEach - public void before() { - executorService = Executors.newCachedThreadPool(); - } - - @AfterEach - public void after() { - executorService.shutdownNow(); - } - - @Override - protected String getProfilerCommand() { - return "wall=~1ms,filter=0"; - } - - @Test - public void smokeTest() throws Exception { - doThreadFiltering(); - } - - @SuppressWarnings("deprecation") - private void doThreadFiltering() throws Exception { - Future[] futures = new Future[1000]; - for (int i = 0; i < futures.length; i++) { - int id = i; - futures[i] = executorService.submit(() -> { - profiler.addThread(); - profiler.setContext(id, 42); - try { - Thread.sleep(2); - } catch(InterruptedException e) { - Thread.currentThread().interrupt(); - } - profiler.removeThread(); - }); - } - for (Future future : futures) { - future.get(); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/CpuDumpSmokeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/CpuDumpSmokeTest.java deleted file mode 100644 index 9a7c6d1fe..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/CpuDumpSmokeTest.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.datadoghq.profiler.jfr; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.params.provider.ValueSource; -import org.junit.jupiter.api.TestTemplate; - -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; - -public class CpuDumpSmokeTest extends JfrDumpTest { - public CpuDumpSmokeTest(@CStack String cstack) { - super(cstack); - } - @Override - protected String getProfilerCommand() { - return "cpu=1ms"; - } - - @RetryTest(3) - @Timeout(value = 60) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void test(@CStack String cstack) throws Exception { - runTest("datadog.ExecutionSample"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/DumpWhileChurningThreadsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/DumpWhileChurningThreadsTest.java deleted file mode 100644 index 67d4a4265..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/DumpWhileChurningThreadsTest.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.datadoghq.profiler.jfr; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; - -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.params.provider.ValueSource; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.CountDownLatch; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Regression test for PROF-14548: SIGSEGV in Profiler::updateThreadName when a thread - * terminates between GetAllThreads and the eetop dereference during dump(). - * - * Exercises the race by churning short-lived threads while calling dump() repeatedly. - * A SIGSEGV would abort the JVM and fail the test with a non-zero exit code. - */ -public class DumpWhileChurningThreadsTest extends CStackAwareAbstractProfilerTest { - - private static final int TEST_DURATION_SECS = 10; - private static final int CHURN_THREADS = 50; - - public DumpWhileChurningThreadsTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected boolean isPlatformSupported() { - return !Platform.isJ9() && Platform.isJavaVersionAtLeast(11); - } - - @Override - protected String getProfilerCommand() { - return "wall=5ms"; - } - - @RetryTest(2) - @Timeout(value = 60) - @TestTemplate - @ValueSource(strings = {"vm"}) - public void dumpWhileChurning(@CStack String cstack) throws Exception { - Assumptions.assumeTrue(!Platform.isZing(), "Zing forces cstack=no, skipping"); - - AtomicInteger dumpCount = new AtomicInteger(0); - AtomicInteger threadStartCount = new AtomicInteger(0); - CountDownLatch churnStarted = new CountDownLatch(1); - - ExecutorService executor = Executors.newFixedThreadPool(CHURN_THREADS); - long deadline = System.currentTimeMillis() + TEST_DURATION_SECS * 1000L; - - // Start the churn task; each pool thread spawns short-lived tasks until deadline - for (int i = 0; i < CHURN_THREADS; i++) { - executor.submit(() -> { - churnStarted.countDown(); - while (System.currentTimeMillis() < deadline) { - Thread t = new Thread(() -> threadStartCount.incrementAndGet()); - t.start(); - try { - t.join(5); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - }); - } - - // Wait for at least one churn thread to be running before starting dumps - assertTrue(churnStarted.await(5, TimeUnit.SECONDS), - "Churn tasks did not start within 5 seconds"); - - // Dump repeatedly while churn runs - try { - while (System.currentTimeMillis() < deadline) { - Path recording = Files.createTempFile("dump-churn-", ".jfr"); - try { - dump(recording); - dumpCount.incrementAndGet(); - } finally { - Files.deleteIfExists(recording); - } - Thread.sleep(200); - } - } finally { - executor.shutdown(); - if (!executor.awaitTermination(5, TimeUnit.SECONDS)) { - executor.shutdownNow(); - } - } - - assertTrue(dumpCount.get() >= 10, - "Expected at least 10 dumps during churn, got " + dumpCount.get() - + " (threadStarts=" + threadStartCount.get() + ")"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/JfrDumpTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/JfrDumpTest.java deleted file mode 100644 index d2c4cb9c6..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/JfrDumpTest.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.datadoghq.profiler.jfr; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.junit.RetryTest; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.Platform; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; - -import org.junit.jupiter.api.Assumptions; - -public abstract class JfrDumpTest extends CStackAwareAbstractProfilerTest { - public JfrDumpTest(@CStack String cstack) { - super(cstack); - } - - public void runTest(String eventName) throws Exception { - runTest(eventName, "method1", "method2", "method3"); - } - - public void runTest(String eventName, String ... patterns) throws Exception { - runTest(eventName, 10, patterns); - } - - public void runTest(String eventName, int dumpCnt, String ... patterns) throws Exception { - Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); - Assumptions.assumeFalse(Platform.isJ9()); - - // Wait for profiler to reach RUNNING state before workload begins - // Use 2000ms timeout to account for slow systems and CI load - waitForProfilerReady(2000); - - for (int j = 0; j < dumpCnt; j++) { - Path recording = Files.createTempFile("dump-", ".jfr"); - try { - for (int i = 0; i < 50; i++) { - method1(); - method2(); - method3(); - } - dump(recording); - verifyStackTraces(recording, eventName, patterns); - } finally { - Files.deleteIfExists(recording); - } - } - for (int i = 0; i < 500; i++) { - method1(); - method2(); - method3(); - } - stopProfiler(); - verifyStackTraces(eventName, patterns); - } - - protected static volatile int value; - - /** - * Override this method in subclasses to provide profiler-specific workload. - * Default implementation provides CPU-bound work suitable for CPU profiling. - */ - protected void method1() { - for (int i = 0; i < 1000000; ++i) { - ++value; - } - } - - /** - * Override this method in subclasses to provide profiler-specific workload. - * Default implementation provides CPU-bound work suitable for CPU profiling. - */ - protected void method2() { - for (int i = 0; i < 1000000; ++i) { - ++value; - } - } - - /** - * Override this method in subclasses to provide profiler-specific workload. - * Default implementation provides CPU-bound work suitable for CPU profiling. - */ - protected void method3() { - for (int i = 0; i < 1000000; ++i) { - ++value; - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/ObjectSampleDumpSmokeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/ObjectSampleDumpSmokeTest.java deleted file mode 100644 index d09491620..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/ObjectSampleDumpSmokeTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.datadoghq.profiler.jfr; - -import com.datadoghq.profiler.Platform; - -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.params.provider.ValueSource; -import org.junit.jupiter.api.TestTemplate; - -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; - -public class ObjectSampleDumpSmokeTest extends JfrDumpTest { - public ObjectSampleDumpSmokeTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected boolean isPlatformSupported() { - return !Platform.isJavaVersion(8) && !Platform.isJ9(); - } - - @Override - protected String getProfilerCommand() { - return "memory=32:a"; - } - - @Override - protected void method3() { - // Allocation profiling: Create many String objects to trigger allocation sampling - // Simulates the original File.list() pattern without I/O dependency - for (int i = 0; i < 500; ++i) { - int cntr = 10; - // Create String array and perform substring operations (allocation-heavy) - String[] files = - new String[] { - "file_" + i + "_0.txt", - "file_" + i + "_1.txt", - "file_" + i + "_2.txt", - "file_" + i + "_3.txt", - "file_" + i + "_4.txt" - }; - for (String s : files) { - if (s != null && !s.isEmpty()) { - value += s.substring(0, Math.min(s.length(), 16)).hashCode(); - if (--cntr < 0) { - break; - } - } - } - } - } - - @RetryTest(5) - @Timeout(value = 300) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void test(@CStack String cstack) throws Exception { - runTest("datadog.ObjectSample", 3, "method3"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/ProcessCallTracesRaceTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/ProcessCallTracesRaceTest.java deleted file mode 100644 index 4f5c4ee4d..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/ProcessCallTracesRaceTest.java +++ /dev/null @@ -1,159 +0,0 @@ -package com.datadoghq.profiler.jfr; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; - -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.params.provider.ValueSource; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.CountDownLatch; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Regression test for PROF-14889: SIGSEGV in Profiler::processCallTraces during JFR - * snapshot dump under high-rate wall-clock profiling. - * - * Root cause: CallTraceHashTable::clearTableOnly() used a global refcount wait - * (waitForAllRefCountsToClear) that timed out under sustained signal-handler load, - * leaving collect() racing with an in-flight put(). The bug surfaces only when: - * - Wall-clock profiling is active at high rate (<=5 ms) - * - Many threads are generating call traces simultaneously - * - dump() is called explicitly while the profiler is running - * - * A SIGSEGV from this crash aborts the JVM, which Gradle/JUnit detects as a - * non-zero exit code and reports as a test failure. - */ -public class ProcessCallTracesRaceTest extends CStackAwareAbstractProfilerTest { - - /** Duration long enough to trigger the race; keep under the @Timeout budget. */ - private static final int TEST_DURATION_SECS = 20; - /** Number of worker threads generating call traces (drives signal-handler load). */ - private static final int WORKER_THREADS = 64; - /** Number of threads calling dump() concurrently. */ - private static final int DUMP_THREADS = 4; - - public ProcessCallTracesRaceTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected boolean isPlatformSupported() { - // The crash is only observable with a real VM stack walker; J9 uses a - // different internal profiling path. Require JDK 11+ to match the customer - // JDK range where the crash is reported. - return !Platform.isJ9() && Platform.isJavaVersionAtLeast(11); - } - - @Override - protected String getProfilerCommand() { - // High-rate wall-clock profiling maximises signal-handler frequency, - // increasing the probability of refcount contention in clearTableOnly(). - return "wall=1ms,cpu=1ms"; - } - - @RetryTest(2) - @Timeout(value = 90) - @TestTemplate - @ValueSource(strings = {"vm"}) - public void dumpUnderHighLoad() throws Exception { - Assumptions.assumeTrue(!Platform.isZing(), "Zing forces cstack=no, skipping"); - - AtomicInteger dumpCount = new AtomicInteger(0); - AtomicInteger workCount = new AtomicInteger(0); - CountDownLatch workersReady = new CountDownLatch(WORKER_THREADS); - - long deadline = System.currentTimeMillis() + TEST_DURATION_SECS * 1000L; - - // Worker pool: keeps all threads busy with CPU work to generate many call - // traces per profiling tick, maximising put() rate into CallTraceStorage. - ExecutorService workers = Executors.newFixedThreadPool(WORKER_THREADS); - for (int i = 0; i < WORKER_THREADS; i++) { - workers.submit(() -> { - workersReady.countDown(); - long x = 0; - while (System.currentTimeMillis() < deadline && !Thread.currentThread().isInterrupted()) { - // Tight loop: generates identifiable call-stack frames and - // keeps the CPU hot so wall-clock signals fire frequently. - for (int j = 0; j < 10_000; j++) { - x = x * 6364136223846793005L + 1442695040888963407L; - } - workCount.incrementAndGet(); - } - return x; // prevent optimisation - }); - } - - // Wait for all workers to start before measuring dump() safety. - assertTrue(workersReady.await(10, TimeUnit.SECONDS), - "Worker threads did not start within 10 s"); - - // Dump threads: repeatedly call dump() while profiling is hot. - // A SIGSEGV in processCallTraces here aborts the JVM and fails the test. - // Each dump thread pre-allocates a single temp file and reuses it across - // all iterations to avoid per-dump filesystem churn that can exhaust /tmp - // or cause awaitTermination to expire on slow CI. - ExecutorService dumpers = Executors.newFixedThreadPool(DUMP_THREADS); - for (int i = 0; i < DUMP_THREADS; i++) { - dumpers.submit(() -> { - Path tmp; - try { - tmp = Files.createTempFile("prof-race-test-", ".jfr"); - tmp.toFile().deleteOnExit(); - } catch (Exception e) { - System.err.println("Failed to create temp file: " + e); - return; - } - try { - while (System.currentTimeMillis() < deadline) { - try { - dump(tmp); - dumpCount.incrementAndGet(); - // Brief pause so dump() interleaves with ongoing profiling - // signals rather than running back-to-back in the same - // profiling quiescent window. - Thread.sleep(5); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } catch (Exception e) { - // dump() failures other than SIGSEGV are logged but do not - // fail the test on their own; the JVM crash is the signal. - System.err.println("dump() threw: " + e); - } - } - } finally { - try { Files.deleteIfExists(tmp); } catch (Exception ignored) {} - } - }); - } - - try { - workers.shutdown(); - dumpers.shutdown(); - // Widen termination budget beyond the deadline to accommodate slow CI I/O. - assertTrue(workers.awaitTermination(TEST_DURATION_SECS + 30L, TimeUnit.SECONDS), - "Worker threads did not finish"); - assertTrue(dumpers.awaitTermination(TEST_DURATION_SECS + 30L, TimeUnit.SECONDS), - "Dump threads did not finish"); - } finally { - workers.shutdownNow(); - dumpers.shutdownNow(); - } - - assertTrue(dumpCount.get() > 0, - "No successful dump() calls recorded; test did not exercise the race"); - System.out.printf("ProcessCallTracesRaceTest: %d dumps, %d work units%n", - dumpCount.get(), workCount.get()); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/WallclockDumpSmokeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/WallclockDumpSmokeTest.java deleted file mode 100644 index 17b5093b5..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/jfr/WallclockDumpSmokeTest.java +++ /dev/null @@ -1,77 +0,0 @@ -package com.datadoghq.profiler.jfr; - -import org.junit.jupiter.api.Timeout; -import org.junit.jupiter.params.provider.ValueSource; -import org.junit.jupiter.api.TestTemplate; - -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; - -public class WallclockDumpSmokeTest extends JfrDumpTest { - public WallclockDumpSmokeTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected boolean isPlatformSupported() { - // Zing forces cstack=no which prevents proper stack trace capture for wall clock profiling - return !Platform.isZing(); - } - - @Override - protected String getProfilerCommand() { - return "wall=5ms"; - } - - @Override - protected void method1() { - // CPU work for wall clock sampling - for (int i = 0; i < 1000000; ++i) { - ++value; - } - // Add brief sleep to ensure wall clock capture - try { - Thread.sleep(1); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - @Override - protected void method2() { - // CPU work for wall clock sampling - for (int i = 0; i < 1000000; ++i) { - ++value; - } - // Add brief sleep to ensure wall clock capture - try { - Thread.sleep(1); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - @Override - protected void method3() { - // CPU work for wall clock sampling - for (int i = 0; i < 1000000; ++i) { - ++value; - } - // Add brief sleep to ensure wall clock capture - try { - Thread.sleep(1); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - @RetryTest(3) - @Timeout(value = 60) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void test(@CStack String cstack) throws Exception { - registerCurrentThreadForWallClockProfiling(); - runTest("datadog.MethodSample"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/junit/CStack.java b/ddprof-test/src/test/java/com/datadoghq/profiler/junit/CStack.java deleted file mode 100644 index de1cf009d..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/junit/CStack.java +++ /dev/null @@ -1,11 +0,0 @@ -package com.datadoghq.profiler.junit; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target(ElementType.PARAMETER) -@Retention(RetentionPolicy.RUNTIME) -public @interface CStack { -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/junit/CStackInjector.java b/ddprof-test/src/test/java/com/datadoghq/profiler/junit/CStackInjector.java deleted file mode 100644 index 7f4c9534f..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/junit/CStackInjector.java +++ /dev/null @@ -1,215 +0,0 @@ -package com.datadoghq.profiler.junit; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; - -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.TestInfo; -import org.junit.jupiter.api.extension.Extension; -import org.junit.jupiter.api.extension.ExtensionContext; -import org.junit.jupiter.api.extension.InvocationInterceptor; -import org.junit.jupiter.api.extension.ParameterContext; -import org.junit.jupiter.api.extension.ParameterResolutionException; -import org.junit.jupiter.api.extension.ParameterResolver; -import org.junit.jupiter.api.extension.TestExecutionExceptionHandler; -import org.junit.jupiter.api.extension.TestTemplateInvocationContext; -import org.junit.jupiter.api.extension.TestTemplateInvocationContextProvider; -import org.junit.jupiter.params.provider.ValueSource; -import org.opentest4j.TestAbortedException; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; -import java.util.stream.Stream; - -public class CStackInjector implements TestTemplateInvocationContextProvider { - - @Override - public boolean supportsTestTemplate(ExtensionContext context) { - return context.getTestMethod().isPresent() && - context.getTestMethod().get().isAnnotationPresent(ValueSource.class); - } - - @Override - public Stream provideTestTemplateInvocationContexts(ExtensionContext context) { - ValueSource valueSource = context.getTestMethod().get().getAnnotation(ValueSource.class); - // Get retry count from @RetryTest (default = 0) - int retryCount = context.getTestMethod() - .flatMap(method -> Optional.ofNullable(method.getAnnotation(RetryTest.class))) - .map(RetryTest::value) - .orElse(0); - if (Platform.isZing()) { - // zing is a bit iffy when using anything but 'no' for cstack - return Stream.of(new ParameterizedTestContext("no", retryCount)); - } else { - List safeValues = Stream.of(valueSource.strings()) - .filter(CStackInjector::isModeSafe) - .collect(Collectors.toList()); - if (safeValues.isEmpty()) { - // No mode passed the filter (e.g. J9 with a {"vm","vmx"} @ValueSource). - // Fall back to the first declared value so the test can reach - // isPlatformSupported() and skip cleanly rather than failing with - // PreconditionViolationException (JUnit 5 requires ≥1 invocation context). - safeValues = Collections.singletonList(valueSource.strings()[0]); - } - return safeValues.stream().map(param -> new ParameterizedTestContext(param, retryCount)); - } - } - - private static boolean isModeSafe(String mode) { - if (Platform.isJ9()) { - return "dwarf".equals(mode); - } - if (Platform.isAarch64() && !Platform.isJavaVersionAtLeast(17)) { - return mode.startsWith("vm"); - } - if (AbstractProfilerTest.isInCI()) { - if (Platform.isMusl() && !Platform.isAarch64()) { - // our CI runner for musl on x64 is iffy and inexplicably locks up - // randomly when doing vm stackwalking - return !mode.startsWith("vm"); - } - if (Platform.isMusl() && Platform.isAarch64() && "vmx".equals(mode)) { - // vmx mode has intermittent initialization timing issues on musl aarch64 - // causing 0 events in intermediate JFR dumps - return false; - } - } - return true; - } - - public static class TestInfoAdapter implements TestInfo { - private final String displayName; - private final Set tags; - private final Class testClass; - private final Method testMethod; - - public TestInfoAdapter(String displayName, Set tags, Class testClass, Method testMethod) { - this.displayName = displayName; - this.tags = tags; - this.testClass = testClass; - this.testMethod = testMethod; - } - - @Override - public String getDisplayName() { - return displayName; - } - - @Override - public Set getTags() { - return tags; - } - - @Override - public Optional> getTestClass() { - return Optional.ofNullable(testClass); - } - - @Override - public Optional getTestMethod() { - return Optional.ofNullable(testMethod); - } - } - - private static class ParameterizedTestContext implements TestTemplateInvocationContext { - private final String parameter; - private final int retryCount; - - ParameterizedTestContext(String parameter, int retryCount) { - this.parameter = parameter; - this.retryCount = retryCount; - } - - @Override - public String getDisplayName(int invocationIndex) { - return "cstack=" + parameter; - } - - @Override - public List getAdditionalExtensions() { - List extensions = new ArrayList<>(TestTemplateInvocationContext.super.getAdditionalExtensions()); - extensions.add( - new ParameterResolver() { - @Override - public boolean supportsParameter(ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return parameterContext.getParameter().getType() == String.class && parameterContext.getParameter().isAnnotationPresent(CStack.class); - } - - @Override - public Object resolveParameter(ParameterContext parameterContext, ExtensionContext extensionContext) - throws ParameterResolutionException { - return parameter; - } - } - ); - extensions.add( - (TestExecutionExceptionHandler) (extensionContext, throwable) -> { - // Don't retry on assumption failures - they should skip the test - if (throwable instanceof TestAbortedException) { - throw throwable; - } - - int attempt = 0; - while (++attempt < retryCount) { - System.out.println("[Retrying] Attempt " + attempt + "/" + retryCount + " due to failure: " + throwable.getMessage()); - // Manually reinvoke the test method - Method testMethod = extensionContext.getRequiredTestMethod(); - Object testInstance = extensionContext.getRequiredTestInstance(); - Class testClass = extensionContext.getRequiredTestClass(); - String displayName = extensionContext.getDisplayName(); - TestInfoAdapter testInfo = new TestInfoAdapter(displayName, Collections.emptySet(), testClass, testMethod); - - List afterEachMethods = Arrays.stream(testInstance.getClass().getMethods()) - .filter(method -> method.isAnnotationPresent(AfterEach.class)) - .collect(Collectors.toList()); - List beforeEachMethods = Arrays.stream(testInstance.getClass().getMethods()) - .filter(method -> method.isAnnotationPresent(BeforeEach.class)) - .collect(Collectors.toList()); - // Retry the test method - try { - for (Method method : afterEachMethods) { - if (method.getParameterTypes().length == 1 && method.getParameterTypes()[0].isAssignableFrom(TestInfo.class)) { - method.invoke(testInstance, testInfo); - } else { - method.invoke(testInstance); - } - } - for (Method method : beforeEachMethods) { - if (method.getParameterTypes().length == 1 && method.getParameterTypes()[0].isAssignableFrom(TestInfo.class)) { - method.invoke(testInstance, testInfo); - } else { - method.invoke(testInstance); - } - } - if (testMethod.getParameterCount() == 0) { - testMethod.invoke(testInstance); - } else { - testMethod.invoke(testInstance, parameter); - } - return; // If the test passes, stop retrying - } catch (InvocationTargetException e) { - throwable = e.getTargetException(); - } catch (Throwable t) { - throwable = t; - } - } - - System.out.println("[Failure] Test failed after " + retryCount + " attempts."); - throw throwable; - } - ); - return extensions; - } - - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/junit/RetryTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/junit/RetryTest.java deleted file mode 100644 index 286752518..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/junit/RetryTest.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.datadoghq.profiler.junit; - -import java.lang.annotation.ElementType; -import java.lang.annotation.Retention; -import java.lang.annotation.RetentionPolicy; -import java.lang.annotation.Target; - -@Target(ElementType.METHOD) -@Retention(RetentionPolicy.RUNTIME) -public @interface RetryTest { - int value() default 5; -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/loadlib/LoadLibraryTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/loadlib/LoadLibraryTest.java deleted file mode 100644 index ea323873e..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/loadlib/LoadLibraryTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.datadoghq.profiler.loadlib; - -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; - -import java.lang.management.ClassLoadingMXBean; -import java.lang.management.ManagementFactory; - -public class LoadLibraryTest extends AbstractProfilerTest { - - @RetryingTest(3) - public void testLoadLibrary() throws InterruptedException { - Assumptions.assumeFalse(Platform.isJ9()); - for (int i = 0; i < 200; i++) { - Thread.sleep(10); - } - // Late load of libmanagement.so - ClassLoadingMXBean bean = ManagementFactory.getClassLoadingMXBean(); - - long n = 0; - long tsLimit = System.nanoTime() + 3_000_000_000L; // 3 seconds - while (System.nanoTime() < tsLimit) { - n += bean.getLoadedClassCount(); - n += bean.getTotalLoadedClassCount(); - n += bean.getUnloadedClassCount(); - } - System.out.println("=== accumulated: " + n); - stopProfiler(); - verifyStackTraces("datadog.ExecutionSample", "Java_sun_management"); - } - - @Override - protected String getProfilerCommand() { - return "cpu=1ms"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/AbstractDynamicClassTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/AbstractDynamicClassTest.java deleted file mode 100644 index 38ff6fb9e..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/AbstractDynamicClassTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.memleak; - -import com.datadoghq.profiler.AbstractProfilerTest; -import org.objectweb.asm.ClassWriter; -import org.objectweb.asm.Label; -import org.objectweb.asm.MethodVisitor; -import org.objectweb.asm.Opcodes; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -abstract class AbstractDynamicClassTest extends AbstractProfilerTest { - - protected int classCounter = 0; - - /** - * Generates ASM bytecode for a class with a constructor and {@code methodCount} public - * int-returning no-arg methods, each with multiple line-number table entries. - */ - protected byte[] generateClassBytecode(String className, int methodCount) { - ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); - cw.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC, className, null, "java/lang/Object", null); - - MethodVisitor ctor = cw.visitMethod(Opcodes.ACC_PUBLIC, "", "()V", null, null); - ctor.visitCode(); - ctor.visitVarInsn(Opcodes.ALOAD, 0); - ctor.visitMethodInsn(Opcodes.INVOKESPECIAL, "java/lang/Object", "", "()V", false); - ctor.visitInsn(Opcodes.RETURN); - ctor.visitMaxs(0, 0); - ctor.visitEnd(); - - for (int i = 0; i < methodCount; i++) { - MethodVisitor mv = cw.visitMethod(Opcodes.ACC_PUBLIC, "method" + i, "()I", null, null); - mv.visitCode(); - - Label l1 = new Label(); - mv.visitLabel(l1); - mv.visitLineNumber(100 + i * 3, l1); - mv.visitInsn(Opcodes.ICONST_0); - mv.visitVarInsn(Opcodes.ISTORE, 1); - - Label l2 = new Label(); - mv.visitLabel(l2); - mv.visitLineNumber(101 + i * 3, l2); - mv.visitVarInsn(Opcodes.ILOAD, 1); - mv.visitInsn(Opcodes.ICONST_1); - mv.visitInsn(Opcodes.IADD); - mv.visitVarInsn(Opcodes.ISTORE, 1); - - Label l3 = new Label(); - mv.visitLabel(l3); - mv.visitLineNumber(102 + i * 3, l3); - mv.visitVarInsn(Opcodes.ILOAD, 1); - mv.visitInsn(Opcodes.IRETURN); - - mv.visitMaxs(0, 0); - mv.visitEnd(); - } - - cw.visitEnd(); - return cw.toByteArray(); - } - - protected static Path tempFile(String prefix) throws IOException { - Path dir = Paths.get(System.getProperty("java.io.tmpdir"), "recordings"); - Files.createDirectories(dir); - return Files.createTempFile(dir, prefix + "-", ".jfr"); - } - - protected static class IsolatedClassLoader extends ClassLoader { - public Class defineClass(String name, byte[] bytecode) { - return defineClass(name, bytecode, 0, bytecode.length); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/CleanupAfterClassUnloadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/CleanupAfterClassUnloadTest.java deleted file mode 100644 index 12ea41ccf..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/CleanupAfterClassUnloadTest.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.memleak; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.lang.reflect.Method; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -/** - * Regression test for PROF-14545: SIGSEGV in Recording::cleanupUnreferencedMethods. - * - *

The bug: cleanupUnreferencedMethods() was called after finishChunk() released its - * GetLoadedClasses pins. ~SharedLineNumberTable() called jvmti->Deallocate() on - * JVMTI-allocated line-number-table memory that the JVM had already freed on class - * unload → SIGSEGV. - * - *

The fix: two complementary changes: - *

    - *
  1. cleanupUnreferencedMethods() is now called inside finishChunk(), before - * DeleteLocalRef releases the GetLoadedClasses pins.
  2. - *
  3. fillJavaMethodInfo() copies the JVMTI line-number-table into a malloc'd buffer - * and immediately calls jvmti->Deallocate() on the original; cleanup calls free() - * on the malloc'd copy, which is safe regardless of class-unload state.
  4. - *
- * - *

This test reproduces the crash scenario: - *

    - *
  1. A dynamically-loaded class with line number tables appears in profiling stack traces
  2. - *
  3. All references to the class and its class loader are dropped
  4. - *
  5. The JVM is encouraged to GC and unload the class (System.gc())
  6. - *
  7. AGE_THRESHOLD+1 dump cycles are triggered to age the method out of the method_map
  8. - *
  9. cleanupUnreferencedMethods() must not SIGSEGV when freeing the line number table
  10. - *
- */ -public class CleanupAfterClassUnloadTest extends AbstractDynamicClassTest { - - // AGE_THRESHOLD in C++ is 3; run 4 dumps to ensure cleanup fires - private static final int DUMPS_TO_AGE_OUT = 4; - - @Override - protected String getProfilerCommand() { - return "cpu=1ms"; - } - - @Test - @Timeout(60) - public void testNoSigsegvAfterClassUnloadAndMethodCleanup() throws Exception { - stopProfiler(); - - Path baseFile = tempFile("prof14545-base"); - Path dumpFile = tempFile("prof14545-dump"); - - try { - profiler.execute( - "start,cpu=1ms,jfr,mcleanup=true,file=" + baseFile.toAbsolutePath()); - Thread.sleep(200); // Let the profiler stabilize - - // 1. Load a class, execute its methods to appear in CPU profiling stack traces, - // then drop all strong references so the class can be GC'd. - WeakReference loaderRef = loadAndProfileDynamicClass(); - - // 2. Trigger one dump so the profiler captures the class in method_map. - profiler.dump(dumpFile); - Thread.sleep(50); - - // 3. Drop all references and aggressively GC to encourage class unloading. - // We poll until the WeakReference is cleared or we time out. - long deadline = System.currentTimeMillis() + 8_000; - while (loaderRef.get() != null && System.currentTimeMillis() < deadline) { - System.gc(); - Thread.sleep(30); - } - assumeTrue(loaderRef.get() == null, - "Skipping: class loader not GC'd within 8 s — class unloading is required for this test to be meaningful"); - - // 4. Run AGE_THRESHOLD+1 dump cycles so the method ages out and cleanup fires. - // The method is no longer referenced (no stack traces) → age increments each cycle. - // After age >= 3 and the next cleanup pass, the entry is erased and - // SharedLineNumberTable::~SharedLineNumberTable() → jvmti->Deallocate() runs. - // Without the fix this is after DeleteLocalRef; with the fix it is before. - for (int i = 0; i < DUMPS_TO_AGE_OUT; i++) { - profiler.dump(dumpFile); - Thread.sleep(50); - } - - // 5. The profiler must still be alive. If it SIGSEGV'd during cleanup the JVM - // process would have died and this line would never be reached. - profiler.stop(); - - assertTrue(Files.size(dumpFile) > 0, - "Profiler produced no output — it may have crashed during method_map cleanup"); - - } finally { - try { profiler.stop(); } catch (Exception ignored) {} - try { Files.deleteIfExists(baseFile); } catch (IOException ignored) {} - try { Files.deleteIfExists(dumpFile); } catch (IOException ignored) {} - } - } - - /** - * Loads a dynamically-generated class with line number tables in a fresh ClassLoader, - * invokes its methods on the current thread while CPU profiling is active so that - * the profiler calls GetLineNumberTable and stores it in method_map, then returns a - * WeakReference to the ClassLoader so callers can detect unloading. - */ - private WeakReference loadAndProfileDynamicClass() throws Exception { - String className = "com/datadoghq/profiler/generated/Prof14545Class" + (classCounter++); - byte[] bytecode = generateClassBytecode(className, 5); - - IsolatedClassLoader loader = new IsolatedClassLoader(); - Class clazz = loader.defineClass(className.replace('/', '.'), bytecode); - - // Spin-invoke the class methods for ~200ms so the CPU profiler has time to - // capture this thread and include the dynamic class in a stack trace. - long endTime = System.currentTimeMillis() + 200; - Object instance = clazz.getDeclaredConstructor().newInstance(); - Method[] methods = clazz.getDeclaredMethods(); - while (System.currentTimeMillis() < endTime) { - for (Method m : methods) { - if (m.getParameterCount() == 0 && m.getReturnType() == int.class) { - m.invoke(instance); - } - } - } - - // Drop all strong references. Only the WeakReference remains. - WeakReference ref = new WeakReference<>(loader); - //noinspection UnusedAssignment - loader = null; - //noinspection UnusedAssignment - clazz = null; - //noinspection UnusedAssignment - instance = null; - return ref; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/GCGenerationsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/GCGenerationsTest.java deleted file mode 100644 index a7bcb5cf6..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/GCGenerationsTest.java +++ /dev/null @@ -1,89 +0,0 @@ -package com.datadoghq.profiler.memleak; - -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.Aggregators; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.ItemFilters; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicLong; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.Assumptions; - -public class GCGenerationsTest extends AbstractProfilerTest { - @Override - protected String getProfilerCommand() { - return "memory=128,generations=true"; - } - - @Override - protected boolean isPlatformSupported() { - return !(Platform.isJavaVersion(8) || Platform.isJ9() || Platform.isZing()); - } - - @RetryingTest(10) - public void shouldGetLiveObjectSamples() throws InterruptedException { - MemLeakTarget target1 = new MemLeakTarget(); - MemLeakTarget target2 = new MemLeakTarget(); - runTests(target1, target2); - verifyEvents("datadog.HeapLiveObject"); - } - - public static class MemLeakTarget extends ClassValue implements Runnable { - private static byte[] leeway = new byte[32 * 1024 * 1024]; // 32MB to release on OOME - public static volatile List sink = new ArrayList<>(); - - @Override - public void run() { - ThreadLocalRandom random = ThreadLocalRandom.current(); - try { - for (int i = 0; i < 100_000; i++) { - allocate(random, random.nextInt(256)); - } - } catch (OutOfMemoryError e) { - leeway = null; - System.gc(); - } finally { - sink.clear(); - } - } - - long getAllocated(Class clazz) { - return get(clazz).get(); - } - - private static void allocate(ThreadLocalRandom random, int depth) { - if (depth > 0) { - allocate(random, depth - 1); - return; - } - - Object obj; - if (random.nextBoolean()) { - obj = new int[random.nextInt(64, 192) * 1000]; - } else { - obj = new Integer[random.nextInt(64, 192) * 1000]; - } - - if (random.nextInt(100) <= 30 && sink.size() < 100_000) { - sink.add(obj); - } - if (random.nextInt(10000) == 0) { - System.out.println("Triggering GC"); - System.gc(); - } - } - - @Override - protected AtomicLong computeValue(Class type) { - return new AtomicLong(); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/GetLineNumberTableLeakTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/GetLineNumberTableLeakTest.java deleted file mode 100644 index 32a5ede71..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/GetLineNumberTableLeakTest.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright 2025, 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.memleak; - -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.lang.reflect.Method; -import java.nio.file.Files; -import java.nio.file.Path; - -/** - * Test to validate that method_map cleanup prevents unbounded memory growth in continuous profiling. - * - *

This test focuses on the production scenario where the profiler runs continuously for - * extended periods without stop/restart cycles. In production, Recording objects live for the - * entire application lifetime (days/weeks), and without cleanup, _method_map would accumulate - * ALL methods encountered, causing unbounded growth (observed: 1.2 GB line number table leak). - * - *

What This Test Validates: - *

    - *
  • Age-based cleanup removes methods unused for 3+ consecutive chunks
  • - *
  • method_map size stays bounded (~200-400 methods) with cleanup vs unbounded without
  • - *
  • Cleanup runs during switchChunk() triggered by dump() to different file
  • - *
  • Line number table count tracked via Counters infrastructure
  • - *
  • Class unloading frees SharedLineNumberTable memory naturally
  • - *
- * - *

Test Strategy: - *

    - *
  • Continuous profiling (NO stop/restart cycles)
  • - *
  • Generate transient methods across multiple chunk boundaries
  • - *
  • Dump to different file from startup file to trigger switchChunk()
  • - *
  • Validate cleanup via TEST_LOG output showing bounded method_map
  • - *
  • Allow natural class unloading (no strong references held)
  • - *
  • Combined cleanup: method_map cleanup + class unloading
  • - *
- */ -public class GetLineNumberTableLeakTest extends AbstractDynamicClassTest { - - @Override - protected String getProfilerCommand() { - // Aggressive sampling to maximize method encounters and GetLineNumberTable calls - return "cpu=1ms,alloc=512k"; - } - - /** - * Generates and loads truly unique classes using ASM bytecode generation. - * Each class has multiple methods with line number tables to trigger GetLineNumberTable calls. - * - * @return array of generated classes for later reuse - */ - private Class[] generateUniqueMethodCalls() throws Exception { - // Generate 5 unique classes per iteration - // Each class has 20 methods with line number tables - Class[] generatedClasses = new Class[5]; - - for (int i = 0; i < 5; i++) { - String className = "com/datadoghq/profiler/generated/TestClass" + (classCounter++); - byte[] bytecode = generateClassBytecode(className, 20); - - // Use a fresh ClassLoader to ensure truly unique Class object and jmethodIDs - IsolatedClassLoader loader = new IsolatedClassLoader(); - Class clazz = loader.defineClass(className.replace('/', '.'), bytecode); - generatedClasses[i] = clazz; - - // Invoke methods to ensure they're JIT-compiled and show up in stack traces - invokeClassMethods(clazz); - } - - return generatedClasses; - } - - /** - * Invokes all int-returning no-arg methods in a class to trigger profiling samples. - * - * @param clazz the class whose methods to invoke - */ - private void invokeClassMethods(Class clazz) { - try { - Object instance = clazz.getDeclaredConstructor().newInstance(); - Method[] methods = clazz.getDeclaredMethods(); - - // Call each method to trigger potential profiling samples - for (Method m : methods) { - if (m.getParameterCount() == 0 && m.getReturnType() == int.class) { - m.invoke(instance); - } - } - } catch (Exception ignored) { - // Ignore invocation errors - class/method loading is what matters for GetLineNumberTable - } - } - - /** - * Comparison test that validates cleanup effectiveness by running the same workload twice: once - * without cleanup (mcleanup=false) and once with cleanup (mcleanup=true, default). This provides - * empirical evidence that the cleanup mechanism prevents unbounded growth. - * - *

Key implementation detail: Dumps to a DIFFERENT file from the startup file to trigger - * switchChunk(), which is where cleanup happens. Dumping to the same file as the startup file - * does NOT trigger switchChunk. - * - *

Expected results (validated via TEST_LOG): - *

    - *
  • WITHOUT cleanup: method_map grows unbounded (~10k methods generated) - *
  • WITH cleanup: method_map stays bounded at ~200-400 methods (age-based removal) - *
  • TEST_LOG shows "Cleaned up X unreferenced methods" during cleanup phase - *
  • TEST_LOG shows "Live line number tables after cleanup: N" tracking table count - *
- */ - @Test - public void testCleanupEffectivenessComparison() throws Exception { - // Stop the default profiler from AbstractProfilerTest - // We need to manage our own profiler instances for this comparison - stopProfiler(); - - final int iterations = 100; // 100 iterations for fast validation (~10k potential methods) - final int classesPerIteration = 10; // 10 classes × 5 methods = 50 methods per iteration - - // Create temp files that will be cleaned up in finally block - Path noCleanupBaseFile = tempFile("no-cleanup-base"); - Path noCleanupDumpFile = tempFile("no-cleanup-dump"); - Path withCleanupBaseFile = tempFile("with-cleanup-base"); - Path withCleanupDumpFile = tempFile("with-cleanup-dump"); - - try { - // ===== Phase 1: WITHOUT cleanup (mcleanup=false) ===== - System.out.println("\n=== Phase 1: WITHOUT cleanup (mcleanup=false) ==="); - - profiler.execute( - "start," - + getProfilerCommand() - + ",jfr,mcleanup=false,file=" - + noCleanupBaseFile); - - Thread.sleep(300); // Stabilize - - // Run workload without cleanup - // CRITICAL: Dump to DIFFERENT file from startup to trigger switchChunk() - for (int iter = 0; iter < iterations; iter++) { - for (int i = 0; i < classesPerIteration; i++) { - Class[] transientClasses = generateUniqueMethodCalls(); - for (Class clazz : transientClasses) { - invokeClassMethods(clazz); - } - } - - // Dump to different file to trigger switchChunk - profiler.dump(noCleanupDumpFile); - Thread.sleep(50); - - if ((iter + 1) % 3 == 0) { - System.gc(); - Thread.sleep(20); - } - } - - System.out.println( - "Phase 1 complete. Check TEST_LOG: MethodMap should grow unbounded (no cleanup logs expected)"); - - profiler.stop(); - Thread.sleep(300); // Allow cleanup - - // ===== Phase 2: WITH cleanup (mcleanup=true) ===== - System.out.println("\n=== Phase 2: WITH cleanup (mcleanup=true) ==="); - - // Reset class counter to generate same classes - classCounter = 0; - - profiler.execute( - "start," + getProfilerCommand() + ",jfr,mcleanup=true,file=" + withCleanupBaseFile); - - Thread.sleep(300); // Stabilize - - // Run same workload with cleanup - // CRITICAL: Dump to DIFFERENT file from startup to trigger switchChunk() - for (int iter = 0; iter < iterations; iter++) { - for (int i = 0; i < classesPerIteration; i++) { - Class[] transientClasses = generateUniqueMethodCalls(); - for (Class clazz : transientClasses) { - invokeClassMethods(clazz); - } - } - - // Dump to different file to trigger switchChunk - profiler.dump(withCleanupDumpFile); - Thread.sleep(50); - - if ((iter + 1) % 3 == 0) { - System.gc(); - Thread.sleep(20); - } - } - - System.out.println( - "Phase 2 complete. Check TEST_LOG: MethodMap should stay bounded, cleanup logs should appear"); - - profiler.stop(); - - System.out.println( - "\n=== Validation Summary ===\n" - + "✓ Cleanup mechanism runs (check TEST_LOG for 'Cleaned up X unreferenced methods')\n" - + "✓ method_map stays bounded at ~200-400 methods (WITH) vs unbounded (WITHOUT)\n" - + "✓ Line number table tracking shows live tables (check TEST_LOG for 'Live line number tables')\n" - + "✓ Test completed without errors - cleanup is working correctly"); - } finally { - // Clean up temp files - try { - Files.deleteIfExists(noCleanupBaseFile); - } catch (IOException ignored) { - } - try { - Files.deleteIfExists(noCleanupDumpFile); - } catch (IOException ignored) { - } - try { - Files.deleteIfExists(withCleanupBaseFile); - } catch (IOException ignored) { - } - try { - Files.deleteIfExists(withCleanupDumpFile); - } catch (IOException ignored) { - } - } - } - -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/LivenessTrackingTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/LivenessTrackingTest.java deleted file mode 100644 index 4fd9b1e78..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/LivenessTrackingTest.java +++ /dev/null @@ -1,258 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.datadoghq.profiler.memleak; - -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.IMCStackTrace; -import org.openjdk.jmc.common.item.Aggregators; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.ItemFilters; -import org.openjdk.jmc.flightrecorder.JfrLoaderToolkit; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicInteger; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Test that validates liveness tracking functionality with double-buffering and trace preservation. - * This test exercises the complete liveness tracking pipeline: - * - LivenessTracker records live objects with call traces - * - CallTraceStorage preserves traces for live objects during JFR writes - * - GC cleanup properly removes dead objects from tracking - */ -public class LivenessTrackingTest extends AbstractProfilerTest { - - @Override - protected String getProfilerCommand() { - // Enable liveness tracking with memory profiling - // "memory=256:L" configures the profiler as follows: - // 256 - sets the memory sampling interval (in kilobytes) - // :L - enables liveness tracking (records only live objects) - return "memory=256:L"; - } - - @Override - protected boolean isPlatformSupported() { - // Liveness tracking requires Java 11+ and specific JVM types - return !(Platform.isJavaVersion(8) || Platform.isJ9() || Platform.isZing()); - } - - @RetryingTest(5) - public void shouldPreserveLiveObjectTracesAcrossJFRDumps() throws Exception { - // Phase 1: Allocate objects and keep them alive to generate liveness samples - List liveObjects = new ArrayList<>(); - for (int i = 0; i < 1000; i++) { - liveObjects.add(new AllocatingTarget(i)); - } - - // Generate allocation load to trigger liveness sampling - runAllocatingWorkload(liveObjects); - - // Wait for liveness tracking to stabilize - Thread.sleep(100); - for (int i = 0; i < 6; i++) { - System.gc(); // Trigger GC to update liveness state - Thread.sleep(100); - } - // A time buffer to finish any concurrent liveness updates - Thread.sleep(300); - - // Dump first recording - Path firstDump = Paths.get("liveness-test-first.jfr"); - try { - dump(firstDump); - assertTrue(Files.exists(firstDump), "First JFR dump should be created"); - - // Parse first recording and verify liveness samples - IItemCollection firstRecording = JfrLoaderToolkit.loadEvents(Files.newInputStream(firstDump)); - IItemCollection firstLiveObjects = firstRecording.apply( - ItemFilters.type("datadog.HeapLiveObject")); - - assertTrue(firstLiveObjects.hasItems(), "First recording should contain live object samples"); - long firstSampleCount = firstLiveObjects.getAggregate(Aggregators.count()).longValue(); - assertTrue(firstSampleCount > 0, "First recording should have liveness samples"); - - // Verify all live object samples have stack traces with at least one frame - verifyStackTracesPresent(firstLiveObjects); - - System.out.println("First dump: " + firstSampleCount + " liveness samples"); - - // Phase 2: Release half the objects to simulate real application behavior - int objectsToRelease = liveObjects.size() / 2; - for (int i = 0; i < objectsToRelease; i++) { - liveObjects.set(i, null); // Release references - } - - // Force GC multiple times to ensure released objects are collected - // and removed from liveness tracking - for (int i = 0; i < 3; i++) { - System.gc(); - System.runFinalization(); - Thread.sleep(50); - } - - // Generate some more allocation activity to trigger liveness updates - runAllocatingWorkload(liveObjects.subList(objectsToRelease, liveObjects.size())); - Thread.sleep(100); - - // Dump second recording - Path secondDump = Paths.get("liveness-test-second.jfr"); - try { - dump(secondDump); - assertTrue(Files.exists(secondDump), "Second JFR dump should be created"); - - // Parse second recording and verify reduced liveness samples - IItemCollection secondRecording = JfrLoaderToolkit.loadEvents(Files.newInputStream(secondDump)); - IItemCollection secondLiveObjects = secondRecording.apply( - ItemFilters.type("datadog.HeapLiveObject")); - - assertTrue(secondLiveObjects.hasItems(), "Second recording should contain live object samples"); - long secondSampleCount = secondLiveObjects.getAggregate(Aggregators.count()).longValue(); - - // Verify all live object samples have stack traces with at least one frame - verifyStackTracesPresent(secondLiveObjects); - - System.out.println("Second dump: " + secondSampleCount + " liveness samples"); - - System.out.printf("Sample comparison: first=%d, second=%d%n", - firstSampleCount, secondSampleCount); - - // The key validation is that the liveness tracking system is working: - // - Both dumps have liveness samples with valid stack traces - // - The double-buffering and trace preservation system is functional - // - We successfully completed two JFR dump cycles with liveness data - - // Note: In a real application, the second dump would typically have fewer samples - // after GC, but in test conditions with forced allocation and timing variations, - // the exact sample counts can vary. The important thing is that the system works. - - System.out.println("✅ Liveness tracking system validation completed successfully:"); - - } finally { - Files.deleteIfExists(secondDump); - } - } finally { - Files.deleteIfExists(firstDump); - } - } - - /** - * Verify that liveness samples have valid stack traces with at least one frame - * Allow some tolerance for profiling timing issues - */ - private void verifyStackTracesPresent(IItemCollection liveObjects) { - AtomicInteger samplesWithoutStackTrace = new AtomicInteger(0); - AtomicInteger samplesWithEmptyStackTrace = new AtomicInteger(0); - AtomicInteger totalSamples = new AtomicInteger(0); - - for (IItemIterable iterable : liveObjects) { - for (IItem item : iterable) { - totalSamples.incrementAndGet(); - - IMCStackTrace stackTrace = STACK_TRACE.getAccessor(iterable.getType()).getMember(item); - if (stackTrace == null) { - samplesWithoutStackTrace.incrementAndGet(); - } else if (stackTrace.getFrames().isEmpty()) { - samplesWithEmptyStackTrace.incrementAndGet(); - } - } - } - - // Allow some tolerance for profiling timing issues - most samples should have stack traces - int samplesWithIssues = samplesWithoutStackTrace.get() + samplesWithEmptyStackTrace.get(); - int total = totalSamples.get(); - double validPercentage = total > 0 ? (double)(total - samplesWithIssues) / total : 0; - - assertTrue(validPercentage >= 0.7, - String.format("At least 70%% of liveness samples must have valid stack traces, but only %.1f%% do " + - "(total: %d, missing: %d, empty: %d)", - validPercentage * 100, total, samplesWithoutStackTrace.get(), samplesWithEmptyStackTrace.get())); - - System.out.printf("✅ Stack trace validation passed: %d total samples, %.1f%% with valid stack traces%n", - total, validPercentage * 100); - } - - /** - * Generate allocation workload to trigger liveness sampling - */ - private void runAllocatingWorkload(List targets) { - for (AllocatingTarget target : targets) { - if (target != null) { - target.allocateMemory(); - } - } - } - - /** - * Target class for allocation that will be tracked by liveness profiling - */ - public static class AllocatingTarget { - private final int id; - private volatile List allocations = new ArrayList<>(); - - public AllocatingTarget(int id) { - this.id = id; - } - - public void allocateMemory() { - ThreadLocalRandom random = ThreadLocalRandom.current(); - - // Allocate various sizes to create diverse call traces - for (int i = 0; i < 10; i++) { - allocateByteArray(random.nextInt(1024, 4096)); - allocateIntArray(random.nextInt(256, 1024)); - allocateObjectArray(random.nextInt(16, 64)); - } - } - - private void allocateByteArray(int size) { - byte[] array = new byte[size]; - addTrackedAllocation(array); - } - - private void allocateIntArray(int size) { - int[] array = new int[size]; - // Simulate some work with the array - for (int i = 0; i < Math.min(size, 10); i++) { - array[i] = i * id; - } - addTrackedAllocation(array); - } - - private void allocateObjectArray(int size) { - Object[] array = new Object[size]; - // Fill with some objects - for (int i = 0; i < Math.min(size, 5); i++) { - array[i] = id + i; - } - addTrackedAllocation(array); - } - - private void addTrackedAllocation(Object allocation) { - allocations.add(allocation); - // Keep only recent allocations to control memory usage - if (allocations.size() > 100) { - allocations.subList(0, allocations.size() - 50).clear(); - } - } - - @Override - public String toString() { - return "AllocatingTarget{id=" + id + "}"; - } - } -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/MemleakProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/MemleakProfilerTest.java deleted file mode 100644 index adbca5ab6..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/MemleakProfilerTest.java +++ /dev/null @@ -1,105 +0,0 @@ -package com.datadoghq.profiler.memleak; - -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Disabled; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.Aggregators; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.ItemFilters; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicLong; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.Assumptions; - -public class MemleakProfilerTest extends AbstractProfilerTest { - @Override - protected String getProfilerCommand() { - return "memory=524288:L:0.5"; - } - - @Override - protected boolean isPlatformSupported() { - return !(Platform.isJavaVersion(8) || Platform.isJ9() || Platform.isZing()); - } - - @RetryingTest(5) - public void shouldGetLiveObjectSamples() throws InterruptedException { - MemLeakTarget target1 = new MemLeakTarget(); - MemLeakTarget target2 = new MemLeakTarget(); - runTests(target1, target2); - IItemCollection allocations = verifyEvents("datadog.HeapLiveObject"); - IItemCollection heapUsage = verifyEvents("datadog.HeapUsage"); -// assertAllocations(allocations, int[].class, target1, target2); -// assertAllocations(allocations, Integer[].class, target1, target2); - } - - private static void assertAllocations(IItemCollection allocations, Class clazz, MemLeakTarget... targets) { - long allocated = 0; - for (MemLeakTarget target : targets) { - allocated += target.getAllocated(clazz); - } - IItemCollection allocationsByType = allocations.apply(allocatedTypeFilter(clazz.getCanonicalName())); - assertTrue(allocationsByType.hasItems()); - long recorded = allocationsByType.getAggregate(Aggregators.sum(SCALED_SIZE)).longValue(); - long absoluteError = Math.abs(recorded - allocated); - assertTrue(absoluteError < allocated / 10, - String.format("allocation samples should be within 10pct tolerance of allocated memory (recorded %d, allocated %d)", - recorded, allocated)); - } - - public static class MemLeakTarget extends ClassValue implements Runnable { - private static byte[] leeway = new byte[32 * 1024 * 1024]; // 32MB to release on OOME - public static volatile List sink = new ArrayList<>(); - - @Override - public void run() { - ThreadLocalRandom random = ThreadLocalRandom.current(); - try { - for (int i = 0; i < 100_000; i++) { - allocate(random, random.nextInt(256)); - } - } catch (OutOfMemoryError e) { - leeway = null; - System.gc(); - } finally { - sink.clear(); - } - } - - long getAllocated(Class clazz) { - return get(clazz).get(); - } - - private static void allocate(ThreadLocalRandom random, int depth) { - if (depth > 0) { - allocate(random, depth - 1); - return; - } - - Object obj; - if (random.nextBoolean()) { - obj = new int[random.nextInt(64, 192) * 1000]; - } else { - obj = new Integer[random.nextInt(64, 192) * 1000]; - } - - if (random.nextInt(100) == 0 && sink.size() < 100_000) { - sink.add(obj); - } - if (random.nextInt(10000) == 0) { - System.gc(); - } - } - - @Override - protected AtomicLong computeValue(Class type) { - return new AtomicLong(); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java deleted file mode 100644 index 2201f7cac..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/memleak/WriteStackTracesAfterClassUnloadTest.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.memleak; - -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.Timeout; - -import java.io.IOException; -import java.lang.ref.WeakReference; -import java.lang.reflect.Method; -import java.nio.file.Files; -import java.nio.file.Path; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -/** - * Regression test for a SIGSEGV in {@code Profiler::processCallTraces} during - * {@code Recording::writeStackTraces} triggered by class unload. - * - *

Bug: {@code MethodInfo::_line_number_table->_ptr} held a raw pointer to JVMTI-allocated - * memory returned by {@code GetLineNumberTable}. When the underlying class was unloaded, the JVM - * freed that memory, leaving {@code _ptr} dangling. The crash manifested when - * {@code MethodInfo::getLineNumber(bci)} dereferenced the freed memory while - * {@code writeStackTraces} iterated traces that still referenced the now-unloaded method. - * - *

Test scenario (timing-sensitive — catches the bug aggressively under ASan): - *

    - *
  1. Aggressively profile a dynamically-loaded class so its methods land in many CPU samples. - * Each sample creates an entry in {@code call_trace_storage} and a {@code MethodInfo} entry - * in {@code _method_map} that holds a {@code shared_ptr}.
  2. - *
  3. Drop all strong references to the class and its loader; encourage unloading via - * {@code System.gc()} until the {@code WeakReference} is cleared.
  4. - *
  5. Immediately dump — within the same chunk if possible — so the freshly-unloaded method's - * traces are still in {@code call_trace_storage} when {@code writeStackTraces} runs.
  6. - *
  7. The lambda inside {@code processCallTraces} resolves the unloaded method via - * {@code Lookup::resolveMethod} and calls {@code MethodInfo::getLineNumber(bci)} on a - * {@code MethodInfo} whose {@code _line_number_table->_ptr} was freed by the JVM → - * SIGSEGV without the fix.
  8. - *
- * - *

With the fix, {@code SharedLineNumberTable} owns a malloc'd copy of the entries - * (the JVMTI allocation is deallocated immediately at capture time inside - * {@code Lookup::fillJavaMethodInfo}), so {@code _ptr} stays valid regardless of class unload. - * - *

Limitations: - *

    - *
  • Not authored "blind" — the implementation existed before the test was written.
  • - *
  • Without ASan/UBSan, the freed JVMTI region may still hold plausible bytes at read time - * and the test will report green even on a buggy binary. ASan/UBSan builds are the - * authoritative signal here.
  • - *
  • Class unload is JVM-discretionary; if the {@code WeakReference} doesn't clear within - * the deadline the test {@code assumeTrue}-skips rather than passing spuriously.
  • - *
  • {@code mcleanup=false} is set deliberately so a SIGSEGV here exercises the - * {@code getLineNumber} read path, not the {@code ~SharedLineNumberTable} cleanup path - * covered by {@code CleanupAfterClassUnloadTest}.
  • - *
- */ -public class WriteStackTracesAfterClassUnloadTest extends AbstractDynamicClassTest { - - @Override - protected String getProfilerCommand() { - // Aggressive CPU sampling to ensure the dynamic class lands in many traces. - return "cpu=1ms"; - } - - @Test - @Timeout(60) - public void testNoSigsegvInWriteStackTracesAfterClassUnload() throws Exception { - stopProfiler(); - - Path baseFile = tempFile("class-unload-sigsegv-base"); - Path dumpFile = tempFile("class-unload-sigsegv-dump"); - - try { - // mcleanup=false isolates the test to the writeStackTraces read path; the - // SharedLineNumberTable destructor path is covered by CleanupAfterClassUnloadTest. - profiler.execute("start,cpu=1ms,jfr,mcleanup=false,file=" + baseFile.toAbsolutePath()); - try { - Thread.sleep(200); // let the profiler stabilize - - // 1. Load a class, hammer its methods on this thread so the CPU profiler captures - // many traces referencing it. Drop strong refs and return only a WeakReference. - WeakReference loaderRef = loadAndProfileDynamicClass(); - - // 2. Drop refs and GC until the class actually unloads. - long deadline = System.currentTimeMillis() + 8_000; - while (loaderRef.get() != null && System.currentTimeMillis() < deadline) { - System.gc(); - Thread.sleep(20); - } - - // Skip the test (rather than pass spuriously) if the JVM declined to unload. - // Without unload the JVMTI line-number-table memory is never freed and the bug - // cannot manifest — running the dumps would prove nothing. - assumeTrue(loaderRef.get() == null, - "JVM did not unload the dynamic class within the deadline; cannot exercise the use-after-free scenario"); - - // 3. Immediately dump several times. The first dump runs writeStackTraces over - // the trace_buffer that still contains the unloaded method's traces; subsequent - // dumps cover the rotation tail. With the bug, getLineNumber dereferences the - // freed JVMTI memory → SIGSEGV. With the fix, _ptr is a malloc'd copy → safe. - for (int i = 0; i < 4; i++) { - profiler.dump(dumpFile); - Thread.sleep(10); - } - - // 4. If the profiler crashed inside processCallTraces the JVM would have died and we - // would never reach this line. The non-empty-output assertion is secondary — the - // primary signal is reaching profiler.stop() at all. - assertTrue(Files.size(dumpFile) > 0, - "Profiler produced no output — SIGSEGV during writeStackTraces is suspected"); - } finally { - try { - profiler.stop(); - } finally { - profiler.resetThreadContext(); - } - } - - } finally { - try { Files.deleteIfExists(baseFile); } catch (IOException ignored) {} - try { Files.deleteIfExists(dumpFile); } catch (IOException ignored) {} - } - } - - private WeakReference loadAndProfileDynamicClass() throws Exception { - String className = "com/datadoghq/profiler/generated/DynamicUnloadClass" + (classCounter++); - byte[] bytecode = generateClassBytecode(className, 5); - - IsolatedClassLoader loader = new IsolatedClassLoader(); - Class clazz = loader.defineClass(className.replace('/', '.'), bytecode); - - // Hammer methods for ~300ms so the CPU profiler at 1ms captures plenty of samples - // referencing this class. The longer this runs, the more traces survive into the - // dump-after-unload window. - long endTime = System.currentTimeMillis() + 300; - Object instance = clazz.getDeclaredConstructor().newInstance(); - Method[] methods = clazz.getDeclaredMethods(); - while (System.currentTimeMillis() < endTime) { - for (Method m : methods) { - if (m.getParameterCount() == 0 && m.getReturnType() == int.class) { - m.invoke(instance); - } - } - } - - WeakReference ref = new WeakReference<>(loader); - //noinspection UnusedAssignment - loader = null; - //noinspection UnusedAssignment - clazz = null; - //noinspection UnusedAssignment - instance = null; - return ref; - } - -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/BoundMethodHandleProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/BoundMethodHandleProfilerTest.java deleted file mode 100644 index 8cb8f6186..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/BoundMethodHandleProfilerTest.java +++ /dev/null @@ -1,61 +0,0 @@ -package com.datadoghq.profiler.metadata; - -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Test; - -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.util.Map; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeFalse; - -public class BoundMethodHandleProfilerTest extends AbstractProfilerTest { - @Override - protected String getProfilerCommand() { - return Platform.isJ9() ? "wall=100ms" : "wall=100us"; - } - - @Test - public void test() throws Throwable { - assumeFalse(Platform.isJ9() && isAsan()); // running this test on j9 and asan is weirdly crashy - assumeFalse(Platform.isJ9() && Platform.isJavaVersion(8)); // geting crash-failur in CI reliable; remove once that is fixed - assumeFalse(Platform.isJ9() && Platform.isJavaVersion(17)); // JVMTI::GetClassSignature() is reliably crashing on a valid 'class' instance - assumeFalse(Platform.isAarch64() && Platform.isMusl() && !Platform.isJavaVersionAtLeast(11)); // aarch64 + musl + jdk 8 will crash very often - registerCurrentThreadForWallClockProfiling(); - // Reduce workload on aarch64+asan: ASAN slows each invocation enough that the test - // takes 3+ minutes, generating a 56MB JFR that OOMs the 512MB test-runner heap. - int numBoundMethodHandles = isAsan() && Platform.isAarch64() ? 1_000 : 10_000; - int x = generateBoundMethodHandles(numBoundMethodHandles); - assertTrue(x != 0); - stopProfiler(); - verifyEvents("datadog.MethodSample"); - Map counters = profiler.getDebugCounters(); - assertFalse(counters.isEmpty(), "profiler debug counters must not be empty after BoundMethodHandle workload"); - } - - - - public static String append(String string, int suffix) { - return string + suffix; - } - - public static int generateBoundMethodHandles(int howMany) throws Throwable { - int total = 0; - MethodHandle append = MethodHandles.lookup() - .findStatic(BoundMethodHandleProfilerTest.class, - "append", - MethodType.methodType(String.class, String.class, int.class)); - for (int i = 0; i < howMany; i++) { - // binding many constants amplifies the effect of class generation below - MethodHandle bound = append.bindTo("string" + i); - for (int j = 0; j < 1024; j++) { - // many invocations has the effect of generate a new class - total += ((String) bound.invokeExact(j)).length(); - } - } - return total; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/DictionaryRotationTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/DictionaryRotationTest.java deleted file mode 100644 index 6b8c40415..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/DictionaryRotationTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2025 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.metadata; - -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.*; -import static org.openjdk.jmc.common.item.Attribute.attr; -import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT; - -/** - * Verifies that the dictionary rotate+clearStandby cycle correctly: - * - Exposes only pre-dump entries in the dump snapshot. - * - Recalibrates the live counter to reflect the active buffer after clearStandby(). - * - Accumulates post-dump entries in the new active buffer. - */ -public class DictionaryRotationTest extends AbstractProfilerTest { - - private static final IAttribute ENDPOINT_ATTR = - attr("endpoint", "endpoint", "endpoint", PLAIN_TEXT); - - @Test - public void dumpCycleSeparatesPreAndPostDumpEntries() throws Exception { - String[] preDump = { "ep_pre_0", "ep_pre_1", "ep_pre_2" }; - String[] postDump = { "ep_post_0", "ep_post_1" }; - int sizeLimit = 100; - - for (int i = 0; i < preDump.length; i++) { - profiler.recordTraceRoot(i, preDump[i], null, sizeLimit); - } - - // dump() triggers: lockAll() → rotate() → jfr.dump(snapshot) → unlockAll() - // → clearStandby() (resets per-dump counters to 0, frees scratch buffer) - Path snapshot = Files.createTempFile("DictionaryRotation_snapshot_", ".jfr"); - try { - dump(snapshot); - - // Counter reset to 0 by clearStandby() — tracks only post-clearStandby inserts - Map afterDump = profiler.getDebugCounters(); - assertEquals(0L, afterDump.getOrDefault("dictionary_endpoints_keys", -1L), - "dictionary_endpoints_keys must be 0 after clearStandby"); - - for (int i = 0; i < postDump.length; i++) { - profiler.recordTraceRoot(preDump.length + i, postDump[i], null, sizeLimit); - } - - // Live counter reflects only post-dump insertions - Map live = profiler.getDebugCounters(); - assertEquals((long) postDump.length, live.get("dictionary_endpoints_keys"), - "Live counter must equal number of post-dump endpoints"); - - stopProfiler(); - - // Snapshot contains pre-dump endpoints only - Set inSnapshot = endpointNames(verifyEvents(snapshot, "datadog.Endpoint", true)); - for (String ep : preDump) { - assertTrue(inSnapshot.contains(ep), "Snapshot must contain pre-dump endpoint: " + ep); - } - for (String ep : postDump) { - assertFalse(inSnapshot.contains(ep), "Snapshot must NOT contain post-dump endpoint: " + ep); - } - - // Main recording contains post-dump endpoints only - Set inRecording = endpointNames(verifyEvents("datadog.Endpoint")); - for (String ep : postDump) { - assertTrue(inRecording.contains(ep), "Recording must contain post-dump endpoint: " + ep); - } - for (String ep : preDump) { - assertFalse(inRecording.contains(ep), "Recording must NOT contain pre-dump endpoint: " + ep); - } - } finally { - Files.deleteIfExists(snapshot); - } - } - - @Override - protected String getProfilerCommand() { - return "wall=~1ms"; - } - - private static Set endpointNames(IItemCollection events) { - Set names = new HashSet<>(); - for (IItemIterable it : events) { - IMemberAccessor accessor = ENDPOINT_ATTR.getAccessor(it.getType()); - if (accessor == null) continue; - for (IItem item : it) { - String v = accessor.getMember(item); - if (v != null) names.add(v); - } - } - return names; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/EmptyConstantPoolTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/EmptyConstantPoolTest.java deleted file mode 100644 index 3df18f0c2..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/EmptyConstantPoolTest.java +++ /dev/null @@ -1,367 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.metadata; - -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Regression test for PROF-15130: the JFR checkpoint must not emit any constant-pool section with - * zero elements. OpenJDK {@code jfr print} rejects such a chunk with - * {@code "Pool X must contain at least one element"}. - * - *

The fix: each variable pool writer ({@code writeThreads}, {@code writeStackTraces}, etc.) - * skips emission when its pool is empty and returns 0; {@code writeCpool} sums the actually-emitted - * pool count and back-patches it into the checkpoint header after flushing. - * - *

Two scenarios are tested: - *

    - *
  1. An early dump (before any profiling samples are collected) — maximises the number of - * empty variable pools (stacktraces, methods, classes, etc.).
  2. - *
  3. A dump after a brief profiling window — exercises the back-patched count with a mix of - * emitted and skipped variable pools.
  4. - *
- */ -public class EmptyConstantPoolTest extends AbstractProfilerTest { - - @Test - public void noEmptyPoolsInEarlyDump() throws Exception { - // dump immediately, before any samples can be collected - Path rec = Files.createTempFile("empty-cpool-early-", ".jfr"); - try { - dump(rec); - stopProfiler(); - assertNoEmptyPoolsAndCountConsistent(rec); - } finally { - Files.deleteIfExists(rec); - } - } - - @Test - public void noEmptyPoolsAfterBriefProfiling() throws Exception { - waitForProfilerReady(2000); - // a short busy-loop so at least some stacks are collected - long acc = 0; - long deadline = System.currentTimeMillis() + 300; - while (System.currentTimeMillis() < deadline) { - acc ^= deadline; - } - // consume acc to prevent DCE - if (acc == Long.MIN_VALUE) throw new IllegalStateException(); - - Path rec = Files.createTempFile("empty-cpool-active-", ".jfr"); - try { - dump(rec); - stopProfiler(); - assertNoEmptyPoolsAndCountConsistent(rec); - } finally { - Files.deleteIfExists(rec); - } - } - - @Override - protected String getProfilerCommand() { - return "cpu=1ms,wall=~1ms"; - } - - // ── oracle ───────────────────────────────────────────────────────────────── - - /** - * Walks every checkpoint in every chunk of the JFR file and asserts: - *
    - *
  1. Every declared pool section has element count {@code > 0}.
  2. - *
  3. The declared pool count in the checkpoint header equals the number of - * sections we can actually walk (i.e. the back-patch is consistent).
  4. - *
- */ - static void assertNoEmptyPoolsAndCountConsistent(Path file) throws IOException { - byte[] all = Files.readAllBytes(file); - long pos = 0; - int chunksChecked = 0; - while (pos + 8 <= all.length) { - if (!(all[(int) pos] == 'F' && all[(int) pos + 1] == 'L' - && all[(int) pos + 2] == 'R' && all[(int) pos + 3] == 0)) { - break; - } - new ChunkChecker(all, (int) pos).assertPoolsValid(); - long chunkSize = beLong(all, (int) pos + 8); - assertTrue(chunkSize > 0, "chunk size must be positive"); - pos += chunkSize; - chunksChecked++; - } - assertTrue(chunksChecked > 0, "no JFR chunks found in recording"); - } - - // ── JFR chunk parser ─────────────────────────────────────────────────────── - - private static final class ChunkChecker { - private final byte[] f; - private final int base; - private final long chunkSize; - private final long cpOffset; - private final long metaOffset; - - // class id -> field layout (parsed from metadata) - private final Map classes = new HashMap<>(); - - ChunkChecker(byte[] f, int base) { - this.f = f; - this.base = base; - this.chunkSize = beLong(f, base + 8); - this.cpOffset = beLong(f, base + 16); - this.metaOffset = beLong(f, base + 24); - parseMetadata(); - } - - void assertPoolsValid() { - Set visited = new HashSet<>(); - long off = base + cpOffset; - while (off >= base && off < base + chunkSize && !visited.contains(off)) { - visited.add(off); - long[] p = {off}; - readVarLong(p); // size - readVarLong(p); // typeId (checkpoint event) - readVarLong(p); // start - readVarLong(p); // duration - long deltaRaw = readVarLong(p); - long delta = (deltaRaw >>> 1) ^ -(deltaRaw & 1); // zig-zag decode - p[0] += 1; // flush byte - long declaredPoolCount = readVarLong(p); - - boolean fullWalk = true; - for (long i = 0; i < declaredPoolCount; i++) { - long classId = readVarLong(p); - long count = readVarLong(p); - assertTrue(count > 0, - "constant pool section for classId=" + classId - + " has zero elements (empty pool was not skipped)"); - ClassDef cd = classes.get(classId); - if (cd == null) { - // Unknown layout — cannot safely skip entries; stop walking. - // We still checked count > 0 for this section. - fullWalk = false; - break; - } - for (long e = 0; e < count; e++) { - readVarLong(p); // entry id - for (FieldDef fd : cd.fields) { - skipField(p, fd); - } - } - } - // Only assert count consistency when we could walk all sections; - // an unknown class layout breaks the walk early but is not a test failure. - if (fullWalk) { - assertTrue(declaredPoolCount >= 5, - "pool count " + declaredPoolCount - + " is less than 5 (frameTypes + threadStates + " - + "executionModes + logLevels + threads are always non-empty)"); - } - - if (delta == 0) break; - off += delta; - } - } - - // ── metadata parsing ────────────────────────────────────────────────── - - private void parseMetadata() { - long[] p = {base + metaOffset}; - readVarLong(p); // size - readVarLong(p); // typeId - readVarLong(p); // start - readVarLong(p); // duration - readVarLong(p); // metadataId - long stringCount = readVarLong(p); - String[] pool = new String[(int) stringCount]; - for (int i = 0; i < stringCount; i++) { - pool[i] = readString(p); - } - collectClasses(readElement(p, pool)); - } - - private void collectClasses(Element el) { - if ("class".equals(el.name)) { - String idStr = el.attrs.get("id"); - if (idStr != null) { - ClassDef cd = new ClassDef(); - cd.id = Long.parseLong(idStr); - cd.name = el.attrs.getOrDefault("name", ""); - for (Element child : el.children) { - if ("field".equals(child.name)) { - FieldDef fd = new FieldDef(); - fd.name = child.attrs.get("name"); - fd.typeId = Long.parseLong(child.attrs.get("class")); - fd.constantPool = "true".equals(child.attrs.get("constantPool")); - String dim = child.attrs.get("dimension"); - fd.dimension = dim != null ? Integer.parseInt(dim) : 0; - cd.fields.add(fd); - } - } - classes.put(cd.id, cd); - } - } - for (Element child : el.children) { - collectClasses(child); - } - } - - // ── field-skipping ──────────────────────────────────────────────────── - - private void skipField(long[] p, FieldDef fd) { - if (fd.dimension == 1) { - long n = readVarLong(p); - for (long i = 0; i < n; i++) { - skipScalar(p, fd); - } - return; - } - skipScalar(p, fd); - } - - private void skipScalar(long[] p, FieldDef fd) { - if (fd.constantPool) { - readVarLong(p); - return; - } - ClassDef t = classes.get(fd.typeId); - String tn = t != null ? t.name : ""; - switch (tn) { - case "java.lang.String": readString(p); break; - case "boolean": - case "byte": p[0] += 1; break; - case "short": - case "char": - case "int": - case "long": readVarLong(p); break; - case "float": p[0] += 4; break; - case "double": p[0] += 8; break; - default: - if (t != null) { - for (FieldDef sub : t.fields) { - skipField(p, sub); - } - } - break; - } - } - - // ── low-level readers ───────────────────────────────────────────────── - - private Element readElement(long[] p, String[] pool) { - Element el = new Element(); - el.name = pool[(int) readVarLong(p)]; - long attrCount = readVarLong(p); - for (long i = 0; i < attrCount; i++) { - String k = pool[(int) readVarLong(p)]; - String v = pool[(int) readVarLong(p)]; - el.attrs.put(k, v); - } - long childCount = readVarLong(p); - for (long i = 0; i < childCount; i++) { - el.children.add(readElement(p, pool)); - } - return el; - } - - private long readVarLong(long[] p) { - long result = 0; - int shift = 0; - int i = (int) p[0]; - for (int b = 0; b < 8; b++) { - int by = f[i++] & 0xff; - result |= ((long) (by & 0x7f)) << shift; - if ((by & 0x80) == 0) { p[0] = i; return result; } - shift += 7; - } - result |= ((long) (f[i++] & 0xff)) << shift; - p[0] = i; - return result; - } - - private String readString(long[] p) { - int i = (int) p[0]; - int enc = f[i++] & 0xff; - p[0] = i; - switch (enc) { - case 0: return null; - case 1: return ""; - case 3: { - long len = readVarLong(p); - String s = new String(f, (int) p[0], (int) len, - java.nio.charset.StandardCharsets.UTF_8); - p[0] += len; - return s; - } - case 4: { - long len = readVarLong(p); - StringBuilder sb = new StringBuilder((int) len); - for (long c = 0; c < len; c++) sb.append((char) readVarLong(p)); - return sb.toString(); - } - case 5: { - long len = readVarLong(p); - String s = new String(f, (int) p[0], (int) len, - java.nio.charset.StandardCharsets.ISO_8859_1); - p[0] += len; - return s; - } - default: - throw new IllegalStateException("unknown JFR string encoding " + enc); - } - } - } - - // ── helpers ──────────────────────────────────────────────────────────────── - - private static long beLong(byte[] f, int off) { - long v = 0; - for (int i = 0; i < 8; i++) v = (v << 8) | (f[off + i] & 0xffL); - return v; - } - - private static final class ClassDef { - long id; - String name; - final List fields = new ArrayList<>(); - } - - private static final class FieldDef { - String name; - long typeId; - boolean constantPool; - int dimension; - } - - private static final class Element { - String name; - final Map attrs = new HashMap<>(); - final List children = new ArrayList<>(); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MetadataNormalisationTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MetadataNormalisationTest.java deleted file mode 100644 index 44ac524eb..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MetadataNormalisationTest.java +++ /dev/null @@ -1,68 +0,0 @@ -package com.datadoghq.profiler.metadata; - -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.lang.reflect.Constructor; -import java.lang.reflect.Method; -import java.util.ArrayList; -import java.util.LinkedList; -import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Stream; - -import static org.junit.jupiter.api.Assertions.assertFalse; - -public class MetadataNormalisationTest extends AbstractProfilerTest { - - @Test - public void test() throws Exception { - Constructor arrayListConstructor = ArrayList.class.getConstructor(); - Method arrayListAdd = ArrayList.class.getDeclaredMethod("add", Object.class); - Constructor linkedListConstructor = LinkedList.class.getConstructor(); - Method linkedListAdd = LinkedList.class.getDeclaredMethod("add", Object.class); - // need to invoke enough times to result in generation of accessors and to record some cpu time - int count = 0; - for (int i = 0; i < 100_000; i++) { - Object list = arrayListConstructor.newInstance(); - arrayListAdd.invoke(list, "element"); - count += ((List) list).size(); - } - for (int i = 0; i < 100_000; i++) { - Object list = linkedListConstructor.newInstance(); - linkedListAdd.invoke(list, "element"); - count += ((List) list).size(); - } - System.out.println(count); - stopProfiler(); - IItemCollection executionSamples = verifyEvents("datadog.ExecutionSample"); - Matcher[] forbiddenPatternMatchers = Stream.of( - "MH.*0x[A-Fa-f0-9]{3}", // method handles - "GeneratedConstructorAccessor\\d+", - "GeneratedMethodAccessor\\d+" - ) - .map(regex -> Pattern.compile(regex).matcher("")) - .toArray(Matcher[]::new); - for (IItemIterable samples : executionSamples) { - IMemberAccessor stacktraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(samples.getType()); - for (IItem item : samples) { - String stacktrace = stacktraceAccessor.getMember(item); - for (Matcher matcher : forbiddenPatternMatchers) { - matcher.reset(stacktrace); - assertFalse(matcher.find(), () -> matcher.pattern() + "\n" + stacktrace); - } - } - } - } - - @Override - protected String getProfilerCommand() { - return "cpu=100us"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MethodIdReuseTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MethodIdReuseTest.java deleted file mode 100644 index 146d28c71..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/metadata/MethodIdReuseTest.java +++ /dev/null @@ -1,618 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.metadata; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; -import org.objectweb.asm.ClassWriter; -import org.objectweb.asm.Label; -import org.objectweb.asm.MethodVisitor; -import org.objectweb.asm.Opcodes; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.nio.channels.FileChannel; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.StandardOpenOption; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -/** - * Regression test for PROF-15130: duplicate {@code jdk.types.Method} constant-pool ids. - * - *

{@code resolveMethod()} used to assign {@code mi->_key = _method_map->size() + 1}, but - * {@code cleanupUnreferencedMethods()} (enabled by default, erases methods unreferenced for - * {@code AGE_THRESHOLD = 3} chunks) shrinks the map. After an erase, {@code size()+1} reissues an - * id still owned by a surviving method, so two live methods share one id in a single chunk's - * method constant pool. First-wins strict parsers (JMC / jafar / the Datadog backend) then render - * a phantom (wrong-but-valid) method, while last-wins OpenJDK {@code jfr} renders correctly. JMC's - * loader is last-wins and therefore CANNOT see the duplicate — so this test parses the raw chunk - * constant pools itself, which is the precise oracle. - * - *

The fix replaced {@code size()+1} with a free-list id allocator - * ({@code MethodMap::allocId()/freeId()}): an id is recycled only after the owning method is - * erased, so no two live methods ever share an id. - * - *

Deterministic erase+reuse driver: each {@code dump()} produces a chunk and runs - * {@code cleanupUnreferencedMethods}. Phase 1 touches a LARGE set of distinct lambda call targets - * (high ids). Phases 2..N (≥ AGE_THRESHOLD dumps) stop touching the phase-1 lambdas (so they age - * out and get erased) while touching a DIFFERENT persistent set plus brand-new lambdas (which draw - * recycled ids). The oracle asserts no {@code jdk.types.Method} id maps to two distinct method - * definitions within any chunk. - */ -public class MethodIdReuseTest extends AbstractProfilerTest { - - // Phases beyond AGE_THRESHOLD (3) so phase-1 methods are definitely erased. - private static final int PHASE1_DUMPS = 3; - private static final int PHASE2_DUMPS = 7; // well beyond AGE_THRESHOLD; many erase+reuse chunks - private static final int DISTINCT_TARGETS_PER_PHASE = 600; - - // Generated distinct-class call targets (one JVM method each). A large, churned set maximises - // method-pool turnover so the free-list / size()+1 schemes diverge. - private final List phase1Targets = new ArrayList<>(); - private final List persistentTargets = new ArrayList<>(); - - // Every temporary JFR dump produced by the test; deleted in after() unless ddprof_test.keep_jfrs. - private final List dumps = new ArrayList<>(); - - // Distinct generated Runnable classes are loaded here; each carries its busy loop in its own - // run() method, so every target is a distinct JVM method (and thus a distinct method-pool id). - private final GeneratingClassLoader targetLoader = new GeneratingClassLoader(); - private int targetCounter = 0; - - // Written by the generated run() methods; public so the generated classes (loaded by a child - // class loader, in a different runtime package) can reach it via GETSTATIC/PUTSTATIC. - public static volatile long sink; - - /** - * Generates and instantiates a fresh class implementing {@link Runnable} whose {@code run()} - * body spins on a CPU-busy loop. Because each call produces a distinct class, every target is a - * distinct JVM method — unlike a shared lambda call site, which HotSpot compiles to a single - * synthetic method (and thus a single method-pool id). Distinct methods are what build the large - * high-id population PROF-15130 needs to age out and recycle ids. - */ - private Runnable makeBusyTarget(final int seed) { - String internalName = "com/datadoghq/profiler/metadata/gen/BusyTarget_" + (targetCounter++); - byte[] bytecode = generateBusyRunnable(internalName, seed); - try { - Class clazz = targetLoader.define(internalName.replace('/', '.'), bytecode); - return (Runnable) clazz.getDeclaredConstructor().newInstance(); - } catch (ReflectiveOperationException e) { - throw new IllegalStateException("failed to generate busy target for seed " + seed, e); - } - } - - private void buildTargets() { - for (int i = 0; i < DISTINCT_TARGETS_PER_PHASE; i++) { - phase1Targets.add(makeBusyTarget(1_000 + i)); - persistentTargets.add(makeBusyTarget(9_000_000 + i)); - } - } - - @Override - protected void after() throws Exception { - if (!Boolean.getBoolean("ddprof_test.keep_jfrs")) { - for (Path dump : dumps) { - Files.deleteIfExists(dump); - } - } - } - - /** - * Emits {@code public final class implements Runnable} whose {@code run()} runs - * the CPU-busy loop the test previously expressed as a lambda: - * {@code long acc = seed; for (int i = 0; i < 50000; i++) acc += (i ^ (acc >> 1)) + seed; sink += acc;} - */ - private static byte[] generateBusyRunnable(String internalName, long seed) { - ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES | ClassWriter.COMPUTE_MAXS); - cw.visit(Opcodes.V1_8, Opcodes.ACC_PUBLIC | Opcodes.ACC_FINAL | Opcodes.ACC_SUPER, - internalName, null, "java/lang/Object", new String[] {"java/lang/Runnable"}); - - MethodVisitor ctor = cw.visitMethod(Opcodes.ACC_PUBLIC, "", "()V", null, null); - ctor.visitCode(); - ctor.visitVarInsn(Opcodes.ALOAD, 0); - ctor.visitMethodInsn(Opcodes.INVOKESPECIAL, "java/lang/Object", "", "()V", false); - ctor.visitInsn(Opcodes.RETURN); - ctor.visitMaxs(0, 0); - ctor.visitEnd(); - - MethodVisitor mv = cw.visitMethod(Opcodes.ACC_PUBLIC, "run", "()V", null, null); - mv.visitCode(); - mv.visitLdcInsn(seed); // acc = seed - mv.visitVarInsn(Opcodes.LSTORE, 1); - mv.visitInsn(Opcodes.ICONST_0); // i = 0 - mv.visitVarInsn(Opcodes.ISTORE, 3); - Label cond = new Label(); - Label end = new Label(); - mv.visitLabel(cond); - mv.visitVarInsn(Opcodes.ILOAD, 3); - mv.visitLdcInsn(50_000); - mv.visitJumpInsn(Opcodes.IF_ICMPGE, end); - mv.visitVarInsn(Opcodes.LLOAD, 1); // acc - mv.visitVarInsn(Opcodes.ILOAD, 3); // i - mv.visitInsn(Opcodes.I2L); // (long) i - mv.visitVarInsn(Opcodes.LLOAD, 1); // acc - mv.visitInsn(Opcodes.ICONST_1); // shift count is an int for LSHR - mv.visitInsn(Opcodes.LSHR); // acc >> 1 - mv.visitInsn(Opcodes.LXOR); // (long) i ^ (acc >> 1) - mv.visitLdcInsn(seed); - mv.visitInsn(Opcodes.LADD); // + seed - mv.visitInsn(Opcodes.LADD); // acc + ... - mv.visitVarInsn(Opcodes.LSTORE, 1); // acc = - mv.visitIincInsn(3, 1); // i++ - mv.visitJumpInsn(Opcodes.GOTO, cond); - mv.visitLabel(end); - mv.visitFieldInsn(Opcodes.GETSTATIC, - "com/datadoghq/profiler/metadata/MethodIdReuseTest", "sink", "J"); - mv.visitVarInsn(Opcodes.LLOAD, 1); - mv.visitInsn(Opcodes.LADD); - mv.visitFieldInsn(Opcodes.PUTSTATIC, - "com/datadoghq/profiler/metadata/MethodIdReuseTest", "sink", "J"); - mv.visitInsn(Opcodes.RETURN); - mv.visitMaxs(0, 0); - mv.visitEnd(); - - cw.visitEnd(); - return cw.toByteArray(); - } - - /** Child loader that exposes defineClass so generated targets become distinct loaded classes. */ - private static final class GeneratingClassLoader extends ClassLoader { - Class define(String binaryName, byte[] bytecode) { - return defineClass(binaryName, bytecode, 0, bytecode.length); - } - } - - private void runFor(List targets, List extra, long millis) { - long deadline = System.currentTimeMillis() + millis; - int idx = 0; - while (System.currentTimeMillis() < deadline) { - targets.get(idx % targets.size()).run(); - if (extra != null && !extra.isEmpty()) { - extra.get(idx % extra.size()).run(); - } - idx++; - } - } - - @Test - public void methodPoolIdsAreUniquePerChunk() throws Exception { - Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); - Assumptions.assumeFalse(Platform.isJ9()); - - buildTargets(); - registerCurrentThreadForWallClockProfiling(); - waitForProfilerReady(2000); - - // Phase 1: touch the large phase-1 lambda set (+ persistent set) so they all get resolved - // and assigned high method-pool ids. - for (int d = 0; d < PHASE1_DUMPS; d++) { - runFor(phase1Targets, persistentTargets, 600); - Path rec = Files.createTempFile("prof15130-reuse-p1-" + d + "-", ".jfr"); - dumps.add(rec); - dump(rec); - System.out.println("[PROF-15130] phase1 dump #" + d + " -> " + rec.toAbsolutePath()); - } - - // Phase 2: STOP touching phase-1 lambdas (they age out and get erased after AGE_THRESHOLD - // dumps) while touching the persistent set + brand-new lambdas that draw recycled ids. - for (int d = 0; d < PHASE2_DUMPS; d++) { - List fresh = new ArrayList<>(); - for (int i = 0; i < DISTINCT_TARGETS_PER_PHASE; i++) { - fresh.add(makeBusyTarget(50_000_000 + d * 100_000 + i)); - } - runFor(persistentTargets, fresh, 600); - Path rec = Files.createTempFile("prof15130-reuse-p2-" + d + "-", ".jfr"); - dumps.add(rec); - dump(rec); - System.out.println("[PROF-15130] phase2 dump #" + d + " -> " + rec.toAbsolutePath()); - } - - stopProfiler(); - - System.out.println("[PROF-15130] produced dump files:"); - for (Path p : dumps) { - System.out.println("[PROF-15130] " + p.toAbsolutePath()); - } - - // Oracle: across every chunk in every dump, no method-pool id maps to two distinct method - // definitions. - int totalDuplicates = 0; - for (Path dump : dumps) { - int dups = countDuplicateMethodIds(dump); - System.out.println("[PROF-15130] " + dump.getFileName() + " duplicate method-pool ids: " + dups); - totalDuplicates += dups; - } - - assertEquals(0, totalDuplicates, - "Found " + totalDuplicates + " jdk.types.Method constant-pool id(s) mapping to two " - + "distinct method definitions (PROF-15130). See stdout for the dump files."); - } - - @Override - protected String getProfilerCommand() { - // wall+cpu maximise sample density so the churned lambdas land in stacktraces and thus the - // method pool. Method cleanup is left ENABLED (default) — that is what triggers the bug. - return "cpu=1ms,wall=~1ms"; - } - - // ───────────────────────── raw JFR method-pool oracle ───────────────────────── - // - // A minimal, dependency-free JFR reader that walks every chunk in the file, parses the - // metadata to learn the field layout of jdk.types.Method / jdk.types.Symbol / java.lang.Class, - // then reads each constant-pool checkpoint and detects any jdk.types.Method id that resolves to - // two DISTINCT method definitions within the same chunk. This is the precise duplicate-id - // oracle (JMC's last-wins loader would silently hide it). - - /** @return number of method-pool ids in the file that map to >1 distinct definition. */ - static int countDuplicateMethodIds(Path file) throws IOException { - byte[] all; - try (FileChannel ch = FileChannel.open(file, StandardOpenOption.READ)) { - long size = ch.size(); - ByteBuffer bb = ByteBuffer.allocate((int) size); - while (bb.hasRemaining() && ch.read(bb) > 0) { /* read fully */ } - all = bb.array(); - } - int duplicates = 0; - long pos = 0; - while (pos + 8 <= all.length) { - // Chunk magic "FLR\0" - if (!(all[(int) pos] == 'F' && all[(int) pos + 1] == 'L' && all[(int) pos + 2] == 'R' - && all[(int) pos + 3] == 0)) { - break; - } - Chunk chunk = new Chunk(all, (int) pos); - duplicates += chunk.countDuplicateMethodIds(); - if (chunk.chunkSize <= 0) { - break; - } - pos += chunk.chunkSize; - } - return duplicates; - } - - private static final class Chunk { - final byte[] f; - final int base; // chunk start offset within the file - final long chunkSize; - final long cpOffset; // checkpoint offset, relative to chunk start - final long metaOffset; // metadata offset, relative to chunk start - - // metadata: classId -> ClassDef - final Map classes = new HashMap<>(); - long methodTypeId = -1, symbolTypeId = -1, classTypeId = -1, stringTypeId = -1; - - Chunk(byte[] f, int base) { - this.f = f; - this.base = base; - // Header (big-endian): magic(4) major(2) minor(2) chunkSize(8) cpOffset(8) metaOffset(8) - this.chunkSize = beLong(base + 8); - this.cpOffset = beLong(base + 16); - this.metaOffset = beLong(base + 24); - parseMetadata(); - } - - // Resolution tables, unioned across all checkpoints in the chunk (last-wins is fine: a - // symbol/class id is stable within a chunk). Used only to render readable defs. - private final Map symbols = new HashMap<>(); - private final Map classNameRef = new HashMap<>(); // class id -> name symbol ref - // id -> set of DISTINCT raw [type,name,descriptor] ref-tuples seen for that method id. - // size() > 1 ⇒ the id carried two different method definitions in this chunk ⇒ the bug. - private final Map> methodRefTuples = new LinkedHashMap<>(); - - int countDuplicateMethodIds() { - // Follow the checkpoint delta chain. - Set visited = new HashSet<>(); - long off = base + cpOffset; - while (off >= base && off < base + chunkSize && !visited.contains(off)) { - visited.add(off); - long[] p = {off}; - readVarLong(p); // size - readVarLong(p); // typeId (EventCheckpoint) - readVarLong(p); // start - readVarLong(p); // duration - long deltaRaw = readVarLong(p); - long delta = (deltaRaw >>> 1) ^ -(deltaRaw & 1); // zig-zag - p[0] += 1; // flush byte - long poolCount = readVarLong(p); - for (long i = 0; i < poolCount; i++) { - long classId = readVarLong(p); - long count = readVarLong(p); - ClassDef cd = classes.get(classId); - if (cd == null) { - // Unknown layout — cannot safely parse further in this checkpoint. - return duplicateCount(); - } - boolean isMethod = classId == methodTypeId; - boolean isSymbol = classId == symbolTypeId; - boolean isClass = classId == classTypeId; - for (long e = 0; e < count; e++) { - long id = readVarLong(p); - if (isSymbol) { - String s = readString(p); - symbols.put(id, s); - } else if (isMethod) { - long typeRef = 0, nameRef = 0, descRef = 0; - for (FieldDef fd : cd.fields) { - Object v = readField(p, fd); - if (v instanceof Long) { - if ("type".equals(fd.name)) typeRef = (Long) v; - else if ("name".equals(fd.name)) nameRef = (Long) v; - else if ("descriptor".equals(fd.name)) descRef = (Long) v; - } - } - // Record the DISTINCT raw ref-tuple for this id. Two different tuples - // for one id within the chunk is exactly the PROF-15130 collision. - String tuple = typeRef + ":" + nameRef + ":" + descRef; - methodRefTuples.computeIfAbsent(id, k -> new HashSet<>()).add(tuple); - } else if (isClass) { - long nameRef = 0; - for (FieldDef fd : cd.fields) { - Object v = readField(p, fd); - if ("name".equals(fd.name) && v instanceof Long) { - nameRef = (Long) v; - } - } - classNameRef.put(id, nameRef); - } else { - // skip every field of this entry - for (FieldDef fd : cd.fields) { - readField(p, fd); - } - } - } - } - if (delta == 0) break; - off += delta; - } - return duplicateCount(); - } - - private int duplicateCount() { - int dups = 0; - for (Map.Entry> en : methodRefTuples.entrySet()) { - if (en.getValue().size() > 1) { - dups++; - System.out.println("[PROF-15130] DUPLICATE method id=" + en.getKey() - + " defs=" + renderTuples(en.getValue())); - } - } - return dups; - } - - private List renderTuples(Set tuples) { - List out = new ArrayList<>(); - for (String t : tuples) { - String[] parts = t.split(":"); - long typeRef = Long.parseLong(parts[0]); - long nameRef = Long.parseLong(parts[1]); - long descRef = Long.parseLong(parts[2]); - String cls = ""; - Long cnr = classNameRef.get(typeRef); - if (cnr != null) { - String s = symbols.get(cnr); - if (s != null) cls = s; - } - String nm = symbols.getOrDefault(nameRef, ""); - String desc = symbols.getOrDefault(descRef, ""); - out.add(cls + "." + nm + desc); - } - return out; - } - - private Object readField(long[] p, FieldDef fd) { - if (fd.dimension == 1) { - long n = readVarLong(p); - for (long i = 0; i < n; i++) { - readScalar(p, fd); - } - return null; - } - return readScalar(p, fd); - } - - private Object readScalar(long[] p, FieldDef fd) { - if (fd.constantPool) { - return readVarLong(p); // a constant-pool reference id - } - ClassDef t = classes.get(fd.typeId); - String tn = t != null ? t.name : null; - if ("java.lang.String".equals(tn)) { - return readString(p); - } - if ("boolean".equals(tn) || "byte".equals(tn)) { - p[0] += 1; - return null; - } - if ("short".equals(tn) || "char".equals(tn) || "int".equals(tn) || "long".equals(tn)) { - return readVarLong(p); - } - if ("float".equals(tn)) { p[0] += 4; return null; } - if ("double".equals(tn)) { p[0] += 8; return null; } - // inline struct: read its fields recursively - if (t != null) { - for (FieldDef sub : t.fields) { - readField(p, sub); - } - } - return null; - } - - // ── metadata parsing ── - private void parseMetadata() { - long[] p = {base + metaOffset}; - readVarLong(p); // size - readVarLong(p); // typeId (Metadata event) - readVarLong(p); // start - readVarLong(p); // duration - readVarLong(p); // metadataId - long stringCount = readVarLong(p); - String[] pool = new String[(int) stringCount]; - for (int i = 0; i < stringCount; i++) { - pool[i] = readString(p); - } - // root element - Element root = readElement(p, pool); - // walk to find elements - collectClasses(root); - for (ClassDef cd : classes.values()) { - switch (cd.name) { - case "jdk.types.Method": methodTypeId = cd.id; break; - case "jdk.types.Symbol": symbolTypeId = cd.id; break; - case "java.lang.Class": classTypeId = cd.id; break; - case "java.lang.String": stringTypeId = cd.id; break; - default: break; - } - } - } - - private void collectClasses(Element el) { - if ("class".equals(el.name)) { - String name = el.attrs.get("name"); - String idStr = el.attrs.get("id"); - if (name != null && idStr != null) { - ClassDef cd = new ClassDef(); - cd.id = Long.parseLong(idStr); - cd.name = name; - for (Element child : el.children) { - if ("field".equals(child.name)) { - FieldDef fd = new FieldDef(); - fd.name = child.attrs.get("name"); - fd.typeId = Long.parseLong(child.attrs.get("class")); - String cp = child.attrs.get("constantPool"); - fd.constantPool = "true".equals(cp); - String dim = child.attrs.get("dimension"); - fd.dimension = dim != null ? Integer.parseInt(dim) : 0; - cd.fields.add(fd); - } - } - classes.put(cd.id, cd); - } - } - for (Element child : el.children) { - collectClasses(child); - } - } - - private Element readElement(long[] p, String[] pool) { - Element el = new Element(); - long nameIdx = readVarLong(p); - el.name = pool[(int) nameIdx]; - long attrCount = readVarLong(p); - for (long i = 0; i < attrCount; i++) { - long k = readVarLong(p); - long v = readVarLong(p); - el.attrs.put(pool[(int) k], pool[(int) v]); - } - long childCount = readVarLong(p); - for (long i = 0; i < childCount; i++) { - el.children.add(readElement(p, pool)); - } - return el; - } - - // ── low-level readers ── - private long beLong(long off) { - long v = 0; - for (int i = 0; i < 8; i++) { - v = (v << 8) | (f[(int) off + i] & 0xffL); - } - return v; - } - - private long readVarLong(long[] p) { - long result = 0; - int shift = 0; - int i = (int) p[0]; - // Up to 8 continuation bytes carry 7 data bits each. - for (int b = 0; b < 8; b++) { - int by = f[i++] & 0xff; - result |= ((long) (by & 0x7f)) << shift; - if ((by & 0x80) == 0) { - p[0] = i; - return result; - } - shift += 7; - } - // 9th byte (shift == 56) carries all 8 bits with no continuation flag. - int by = f[i++] & 0xff; - result |= ((long) by) << shift; - p[0] = i; - return result; - } - - private String readString(long[] p) { - int i = (int) p[0]; - int enc = f[i++] & 0xff; - p[0] = i; - switch (enc) { - case 0: return null; // null - case 1: return ""; // empty - case 3: { // UTF-8 - long len = readVarLong(p); - String s = new String(f, (int) p[0], (int) len, java.nio.charset.StandardCharsets.UTF_8); - p[0] += len; - return s; - } - case 4: { // char array - long len = readVarLong(p); - StringBuilder sb = new StringBuilder((int) len); - for (long c = 0; c < len; c++) { - sb.append((char) readVarLong(p)); - } - return sb.toString(); - } - case 5: { // latin1 - long len = readVarLong(p); - String s = new String(f, (int) p[0], (int) len, java.nio.charset.StandardCharsets.ISO_8859_1); - p[0] += len; - return s; - } - default: - throw new IllegalStateException("Unknown JFR string encoding " + enc); - } - } - } - - private static final class ClassDef { - long id; - String name; - final List fields = new ArrayList<>(); - } - - private static final class FieldDef { - String name; - long typeId; - boolean constantPool; - int dimension; - } - - private static final class Element { - String name; - final Map attrs = new HashMap<>(); - final List children = new ArrayList<>(); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativelibs/NativeLibrariesTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativelibs/NativeLibrariesTest.java deleted file mode 100644 index 5cd62c1b6..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativelibs/NativeLibrariesTest.java +++ /dev/null @@ -1,220 +0,0 @@ -package com.datadoghq.profiler.nativelibs; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.github.luben.zstd.Zstd; -import net.jpountz.lz4.LZ4Compressor; -import net.jpountz.lz4.LZ4Factory; -import net.jpountz.lz4.LZ4FastDecompressor; -import net.jpountz.lz4.LZ4SafeDecompressor; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; -import org.xerial.snappy.Snappy; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.HashMap; -import java.util.Map; -import java.util.Optional; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiConsumer; -import java.util.function.IntFunction; -import java.util.function.ToIntFunction; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Loads and calls into widely used native libraries - * - * we should be able to parse the dwarf info section of these common libraries on linux without segfaulting - * and we should be able to unwind the native stacks - * */ -public class NativeLibrariesTest extends AbstractProfilerTest { - @Override - protected String getProfilerCommand() { - return "cpu=1ms,cstack=" + (Platform.isMac() ? "fp" : "dwarf"); - } - - @RetryingTest(3) - public void test() { - String config = System.getProperty("ddprof_test.config"); - boolean isSanitizer = config.endsWith("san"); - - Assumptions.assumeFalse(Platform.isZing() || Platform.isJ9()); - Assumptions.assumeFalse(Platform.isMusl() && Platform.isAarch64()); - boolean isMusl = Optional.ofNullable(System.getenv("TEST_CONFIGURATION")).orElse("").startsWith("musl"); - boolean isAsan = Optional.ofNullable(System.getenv("TEST_CONFIGURATION")).orElse("").contains("asan"); - int iterations = Platform.isAarch64() && isAsan ? 200 : 100; - int blackhole = 0; - for (int i = 0; i < iterations; i++) { - blackhole ^= lz4Java(); - } - for (int i = 0; i < iterations; i++) { - blackhole ^= zstdJni(); - } - // snappy-java may not load under musl - if (!isMusl) { - for (int i = 0; i < iterations; i++) { - blackhole ^= snappyJava(); - } - } - stopProfiler(); - assertTrue(blackhole != 0); - Map modeCounters = new HashMap<>(); - Map libraryCounters = new HashMap<>(); - for (IItemIterable cpuSamples : verifyEvents("datadog.ExecutionSample")) { - IMemberAccessor stacktraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - IMemberAccessor modeAccessor = THREAD_EXECUTION_MODE.getAccessor(cpuSamples.getType()); - for (IItem item : cpuSamples) { - String stacktrace = stacktraceAccessor.getMember(item); - String mode = modeAccessor.getMember(item); - modeCounters.computeIfAbsent(mode, x -> new AtomicInteger()).incrementAndGet(); - if ("NATIVE".equals(mode)) { - String library = ""; - if (stacktrace.contains("LZ4JNI") || stacktrace.contains(".LZ4HC_")) { - library = "LZ4"; - } else if (stacktrace.contains("Java_org_xerial_snappy_SnappyNative") || stacktrace.contains("libsnappyjava")) { - library = "SNAPPY"; - } else if (stacktrace.contains("Java_com_github_luben_zstd") || stacktrace.contains(".ZSTD_")) { - library = "ZSTD"; - } else if (stacktrace.contains("Compile")) { - library = "JIT"; - } - libraryCounters.computeIfAbsent(library, x -> new AtomicInteger()).incrementAndGet(); - } - } - } - assertTrue(modeCounters.containsKey("JVM"), "no JVM samples"); - assertTrue(modeCounters.containsKey("NATIVE"), "no NATIVE samples"); - assertTrue(libraryCounters.containsKey("LZ4"), "no lz4-java samples"); - // snappy is problematic on musl; we are not running it - // for some reason it is not also appearing in sanitized runs - assertTrue(isMusl || isSanitizer || libraryCounters.containsKey("SNAPPY"), "no snappy-java samples"); - assertTrue(libraryCounters.containsKey("ZSTD"), "no zstd-jni samples"); - modeCounters.forEach((mode, count) -> System.err.println(mode + ": " + count.get())); - libraryCounters.forEach((lib, count) -> System.err.println(lib + ": " + count.get())); - } - - - private int lz4Java() { - LZ4FastDecompressor fastDecompressor = LZ4Factory.nativeInstance().fastDecompressor(); - LZ4SafeDecompressor safeDecompressor = LZ4Factory.nativeInstance().safeDecompressor(); - LZ4Compressor fastCompressor = LZ4Factory.nativeInstance().fastCompressor(); - LZ4Compressor highCompressor = LZ4Factory.nativeInstance().highCompressor(); - int blackhole = ThreadLocalRandom.current().nextInt(); - blackhole ^= roundTrip(fill(ByteBuffer.allocateDirect(256 << 10)), - bb -> fastCompressor.maxCompressedLength(bb.limit()), - ByteBuffer::allocateDirect, - fastCompressor::compress, - fastDecompressor::decompress); - blackhole ^= roundTrip(fill(ByteBuffer.allocate(256 << 10)), - bb -> fastCompressor.maxCompressedLength(bb.limit()), - ByteBuffer::allocate, - fastCompressor::compress, - fastDecompressor::decompress); - blackhole ^= roundTrip(fill(ByteBuffer.allocateDirect(256 << 10)), - bb -> 2 * bb.limit(), - ByteBuffer::allocateDirect, - fastCompressor::compress, - safeDecompressor::decompress); - blackhole ^= roundTrip(fill(ByteBuffer.allocate(256 << 10)), - bb -> 2 * bb.limit(), - ByteBuffer::allocate, - fastCompressor::compress, - safeDecompressor::decompress); - blackhole ^= roundTrip(fill(ByteBuffer.allocateDirect(256 << 10)), - bb -> fastCompressor.maxCompressedLength(bb.limit()), - ByteBuffer::allocateDirect, - highCompressor::compress, - fastDecompressor::decompress); - blackhole ^= roundTrip(fill(ByteBuffer.allocate(256 << 10)), - bb -> fastCompressor.maxCompressedLength(bb.limit()), - ByteBuffer::allocate, - highCompressor::compress, - fastDecompressor::decompress); - blackhole ^= roundTrip(fill(ByteBuffer.allocateDirect(256 << 10)), - bb -> 2 * bb.limit(), - ByteBuffer::allocateDirect, - highCompressor::compress, - safeDecompressor::decompress); - blackhole ^= roundTrip(fill(ByteBuffer.allocate(256 << 10)), - bb -> 2 * bb.limit(), - ByteBuffer::allocate, - highCompressor::compress, - safeDecompressor::decompress); - return blackhole; - } - - private int zstdJni() { - int blackhole = 0; - for (int i = 0; i < 20; i++) { - blackhole ^= roundTrip(fill(ByteBuffer.allocateDirect(256 << 10)), - bb -> Math.toIntExact(Zstd.compressBound(bb.limit())), - ByteBuffer::allocateDirect, - (source, dest) -> { - int limit = Zstd.compress(dest, source); - dest.position(limit); - }, - (source, dest) -> { - int limit = Zstd.decompress(dest, source); - dest.position(limit); - }); - } - return blackhole; - } - - private int snappyJava() { - int blackhole = 0; - for (int i = 0; i < 10; i++) { - blackhole ^= roundTrip(fill(ByteBuffer.allocateDirect(256 << 10)), - bb -> Snappy.maxCompressedLength(bb.limit()), - ByteBuffer::allocateDirect, - (source, dest) -> { - try { - int limit = Snappy.compress(source, dest); - dest.position(limit); - } catch (IOException e) { - e.printStackTrace(System.err); - } - }, - (source, dest) -> { - try { - int limit = Snappy.uncompress(source, dest); - dest.position(limit); - } catch (IOException e) { - e.printStackTrace(System.err); - } - }); - } - return blackhole; - } - - ByteBuffer fill(ByteBuffer buffer) { - byte[] bytes = new byte[buffer.limit()]; - ThreadLocalRandom.current().nextBytes(bytes); - buffer.mark(); - buffer.put(bytes); - buffer.reset(); - return buffer; - } - - private int roundTrip(ByteBuffer data, - ToIntFunction maxCompressedSizer, - IntFunction compressedBufferSupplier, - BiConsumer compress, - BiConsumer decompress) { - int maxCompressedSize = maxCompressedSizer.applyAsInt(data); - ByteBuffer compressedBuffer = compressedBufferSupplier.apply(maxCompressedSize); - ByteBuffer targetBuffer = compressedBufferSupplier.apply(data.limit()); - compress.accept(data, compressedBuffer); - compressedBuffer.flip(); - decompress.accept(compressedBuffer, targetBuffer); - return targetBuffer.position(); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativemem/NativeAllocHelper.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativemem/NativeAllocHelper.java deleted file mode 100644 index e0dbf5352..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativemem/NativeAllocHelper.java +++ /dev/null @@ -1,15 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ -package com.datadoghq.profiler.nativemem; - -final class NativeAllocHelper { - static { - System.loadLibrary("ddproftest"); - } - - static native void nativeMalloc(long size, int count); - - private NativeAllocHelper() {} -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativemem/NativememProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativemem/NativememProfilerTest.java deleted file mode 100644 index c62f0ed43..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativemem/NativememProfilerTest.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ -package com.datadoghq.profiler.nativemem; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.params.provider.ValueSource; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.openjdk.jmc.common.item.Attribute.attr; -import static org.openjdk.jmc.common.unit.UnitLookup.ADDRESS; - -/** - * Smoke tests for native memory (malloc) profiling. - * - *

Runs with {@code cstack=vm}, {@code cstack=vmx}, {@code cstack=dwarf}, and - * {@code cstack=fp}. All modes produce usable Java stacks for malloc events: - * vm/vmx seed from {@code callerPC()}/{@code JavaFrameAnchor} via - * {@code HotspotSupport::walkVM}; dwarf/fp hand a {@code NULL ucontext} to - * {@code AsyncGetCallTrace}, which falls back to the JavaFrameAnchor populated - * by the Java → native transition. - */ -public class NativememProfilerTest extends CStackAwareAbstractProfilerTest { - - private static final IAttribute MALLOC_ADDRESS = attr("address", "address", "", ADDRESS); - - @BeforeAll - static void preloadNativeLib() { - // Ensure libddproftest.so is loaded before the profiler starts in @BeforeEach. - // patchLibraries() only patches libraries already in native_libs at call time; - // if the library loads after start() via dlopen_hook, glibc JVMs may not forward - // the System.loadLibrary dlopen through the patched GOT entry. - NativeAllocHelper.nativeMalloc(0, 0); - } - - public NativememProfilerTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected String getProfilerCommand() { - return "nativemem=0"; // sample every allocation - } - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux() && !Platform.isJ9() && !Platform.isZing(); - } - - @RetryTest(3) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "dwarf", "fp"}) - public void shouldRecordMallocSamples() throws InterruptedException { - // GOT patching conflicts with ASan/TSan interceptors: both replace malloc/free - // symbols, causing undefined behavior or crashes when hooks chain into each other. - Assumptions.assumeFalse(isAsan() || isTsan()); - - triggerAllocations(1000); - - stopProfiler(); - - IItemCollection events = verifyEvents("profiler.Malloc"); - boolean foundMinSize = false; - for (IItemIterable items : events) { - IMemberAccessor sizeAccessor = SIZE.getAccessor(items.getType()); - IMemberAccessor weightAccessor = WEIGHT.getAccessor(items.getType()); - IMemberAccessor addrAccessor = MALLOC_ADDRESS.getAccessor(items.getType()); - if (sizeAccessor == null) { - continue; - } - assertNotNull(addrAccessor, "profiler.Malloc events must carry an address field"); - assertNotNull(weightAccessor, "profiler.Malloc events must carry a weight field"); - for (IItem item : items) { - IQuantity size = sizeAccessor.getMember(item); - assertNotNull(size, "profiler.Malloc event must have a non-null size field"); - assertTrue(size.longValue() > 0, "allocation size must be positive"); - if (size.longValue() >= 1024) { - foundMinSize = true; - } - IQuantity addr = addrAccessor.getMember(item); - assertTrue(addr == null || addr.longValue() != 0, "malloc address must not be zero"); - // nativemem=0 samples every allocation; weight must be exactly 1.0. - IQuantity weight = weightAccessor.getMember(item); - assertNotNull(weight, "profiler.Malloc event must have a non-null weight field"); - assertTrue(Math.abs(weight.doubleValue() - 1.0) < 1e-6, - "weight must be 1.0 for nativemem=0 (all allocations sampled), got " + weight.doubleValue()); - } - } - assertTrue(foundMinSize, "expected at least one malloc event with size >= 1024 bytes"); - - // triggerAllocations is a Java wrapper so it appears in all cstack modes, including fp/dwarf. - verifyStackTraces("profiler.Malloc", "triggerAllocations", "shouldRecordMallocSamples"); - } - - private static void triggerAllocations(int count) { - NativeAllocHelper.nativeMalloc(1024, count); - } - -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativemem/NativememSampledProfilerTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativemem/NativememSampledProfilerTest.java deleted file mode 100644 index 2a8a71cb0..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativemem/NativememSampledProfilerTest.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ -package com.datadoghq.profiler.nativemem; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.params.provider.ValueSource; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Covers the sampled path of the native memory profiler (interval > 1), which - * exercises the Poisson interval generator, PID controller update, and the - * {@code 1 / (1 - exp(-size/interval))} weight formula. The smoke test class - * {@link NativememProfilerTest} only covers {@code nativemem=0}, which bypasses - * these code paths via the {@code _interval <= 1} fast path. - */ -public class NativememSampledProfilerTest extends CStackAwareAbstractProfilerTest { - - @BeforeAll - static void preloadNativeLib() { - // Same as NativememProfilerTest: load libddproftest.so before the profiler starts - // so patchLibraries() finds it in native_libs and patches its malloc GOT entry. - NativeAllocHelper.nativeMalloc(0, 0); - } - - public NativememSampledProfilerTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected String getProfilerCommand() { - return "nativemem=512"; - } - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux() && !Platform.isJ9() && !Platform.isZing(); - } - - @RetryTest(3) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "dwarf", "fp"}) - public void shouldEmitWeightedMallocSamples() throws InterruptedException { - // GOT patching conflicts with ASan/TSan interceptors. - Assumptions.assumeFalse(isAsan() || isTsan()); - - // Drive enough allocation volume through malloc to yield several Poisson samples. - triggerAllocations(20_000); - - stopProfiler(); - - IItemCollection events = verifyEvents("profiler.Malloc"); - int sampleCount = 0; - for (IItemIterable items : events) { - IMemberAccessor sizeAccessor = SIZE.getAccessor(items.getType()); - IMemberAccessor weightAccessor = WEIGHT.getAccessor(items.getType()); - assertNotNull(sizeAccessor, "profiler.Malloc events must carry a size field"); - assertNotNull(weightAccessor, "profiler.Malloc events must carry a weight field"); - for (IItem item : items) { - IQuantity size = sizeAccessor.getMember(item); - IQuantity weight = weightAccessor.getMember(item); - assertNotNull(size, "profiler.Malloc event must have a non-null size field"); - assertNotNull(weight, "profiler.Malloc event must have a non-null weight field"); - // Weight is 1 / (1 - exp(-size/interval)); that function is strictly > 1 - // for all positive sizes, so any Poisson-sampled event must carry weight >= 1. - assertTrue(weight.doubleValue() >= 1.0, - "weight must be >= 1.0 on the sampled path, got " + weight.doubleValue() - + " (size=" + size.longValue() + ")"); - sampleCount++; - } - } - - // With ~20M bytes allocated and a 512-byte interval we expect plenty of samples. - // The assertion is loose to tolerate CI variance but tight enough to catch - // regressions where the sampling path silently produces zero events. - assertTrue(sampleCount >= 8, - "expected at least 8 sampled malloc events, got " + sampleCount); - } - - private static void triggerAllocations(int count) { - NativeAllocHelper.nativeMalloc(1024, count); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketBytesAccuracyTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketBytesAccuracyTest.java deleted file mode 100644 index 92347ca03..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketBytesAccuracyTest.java +++ /dev/null @@ -1,138 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.Attribute; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.common.unit.UnitLookup; - -import java.io.IOException; -import java.io.InputStream; -import java.net.ServerSocket; -import java.net.Socket; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Verifies that time-weighted sampling produces a statistically reasonable - * estimate of total I/O time. With time-weighted inverse-transform sampling - * the invariant is: - *

- *   E[ sum(weight * duration) ] = total_io_time
- * 
- * - * We verify that {@code sum(weight * duration_ns)} is positive and within a - * generous 100x tolerance of the actual test duration, confirming that the - * weight field reflects duration-based probability rather than a degenerate - * value. - */ -public class NativeSocketBytesAccuracyTest extends AbstractProfilerTest { - - private static final IAttribute DURATION_ATTR = - Attribute.attr("duration", "duration", "Duration", UnitLookup.TIMESPAN); - private static final IAttribute WEIGHT_ATTR = - Attribute.attr("weight", "weight", "weight", UnitLookup.NUMBER); - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux() && !Platform.isMusl(); - } - - @Override - protected String getProfilerCommand() { - // 100us period keeps sampling probability high on short localhost I/O. - return "natsock=100us"; - } - - @RetryingTest(5) - public void timeWeightedEstimateIsWithinReasonableBounds() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - int payloadSize = 256 * 1024; - int iterations = 100; - - long wallStart = System.nanoTime(); - doTcpSend(payloadSize, iterations); - long wallEnd = System.nanoTime(); - long wallNs = wallEnd - wallStart; - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent"); - assertTrue(events.hasItems(), "No NativeSocketEvent events found"); - - double scaledDurationNs = 0.0; - long sendEventCount = 0; - for (IItemIterable items : events) { - IMemberAccessor opAccessor = OPERATION.getAccessor(items.getType()); - IMemberAccessor durationAccessor = DURATION_ATTR.getAccessor(items.getType()); - IMemberAccessor weightAccessor = WEIGHT_ATTR.getAccessor(items.getType()); - if (opAccessor == null || durationAccessor == null || weightAccessor == null) continue; - for (IItem item : items) { - String op = opAccessor.getMember(item); - // Outbound direction: SEND (send syscall) or WRITE (write syscall on socket fd). - if ("SEND".equals(op) || "WRITE".equals(op)) { - IQuantity dur = durationAccessor.getMember(item); - IQuantity weight = weightAccessor.getMember(item); - if (dur != null && weight != null) { - double durationNs = dur.doubleValueIn(UnitLookup.NANOSECOND); - scaledDurationNs += durationNs * weight.doubleValue(); - sendEventCount++; - } - } - } - } - - System.out.println("Wall time of transfers: " + wallNs + " ns"); - System.out.println("Scaled I/O time (sum of duration*weight): " + scaledDurationNs + " ns"); - System.out.println("Outbound (SEND/WRITE) event count: " + sendEventCount); - - assertTrue(sendEventCount > 0, "No outbound (SEND/WRITE) events recorded"); - assertTrue(scaledDurationNs > 0.0, "sum(weight * duration) must be positive"); - - // Generous 100x tolerance: scaled estimate must not exceed 100x wall time. - // Lower bound is not enforced because very short I/O calls may all fall - // below the sampling threshold in a brief recording window. - assertTrue(scaledDurationNs <= wallNs * 100.0, - String.format("sum(weight * duration) = %.0f ns is implausibly large (wall=%d ns)", - scaledDurationNs, wallNs)); - } - - private void doTcpSend(int payloadSize, int iterations) throws Exception { - byte[] payload = new byte[payloadSize]; - try (ServerSocket server = new ServerSocket(0)) { - int port = server.getLocalPort(); - Thread serverThread = new Thread(() -> { - for (int iter = 0; iter < iterations; iter++) { - try (Socket conn = server.accept()) { - InputStream in = conn.getInputStream(); - byte[] buf = new byte[payloadSize]; - int read = 0; - while (read < payloadSize) { - int n = in.read(buf, read, payloadSize - read); - if (n < 0) break; - read += n; - } - } catch (IOException ignored) {} - } - }); - serverThread.setDaemon(true); - serverThread.start(); - - for (int iter = 0; iter < iterations; iter++) { - try (Socket client = new Socket("127.0.0.1", port)) { - client.getOutputStream().write(payload); - client.getOutputStream().flush(); - } - } - serverThread.join(5000); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketDisabledTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketDisabledTest.java deleted file mode 100644 index 0db60ec2a..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketDisabledTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItemCollection; - -import static org.junit.jupiter.api.Assertions.assertFalse; - -/** - * Verifies that NativeSocketEvent events are absent when the 'nativesocket' - * profiler argument is not specified. - */ -public class NativeSocketDisabledTest extends NativeSocketTestBase { - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux(); - } - - @Override - protected String getProfilerCommand() { - return "cpu=10ms"; - } - - @RetryingTest(3) - public void noSocketEventsWithoutFeatureEnabled() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - doTcpTransfer(64 * 1024, 8); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent", false); - assertFalse(events.hasItems(), - "NativeSocketEvent events must not appear when nativesocket argument is absent"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketEnabledTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketEnabledTest.java deleted file mode 100644 index 5f0f3e7f6..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketEnabledTest.java +++ /dev/null @@ -1,28 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItemCollection; - -import com.datadoghq.profiler.Platform; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Verifies that the 'nativesocket' profiler argument enables socket I/O tracking - * and that NativeSocketEvent JFR events are produced. - */ -public class NativeSocketEnabledTest extends NativeSocketTestBase { - - @RetryingTest(3) - public void socketEventsProducedWhenFeatureEnabled() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - doTcpTransfer(4096, 128); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent"); - assertTrue(events.hasItems(), "Expected NativeSocketEvent events to be present in JFR recording"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketEventFieldsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketEventFieldsTest.java deleted file mode 100644 index ac5a0271b..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketEventFieldsTest.java +++ /dev/null @@ -1,115 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.IMCThread; -import org.openjdk.jmc.common.IMCStackTrace; -import org.openjdk.jmc.common.item.Attribute; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.common.unit.UnitLookup; -import org.openjdk.jmc.flightrecorder.JfrAttributes; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Verifies that each NativeSocketEvent carries all required fields with valid values: - * eventThread, stackTrace, duration, operation (SEND/RECV), remoteAddress (ip:port), - * bytesTransferred (> 0), weight (> 0). - */ -public class NativeSocketEventFieldsTest extends NativeSocketTestBase { - - private static final IAttribute REMOTE_ADDRESS = - Attribute.attr("remoteAddress", "remoteAddress", "Remote address", UnitLookup.PLAIN_TEXT); - private static final IAttribute BYTES_TRANSFERRED = - Attribute.attr("bytesTransferred", "bytesTransferred", "Bytes transferred", UnitLookup.MEMORY); - private static final IAttribute DURATION = - Attribute.attr("duration", "duration", "Duration", UnitLookup.TIMESPAN); - - @RetryingTest(3) - public void allRequiredFieldsPresentAndValid() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - doTcpTransfer(4096, 128); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent"); - assertTrue(events.hasItems(), "No NativeSocketEvent events found"); - - boolean foundSend = false; - boolean foundRecv = false; - - for (IItemIterable items : events) { - IMemberAccessor operationAccessor = - OPERATION.getAccessor(items.getType()); - IMemberAccessor remoteAddressAccessor = - REMOTE_ADDRESS.getAccessor(items.getType()); - IMemberAccessor bytesAccessor = - BYTES_TRANSFERRED.getAccessor(items.getType()); - IMemberAccessor weightAccessor = - WEIGHT.getAccessor(items.getType()); - IMemberAccessor durationAccessor = - DURATION.getAccessor(items.getType()); - IMemberAccessor threadAccessor = - JfrAttributes.EVENT_THREAD.getAccessor(items.getType()); - IMemberAccessor stackTraceAccessor = - STACK_TRACE.getAccessor(items.getType()); - - assertNotNull(operationAccessor, "operation field accessor must be present"); - assertNotNull(remoteAddressAccessor, "remoteAddress field accessor must be present"); - assertNotNull(bytesAccessor, "bytesTransferred field accessor must be present"); - assertNotNull(weightAccessor, "weight field accessor must be present"); - assertNotNull(durationAccessor, "duration field accessor must be present"); - assertNotNull(threadAccessor, "eventThread field accessor must be present"); - assertNotNull(stackTraceAccessor, "stackTrace field accessor must be present"); - - for (IItem item : items) { - String operation = operationAccessor.getMember(item); - assertNotNull(operation, "operation must not be null"); - // op encodes the underlying syscall: SEND/RECV are emitted by send_hook/recv_hook; - // WRITE/READ are emitted by write_hook/read_hook. Java sockets typically reach - // libc via write()/read(), so foundSend covers SEND and WRITE, foundRecv covers - // RECV and READ — both directions must be observed. - assertTrue(operation.equals("SEND") || operation.equals("RECV") - || operation.equals("WRITE") || operation.equals("READ"), - "operation must be one of SEND/RECV/WRITE/READ, got: " + operation); - if ("SEND".equals(operation) || "WRITE".equals(operation)) foundSend = true; - if ("RECV".equals(operation) || "READ".equals(operation)) foundRecv = true; - - String remoteAddress = remoteAddressAccessor.getMember(item); - assertNotNull(remoteAddress, "remoteAddress must not be null"); - // AF_UNIX SOCK_STREAM sockets produce an empty remoteAddress; skip - // the ip:port format check for those events. - if (!remoteAddress.isEmpty()) { - assertTrue(remoteAddress.contains(":"), - "remoteAddress must be in ip:port format, got: " + remoteAddress); - } - - IQuantity bytes = bytesAccessor.getMember(item); - assertNotNull(bytes, "bytesTransferred must not be null"); - assertTrue(bytes.longValue() > 0, - "bytesTransferred must be > 0, got: " + bytes); - - IQuantity weight = weightAccessor.getMember(item); - assertNotNull(weight, "weight must not be null"); - assertTrue(weight.doubleValue() > 0.0, - "weight must be > 0, got: " + weight); - - IQuantity duration = durationAccessor.getMember(item); - assertNotNull(duration, "duration must not be null"); - - IMCThread thread = threadAccessor.getMember(item); - assertNotNull(thread, "eventThread must not be null"); - } - } - - assertTrue(foundSend, "Expected at least one SEND event"); - assertTrue(foundRecv, "Expected at least one RECV event"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketEventThreadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketEventThreadTest.java deleted file mode 100644 index 0aa1d45c4..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketEventThreadTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.IMCThread; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.JfrAttributes; - -import java.io.IOException; -import java.io.InputStream; -import java.net.ServerSocket; -import java.net.Socket; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Verifies that the eventThread field on NativeSocketEvent is populated and - * names a non-empty thread name, indicating the I/O was attributed to the - * calling thread. - */ -public class NativeSocketEventThreadTest extends AbstractProfilerTest { - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux() && !Platform.isMusl(); - } - - @Override - protected String getProfilerCommand() { - // 100us period keeps sampling probability high on short localhost I/O. - return "natsock=100us"; - } - - @RetryingTest(3) - public void eventThreadIsPopulated() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - doTcpTransfer(64 * 1024, 16); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent"); - assertTrue(events.hasItems(), "No NativeSocketEvent events found"); - - for (IItemIterable items : events) { - IMemberAccessor threadAccessor = - JfrAttributes.EVENT_THREAD.getAccessor(items.getType()); - assertNotNull(threadAccessor, "eventThread accessor must be present"); - for (IItem item : items) { - IMCThread thread = threadAccessor.getMember(item); - assertNotNull(thread, "eventThread must not be null"); - String name = thread.getThreadName(); - assertNotNull(name, "thread name must not be null"); - assertFalse(name.isEmpty(), "thread name must not be empty"); - } - } - } - - private void doTcpTransfer(int payloadSize, int iterations) throws Exception { - byte[] payload = new byte[payloadSize]; - try (ServerSocket server = new ServerSocket(0)) { - int port = server.getLocalPort(); - Thread serverThread = new Thread(() -> { - for (int iter = 0; iter < iterations; iter++) { - try (Socket conn = server.accept()) { - InputStream in = conn.getInputStream(); - byte[] buf = new byte[payloadSize]; - int read = 0; - while (read < payloadSize) { - int n = in.read(buf, read, payloadSize - read); - if (n < 0) break; - read += n; - } - conn.getOutputStream().write(buf, 0, read); - conn.getOutputStream().flush(); - } catch (IOException ignored) {} - } - }); - serverThread.setDaemon(true); - serverThread.start(); - - for (int iter = 0; iter < iterations; iter++) { - try (Socket client = new Socket("127.0.0.1", port)) { - client.getOutputStream().write(payload); - client.getOutputStream().flush(); - byte[] resp = new byte[payloadSize]; - InputStream in = client.getInputStream(); - int read = 0; - while (read < payloadSize) { - int n = in.read(resp, read, payloadSize - read); - if (n < 0) break; - read += n; - } - } - } - serverThread.join(5000); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketMacOsNoOpTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketMacOsNoOpTest.java deleted file mode 100644 index 9ad59ad87..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketMacOsNoOpTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItemCollection; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * On macOS the nativesocket feature is a no-op stub. - * Verifies that the profiler starts without error when 'nativesocket' is specified - * and that no NativeSocketEvent events appear in the recording. - */ -public class NativeSocketMacOsNoOpTest extends NativeSocketTestBase { - - @Override - protected boolean isPlatformSupported() { - return Platform.isMac(); - } - - @Override - protected String getProfilerCommand() { - return "natsock"; - } - - @RetryingTest(3) - public void noEventsOnMacOS() throws Exception { - Assumptions.assumeTrue(Platform.isMac(), "This test targets macOS no-op behaviour"); - - String status = profiler.getStatus(); - assertTrue(status.contains("Running : true"), - "Profiler should be running after start with nativesocket on macOS; status: " + status); - - doTcpTransfer(32 * 1024, 8); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent", false); - assertNotNull(events); - assertFalse(events.hasItems(), - "NativeSocketEvent must not be emitted on macOS (no-op stub)"); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketRateLimitTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketRateLimitTest.java deleted file mode 100644 index 1c3426a6a..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketRateLimitTest.java +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2026 Datadog, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.Attribute; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.common.unit.UnitLookup; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.ServerSocket; -import java.net.Socket; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Verifies that the sampler is active: when a large number of I/O operations are - * performed over a single persistent connection, only a fraction are recorded - * (events << operations), and the weight field is > 1 on at least some events, - * reflecting statistical significance. - * - * The target rate is ~5000 events/min; for a 10-second window that is ~833 events. - * We generate far more byte volume than that and assert the event count is - * substantially less than the operation count, and that at least one event - * carries weight > 1. - * - * A persistent connection is reused across all iterations to avoid the TCP - * handshake overhead that would make the test slow and unreliable on CI. - */ -public class NativeSocketRateLimitTest extends AbstractProfilerTest { - - private static final IAttribute WEIGHT_ATTR = - Attribute.attr("weight", "weight", "weight", UnitLookup.NUMBER); - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux() && !Platform.isMusl(); - } - - @Override - protected String getProfilerCommand() { - return "natsock"; - } - - @RetryingTest(3) - public void eventCountIsSubstantiallyLessThanOperationCount() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - // Generate a high volume of byte-sized writes over a single connection. - // ~5000 iterations * 4 KB = 20 MB total — far above the ~5000-event/min rate limit. - int operations = doHighRateTcpTransfer(4096, 5000); - System.out.println("Total TCP operations performed: " + operations); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent"); - assertTrue(events.hasItems(), "No NativeSocketEvent events found"); - - long eventCount = 0; - boolean foundWeightAboveOne = false; - - for (IItemIterable items : events) { - IMemberAccessor weightAccessor = - WEIGHT_ATTR.getAccessor(items.getType()); - for (IItem item : items) { - eventCount++; - if (weightAccessor != null) { - IQuantity w = weightAccessor.getMember(item); - if (w != null && w.doubleValue() > 1.0) { - foundWeightAboveOne = true; - } - } - } - } - - System.out.println("Recorded NativeSocketEvent count: " + eventCount); - - // Recorded events must be far fewer than raw operations (subsampling is active). - // Target rate is ~5000 events/min (~83/s); for a 10-second window ~830 events expected. - // Ceiling of operations/5 (~4000) gives ~5x headroom and catches broken rate limiting. - assertTrue(eventCount < operations / 5, - "Too many events sampled (rate limiting not working): event count (" + eventCount - + ") should be less than operations/5 (" + operations / 5 + ")"); - - // At least some events must have weight > 1, indicating time-weighted sampling - assertTrue(foundWeightAboveOne, - "Expected at least one event with weight > 1 (time-weighted inverse-transform sampling)"); - } - - /** - * Sends {@code iterations} writes of {@code payloadSize} bytes over a single - * persistent TCP connection and reads the echo back. Reusing the connection - * avoids per-iteration TCP handshake overhead that would otherwise make the - * workload too slow to reliably hit the rate limit within the recording window. - * - * @return total number of individual send/recv calls performed (4 per iteration) - */ - private int doHighRateTcpTransfer(int payloadSize, int iterations) throws Exception { - byte[] payload = new byte[payloadSize]; - - try (ServerSocket server = new ServerSocket(0)) { - int port = server.getLocalPort(); - - Thread serverThread = new Thread(() -> { - try (Socket conn = server.accept()) { - InputStream in = conn.getInputStream(); - OutputStream out = conn.getOutputStream(); - byte[] buf = new byte[payloadSize]; - for (int iter = 0; iter < iterations; iter++) { - int read = 0; - while (read < payloadSize) { - int n = in.read(buf, read, payloadSize - read); - if (n < 0) return; - read += n; - } - out.write(buf, 0, payloadSize); - out.flush(); - } - } catch (IOException ignored) {} - }); - serverThread.setDaemon(true); - serverThread.start(); - - try (Socket client = new Socket("127.0.0.1", port)) { - OutputStream out = client.getOutputStream(); - InputStream in = client.getInputStream(); - byte[] resp = new byte[payloadSize]; - for (int iter = 0; iter < iterations; iter++) { - out.write(payload); - out.flush(); - int read = 0; - while (read < payloadSize) { - int n = in.read(resp, read, payloadSize - read); - if (n < 0) break; - read += n; - } - } - } - serverThread.join(10000); - } - // Each iteration: 1 send + 1 recv on client; 1 recv + 1 send on server - return iterations * 4; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketRemoteAddressTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketRemoteAddressTest.java deleted file mode 100644 index 04b46e6e8..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketRemoteAddressTest.java +++ /dev/null @@ -1,119 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.Attribute; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.UnitLookup; - -import java.io.IOException; -import java.io.InputStream; -import java.net.ServerSocket; -import java.net.Socket; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Verifies that the remoteAddress field in NativeSocketEvent is a non-empty - * string of the form ":" matching the known server endpoint. - */ -public class NativeSocketRemoteAddressTest extends AbstractProfilerTest { - - private static final IAttribute REMOTE_ADDRESS = - Attribute.attr("remoteAddress", "remoteAddress", "Remote address", UnitLookup.PLAIN_TEXT); - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux() && !Platform.isMusl(); - } - - @Override - protected String getProfilerCommand() { - // 100us period keeps sampling probability high enough that the 16 - // short-lived connections reliably produce at least one event whose - // remoteAddress points at the known server port. - return "natsock=100us"; - } - - @RetryingTest(3) - public void remoteAddressIsIpColonPort() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - int serverPort = doTcpTransfer(64 * 1024, 16); - System.out.println("Server port: " + serverPort); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent"); - assertTrue(events.hasItems(), "No NativeSocketEvent events found"); - - boolean foundMatchingAddress = false; - for (IItemIterable items : events) { - IMemberAccessor addrAccessor = - REMOTE_ADDRESS.getAccessor(items.getType()); - assertNotNull(addrAccessor, "remoteAddress accessor must exist"); - for (IItem item : items) { - String addr = addrAccessor.getMember(item); - assertNotNull(addr, "remoteAddress must not be null"); - assertFalse(addr.isEmpty(), "remoteAddress must not be empty"); - // Must match ip:port pattern - assertTrue(addr.matches("^[\\d.]+:\\d+$") || addr.matches("^\\[.*\\]:\\d+$"), - "remoteAddress '" + addr + "' does not match expected ip:port format"); - if (addr.endsWith(":" + serverPort)) { - foundMatchingAddress = true; - } - } - } - assertTrue(foundMatchingAddress, - "Expected at least one event with remoteAddress pointing to server port " + serverPort); - } - - /** Returns the server's bound port. */ - private int doTcpTransfer(int payloadSize, int iterations) throws Exception { - byte[] payload = new byte[payloadSize]; - try (ServerSocket server = new ServerSocket(0)) { - int port = server.getLocalPort(); - Thread serverThread = new Thread(() -> { - for (int iter = 0; iter < iterations; iter++) { - try (Socket conn = server.accept()) { - InputStream in = conn.getInputStream(); - byte[] buf = new byte[payloadSize]; - int read = 0; - while (read < payloadSize) { - int n = in.read(buf, read, payloadSize - read); - if (n < 0) break; - read += n; - } - conn.getOutputStream().write(buf, 0, read); - conn.getOutputStream().flush(); - } catch (IOException ignored) {} - } - }); - serverThread.setDaemon(true); - serverThread.start(); - - for (int iter = 0; iter < iterations; iter++) { - try (Socket client = new Socket("127.0.0.1", port)) { - client.getOutputStream().write(payload); - client.getOutputStream().flush(); - byte[] resp = new byte[payloadSize]; - InputStream in = client.getInputStream(); - int read = 0; - while (read < payloadSize) { - int n = in.read(resp, read, payloadSize - read); - if (n < 0) break; - read += n; - } - } - } - serverThread.join(5000); - return port; - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketRestartTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketRestartTest.java deleted file mode 100644 index 375bc0140..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketRestartTest.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItemCollection; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Verifies that nativesocket profiling survives a stop/restart cycle. - * Events must be recorded in the second profiling session, confirming - * that lifecycle management (hook install/uninstall/reinstall) is correct. - */ -public class NativeSocketRestartTest extends NativeSocketTestBase { - - @RetryingTest(3) - public void testNativeSocketProfilerRestart() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - // First session: framework already started the profiler in @BeforeEach. - doTcpTransfer(4096, 64); - stopProfiler(); - - // Second session: start manually with a fresh JFR file. - Files.createDirectories(Paths.get("/tmp/recordings")); - Path jfr2 = Files.createTempFile(Paths.get("/tmp/recordings"), "NativeSocketRestartTest_restart", ".jfr"); - try { - profiler.execute("start,natsock=100us,jfr,file=" + jfr2.toAbsolutePath()); - - doTcpTransfer(4096, 64); - - profiler.stop(); - - IItemCollection events = verifyEvents(jfr2, "datadog.NativeSocketEvent", true); - assertTrue(events.hasItems(), - "NativeSocketEvent events must be recorded in the second profiling session"); - } finally { - Files.deleteIfExists(jfr2); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketSendRecvSeparateTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketSendRecvSeparateTest.java deleted file mode 100644 index 7eb7d688e..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketSendRecvSeparateTest.java +++ /dev/null @@ -1,107 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; - -import java.io.IOException; -import java.io.InputStream; -import java.net.ServerSocket; -import java.net.Socket; -import java.util.concurrent.CountDownLatch; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Verifies that SEND and RECV operations are tracked as independent events. - * Generates a workload where only one side performs writes so that events - * can be attributed unambiguously to the sending or receiving thread. - */ -public class NativeSocketSendRecvSeparateTest extends AbstractProfilerTest { - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux() && !Platform.isMusl(); - } - - @Override - protected String getProfilerCommand() { - // 100us initial sampling period: a single 256KB localhost write - // completes in ~100-500us giving P ~= 0.6-1.0 per call, so both - // SEND and RECV directions are sampled reliably over 32 iterations. - return "natsock=100us"; - } - - @RetryingTest(3) - public void sendAndRecvTrackedWithSeparateCounts() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - // Large volume of data to ensure sampler captures both directions - doUnidirectionalTransfer(256 * 1024, 32); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent"); - assertTrue(events.hasItems(), "No NativeSocketEvent events found"); - - long sendCount = 0; - long recvCount = 0; - - for (IItemIterable items : events) { - IMemberAccessor opAccessor = OPERATION.getAccessor(items.getType()); - assertNotNull(opAccessor); - for (IItem item : items) { - String op = opAccessor.getMember(item); - // Java sockets reach libc via write()/read(); send()/recv() also possible. - // Group by direction: outbound (SEND, WRITE) vs inbound (RECV, READ). - if ("SEND".equals(op) || "WRITE".equals(op)) sendCount++; - else if ("RECV".equals(op) || "READ".equals(op)) recvCount++; - } - } - - System.out.println("Outbound (SEND/WRITE) events: " + sendCount - + ", Inbound (RECV/READ) events: " + recvCount); - assertTrue(sendCount > 0, "Expected at least one outbound (SEND/WRITE) event, got 0"); - assertTrue(recvCount > 0, "Expected at least one inbound (RECV/READ) event, got 0"); - } - - private void doUnidirectionalTransfer(int payloadSize, int iterations) throws Exception { - byte[] payload = new byte[payloadSize]; - CountDownLatch serverReady = new CountDownLatch(1); - - try (ServerSocket server = new ServerSocket(0)) { - int port = server.getLocalPort(); - Thread serverThread = new Thread(() -> { - serverReady.countDown(); - for (int iter = 0; iter < iterations; iter++) { - try (Socket conn = server.accept()) { - InputStream in = conn.getInputStream(); - byte[] buf = new byte[payloadSize]; - int read = 0; - while (read < payloadSize) { - int n = in.read(buf, read, payloadSize - read); - if (n < 0) break; - read += n; - } - } catch (IOException ignored) {} - } - }); - serverThread.setDaemon(true); - serverThread.start(); - serverReady.await(); - - for (int iter = 0; iter < iterations; iter++) { - try (Socket client = new Socket("127.0.0.1", port)) { - client.getOutputStream().write(payload); - client.getOutputStream().flush(); - } - } - serverThread.join(5000); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketStackTraceTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketStackTraceTest.java deleted file mode 100644 index d966b4538..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketStackTraceTest.java +++ /dev/null @@ -1,120 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.params.provider.ValueSource; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.io.IOException; -import java.io.InputStream; -import java.net.ServerSocket; -import java.net.Socket; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Verifies that NativeSocketEvent events carry non-empty stack traces, - * and that a recognizable Java call site from the test workload appears - * in at least one event's stack trace. - * - *

Parameterized over cstack modes (vm, vmx, dwarf, fp) to exercise both - * the walkVM path (cstack >= CSTACK_VM) and the AsyncGetCallTrace path - * (dwarf/fp) through the BCI_NATIVE_SOCKET code. - */ -public class NativeSocketStackTraceTest extends CStackAwareAbstractProfilerTest { - - public NativeSocketStackTraceTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux() && !Platform.isMusl(); - } - - @Override - protected String getProfilerCommand() { - // 100us period keeps sampling probability high on short localhost I/O. - return "natsock=100us"; - } - - @RetryTest(3) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "dwarf", "fp"}) - public void stackTraceIsCapturedForSocketEvents() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - doTcpTransfer(64 * 1024, 20); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent"); - assertTrue(events.hasItems(), "No NativeSocketEvent events found"); - - boolean foundNonEmptyStackTrace = false; - for (IItemIterable items : events) { - IMemberAccessor stackTraceAccessor = - JdkAttributes.STACK_TRACE_STRING.getAccessor(items.getType()); - if (stackTraceAccessor == null) continue; - for (IItem item : items) { - String st = stackTraceAccessor.getMember(item); - if (st != null && !st.isEmpty()) { - foundNonEmptyStackTrace = true; - break; - } - } - if (foundNonEmptyStackTrace) break; - } - assertTrue(foundNonEmptyStackTrace, "Expected at least one NativeSocketEvent with a non-empty stack trace"); - } - - // Named method so it can appear as a recognizable frame - private void doTcpTransfer(int payloadSize, int iterations) throws Exception { - byte[] payload = new byte[payloadSize]; - try (ServerSocket server = new ServerSocket(0)) { - int port = server.getLocalPort(); - Thread serverThread = new Thread(() -> { - for (int iter = 0; iter < iterations; iter++) { - try (Socket conn = server.accept()) { - InputStream in = conn.getInputStream(); - byte[] buf = new byte[payloadSize]; - int read = 0; - while (read < payloadSize) { - int n = in.read(buf, read, payloadSize - read); - if (n < 0) break; - read += n; - } - conn.getOutputStream().write(buf, 0, read); - conn.getOutputStream().flush(); - } catch (IOException ignored) {} - } - }); - serverThread.setDaemon(true); - serverThread.start(); - - for (int iter = 0; iter < iterations; iter++) { - try (Socket client = new Socket("127.0.0.1", port)) { - client.getOutputStream().write(payload); - client.getOutputStream().flush(); - byte[] resp = new byte[payloadSize]; - InputStream in = client.getInputStream(); - int read = 0; - while (read < payloadSize) { - int n = in.read(resp, read, payloadSize - read); - if (n < 0) break; - read += n; - } - } - } - serverThread.join(5000); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketTestBase.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketTestBase.java deleted file mode 100644 index c45307959..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketTestBase.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.ServerSocket; -import java.net.Socket; - -/** Base class for native-socket profiler tests that need a persistent TCP workload. */ -abstract class NativeSocketTestBase extends AbstractProfilerTest { - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux(); - } - - @Override - protected String getProfilerCommand() { - // 100us initial period keeps P high enough that fast localhost I/O - // reliably produces events across small test workloads. - return "natsock=100us"; - } - - /** - * Sends {@code iterations} writes of {@code payloadSize} bytes over a single - * persistent TCP connection and reads the echo back. Reusing the connection - * fills the TCP send buffer after ~32 iterations (at 4 KB/write, 128 KB OS buffer), - * causing subsequent writes to block for >1 ms and driving the Poisson sampler - * probability close to 1.0 for those calls. - */ - protected void doTcpTransfer(int payloadSize, int iterations) throws Exception { - byte[] payload = new byte[payloadSize]; - for (int i = 0; i < payloadSize; i++) payload[i] = (byte) (i & 0xFF); - - try (ServerSocket server = new ServerSocket(0)) { - int port = server.getLocalPort(); - Thread serverThread = new Thread(() -> { - try (Socket conn = server.accept()) { - InputStream in = conn.getInputStream(); - OutputStream out = conn.getOutputStream(); - byte[] buf = new byte[payloadSize]; - for (int iter = 0; iter < iterations; iter++) { - int read = 0; - while (read < payloadSize) { - int n = in.read(buf, read, payloadSize - read); - if (n < 0) return; - read += n; - } - out.write(buf, 0, payloadSize); - out.flush(); - } - } catch (IOException ignored) {} - }); - serverThread.setDaemon(true); - serverThread.start(); - - try (Socket client = new Socket("127.0.0.1", port)) { - OutputStream out = client.getOutputStream(); - InputStream in = client.getInputStream(); - byte[] resp = new byte[payloadSize]; - for (int iter = 0; iter < iterations; iter++) { - out.write(payload); - out.flush(); - int read = 0; - while (read < payloadSize) { - int n = in.read(resp, read, payloadSize - read); - if (n < 0) break; - read += n; - } - } - } - serverThread.join(10000); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketUdpExcludedTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketUdpExcludedTest.java deleted file mode 100644 index 3007d4f56..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativesocket/NativeSocketUdpExcludedTest.java +++ /dev/null @@ -1,78 +0,0 @@ -package com.datadoghq.profiler.nativesocket; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItemCollection; - -import java.net.DatagramPacket; -import java.net.DatagramSocket; -import java.net.InetAddress; - -import static org.junit.jupiter.api.Assertions.*; - -/** - * Verifies that UDP (DatagramSocket / sendto / recvfrom) transfers do NOT - * produce NativeSocketEvent events. Only TCP blocking send/recv are in scope. - * - * This test performs only UDP transfers and expects zero NativeSocketEvent - * events in the recording. - */ -public class NativeSocketUdpExcludedTest extends AbstractProfilerTest { - - @Override - protected boolean isPlatformSupported() { - return Platform.isLinux() && !Platform.isMusl(); - } - - @Override - protected String getProfilerCommand() { - // 100us period strengthens the negative assertion: any UDP traffic - // that accidentally leaks through the TCP filter would be far more - // likely to produce a sampled event at this tighter interval. - return "natsock=100us"; - } - - @RetryingTest(3) - public void udpTransfersProduceNoSocketEvents() throws Exception { - Assumptions.assumeTrue(Platform.isLinux(), "nativesocket tracking is Linux-only"); - - doUdpTransfer(1024, 500); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.NativeSocketEvent", false); - assertNotNull(events); - assertFalse(events.hasItems(), - "NativeSocketEvent must not be produced for UDP (sendto/recvfrom) transfers"); - } - - private void doUdpTransfer(int payloadSize, int iterations) throws Exception { - byte[] payload = new byte[payloadSize]; - InetAddress loopback = InetAddress.getLoopbackAddress(); - - try (DatagramSocket server = new DatagramSocket(0, loopback)) { - int port = server.getLocalPort(); - Thread serverThread = new Thread(() -> { - byte[] buf = new byte[payloadSize]; - DatagramPacket pkt = new DatagramPacket(buf, buf.length); - for (int iter = 0; iter < iterations; iter++) { - try { - server.receive(pkt); - } catch (Exception ignored) {} - } - }); - serverThread.setDaemon(true); - serverThread.start(); - - try (DatagramSocket client = new DatagramSocket()) { - DatagramPacket pkt = new DatagramPacket(payload, payload.length, loopback, port); - for (int iter = 0; iter < iterations; iter++) { - client.send(pkt); - } - } - serverThread.join(5000); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/DynamicNativeThread.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/DynamicNativeThread.java deleted file mode 100644 index ddea4f8c1..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/DynamicNativeThread.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.nativethread; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.nativethread.NativeThreadCreator; - -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class DynamicNativeThread extends AbstractProfilerTest { - private ThreadCreator threadCreator; - - @Override - protected String getProfilerCommand() { - // set cstack=dwarf to enable dlopen hook - return "cpu=1ms,cstack=dwarf"; - } - - @Override - public void before() { - try { - Class clz = Class.forName("com.datadoghq.profiler.nativethread.NativeThreadCreator"); - threadCreator = (ThreadCreator)clz.newInstance(); - } catch (Exception e) { - e.printStackTrace(); - } - } - - @RetryingTest(3) - public void test() { - long[] threads = new long[8]; - for (int index = 0; index < threads.length; index++) { - threads[index] = threadCreator.createThread(); - } - - for (int index = 0; index < threads.length; index++) { - threadCreator.waitThread(threads[index]); - } - stopProfiler(); - int count = 0; - boolean stacktrace_printed = false; - for (IItemIterable cpuSamples : verifyEvents("datadog.ExecutionSample")) { - IMemberAccessor stacktraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - IMemberAccessor modeAccessor = THREAD_EXECUTION_MODE.getAccessor(cpuSamples.getType()); - for (IItem item : cpuSamples) { - String stacktrace = stacktraceAccessor.getMember(item); - if (stacktrace.indexOf("do_primes()") != -1) { - if (!stacktrace_printed) { - stacktrace_printed = true; - System.out.println("Native thread stack:"); - System.out.println(stacktrace); - } - count++; - } - } - } - assertTrue(count > 0, "no native thread sample"); - } -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadCreator.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadCreator.java deleted file mode 100644 index f122bfe90..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadCreator.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.nativethread; - -public class NativeThreadCreator implements ThreadCreator { - static { - System.loadLibrary("ddproftest"); - } - - public long createThread() { - return createNativeThread(); - } - - public void waitThread(long threadId) { - waitNativeThread(threadId); - } - - public static native long createNativeThread(); - public static native void waitNativeThread(long threadId); -} - diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java deleted file mode 100644 index 5547b84d7..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/NativeThreadTest.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.nativethread; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.nativethread.NativeThreadCreator; - -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.HashMap; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Tests native thread profiling to ensure proper stack unwinding. - * Validates that native threads produce usable samples with do_primes() and - * that non-Java thread samples are not misclassified as break_no_anchor - * (which implies a missing JavaFrameAnchor, inapplicable for pure pthreads). - */ -public class NativeThreadTest extends AbstractProfilerTest { - - - @Override - protected String getProfilerCommand() { - return "cpu=1ms"; - } - - @RetryingTest(3) - public void test() { - long[] threads = new long[8]; - for (int index = 0; index < threads.length; index++) { - threads[index] = NativeThreadCreator.createNativeThread(); - } - - for (int index = 0; index < threads.length; index++) { - NativeThreadCreator.waitNativeThread(threads[index]); - } - stopProfiler(); - - int count = 0; - int totalSamples = 0; - boolean stacktrace_printed = false; - - for (IItemIterable cpuSamples : verifyEvents("datadog.ExecutionSample")) { - IMemberAccessor stacktraceAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - IMemberAccessor modeAccessor = THREAD_EXECUTION_MODE.getAccessor(cpuSamples.getType()); - - for (IItem item : cpuSamples) { - String stacktrace = stacktraceAccessor.getMember(item); - totalSamples++; - - if (stacktrace.indexOf("do_primes()") != -1) { - // Native thread sample: must not contain break_no_anchor - // (non-Java threads have no JavaFrameAnchor — that error is inapplicable) - // break_no_symbol is expected when DWARF CFI is incomplete - assertFalse(stacktrace.contains("break_no_anchor"), - "Found break_no_anchor in native thread sample: " + stacktrace); - - if (!stacktrace_printed) { - stacktrace_printed = true; - System.out.println("Native thread stack:"); - System.out.println(stacktrace); - } - count++; - } - } - } - - System.out.println("Total samples: " + totalSamples + ", samples with do_primes(): " + count); - assertTrue(count > 0, "no native thread sample"); - } - -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/ThreadCreator.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/ThreadCreator.java deleted file mode 100644 index 187313d48..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/ThreadCreator.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright 2025, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.nativethread; - -interface ThreadCreator { - long createThread(); - void waitThread(long id); -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/ThreadEntryDetectionTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/ThreadEntryDetectionTest.java deleted file mode 100644 index 76e5e08ac..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/nativethread/ThreadEntryDetectionTest.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.datadoghq.profiler.nativethread; - -import com.datadoghq.profiler.AbstractProfilerTest; - -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Test for thread entry point detection on native threads. - * - * Validates that when profiling native threads the profiler produces usable - * samples and that non-Java thread samples are not misclassified as - * break_no_anchor (which implies a missing JavaFrameAnchor, inapplicable - * for pure pthreads). break_no_symbol is acceptable — it indicates the - * DWARF unwind could not resolve the next frame, which is expected when - * the test library or libc lacks complete CFI. - */ -public class ThreadEntryDetectionTest extends AbstractProfilerTest { - - @Override - protected String getProfilerCommand() { - // Use fast sampling to capture more thread startup samples - return "cpu=1ms"; - } - - @RetryingTest(3) - public void testThreadEntryDetection() throws Exception { - // Create more threads than standard test to increase likelihood - // of capturing thread startup samples - int numThreads = 16; - long[] threads = new long[numThreads]; - - System.out.println("Creating " + numThreads + " native threads..."); - for (int i = 0; i < numThreads; i++) { - threads[i] = NativeThreadCreator.createNativeThread(); - } - - // Wait for all threads to complete - for (long threadId : threads) { - NativeThreadCreator.waitNativeThread(threadId); - } - - stopProfiler(); - - // Verify events - IItemCollection events = verifyEvents("datadog.ExecutionSample"); - assertNoErrorFrames(events); - - // Verify we captured some samples - int totalSamples = countTotalSamples(events); - System.out.println("Total samples captured: " + totalSamples); - assertTrue(totalSamples > 0, "Expected to capture at least some profiling samples"); - } - - /** - * Verifies that native thread samples do not contain break_no_anchor. - * break_no_anchor implies a missing JavaFrameAnchor which is inapplicable - * for pure pthreads. break_no_symbol is acceptable — it means the DWARF - * unwind hit an unresolvable PC, expected when CFI is incomplete. - */ - private void assertNoErrorFrames(IItemCollection events) { - int samplesChecked = 0; - - for (IItemIterable samples : events) { - IMemberAccessor stackTraceAccessor = - JdkAttributes.STACK_TRACE_STRING.getAccessor(samples.getType()); - - for (IItem sample : samples) { - String stackTrace = stackTraceAccessor.getMember(sample); - samplesChecked++; - - if (stackTrace.contains("do_primes()")) { - // Native thread sample: must not be misclassified as break_no_anchor - assertFalse(stackTrace.contains("break_no_anchor"), - String.format("Found break_no_anchor in native thread sample %d:\n%s", - samplesChecked, stackTrace)); - } - } - } - - System.out.println("Verified " + samplesChecked + " samples"); - } - - /** - * Counts total number of samples in the event collection. - */ - private int countTotalSamples(IItemCollection events) { - int count = 0; - for (IItemIterable samples : events) { - for (IItem sample : samples) { - count++; - } - } - return count; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java deleted file mode 100644 index 88cc6a472..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/queue/QueueTimeTest.java +++ /dev/null @@ -1,103 +0,0 @@ -package com.datadoghq.profiler.queue; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.JavaProfiler; -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.IMCThread; -import org.openjdk.jmc.common.IMCType; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.common.unit.IRange; -import org.openjdk.jmc.flightrecorder.JfrAttributes; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.concurrent.ArrayBlockingQueue; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.openjdk.jmc.common.item.Attribute.attr; -import static org.openjdk.jmc.common.unit.UnitLookup.*; - -public class QueueTimeTest extends AbstractProfilerTest { - @Override - protected String getProfilerCommand() { - return "cpu=10ms"; - } - - private static final class Task implements Runnable { - - private final JavaProfiler profiler; - private final long start; - private final Thread origin; - - private Task(JavaProfiler profiler) { - this.profiler = profiler; - this.start = profiler.getCurrentTicks(); - this.origin = Thread.currentThread(); - } - - @Override - @SuppressWarnings("deprecation") - public void run() { - profiler.setContext(1, 2); - long now = profiler.getCurrentTicks(); - if (profiler.isThresholdExceeded(9, start, now)) { - profiler.recordQueueTime(start, now, getClass(), QueueTimeTest.class, ArrayBlockingQueue.class, 10, origin); - } - profiler.clearContext(); - } - } - - @Test - public void testRecordQueueTime() throws Exception { - Thread origin = Thread.currentThread(); - origin.setName("origin"); - Task task = new Task(profiler); - Thread thread = new Thread(task, "destination"); - Thread.sleep(10); - thread.start(); - thread.join(); - stopProfiler(); - - IAttribute startTimeAttr = attr("startTime", "", "", TIMESTAMP); - IAttribute originAttr = attr("origin", "", "", THREAD); - IAttribute taskAttr = attr("task", "", "", CLASS); - IAttribute schedulerAttr = attr("scheduler", "", "", CLASS); - IAttribute queueTypeAttr = attr("queueType", "", "", CLASS); - IAttribute queueLengthAttr = attr("queueLength", "", "", NUMBER); - - IItemCollection activeSettings = verifyEvents("jdk.ActiveSetting"); - for (IItemIterable activeSetting : activeSettings) { - IMemberAccessor nameAccessor = JdkAttributes.REC_SETTING_NAME.getAccessor(activeSetting.getType()); - IMemberAccessor valueAccessor = JdkAttributes.REC_SETTING_VALUE.getAccessor(activeSetting.getType()); - for (IItem item : activeSetting) { - String name = nameAccessor.getMember(item); - if ("tscfrequency".equals(name)) { - String frequency = valueAccessor.getMember(item); - assertTrue(Long.valueOf(frequency) > 0, frequency); - } - } - } - - IItemCollection events = verifyEvents("datadog.QueueTime"); - for (IItemIterable it : events) { - for (IItem item : it) { - assertTrue(startTimeAttr.getAccessor(it.getType()).getMember(item).longValueIn(EPOCH_NS) > 0); - IRange lifetime = JfrAttributes.LIFETIME.getAccessor(it.getType()).getMember(item); - long duration = lifetime.getEnd().longValueIn(EPOCH_MS) - lifetime.getStart().longValueIn(EPOCH_MS); - assertTrue(duration >= 9); - assertEquals(task.getClass().getName(), taskAttr.getAccessor(it.getType()).getMember(item).getTypeName()); - assertEquals(getClass().getName(), schedulerAttr.getAccessor(it.getType()).getMember(item).getTypeName()); - assertEquals(1, SPAN_ID.getAccessor(it.getType()).getMember(item).longValue()); - assertEquals(2, LOCAL_ROOT_SPAN_ID.getAccessor(it.getType()).getMember(item).longValue()); - assertEquals("origin", originAttr.getAccessor(it.getType()).getMember(item).getThreadName()); - assertEquals(ArrayBlockingQueue.class.getName(), queueTypeAttr.getAccessor(it.getType()).getMember(item).getTypeName()); - assertEquals(10, queueLengthAttr.getAccessor(it.getType()).getMember(item).longValue()); - } - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/settings/DatadogSettingsTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/settings/DatadogSettingsTest.java deleted file mode 100644 index ac3064eff..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/settings/DatadogSettingsTest.java +++ /dev/null @@ -1,73 +0,0 @@ -package com.datadoghq.profiler.settings; - -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; - -import java.util.Arrays; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.openjdk.jmc.common.item.Attribute.attr; -import static org.openjdk.jmc.common.unit.UnitLookup.PLAIN_TEXT; - -public class DatadogSettingsTest extends AbstractProfilerTest { - @Override - protected String getProfilerCommand() { - return "cpu=1ms"; - } - - @RetryingTest(5) - public void testRecordDatadogSetting() { - profiler.recordSetting("dimensionless", "value"); - profiler.recordSetting("withUnit", "60", "seconds"); - byte[] longValueBytes = new byte[8191]; - Arrays.fill(longValueBytes, (byte) 'a'); - for (int i = 0; i < 10000; i++) { - profiler.recordSetting("long value " + i, new String(longValueBytes)); - } - stopProfiler(); - IItemCollection events = verifyEvents("datadog.ProfilerSetting"); - final IAttribute nameAttr = - attr("name", "", "", PLAIN_TEXT); - final IAttribute valueAttr = - attr("value", "", "", PLAIN_TEXT); - final IAttribute unitAttr = - attr("unit", "", "", PLAIN_TEXT); - boolean dimensionlessChecked = false; - boolean withUnitChecked = false; - int longValuesChecked = 0; - for (IItemIterable settings : events) { - IMemberAccessor nameAccessor = nameAttr.getAccessor(settings.getType()); - IMemberAccessor valueAccessor = valueAttr.getAccessor(settings.getType()); - IMemberAccessor unitAccessor = unitAttr.getAccessor(settings.getType()); - for (IItem setting : settings) { - String name = nameAccessor.getMember(setting); - String value = valueAccessor.getMember(setting); - String unit = unitAccessor.getMember(setting); - if (!dimensionlessChecked && name.equals("dimensionless")) { - assertEquals("value", value); - assertEquals("", unit); - dimensionlessChecked = true; - } else if (!withUnitChecked && "withUnit".equals(name)) { - assertEquals("60", value); - assertEquals("seconds", unit); - withUnitChecked = true; - } else { - assertTrue(name.startsWith("long value")); - assertEquals(longValueBytes.length, value.length()); - assertEquals("", unit); - longValuesChecked++; - } - } - } - assertTrue(dimensionlessChecked); - assertTrue(withUnitChecked); - assertEquals(10000, longValuesChecked); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/shutdown/ShutdownTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/shutdown/ShutdownTest.java deleted file mode 100644 index a7d2cbf3a..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/shutdown/ShutdownTest.java +++ /dev/null @@ -1,81 +0,0 @@ -package com.datadoghq.profiler.shutdown; - -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Queue; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.LinkedBlockingQueue; - -import com.datadoghq.profiler.JavaProfiler; - -import static org.junit.jupiter.api.Assertions.fail; - -public class ShutdownTest { - - - @Test - public void testShutdownCpu() throws IOException { - System.out.println("=== testShutdownCpu()"); - JavaProfiler profiler = JavaProfiler.getInstance(); - runTest(profiler, "start,cpu=10us,filter=0"); - } - - @Test - public void testShutdownWall() throws IOException { - System.out.println("=== testShutdownWall()"); - JavaProfiler profiler = JavaProfiler.getInstance(); - profiler.addThread(); - runTest(profiler, "start,wall=10us,filter=0"); - } - - @Test - public void testShutdownCpuAndWall() throws IOException { - System.out.println("=== testShutdownCpuAndWall()"); - JavaProfiler profiler = JavaProfiler.getInstance(); - profiler.addThread(); - runTest(profiler, "start,cpu=10us,wall=~10us,filter=0"); - } - - private static void runTest(JavaProfiler profiler, String command) throws IOException { - Path rootDir = Paths.get("/tmp/recordings"); - Files.createDirectories(rootDir); - Path jfrDump = Files.createTempFile(rootDir, "shutdown-test", ".jfr"); - String commandWithDump = command + ",jfr,file=" + jfrDump.toAbsolutePath(); - ExecutorService executor = Executors.newSingleThreadExecutor(); - Queue errors = new LinkedBlockingQueue<>(); - try { - executor.submit(new Runnable() { - @Override - public void run() { - for (int i = 0; i < 100; i++) { - try { - profiler.execute(commandWithDump); - try { - Thread.sleep(20); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } finally { - profiler.stop(); - } - } catch (Throwable error) { - errors.offer(error); - return; - } - } - } - }).get(); - } catch (Throwable t) { - fail(t.getMessage()); - } finally { - executor.shutdownNow(); - } - if (!errors.isEmpty()) { - fail(errors.poll()); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/test/ProfilerTestRunner.java b/ddprof-test/src/test/java/com/datadoghq/profiler/test/ProfilerTestRunner.java deleted file mode 100644 index 268925644..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/test/ProfilerTestRunner.java +++ /dev/null @@ -1,229 +0,0 @@ -package com.datadoghq.profiler.test; - -import org.junit.platform.engine.TestExecutionResult; -import org.junit.platform.engine.TestSource; -import org.junit.platform.engine.discovery.ClassNameFilter; -import org.junit.platform.engine.discovery.DiscoverySelectors; -import org.junit.platform.engine.support.descriptor.MethodSource; -import org.junit.platform.launcher.Launcher; -import org.junit.platform.launcher.LauncherDiscoveryRequest; -import org.junit.platform.launcher.TestExecutionListener; -import org.junit.platform.launcher.TestIdentifier; -import org.junit.platform.launcher.core.LauncherDiscoveryRequestBuilder; -import org.junit.platform.launcher.core.LauncherFactory; -import org.junit.platform.launcher.listeners.SummaryGeneratingListener; - -import java.io.PrintWriter; -import java.util.Optional; -import java.util.concurrent.ConcurrentHashMap; - -/** - * Custom test runner using JUnit Platform Launcher API. - * - * This runner bypasses JUnit Platform Console Launcher, avoiding known issues with: - * - Assertion handling differences - * - JVM argument forwarding - * - Classpath mounting problems - * - NoSuchMethodError on musl + JDK 11 - * - * Uses the same Launcher API that Gradle's Test task and IDEs use internally, - * ensuring consistent test execution across all platforms. - * - * Test Filtering: - * - -Dtest.filter=ClassName - Run all tests in a class - * - -Dtest.filter=ClassName#method - Run specific test method - * - -Dtest.filter=*.Pattern* - Pattern matching on class names - */ -public class ProfilerTestRunner { - public static void main(String[] args) { - try { - runTests(); - } catch (Throwable t) { - System.err.println("FATAL ERROR in ProfilerTestRunner:"); - t.printStackTrace(System.err); - System.exit(2); - } - } - - private static void runTests() { - // Parse test filter from system property - String testFilter = System.getProperty("test.filter"); - - // Build discovery request - LauncherDiscoveryRequestBuilder requestBuilder = LauncherDiscoveryRequestBuilder.request(); - - if (testFilter != null && !testFilter.isEmpty()) { - // Apply test filter based on format - // Support both # and . as method separators for consistency with Gradle Test tasks - if (testFilter.contains("#")) { - // Method filter with # separator: ClassName#methodName - String[] parts = testFilter.split("#", 2); - requestBuilder.selectors( - DiscoverySelectors.selectMethod(parts[0], parts[1]) - ); - } else if (testFilter.contains(".") && !testFilter.startsWith("*") && isMethodFilter(testFilter)) { - // Method filter with . separator: ClassName.methodName - // Heuristic: if last segment starts with lowercase, it's probably a method name - int lastDot = testFilter.lastIndexOf('.'); - String className = testFilter.substring(0, lastDot); - String methodName = testFilter.substring(lastDot + 1); - requestBuilder.selectors( - DiscoverySelectors.selectMethod(className, methodName) - ); - } else if (testFilter.contains("*")) { - // Pattern filter: *.ClassName or package.* - // Scan entire classpath and apply pattern filter - requestBuilder.selectors( - DiscoverySelectors.selectPackage("") - ); - requestBuilder.filters( - ClassNameFilter.includeClassNamePatterns(testFilter) - ); - } else { - // Class filter - support both fully qualified and short names - // JUnit uses regex patterns, not globs: - // - Fully qualified: com.foo.Bar -> com\.foo\.Bar (escape dots) - // - Short name: Bar -> .*\.Bar (match any package) - String classPattern; - if (testFilter.contains(".")) { - // Fully qualified name - escape dots for regex - classPattern = testFilter.replace(".", "\\."); - } else { - // Short name - prefix with .* to match any package - classPattern = ".*\\." + testFilter; - } - requestBuilder.selectors( - DiscoverySelectors.selectPackage("") - ); - requestBuilder.filters( - ClassNameFilter.includeClassNamePatterns(classPattern) - ); - } - } else { - // No filter: scan all classes in classpath - requestBuilder.selectors( - DiscoverySelectors.selectPackage("") - ); - } - - LauncherDiscoveryRequest request = requestBuilder.build(); - - // Create launcher and register listeners - Launcher launcher = LauncherFactory.create(); - SummaryGeneratingListener listener = new SummaryGeneratingListener(); - launcher.registerTestExecutionListeners(new GradleStyleTestListener(), listener); - - // Execute tests - launcher.execute(request); - - // Print summary to console - listener.getSummary().printTo(new PrintWriter(System.out)); - - // Exit with appropriate code (0 = success, 1 = failures) - long failures = listener.getSummary().getFailures().size(); - System.exit(failures > 0 ? 1 : 0); - } - - /** - * Heuristic to determine if a filter string is a method filter (ClassName.methodName) - * rather than just a class name. - * - * Convention: method names start with lowercase, class names start with uppercase. - * - * IMPORTANT: When using the dot-separator for method filtering, you MUST provide a - * fully qualified class name. JUnit Platform's selectMethod() requires a FQCN. - * - * Examples: - * - "com.foo.Bar" -> false (class name) - * - "com.foo.Bar.testSomething" -> true (method filter, FQCN + lowercase method) - * - "com.foo.Bar.InnerClass" -> false (inner class, "InnerClass" starts with uppercase) - * - "Bar.testSomething" -> true (but will FAIL - "Bar" is not a FQCN) - * - * For short class names with methods, use the # separator instead: "Bar#testSomething" - * Or provide the FQCN: "com.foo.Bar.testSomething" - */ - private static boolean isMethodFilter(String filter) { - int lastDot = filter.lastIndexOf('.'); - if (lastDot < 0 || lastDot >= filter.length() - 1) { - return false; // No dot or dot is at the end - } - - String lastSegment = filter.substring(lastDot + 1); - if (lastSegment.isEmpty()) { - return false; - } - - // Method names conventionally start with lowercase - return Character.isLowerCase(lastSegment.charAt(0)); - } - - /** - * Emits per-test STARTED / PASSED / FAILED / SKIPPED markers to stdout in the same - * format as Gradle's Test task, so that filter_gradle_log.py can compress the output - * identically on both glibc and musl paths. - * - * Output format (matches Gradle's testLogging output): - * com.example.FooTest > testBar STARTED - * com.example.FooTest > testBar PASSED (42ms) - * com.example.FooTest > testBar SKIPPED - * com.example.FooTest > testBar FAILED - * java.lang.AssertionError: ... - */ - private static final class GradleStyleTestListener implements TestExecutionListener { - private final ConcurrentHashMap startTimes = new ConcurrentHashMap<>(); - - @Override - public void executionStarted(TestIdentifier testIdentifier) { - if (!testIdentifier.isTest()) return; - startTimes.put(testIdentifier.getUniqueId(), System.currentTimeMillis()); - String name = formatName(testIdentifier); - if (name != null) { - System.out.println(name + " STARTED"); - System.out.flush(); - } - } - - @Override - public void executionFinished(TestIdentifier testIdentifier, TestExecutionResult result) { - if (!testIdentifier.isTest()) return; - String name = formatName(testIdentifier); - if (name == null) return; - - Long start = startTimes.remove(testIdentifier.getUniqueId()); - long ms = start != null ? System.currentTimeMillis() - start : 0; - - switch (result.getStatus()) { - case SUCCESSFUL: - System.out.printf("%s PASSED (%dms)%n", name, ms); - break; - case ABORTED: - System.out.printf("%s SKIPPED%n", name); - result.getThrowable().ifPresent(t -> - System.out.println(t.getMessage())); - break; - case FAILED: - System.out.printf("%s FAILED%n", name); - result.getThrowable().ifPresent(t -> t.printStackTrace(System.out)); - break; - } - System.out.flush(); - } - - @Override - public void executionSkipped(TestIdentifier testIdentifier, String reason) { - if (!testIdentifier.isTest()) return; - String name = formatName(testIdentifier); - if (name != null) { - System.out.println(name + " SKIPPED"); - System.out.flush(); - } - } - - private static String formatName(TestIdentifier testIdentifier) { - Optional source = testIdentifier.getSource(); - if (!source.isPresent() || !(source.get() instanceof MethodSource)) return null; - MethodSource ms = (MethodSource) source.get(); - return ms.getClassName() + " > " + ms.getMethodName(); - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/BaseContextWallClockTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/BaseContextWallClockTest.java deleted file mode 100644 index 1ff6bbb7c..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/BaseContextWallClockTest.java +++ /dev/null @@ -1,293 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.JavaProfiler; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.context.ContextExecutor; -import com.datadoghq.profiler.context.Tracing; -import org.junit.jupiter.api.Assumptions; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.function.Supplier; - -import static com.datadoghq.profiler.AbstractProfilerTest.*; -import static com.datadoghq.profiler.MoreAssertions.assertInRange; -import static org.junit.jupiter.api.Assertions.*; - -final class BaseContextWallClockTest { - private final Supplier profilerRef; - private volatile JavaProfiler profiler; - - private ContextExecutor executor; - - private Map> methodsToSpanIds; - - BaseContextWallClockTest(Supplier profilerRef) { - this.profilerRef = profilerRef; - } - - void before() { - profiler = profilerRef.get(); - executor = new ContextExecutor(1, profiler); - methodsToSpanIds = new ConcurrentHashMap<>(); - } - - void after() throws InterruptedException { - if (executor == null) { - return; - } - executor.shutdownNow(); - boolean terminated = executor.awaitTermination(30, TimeUnit.SECONDS); - if (!terminated) { - throw new IllegalStateException("Executor failed to terminate within 30 seconds"); - } - profiler = null; - } - - void test(AbstractProfilerTest test) throws ExecutionException, InterruptedException { - test(test, true); - } - - void test(AbstractProfilerTest test, String cstack) throws ExecutionException, InterruptedException { - test(test, true, cstack); - } - - void test(AbstractProfilerTest test, boolean assertContext) throws ExecutionException, InterruptedException { - test(test, assertContext, null); - } - - void test(AbstractProfilerTest test, boolean assertContext, String cstack) throws ExecutionException, InterruptedException { - String config = System.getProperty("ddprof_test.config"); - - Assumptions.assumeTrue(!Platform.isJ9() && !Platform.isZing()); - - test.registerCurrentThreadForWallClockProfiling(); - for (int i = 0, id = 1; i < 100; i++, id += 3) { - method1(id); - } - test.stopProfiler(); - Set method1SpanIds = new HashSet<>(methodsToSpanIds.get("method1Impl")); - Set method2SpanIds = new HashSet<>(methodsToSpanIds.get("method2Impl")); - Set method3SpanIds = new HashSet<>(methodsToSpanIds.get("method3Impl")); - IItemCollection events = test.verifyEvents("datadog.MethodSample"); - Set states = new HashSet<>(); - Set modes = new HashSet<>(); - // we have 100 method1, method2, and method3 calls, but can't guarantee we sampled them all - long method1Weight = 0; - long method2Weight = 0; - long method3Weight = 0; - long unattributedWeight = 0; - for (IItemIterable wallclockSamples : events) { - IMemberAccessor frameAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(wallclockSamples.getType()); - IMemberAccessor spanIdAccessor = SPAN_ID.getAccessor(wallclockSamples.getType()); - IMemberAccessor rootSpanIdAccessor = LOCAL_ROOT_SPAN_ID.getAccessor(wallclockSamples.getType()); - IMemberAccessor weightAccessor = WEIGHT.getAccessor(wallclockSamples.getType()); - IMemberAccessor stateAccessor = THREAD_STATE.getAccessor(wallclockSamples.getType()); - IMemberAccessor modeAccessor = THREAD_EXECUTION_MODE.getAccessor(wallclockSamples.getType()); - for (IItem sample : wallclockSamples) { - String stackTrace = frameAccessor.getMember(sample); - long spanId = spanIdAccessor.getMember(sample).longValue(); - long rootSpanId = rootSpanIdAccessor.getMember(sample).longValue(); - long weight = weightAccessor.getMember(sample).longValue(); - modes.add(modeAccessor.getMember(sample)); - String state = stateAccessor.getMember(sample); - assertNotNull(state); - states.add(state); - - // a lot fo care needs to be taken here with samples that fall between a context activation and - // a method call. E.g. not finding method2Impl in the stack trace doesn't mean the sample wasn't - // taken in the part of method2 between activation and invoking method2Impl, which complicates - // assertions when we only find method1Impl - boolean attributed = false; - if (stackTrace.contains("method3Impl")) { - if (assertContext) { - // method3 is scheduled after method2, and method1 blocks on it, so spanId == rootSpanId + 2 - assertEquals(rootSpanId + 2, spanId, stackTrace); - assertTrue(spanId == 0 || method3SpanIds.contains(spanId), stackTrace); - } - method3Weight += weight; - attributed = true; - } else if (stackTrace.contains("method2Impl")) { - if (assertContext) { - // method2 is called next, so spanId == rootSpanId + 1 - assertEquals(rootSpanId + 1, spanId, stackTrace); - assertTrue(spanId == 0 || method2SpanIds.contains(spanId), stackTrace); - } - method2Weight += weight; - attributed = true; - } else if (stackTrace.contains("method1Impl") - && !stackTrace.contains("method2") && !stackTrace.contains("method3") - && !stackTrace.contains("Object.wait")) { - // Exclude Object.wait frames: while method1Impl is blocked in monitor.wait(), - // method3 runs concurrently on the executor thread. The wall-clock profiler - // samples all threads, so that same window produces method3Weight samples on - // the executor AND method1Weight samples on the main thread. Counting the - // main-thread double-dip inflates method1's share to ~40-55% instead of ~33%. - if (assertContext) { - // need to check this after method2 because method1 calls method2 - // it's the root so spanId == rootSpanId - assertEquals(rootSpanId, spanId, stackTrace); - assertTrue(spanId == 0 || method1SpanIds.contains(spanId), stackTrace); - } - method1Weight += weight; - attributed = true; - } - assertTrue(weight <= 10 && weight > 0); - // Only count as unattributed if spanId is 0 AND we couldn't attribute by stack trace - // This prevents double-counting samples that have valid stack traces but no context - // (e.g., JVMTI samples when using TLS context which can't be read cross-thread) - if (spanId == 0 && !attributed) { - unattributedWeight += weight; - } - } - } - - if (!Platform.isJ9() && Platform.isJavaVersionAtLeast(11)) { - assertTrue(states.contains("WAITING"), "no WAITING samples"); - assertTrue(states.contains("PARKED"), "no PARKED samples"); - assertTrue(states.contains("CONTENDED"), "no CONTENDED samples"); - } else { - assertTrue(states.contains("SYSCALL"), "no SYSCALL samples"); - } - - if (!Platform.isZing() && !Platform.isJ9()) { - assertTrue(modes.contains("JAVA") || modes.contains("JVM") || modes.contains("NATIVE") || modes.contains("SAFEPOINT"), - "no JAVA|JVM|NATIVE|SAFEPOINT samples"); - assertFalse(modes.contains("UNKNOWN"), "UNKNOWN wallclock samples on HotSpot"); - } else { - assertTrue(modes.contains("UNKNOWN"), "no UNKNOWN samples"); - } - - // TODO: vmstructs unwinding on Liberica and aarch64 creates a higher number of broken frames - // it is under investigation but until it gets resolved we will just relax the error margin - double allowedError = Platform.isAarch64() && "BellSoft".equals(System.getProperty("java.vendor")) ? 0.4d : 0.2d; - - // DWARF collects 10-20 native frames per sample (vs 2-5 for FP/VM). Those native PCs vary - // slightly between samples, fragmenting trace IDs and causing some method2/method3 samples - // to be lost or misattributed. The 0.3 tolerance accommodates that fragmentation without - // masking genuine context-attribution bugs. - if (cstack != null && (cstack.equals("vm") || cstack.equals("dwarf") || cstack.equals("fp") || cstack.equals("vmx"))) { - allowedError = 0.3d; - } - - // context filtering should prevent these - assertFalse(states.contains("NEW")); - assertFalse(states.contains("TERMINATED")); - // the sanitizer configurations are not playing that well with the sample distribution - // still useful to run the profiler, though - so just skipping the assertions here - if (config.equals("release") || config.equals("debug")) { - double totalWeight = method1Weight + method2Weight + method3Weight + unattributedWeight; - // Each method sleeps for the same duration (10ms), so each should contribute ~33%. - // method1Impl's monitor.wait time is excluded from method1Weight (see attribution above) - // to measure self-time only, not elapsed time waiting for method3. - assertWeight("method1Impl", totalWeight, method1Weight, 0.33, allowedError); - assertWeight("method2Impl", totalWeight, method2Weight, 0.33, allowedError); - assertWeight("method3Impl", totalWeight, method3Weight, 0.33, allowedError); - // The recording captures counter values before the final cleanup (before processTraces - // runs and frees all traces). Verify the recording contains meaningful data. - assertInRange(test.getRecordedCounterValue("calltrace_storage_traces"), 1, 100); - assertInRange(test.getRecordedCounterValue("calltrace_storage_bytes"), 1024, 8 * 1024 * 1024); - // live counters are 0 after stop (all traces freed - correct, non-leaking behaviour) - Map debugCounters = profiler.getDebugCounters(); - assertEquals(0, debugCounters.get("calltrace_storage_traces")); - assertEquals(0, debugCounters.get("calltrace_storage_bytes")); - assertEquals(0, debugCounters.get("linear_allocator_bytes")); - assertEquals(0, debugCounters.get("linear_allocator_chunks")); - assertInRange(debugCounters.get("thread_ids_count"), 1, 100); - assertInRange(debugCounters.get("thread_names_count"), 1, 100); - } - } - - private void assertWeight(String name, double total, long weight, double expected, double allowedError) { - assertTrue(Math.abs(weight / total - expected) < allowedError, String.format("expect %f weight for %s but have %f", expected, name, weight / total)); - } - - public void method1(int id) throws ExecutionException, InterruptedException { - try (Tracing.Context context = Tracing.newContext(() -> id, profiler)) { - method1Impl(id, context); - } - } - - public void method1Impl(int id, Tracing.Context context) throws ExecutionException, InterruptedException { - sleep(10); - Object monitor = new Object(); - Future wait = executor.submit(() -> method3(id, monitor)); - method2(id, monitor); - synchronized (monitor) { - // Increased timeout from 10ms to 150ms to accommodate: - // - method2Impl sleep: 10ms - // - method3Impl sleep: 10ms - // - Thread scheduling/lock contention buffer: 130ms - monitor.wait(150); - } - wait.get(); - record("method1Impl", context); - } - - public void method2(long id, Object monitor) { - synchronized (monitor) { - try (Tracing.Context context = Tracing.newContext(() -> id + 1, profiler)) { - method2Impl(context); - monitor.notify(); - } - } - } - - - public void method2Impl(Tracing.Context context) { - sleep(10); - record("method2Impl", context); - } - - public void method3(long id, Object monitor) { - synchronized (monitor) { - try (Tracing.Context context = Tracing.newContext(() -> id + 2, profiler)) { - method3Impl(context); - monitor.notify(); - } - } - } - - public void method3Impl(Tracing.Context context) { - sleep(10); - record("method3Impl", context); - } - - - private void record(String methodName, com.datadoghq.profiler.context.Tracing.Context context) { - methodsToSpanIds.computeIfAbsent(methodName, k -> new CopyOnWriteArrayList<>()) - .add(context.getSpanId()); - } - - - private void sleep(long millis) { - long target = System.nanoTime() + millis * 1_000_000L; - while (System.nanoTime() < target) { - try { - long remaining = (target - System.nanoTime()) / 1_000_000L; - if (remaining <= 0) { - break; - } - Thread.sleep(remaining); - // Continue loop to handle spurious wakeups - ensures full sleep duration - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; // Exit immediately on interrupt - } - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/CollapsingSleepTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/CollapsingSleepTest.java deleted file mode 100644 index ff362085f..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/CollapsingSleepTest.java +++ /dev/null @@ -1,38 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import static org.junit.jupiter.api.Assertions.assertTrue; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.item.Aggregators; -import org.openjdk.jmc.common.item.IItemCollection; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; - -import java.util.concurrent.locks.LockSupport; - -public class CollapsingSleepTest extends AbstractProfilerTest { - - @Test - public void testSleep() { - Assumptions.assumeTrue(!Platform.isJ9()); - registerCurrentThreadForWallClockProfiling(); - long ts = System.nanoTime(); - long waitTime = 1_000_000_000L; // 1mil ns == 1s - do { - LockSupport.parkNanos(waitTime); - waitTime -= (System.nanoTime() - ts); - ts = System.nanoTime(); - } while (waitTime > 1_000); - stopProfiler(); - IItemCollection events = verifyEvents("datadog.MethodSample"); - assertTrue(events.hasItems()); - assertTrue(events.getAggregate(Aggregators.sum(WEIGHT)).longValue() > 700); - assertTrue(events.getAggregate(Aggregators.count()).longValue() > 9); - } - - @Override - protected String getProfilerCommand() { - return "wall=~1ms"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/ContendedWallclockSamplesTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/ContendedWallclockSamplesTest.java deleted file mode 100644 index bf8591ca6..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/ContendedWallclockSamplesTest.java +++ /dev/null @@ -1,127 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.context.ContextExecutor; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.params.provider.ValueSource; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ThreadLocalRandom; -import java.util.concurrent.atomic.AtomicLong; - -import static org.junit.jupiter.api.Assertions.assertTrue; - - -public class ContendedWallclockSamplesTest extends CStackAwareAbstractProfilerTest { - public ContendedWallclockSamplesTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms"; - } - - private ContextExecutor executor; - - @Override - protected void before() { - executor = new ContextExecutor(10, profiler); - } - - @Override - public void after() { - executor.shutdownNow(); - } - - @RetryTest(10) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void test(@CStack String cstack) { - // Skip test entirely on unsupported JVMs (don't use assumeFalse which gets retried) - if (Platform.isZing() || Platform.isJ9() || - (isInCI() && isAsan() && Platform.isGraal() && Platform.isAarch64() && - ("vm".equals(cstack) || "vmx".equals(cstack)))) { - return; - } - - // Running vm stackwalker tests on JVMCI (Graal), JDK 24, aarch64 and with a sanitizer is crashing in a weird place - // This looks like the sanitizer instrumentation is breaking the longjump based crash recovery :( - String config = System.getProperty("ddprof_test.config"); - boolean isJvmci = System.getProperty("java.vm.version", "").contains("jvmci"); - boolean isSanitizer = config.endsWith("san"); - if (Platform.isJavaVersionAtLeast(24) && isJvmci && Platform.isAarch64() && cstack.startsWith("vm") && isSanitizer) { - return; - } - - long result = 0; - for (int i = 0; i < 10; i++) { - result += pingPong(); - } - assertTrue(result != 0); - stopProfiler(); - - verifyCStackSettings(); - - String lambdaName = getClass().getName() + LAMBDA_QUALIFIER; - String lambdaStateName = getClass().getName() + ".lambda$pingPong$"; - for (IItemIterable wallclockSamples : verifyEvents("datadog.MethodSample")) { - IMemberAccessor frameAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(wallclockSamples.getType()); - IMemberAccessor stateAccessor = THREAD_STATE.getAccessor(wallclockSamples.getType()); - for (IItem sample : wallclockSamples) { - String state = stateAccessor.getMember(sample); - if ("CONTENDED".equals(state)) { - String stackTrace = frameAccessor.getMember(sample); - if (!stackTrace.endsWith(".GC_active()")) { - // shortcut the assertions for sanitized runs - // the samples are not that good, but it still makes sense to run this load under sanitizers - assertTrue(isSanitizer || stackTrace.contains(lambdaStateName), () -> stackTrace + " missing " + lambdaStateName); - assertTrue(isSanitizer || stackTrace.contains(lambdaName), () -> stackTrace + " missing " + lambdaName); - } - } - } - } - } - - - private long pingPong() { - Object monitor = new Object(); - AtomicLong counter = new AtomicLong(); - long startTime = System.currentTimeMillis(); - List> futures = new ArrayList<>(); - for (int i = 0; i < 2; i++) { - futures.add(CompletableFuture.supplyAsync(() -> { - while (System.currentTimeMillis() - startTime < 500) { - synchronized (monitor) { - counter.addAndGet(busyWork(Duration.ofMillis(100))); - } - counter.addAndGet(busyWork(Duration.ofMillis(10))); - } - return null; - }, - executor)); - } - - CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); - return counter.get(); - } - - private static long busyWork(Duration duration) { - long startTime = System.nanoTime(); - long counter = ThreadLocalRandom.current().nextLong(); - while (System.nanoTime() - startTime < duration.toNanos()) { - counter ^= ThreadLocalRandom.current().nextLong(); - } - return counter; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/ContextWallClockTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/ContextWallClockTest.java deleted file mode 100644 index 41a1ac606..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/ContextWallClockTest.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import java.util.concurrent.ExecutionException; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.params.provider.ValueSource; - -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -public class ContextWallClockTest extends CStackAwareAbstractProfilerTest { - private final BaseContextWallClockTest base = new BaseContextWallClockTest(() -> profiler); - - public ContextWallClockTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected void before() { - base.before(); - } - - @Override - protected void after() throws InterruptedException { - base.after(); - } - - @RetryTest(5) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void test(@CStack String cstack) throws ExecutionException, InterruptedException, Exception { - base.test(this, cstack); - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms,filter=0,loglevel=warn"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/JvmtiBasedContextWallClockTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/JvmtiBasedContextWallClockTest.java deleted file mode 100644 index 61b7af469..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/JvmtiBasedContextWallClockTest.java +++ /dev/null @@ -1,40 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; - -import java.util.concurrent.ExecutionException; - -public class JvmtiBasedContextWallClockTest extends AbstractProfilerTest { - private final BaseContextWallClockTest base = new BaseContextWallClockTest(() -> profiler); - - @Override - protected void before() { - base.before(); - } - - @Override - protected void after() throws InterruptedException { - base.after(); - } - - @Override - protected boolean isPlatformSupported() { - return Platform.isJ9(); - } - - @RetryingTest(5) - public void test() throws ExecutionException, InterruptedException { - // thread local handshake available only since Java 15 - Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(15)); - // do not assert context because of sampling skid - base.test(this, false); - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms,loglevel=warn,wallsampler=jvmti"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/JvmtiBasedPrecheckTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/JvmtiBasedPrecheckTest.java deleted file mode 100644 index 5190e7da6..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/JvmtiBasedPrecheckTest.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; - -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Runs the wallprecheck regression suite through HotSpot's delegated - * RequestStackTrace wall-clock engine. - */ -public class JvmtiBasedPrecheckTest extends PrecheckTest { - private boolean jvmtiDelegationAvailable; - private long requestedBefore; - - @Override - protected void before() throws Exception { - Map counters = profiler.getDebugCounters(); - Assumptions.assumeTrue( - counters.getOrDefault("jvmti_stacks_init_ok", 0L) > 0, - "HotSpot RequestStackTrace JVMTI extension is not available"); - jvmtiDelegationAvailable = true; - requestedBefore = counters.getOrDefault("jvmti_stacks_requested", 0L); - } - - @Override - protected void after() throws Exception { - if (!jvmtiDelegationAvailable) { - return; - } - long requestedAfter = profiler.getDebugCounters() - .getOrDefault("jvmti_stacks_requested", 0L); - assertTrue( - requestedAfter > requestedBefore, - "Expected wallclock jvmtistacks path to request delegated stack traces"); - } - - @Override - protected boolean isPlatformSupported() { - return !Platform.isJ9(); - } - - @Override - protected void withTestAssumptions() { - Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms,wallprecheck=true,jvmtistacks=true"; - } - - @Override - protected String getPrecheckDisabledProfilerCommand() { - return "wall=1ms,wallprecheck=false,filter=0,jvmtistacks=true"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/JvmtiBasedWallClockThreadFilterTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/JvmtiBasedWallClockThreadFilterTest.java deleted file mode 100644 index fe40da894..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/JvmtiBasedWallClockThreadFilterTest.java +++ /dev/null @@ -1,15 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.Platform; - -public class JvmtiBasedWallClockThreadFilterTest extends WallClockThreadFilterTest { - @Override - protected boolean isPlatformSupported() { - return Platform.isJ9(); - } - - @Override - protected String getProfilerCommand() { - return super.getProfilerCommand() + ";wallsampler=jvmti"; - } -} \ No newline at end of file diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/MegamorphicCallTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/MegamorphicCallTest.java deleted file mode 100644 index 4c0c776bf..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/MegamorphicCallTest.java +++ /dev/null @@ -1,101 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.ThreadLocalRandom; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class MegamorphicCallTest extends AbstractProfilerTest { - @Override - protected String getProfilerCommand() { - return "wall=100us"; - } - - private static int calculation() { - return ThreadLocalRandom.current().nextInt(); - } - - interface Calculator { - /** - * This call needs to be cheap enough for a loop to be dominated by the stub - */ - int calculate(); - } - - // This is OpenJDK specific but we choose 3 of these to prevent inlining, - // which means we will get an itable stub frame - static class Calculator1 implements Calculator { - - @Override - public int calculate() { - return calculation(); - } - } - - static class Calculator2 implements Calculator { - - @Override - public int calculate() { - return calculation(); - } - } - - static class Calculator3 implements Calculator { - - @Override - public int calculate() { - return calculation(); - } - } - - private int profiledWork(Calculator... calculators) { - int result = 0; - for (int i = 0; i < 1_000_000; i++) { - for (Calculator calculator : calculators) { - result += calculator.calculate(); - } - } - return result; - } - - @RetryingTest(5) - public void testITableStubs() { - Assumptions.assumeFalse(Platform.isZing() || Platform.isJ9()); - registerCurrentThreadForWallClockProfiling(); - int result = profiledWork(new Calculator1(), new Calculator2(), new Calculator3()); - System.err.println(result); - stopProfiler(); - IItemCollection events = verifyEvents("datadog.MethodSample"); - System.err.println(events.stream().count()); - List itableStubStacktraces = new ArrayList<>(); - for (IItemIterable cpuSamples : events) { - IMemberAccessor frameAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - for (IItem sample : cpuSamples) { - String stackTrace = frameAccessor.getMember(sample); - if (stackTrace.contains(".itable stub()")) { - itableStubStacktraces.add(stackTrace); - } - } - } - assertFalse(itableStubStacktraces.isEmpty()); - boolean foundProfiledWork = false; - for (String stacktrace : itableStubStacktraces) { - foundProfiledWork = stacktrace.contains("MegamorphicCallTest.profiledWork"); - if (foundProfiledWork) - break; - } - assertTrue(foundProfiledWork); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckEfficiencyTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckEfficiencyTest.java deleted file mode 100644 index 3582cf74d..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckEfficiencyTest.java +++ /dev/null @@ -1,314 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicBoolean; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.locks.LockSupport; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Measures the theoretical upper bound on {@code SIGVTALRM} suppression by running with - * {@code wallprecheck=false} and classifying sample states. The once-per-run filter - * ({@code wallprecheck=true}) suppresses {@code SLEEPING}, {@code CONDVAR_WAIT}, and - * {@code OBJECT_WAIT} after the entry sample; {@code RUNNABLE} is not skipped. Monitor - * contention ({@code MONITOR_WAIT}) is also suppressible when monitor hooks identify the blocked - * interval. - */ -public class PrecheckEfficiencyTest extends AbstractProfilerTest { - - private static final String EFFICIENCY_SLEEPING = "efficiency-sleeping"; - private static final String EFFICIENCY_PARKED = "efficiency-parked"; - private static final String EFFICIENCY_WAITING = "efficiency-waiting"; - private static final String EFFICIENCY_WORKING = "efficiency-working"; - - @Test - public void compareSuppressionRates() throws Exception { - Assumptions.assumeTrue(!Platform.isJ9()); - - CountDownLatch ready = new CountDownLatch(4); - AtomicBoolean stop = new AtomicBoolean(false); - Object monitor = new Object(); - - // SLEEPING / CONDVAR_WAIT — suppressed by once-per-run filter - Thread sleeping = new Thread(() -> { - registerCurrentThreadForWallClockProfiling(); - ready.countDown(); - try { Thread.sleep(10_000); } catch (InterruptedException ignored) {} - }, EFFICIENCY_SLEEPING); - - // CONDVAR_WAIT — suppressed by once-per-run filter - Thread parked = new Thread(() -> { - registerCurrentThreadForWallClockProfiling(); - ready.countDown(); - LockSupport.parkNanos(10_000_000_000L); - }, EFFICIENCY_PARKED); - - // OBJECT_WAIT — suppressed by the once-per-run filter. - Thread waiting = new Thread(() -> { - registerCurrentThreadForWallClockProfiling(); - ready.countDown(); - synchronized (monitor) { - try { monitor.wait(10_000); } catch (InterruptedException ignored) {} - } - }, EFFICIENCY_WAITING); - - // RUNNABLE — not skipped - Thread working = new Thread(() -> { - registerCurrentThreadForWallClockProfiling(); - ready.countDown(); - long x = 0; - while (!stop.get()) { x++; } - }, EFFICIENCY_WORKING); - - sleeping.setDaemon(true); - parked.setDaemon(true); - waiting.setDaemon(true); - working.setDaemon(true); - - sleeping.start(); - parked.start(); - waiting.start(); - working.start(); - - ready.await(); - Thread.sleep(500); - - stop.set(true); - sleeping.interrupt(); - LockSupport.unpark(parked); - synchronized (monitor) { monitor.notifyAll(); } - - sleeping.join(1000); - parked.join(1000); - waiting.join(1000); - working.join(1000); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.MethodSample", false); - - long sleepSamples = 0, parkSamples = 0, objectWaitSamples = 0, runnableSamples = 0; - - for (IItemIterable batch : events) { - IMemberAccessor stackAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(batch.getType()); - IMemberAccessor stateAccessor = THREAD_STATE.getAccessor(batch.getType()); - IMemberAccessor threadNameAccessor = - JdkAttributes.EVENT_THREAD_NAME.getAccessor(batch.getType()); - if (stackAccessor == null && stateAccessor == null && threadNameAccessor == null) { - continue; - } - for (IItem item : batch) { - if (threadNameAccessor != null) { - String threadName = threadNameAccessor.getMember(item); - if (EFFICIENCY_SLEEPING.equals(threadName)) { - sleepSamples++; - continue; - } - if (EFFICIENCY_PARKED.equals(threadName)) { - parkSamples++; - continue; - } - if (EFFICIENCY_WAITING.equals(threadName)) { - objectWaitSamples++; - continue; - } - if (EFFICIENCY_WORKING.equals(threadName)) { - runnableSamples++; - continue; - } - } - String state = stateAccessor != null ? stateAccessor.getMember(item) : null; - // CONDVAR_WAIT is written as "PARKED" in JFR metadata. - if (state != null && !state.isEmpty()) { - switch (state) { - case "SLEEPING": - sleepSamples++; - continue; - case "PARKED": - parkSamples++; - continue; - case "WAITING": - objectWaitSamples++; - continue; - default: - break; - } - } - String stack = stackAccessor != null ? stackAccessor.getMember(item) : null; - if (stack != null && (stack.contains("Thread.sleep") || stack.contains("sleep0"))) { - sleepSamples++; - } else if (stack != null && (stack.contains("LockSupport.park") || stack.contains("Unsafe.park") - || stack.contains("parkNanos"))) { - parkSamples++; - } else if (stack != null && (stack.contains("Object.wait") || stack.contains("wait0"))) { - objectWaitSamples++; - } else { - runnableSamples++; - } - } - } - - long total = sleepSamples + parkSamples + objectWaitSamples + runnableSamples; - if (total == 0) { - System.out.println("No samples collected — skipping efficiency report"); - return; - } - - double pctSleep = 100.0 * sleepSamples / total; - double pctPark = 100.0 * parkSamples / total; - double pctObjectWait = 100.0 * objectWaitSamples / total; - double pctRunnable = 100.0 * runnableSamples / total; - - double oncePerRunSuppression = pctSleep + pctPark + pctObjectWait; - - System.out.printf("%nPrecheck efficiency report (wallprecheck=false baseline, %d total samples):%n", total); - System.out.printf(" SLEEPING (Thread.sleep): %4d samples (%5.1f%%)%n", sleepSamples, pctSleep); - System.out.printf(" CONDVAR_WAIT (LockSupport.park): %4d samples (%5.1f%%)%n", parkSamples, pctPark); - System.out.printf(" OBJECT_WAIT (Object.wait): %4d samples (%5.1f%%)%n", objectWaitSamples, pctObjectWait); - System.out.printf(" RUNNABLE / other: %4d samples (%5.1f%%)%n", runnableSamples, pctRunnable); - System.out.printf( - "Once-per-run filter (SLEEPING + CONDVAR_WAIT + OBJECT_WAIT): %.1f%% of signals suppressed (upper bound)%n", - oncePerRunSuppression); - - assertTrue(sleepSamples > 0, "Expected samples from sleeping thread"); - assertTrue(parkSamples + objectWaitSamples > 0, - "Expected WAITING/PARKED samples from parked or object-waiting threads"); - assertTrue(oncePerRunSuppression > 0.0, - "Expected some suppressible SLEEPING/PARKED/WAITING samples"); - assertTrue(runnableSamples > 0, "Expected RUNNABLE samples (working thread or unidentified)"); - } - - /** Thread pool mostly idle (park), plus a sleep-driven scheduler and a CPU-bound thread. */ - @Test - public void realisticServiceWorkload() throws Exception { - Assumptions.assumeTrue(!Platform.isJ9()); - - final int POOL_SIZE = 8; - final int TASK_DURATION_MS = 20; // each submitted task takes ~20 ms - final int SCHEDULE_INTERVAL_MS = 50; // scheduler fires every 50 ms - - AtomicBoolean stop = new AtomicBoolean(false); - AtomicInteger threadIndex = new AtomicInteger(0); - - ExecutorService pool = Executors.newFixedThreadPool(POOL_SIZE, r -> { - Thread t = new Thread(() -> { - registerCurrentThreadForWallClockProfiling(); - r.run(); - }); - t.setName("realistic-pool-" + threadIndex.incrementAndGet()); - t.setDaemon(true); - return t; - }); - - CountDownLatch primed = new CountDownLatch(POOL_SIZE); - for (int i = 0; i < POOL_SIZE; i++) { - pool.submit(primed::countDown); - } - primed.await(); - Thread.sleep(50); - - Thread scheduler = new Thread(() -> { - registerCurrentThreadForWallClockProfiling(); - while (!stop.get()) { - try { - Thread.sleep(SCHEDULE_INTERVAL_MS); - } catch (InterruptedException e) { - break; - } - pool.submit(() -> { - long x = 0; - long deadline = System.nanoTime() + TASK_DURATION_MS * 1_000_000L; - while (System.nanoTime() < deadline) { x++; } - return x; - }); - } - }, "realistic-scheduler"); - scheduler.setDaemon(true); - scheduler.start(); - - Thread hotThread = new Thread(() -> { - registerCurrentThreadForWallClockProfiling(); - long x = 0; - while (!stop.get()) { x++; } - }, "realistic-hot"); - hotThread.setDaemon(true); - hotThread.start(); - - Thread.sleep(500); - - stop.set(true); - scheduler.interrupt(); - pool.shutdownNow(); - pool.awaitTermination(2, TimeUnit.SECONDS); - hotThread.join(1000); - - stopProfiler(); - - IItemCollection events = verifyEvents("datadog.MethodSample", false); - - long sleepSamples = 0, parkSamples = 0, otherSamples = 0; - - for (IItemIterable batch : events) { - IMemberAccessor stackAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(batch.getType()); - if (stackAccessor == null) continue; - for (IItem item : batch) { - String stack = stackAccessor.getMember(item); - if (stack == null) { - otherSamples++; - } else if (stack.contains("Thread.sleep") || stack.contains("sleep0")) { - sleepSamples++; - } else if (stack.contains("LockSupport.park") || stack.contains("Unsafe.park")) { - parkSamples++; - } else { - otherSamples++; - } - } - } - - long total = sleepSamples + parkSamples + otherSamples; - if (total == 0) { - System.out.println("No samples collected — skipping realistic workload report"); - return; - } - - double pctSleep = 100.0 * sleepSamples / total; - double pctPark = 100.0 * parkSamples / total; - double pctOther = 100.0 * otherSamples / total; - - double oncePerRunSuppression = pctSleep + pctPark; - - System.out.printf("%nRealistic service workload report (%d pool threads, 1 scheduler, 1 hot thread, 500ms):%n", POOL_SIZE); - System.out.printf( - " Scheduler sleep (Thread.sleep): %4d samples (%5.1f%%)%n", sleepSamples, pctSleep); - System.out.printf( - " Idle pool park (LockSupport.park): %4d samples (%5.1f%%)%n", parkSamples, pctPark); - System.out.printf( - " RUNNABLE / other (active work): %4d samples (%5.1f%%)%n", otherSamples, pctOther); - System.out.printf( - "Once-per-run filter (SLEEPING + CONDVAR_WAIT): %.1f%% of signals suppressed (upper bound)%n", - oncePerRunSuppression); - - assertTrue(parkSamples > otherSamples, - String.format("Expected idle pool threads (park=%d) to dominate active threads (other=%d)", parkSamples, otherSamples)); - assertTrue(sleepSamples > 0, "Expected samples from scheduler's Thread.sleep"); - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java deleted file mode 100644 index 43d840b4a..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/PrecheckTest.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.ProfilerOwnedBlockHooks; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.item.Attribute; -import org.openjdk.jmc.common.item.IAttribute; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.item.Aggregators; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.common.unit.UnitLookup; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; - -/** - * Verifies once-per-run signal suppression ({@code wallprecheck=true}): a sleeping thread - * should produce a handful of {@code MethodSample} events (entry + boundary jitter), not ~300. - * Requires JDK 11+ — JDK 8 HotSpot reports inconsistent OSThread states for sleep. - */ -public class PrecheckTest extends AbstractProfilerTest { - private static final int OSTHREAD_STATE_SLEEPING = 7; - private static final String TAIL_WEIGHT_THREAD = "precheck-tail-weight"; - private static final int TAIL_WEIGHT_ITERATIONS = 50; - private static final int TAIL_WEIGHT_SLEEP_MILLIS = 6; - private static final long TAIL_WEIGHT_RUNNABLE_NANOS = 2_000_000L; - private static final IAttribute WEIGHT = - Attribute.attr("weight", "Sample weight", UnitLookup.NUMBER); - private static volatile int tailWeightSpinSink; - - @Test - public void testSleepingThreadIsNotSampled() throws InterruptedException { - Assumptions.assumeTrue(!Platform.isJ9()); - Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); - leaveClearedInitializedContext(); - registerCurrentThreadForWallClockProfiling(); - - long token = ProfilerOwnedBlockHooks.blockEnter(profiler, OSTHREAD_STATE_SLEEPING); - assertTrue(token != 0, "Expected native blockEnter to arm SLEEPING state"); - try { - Thread.sleep(300); - } finally { - ProfilerOwnedBlockHooks.blockExit(profiler, token); - } - - stopProfiler(); - - long sampleCount = verifyEvents("datadog.MethodSample", false) - .getAggregate(Aggregators.count()).longValue(); - // Explicitly owned once-per-run filter: entry signal emits, subsequent signals are - // suppressed until blockExit clears the owned run. - assertTrue(sampleCount < 10, - "Expected nearly no MethodSample events for a sleeping thread with wallprecheck=true, got: " + sampleCount); - - Map counters = profiler.getDebugCounters(); - if (counters.containsKey("wc_signals_suppressed_sampled_run")) { - assertTrue(counters.get("wc_signals_suppressed_sampled_run") > 0, - "wc_signals_suppressed_sampled_run should be > 0 for a 300 ms Thread.sleep()"); - } - } - - @Test - public void unownedSleepingThreadIsNotExactOncePerRunSuppressed() throws Exception { - Assumptions.assumeTrue(!Platform.isJ9()); - Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); - leaveClearedInitializedContext(); - registerCurrentThreadForWallClockProfiling(); - - Thread.sleep(300); - - stopProfiler(); - - long sampleCount = verifyEvents("datadog.MethodSample", false) - .getAggregate(Aggregators.count()).longValue(); - assertTrue(sampleCount >= 10, - "Unowned Thread.sleep must not be exact once-per-run suppressed; got: " + sampleCount); - } - - @Test - public void unownedSleepingTailWeightIsPreserved() throws Exception { - Assumptions.assumeTrue(!Platform.isJ9()); - Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); - - Thread sleeper = new Thread(() -> { - registerCurrentThreadForWallClockProfiling(); - try { - for (int i = 0; i < TAIL_WEIGHT_ITERATIONS; i++) { - Thread.sleep(TAIL_WEIGHT_SLEEP_MILLIS); - long runnableUntil = System.nanoTime() + TAIL_WEIGHT_RUNNABLE_NANOS; - while (System.nanoTime() < runnableUntil) { - tailWeightSpinSink++; - // Brief runnable gap forces the unowned blocked state to flush. - } - } - } catch (InterruptedException ignored) { - } - }, TAIL_WEIGHT_THREAD); - - sleeper.start(); - sleeper.join(); - - stopProfiler(); - - WeightedSamples weightedSamples = weightedSamplesForThread(TAIL_WEIGHT_THREAD); - assertTrue(weightedSamples.count > 0, - "Expected MethodSample events for " + TAIL_WEIGHT_THREAD); - long expectedTailContribution = TAIL_WEIGHT_ITERATIONS; - assertTrue(weightedSamples.weight >= weightedSamples.count + expectedTailContribution, - "Expected preserved suppressed tail weight for " + TAIL_WEIGHT_THREAD - + ", count=" + weightedSamples.count - + ", weight=" + weightedSamples.weight - + ", expectedTailContribution=" + expectedTailContribution); - } - - @Test - public void tracedSleepingThreadIsSampled() throws InterruptedException { - Assumptions.assumeTrue(!Platform.isJ9()); - Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); - registerCurrentThreadForWallClockProfiling(); - - profiler.setContext(0x5100L, 0x5101L, 0L, 0x5101L); - try { - Thread.sleep(300); - } finally { - profiler.clearContext(); - } - - stopProfiler(); - - long sampleCount = verifyEvents("datadog.MethodSample", false) - .getAggregate(Aggregators.count()).longValue(); - assertTrue(sampleCount >= 10, - "Expected normal MethodSample volume for traced sleep, got: " + sampleCount); - - Map counters = profiler.getDebugCounters(); - if (counters.containsKey("wc_signals_suppressed_sampled_run")) { - assertEquals(0L, counters.get("wc_signals_suppressed_sampled_run"), - "wc_signals_suppressed_sampled_run must not increment for traced sleep"); - } - } - - @Test - public void suppressionCounterIsZeroWhenPrecheckDisabled() throws Exception { - Assumptions.assumeTrue(!Platform.isJ9()); - Assumptions.assumeTrue(Platform.isJavaVersionAtLeast(11)); - registerCurrentThreadForWallClockProfiling(); - - // Stop the wallprecheck=true recording started by @BeforeEach before starting a new one. - stopProfiler(); - - Map before = profiler.getDebugCounters(); - if (!before.containsKey("wc_signals_suppressed_sampled_run")) { - return; // counter not available in this build - } - long suppressedBefore = before.get("wc_signals_suppressed_sampled_run"); - - Path recordingB = Files.createTempFile(Paths.get("/tmp/recordings"), - "PrecheckTest_disabled_", ".jfr"); - profiler.execute("start," + getPrecheckDisabledProfilerCommand() - + ",attributes=tag1;tag2;tag3,jfr,file=" + recordingB.toAbsolutePath()); - Thread.sleep(300); - profiler.stop(); - - long suppressedAfter = profiler.getDebugCounters() - .getOrDefault("wc_signals_suppressed_sampled_run", 0L); - Files.deleteIfExists(recordingB); - - assertEquals(suppressedBefore, suppressedAfter, - "wc_signals_suppressed_sampled_run must not increment when wallprecheck=false"); - } - - /** - * Recreates the steady state left after a previous test initialized and then removed the Java - * ThreadContext: the native ProfiledThread still owns an initialized OTEP record, but the - * record is cleared and invalid. - */ - private void leaveClearedInitializedContext() { - profiler.setContext(0x7700L, 0x7701L, 0L, 0x7701L); - profiler.clearContext(); - profiler.resetThreadContext(); - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms,wallprecheck=true"; - } - - protected String getPrecheckDisabledProfilerCommand() { - return "wall=1ms,wallprecheck=false,filter=0"; - } - - private WeightedSamples weightedSamplesForThread(String threadName) { - long count = 0; - long weight = 0; - IItemCollection events = verifyEvents("datadog.MethodSample", false); - for (IItemIterable batch : events) { - IMemberAccessor threadNameAccessor = - JdkAttributes.EVENT_THREAD_NAME.getAccessor(batch.getType()); - IMemberAccessor weightAccessor = WEIGHT.getAccessor(batch.getType()); - if (threadNameAccessor == null || weightAccessor == null) { - continue; - } - for (IItem item : batch) { - if (threadName.equals(threadNameAccessor.getMember(item))) { - count++; - weight += weightAccessor.getMember(item).longValue(); - } - } - } - return new WeightedSamples(count, weight); - } - - private static final class WeightedSamples { - final long count; - final long weight; - - WeightedSamples(long count, long weight) { - this.count = count; - this.weight = weight; - } - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/SleepTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/SleepTest.java deleted file mode 100644 index 5cc63689e..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/SleepTest.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.AbstractProfilerTest; -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.item.Aggregators; - -import java.util.concurrent.locks.LockSupport; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -public class SleepTest extends AbstractProfilerTest { - - @Test - public void testSleep() { - registerCurrentThreadForWallClockProfiling(); - long ts = System.nanoTime(); - long waitTime = 1_000_000_000L; // 1mil ns == 1s - do { - LockSupport.parkNanos(waitTime); - waitTime -= (System.nanoTime() - ts); - ts = System.nanoTime(); - } while (waitTime > 1_000); - stopProfiler(); - assertTrue(verifyEvents("datadog.MethodSample").getAggregate(Aggregators.count()).longValue() > 90); - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/SmokeWallTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/SmokeWallTest.java deleted file mode 100644 index cbcea4ea8..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/SmokeWallTest.java +++ /dev/null @@ -1,64 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.cpu.ProfiledCode; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.TestTemplate; -import org.junit.jupiter.params.provider.ValueSource; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.concurrent.ExecutionException; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assumptions.assumeFalse; - -public class SmokeWallTest extends CStackAwareAbstractProfilerTest { - private ProfiledCode profiledCode; - - public SmokeWallTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected void before() { - profiledCode = new ProfiledCode(profiler); - } - - @RetryTest(10) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void test(@CStack String cstack) throws ExecutionException, InterruptedException { - for (int i = 0, id = 1; i < 100; i++, id += 3) { - profiledCode.method1(id); - } - stopProfiler(); - - verifyCStackSettings(); - - IItemCollection events = verifyEvents("datadog.MethodSample"); - - for (IItemIterable cpuSamples : events) { - IMemberAccessor frameAccessor = JdkAttributes.STACK_TRACE_STRING.getAccessor(cpuSamples.getType()); - for (IItem sample : cpuSamples) { - String stackTrace = frameAccessor.getMember(sample); - assertFalse(stackTrace.contains("jvmtiError")); - } - } - } - - @Override - protected void after() throws Exception { - profiledCode.close(); - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/VirtualThreadWallClockTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/VirtualThreadWallClockTest.java deleted file mode 100644 index e4a51ac3c..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/VirtualThreadWallClockTest.java +++ /dev/null @@ -1,262 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import com.datadoghq.profiler.CStackAwareAbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.junit.CStack; -import com.datadoghq.profiler.junit.RetryTest; -import org.junit.jupiter.api.TestTemplate; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import org.junit.jupiter.params.provider.ValueSource; - -import java.lang.reflect.Method; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.locks.LockSupport; - -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assumptions.assumeTrue; - -/** - * Integration tests for the virtual thread continuation unwind paths in walkVM. - * - * CPU-bound VT: never suspends, so all frames are thawed. The profiler detects - * cont_entry_return_pc as the return PC of the bottommost thawed frame and traverses - * enterSpecial to reach carrier frames. - * - * Blocking VT: parks/unparks repeatedly; when remounted with frozen frames still in the - * StackChunk, cont_returnBarrier is the return PC of the bottommost thawed frame. - * - * Skipped entirely on JDK < 21 via {@link #isPlatformSupported()}. - * - *

Negative-test mode

- * Set {@code DDPROF_DISABLE_CONT_UNWIND=1} to skip both native unwind paths at runtime. - * With this flag the test will fail (carrier frames absent), confirming the fix is necessary. - */ -public class VirtualThreadWallClockTest extends CStackAwareAbstractProfilerTest { - - /** DCE sink for the CPU-bound spin loop. */ - private volatile long sink; - - public VirtualThreadWallClockTest(@CStack String cstack) { - super(cstack); - } - - @Override - protected boolean isPlatformSupported() { - return Platform.isJavaVersionAtLeast(21); - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms,filter="; - } - - /** - * Starts a virtual thread using reflection so the class compiles with {@code --release 8}. - * Equivalent to {@code Thread.ofVirtual().start(task)}. - */ - private static Thread startVirtualThread(Runnable task) throws Exception { - // Thread.ofVirtual() -> Thread.Builder.OfVirtual - Method ofVirtual = Thread.class.getMethod("ofVirtual"); - Object builder = ofVirtual.invoke(null); - // Look up start(Runnable) via the public Thread.Builder interface to avoid - // IllegalAccessException on the internal ThreadBuilders$VirtualThreadBuilder class - Class builderInterface = Class.forName("java.lang.Thread$Builder"); - Method start = builderInterface.getMethod("start", Runnable.class); - return (Thread) start.invoke(builder, task); - } - - /** - * Asserts that carrier frames (ForkJoinWorkerThread) are visible in the stack traces, - * confirming that continuation unwinding is working correctly. - */ - private void assertCarrierFramesVisible(IItemCollection events) { - boolean carrierVisible = false; - for (IItemIterable samples : events) { - IMemberAccessor frameAccessor = - JdkAttributes.STACK_TRACE_STRING.getAccessor(samples.getType()); - if (frameAccessor == null) continue; - for (IItem sample : samples) { - String stackTrace = frameAccessor.getMember(sample); - if (stackTrace == null || !stackTrace.contains("VirtualThreadWallClockTest")) continue; - // Standard JDK VTs run on ForkJoinWorkerThread carriers. - // If the JVM ever changes the default carrier pool this check must be updated. - if (stackTrace.contains("ForkJoinWorkerThread")) { - carrierVisible = true; - break; - } - } - if (carrierVisible) break; - } - if (!carrierVisible) { - System.out.println("=== MISSING CARRIER — sample stack traces ==="); - int printed = 0; - outer: - for (IItemIterable dump : events) { - IMemberAccessor fa = - JdkAttributes.STACK_TRACE_STRING.getAccessor(dump.getType()); - if (fa == null) continue; - for (IItem sample : dump) { - String st = fa.getMember(sample); - if (st != null && st.contains("VirtualThreadWallClockTest")) { - System.out.println("--- vt sample " + (++printed) + " ---"); - System.out.println(st); - if (printed >= 5) break outer; - } - } - } - // Carrier-only samples: ForkJoinWorkerThread without VT frames - int carrierPrinted = 0; - System.out.println("=== CARRIER-ONLY samples (no VT frames) ==="); - outer2: - for (IItemIterable dump : events) { - IMemberAccessor fa = - JdkAttributes.STACK_TRACE_STRING.getAccessor(dump.getType()); - if (fa == null) continue; - for (IItem sample : dump) { - String st = fa.getMember(sample); - if (st != null && st.contains("ForkJoinWorkerThread") && !st.contains("VirtualThreadWallClockTest")) { - System.out.println("--- carrier " + (++carrierPrinted) + " ---"); - System.out.println(st); - if (carrierPrinted >= 3) break outer2; - } - } - } - if (carrierPrinted == 0) { - // No carrier samples at all — print first 3 arbitrary samples - System.out.println("=== No carrier samples — first 3 arbitrary samples ==="); - int anyPrinted = 0; - outer3: - for (IItemIterable dump : events) { - IMemberAccessor fa = - JdkAttributes.STACK_TRACE_STRING.getAccessor(dump.getType()); - if (fa == null) continue; - for (IItem sample : dump) { - String st = fa.getMember(sample); - if (st != null && !st.isEmpty()) { - System.out.println("--- any " + (++anyPrinted) + " ---"); - System.out.println(st); - if (anyPrinted >= 3) break outer3; - } - } - } - } - } - assertTrue(carrierVisible, - "No sample showed carrier-thread frames (ForkJoinWorkerThread) — continuation unwind may be broken"); - } - - /** - * CPU-bound virtual thread (all frames thawed). - * - * The VT runs a pure spin loop for ~2 seconds and never parks, so all frames are always - * thawed. The profiler detects cont_entry_return_pc and traverses enterSpecial to reach - * carrier frames. - */ - @RetryTest(5) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void samplesCarrierFramesFromCpuBoundVT(@CStack String cstack) throws Exception { - // Carrier-frame unwinding is only enabled for vmx (cstack=vm does not set carrier_frames). - // fp/dwarf use ASGCT and cannot cross the continuation boundary at all. - assumeTrue(cstack.equals("vmx"), - "carrier-frame unwind requires cstack=vmx"); - waitForProfilerReady(2_000); - Thread vt = startVirtualThread(() -> { - registerCurrentThreadForWallClockProfiling(); - long sum = 0; - long deadline = System.nanoTime() + 2_000_000_000L; - while (System.nanoTime() < deadline) { - sum ^= System.nanoTime(); - } - sink = sum; - }); - vt.join(); - stopProfiler(); - - Map countersA = profiler.getDebugCounters(); - System.out.println("=== COUNTERS CpuBound cstack=" + cstack + " ==="); - System.out.println(" enter_special_hit : " + countersA.getOrDefault("walkvm_enter_special_hit", 0L)); - System.out.println(" cont_barrier_hit : " + countersA.getOrDefault("walkvm_cont_barrier_hit", 0L)); - System.out.println(" cont_entry_null : " + countersA.getOrDefault("walkvm_cont_entry_null", 0L)); - System.out.println(" break_compiled : " + countersA.getOrDefault("walkvm_break_compiled", 0L)); - System.out.println(" hit_codeheap : " + countersA.getOrDefault("walkvm_hit_codeheap", 0L)); - System.out.println(" java_frame_ok : " + countersA.getOrDefault("walkvm_java_frame_ok", 0L)); - System.out.println(" no_vmthread : " + countersA.getOrDefault("walkvm_no_vmthread", 0L)); - System.out.println(" cached_not_java : " + countersA.getOrDefault("walkvm_cached_not_java", 0L)); - System.out.println(" break_interpreted : " + countersA.getOrDefault("walkvm_break_interpreted", 0L)); - System.out.println(" depth_zero : " + countersA.getOrDefault("walkvm_depth_zero", 0L)); - - verifyCStackSettings(); - - assertCarrierFramesVisible(verifyEvents("datadog.MethodSample")); - } - - /** - * Blocking virtual thread (frozen frames in StackChunk). - * - * The VT parks itself 100 times. The main thread sleeps 10ms between each unpark so the - * wall-clock sampler (1ms period) has time to fire while the VT is mounted with frozen - * frames still in the StackChunk (cont_returnBarrier as the return PC). - */ - @RetryTest(5) - @TestTemplate - @ValueSource(strings = {"vm", "vmx", "fp", "dwarf"}) - public void samplesCarrierFramesFromBlockingVT(@CStack String cstack) throws Exception { - // Carrier-frame unwinding is only enabled for vmx (cstack=vm does not set carrier_frames). - // fp/dwarf use ASGCT and cannot cross the continuation boundary at all. - assumeTrue(cstack.equals("vmx"), - "carrier-frame unwind requires cstack=vmx"); - // cont_returnBarrier detection is not yet verified on JDK 25+ where the stub - // may have changed; skip rather than fail until verified. - assumeTrue(!Platform.isJavaVersionAtLeast(25), - "cont_returnBarrier unwind not yet verified on JDK 25+"); - waitForProfilerReady(2_000); - final CountDownLatch started = new CountDownLatch(1); - Thread vt = startVirtualThread(() -> { - registerCurrentThreadForWallClockProfiling(); - started.countDown(); - for (int i = 0; i < 100; i++) { - LockSupport.park(); - // Do some work after unpark to give profiler a chance to sample - // while mounted with potentially frozen frames - long sum = 0; - for (int j = 0; j < 10000; j++) { - sum += System.nanoTime(); - } - sink = sum; - } - }); - started.await(); - for (int i = 0; i < 100; i++) { - Thread.sleep(10); // give wall-clock sampler time to fire during remount - LockSupport.unpark(vt); - } - vt.join(15_000); - assertFalse(vt.isAlive(), "Virtual thread did not complete within timeout"); - stopProfiler(); - - Map countersB = profiler.getDebugCounters(); - System.out.println("=== COUNTERS Blocking cstack=" + cstack + " ==="); - System.out.println(" enter_special_hit : " + countersB.getOrDefault("walkvm_enter_special_hit", 0L)); - System.out.println(" cont_barrier_hit : " + countersB.getOrDefault("walkvm_cont_barrier_hit", 0L)); - System.out.println(" cont_entry_null : " + countersB.getOrDefault("walkvm_cont_entry_null", 0L)); - System.out.println(" break_compiled : " + countersB.getOrDefault("walkvm_break_compiled", 0L)); - System.out.println(" hit_codeheap : " + countersB.getOrDefault("walkvm_hit_codeheap", 0L)); - System.out.println(" java_frame_ok : " + countersB.getOrDefault("walkvm_java_frame_ok", 0L)); - System.out.println(" no_vmthread : " + countersB.getOrDefault("walkvm_no_vmthread", 0L)); - System.out.println(" cached_not_java : " + countersB.getOrDefault("walkvm_cached_not_java", 0L)); - System.out.println(" break_interpreted : " + countersB.getOrDefault("walkvm_break_interpreted", 0L)); - System.out.println(" depth_zero : " + countersB.getOrDefault("walkvm_depth_zero", 0L)); - - verifyCStackSettings(); - - assertCarrierFramesVisible(verifyEvents("datadog.MethodSample")); - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallClockThreadFilterTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallClockThreadFilterTest.java deleted file mode 100644 index 3a7ad7208..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallClockThreadFilterTest.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.datadoghq.profiler.wallclock; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import org.junit.jupiter.api.Assumptions; -import org.junitpioneer.jupiter.RetryingTest; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.common.unit.IQuantity; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; - -public class WallClockThreadFilterTest extends AbstractProfilerTest { - - @RetryingTest(5) - public void test() throws InterruptedException { - Assumptions.assumeTrue(!Platform.isJ9()); - registerCurrentThreadForWallClockProfiling(); - Thread.sleep(100); - stopProfiler(); - IItemCollection events = verifyEvents("datadog.MethodSample"); - for (IItemIterable wallclockSamples : events) { - IMemberAccessor javaThreadNameAccessor = JdkAttributes.EVENT_THREAD_NAME - .getAccessor(wallclockSamples.getType()); - IMemberAccessor javaThreadIdAccessor = JdkAttributes.EVENT_THREAD_ID - .getAccessor(wallclockSamples.getType()); - for (IItem sample : wallclockSamples) { - String javaThreadName = javaThreadNameAccessor.getMember(sample); - assertEquals(Thread.currentThread().getName(), javaThreadName); - long javaThreadId = javaThreadIdAccessor.getMember(sample).longValue(); - assertEquals(Thread.currentThread().getId(), javaThreadId); - } - } - } - - @Override - protected String getProfilerCommand() { - return "wall=~1ms,filter=0"; - } -} diff --git a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java b/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java deleted file mode 100644 index a2caa1b8a..000000000 --- a/ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/WallclockMitigationsCombinedTest.java +++ /dev/null @@ -1,151 +0,0 @@ -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - */ - -package com.datadoghq.profiler.wallclock; - -import static org.junit.jupiter.api.Assertions.assertTrue; - -import com.datadoghq.profiler.AbstractProfilerTest; -import com.datadoghq.profiler.Platform; -import com.datadoghq.profiler.ProfilerOwnedBlockHooks; -import org.junit.jupiter.api.Assumptions; -import org.junit.jupiter.api.Test; -import org.openjdk.jmc.common.item.IItem; -import org.openjdk.jmc.common.item.IItemCollection; -import org.openjdk.jmc.common.item.IItemIterable; -import org.openjdk.jmc.common.item.IMemberAccessor; -import org.openjdk.jmc.flightrecorder.jdk.JdkAttributes; - -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; - -/** - * Verifies once-per-run suppression ({@code wallprecheck=true}) with a mix of sleeping, - * parked, and runnable threads. - */ -public class WallclockMitigationsCombinedTest extends AbstractProfilerTest { - private static final int OSTHREAD_STATE_SLEEPING = 7; - - @Test - public void precheckAndParkSuppressionWorkTogether() throws Exception { - Assumptions.assumeTrue(!Platform.isJ9()); - Assumptions.assumeTrue( - Platform.isJavaVersionAtLeast(11), - "Sleeping-state precheck assertions are stable on JDK 11+"); - - CountDownLatch ready = new CountDownLatch(3); - AtomicBoolean stop = new AtomicBoolean(false); - - Thread sleeping = - new Thread( - () -> { - registerCurrentThreadForWallClockProfiling(); - ready.countDown(); - long token = ProfilerOwnedBlockHooks.blockEnter( - profiler, OSTHREAD_STATE_SLEEPING); - try { - Thread.sleep(280); - } catch (InterruptedException ignored) { - } finally { - ProfilerOwnedBlockHooks.blockExit(profiler, token); - } - }, - "combined-sleeping"); - - Thread parkedBusy = - new Thread( - () -> { - registerCurrentThreadForWallClockProfiling(); - long spanId = 0x1111L; - long rootSpanId = 0x2222L; - profiler.setContext(rootSpanId, spanId, 0, 0); - ready.countDown(); - ProfilerOwnedBlockHooks.parkEnter(profiler); - long parkedUntil = System.nanoTime() + 280_000_000L; - while (System.nanoTime() < parkedUntil) { - // spin while flagged parked - } - ProfilerOwnedBlockHooks.parkExit( - profiler, System.identityHashCode(this), 0L); - profiler.clearContext(); - }, - "combined-parked"); - - Thread runnable = - new Thread( - () -> { - registerCurrentThreadForWallClockProfiling(); - ready.countDown(); - while (!stop.get()) { - // keep runnable - } - }, - "combined-runnable"); - - sleeping.setDaemon(true); - parkedBusy.setDaemon(true); - runnable.setDaemon(true); - sleeping.start(); - parkedBusy.start(); - runnable.start(); - - ready.await(); - Thread.sleep(350); - stop.set(true); - - sleeping.interrupt(); - sleeping.join(1000); - parkedBusy.join(1000); - runnable.join(1000); - - stopProfiler(); - - Map samplesByThread = samplesByThreadName(); - long sleepingSamples = samplesByThread.getOrDefault("combined-sleeping", 0L); - long parkedSamples = samplesByThread.getOrDefault("combined-parked", 0L); - long runnableSamples = samplesByThread.getOrDefault("combined-runnable", 0L); - - assertTrue(sleepingSamples < 10, - "Expected nearly no samples from owned sleeping thread, got: " + sleepingSamples); - assertTrue(parkedSamples > 0, - "Expected samples from traced parked thread, got: " + parkedSamples); - assertTrue(runnableSamples > 0, - "Expected samples from runnable thread, got: " + runnableSamples); - - // Sleeping thread's suppression counter must have incremented. - Map counters = profiler.getDebugCounters(); - if (counters.containsKey("wc_signals_suppressed_sampled_run")) { - assertTrue( - counters.get("wc_signals_suppressed_sampled_run") > 0, - "Expected once-per-run suppression counter to increase"); - } - } - - @Override - protected String getProfilerCommand() { - return "wall=1ms,filter=0,wallprecheck=true"; - } - - private Map samplesByThreadName() { - Map samplesByThread = new HashMap<>(); - IItemCollection events = verifyEvents("datadog.MethodSample", false); - for (IItemIterable batch : events) { - IMemberAccessor threadNameAccessor = - JdkAttributes.EVENT_THREAD_NAME.getAccessor(batch.getType()); - if (threadNameAccessor == null) { - continue; - } - for (IItem item : batch) { - String threadName = threadNameAccessor.getMember(item); - if (threadName != null) { - samplesByThread.merge(threadName, 1L, Long::sum); - } - } - } - return samplesByThread; - } -} diff --git a/doc/README.md b/doc/README.md deleted file mode 100644 index 466231cc4..000000000 --- a/doc/README.md +++ /dev/null @@ -1,39 +0,0 @@ -# Java Profiler Documentation - -## Directory Structure - -| Directory | Purpose | -|-----------|---------| -| `architecture/` | Profiler internal architecture and design documents | -| `build/` | Build system documentation (Gradle, native compilation) | -| `reference/` | Reference documentation for profiler features | -| `temp/` | Work logs, session state, and analysis reports | - -## Naming Convention - -All documentation files use **PascalCase** naming (e.g., `BuildSystemGuide.md`). - -## Quick Navigation - -### Architecture -- [CallTraceStorage](architecture/CallTraceStorage.md) - Triple-buffer architecture for call traces -- [RefCountGuard](architecture/RefCountGuard.md) - Lock-free RAII reference-counting primitive used to drain readers before resource reclamation -- [StringDictionary](architecture/StringDictionary.md) - Concurrency model: RefCountGuard, clearAll, and rotation protocols -- [TLSContext](architecture/TLSContext.md) - Thread-local context for distributed tracing -- [TLSPriming](architecture/TLSPriming.md) - Signal-safe TLS initialization - -### Build System -- [BuildSystemGuide](build/BuildSystemGuide.md) - Comprehensive build system documentation -- [GradleTasks](build/GradleTasks.md) - Available Gradle tasks reference -- [NativeBuildPlugin](build/NativeBuildPlugin.md) - Native C++ compilation plugin -- [TestingGuide](build/TestingGuide.md) - Test strategy: tiers, sanitizers, CI layout, and local workflows - -### Reference -- [ProfilerMemoryRequirements](reference/ProfilerMemoryRequirements.md) - Memory usage and limits -- [EventTypeSystem](reference/EventTypeSystem.md) - Profiler event types -- [RemoteSymbolication](reference/RemoteSymbolication.md) - Symbol resolution -- [RemoteSymbolicationFrameTypes](reference/RemoteSymbolicationFrameTypes.md) - Frame type design for symbolication -- [TestFlakinessAnalysis](reference/TestFlakinessAnalysis.md) - Test flakiness investigation results - -### Work State (temp/) -Session logs and analysis reports for ongoing work. Not considered permanent documentation. diff --git a/doc/architecture/CallTraceStorage.md b/doc/architecture/CallTraceStorage.md deleted file mode 100644 index 3dabb9120..000000000 --- a/doc/architecture/CallTraceStorage.md +++ /dev/null @@ -1,893 +0,0 @@ -# CallTraceStorage Triple-Buffer Architecture - -## Overview - -The CallTraceStorage system implements a sophisticated triple-buffered architecture designed for lock-free, signal-handler-safe profiling data collection. This design enables concurrent trace collection from signal handlers while allowing safe background processing for JFR (Java Flight Recorder) serialization. - -Each collected call trace receives a globally unique 64-bit identifier composed of a 32-bit instance epoch ID and a 32-bit slot index. This dual-component design ensures collision-free trace identification across buffer rotations and supports stable JFR constant pool references. - -## Core Design Principles - -1. **Signal Handler Safety**: All operations in signal handlers use lock-free atomic operations -2. **Globally Unique Trace IDs**: 64-bit identifiers (instance epoch + slot index) prevent collisions across buffer rotations -3. **Memory Continuity**: Traces can be preserved across collection cycles for liveness tracking -4. **Zero-Copy Collection**: Uses atomic pointer swapping instead of data copying -5. **ABA Protection**: Generation counters and RefCountGuard prevent use-after-free -6. **Lock-Free Concurrency**: Multiple threads can collect traces without blocking each other - -## Triple-Buffer States - -The system maintains three `CallTraceHashTable` instances with distinct roles: - -``` -┌─────────────┐ ┌─────────────┐ ┌─────────────┐ -│ ACTIVE │ │ STANDBY │ │ SCRATCH │ -│ │ │ │ │ │ -│ New traces │ │ Preserved │ │ Processing │ -│ from signal │ │ traces from │ │ old traces │ -│ handlers │ │ prev cycle │ │ before clear│ -└─────────────┘ └─────────────┘ └─────────────┘ -``` - -### Buffer Roles - -- **ACTIVE**: Receives new traces from signal handlers (lock-free puts) -- **STANDBY**: Contains preserved traces from the previous collection cycle -- **SCRATCH**: Temporary storage during rotation, gets cleared after processing - -## Triple-Buffer Rotation Algorithm - -The rotation follows a carefully orchestrated 6-step sequence: - -### Phase Diagram - -``` -BEFORE ROTATION: -┌─────────────────────────────────────────────────────────────┐ -│ Thread A (Signal Handler) │ Thread B (JFR Processing) │ -├─────────────────────────────────────────────────────────────┤ -│ │ │ -│ put() → ACTIVE │ processTraces() │ -│ ↓ │ ↓ │ -│ [New Traces] │ Step 1: Collect STANDBY │ -│ │ Step 2: Clear STANDBY │ -│ │ Step 3: ATOMIC SWAP │ -└─────────────────────────────────────────────────────────────┘ - -DURING ROTATION (Atomic Swap): -┌─────────────────────────────────────────────────────────────┐ -│ OLD STATE │ ATOMIC SWAP │ NEW STATE │ -├─────────────────────────────────────────────────────────────┤ -│ ACTIVE = A │ │ ACTIVE = B │ -│ STANDBY = B │ ──── SWAP ────→ │ STANDBY = C │ -│ SCRATCH = C │ │ SCRATCH = A │ -└─────────────────────────────────────────────────────────────┘ - -AFTER ROTATION: -┌────────────────────────────────────────────────────────────┐ -│ put() → NEW ACTIVE (B) │ Step 4: Collect SCRATCH │ -│ │ Step 5: Process All │ -│ [Safe to continue] │ Step 6: Preserve & Clear │ -└────────────────────────────────────────────────────────────┘ -``` - -### Detailed Steps - -```cpp -void processTraces() { - // PHASE 1: Liveness Analysis - // Determine which traces need preservation - - // PHASE 2: Collection Sequence - - // Step 1: Collect from STANDBY (preserved traces) - current_standby->collect(standby_traces); - - // Step 2: Clear STANDBY, prepare for new role as ACTIVE - current_standby->clear(); - current_standby->setInstanceId(new_instance_id); - - // Step 3: ATOMIC ROTATION - // STANDBY (empty) → ACTIVE (receives new traces) - old_active = _active_storage.exchange(current_standby); - - // ACTIVE (full) → SCRATCH (for processing) - old_scratch = _scratch_storage.exchange(old_active); - - // SCRATCH (processed) → STANDBY (for next cycle) - _standby_storage.store(old_scratch); - - // Step 4: Collect from SCRATCH (old active, now read-only) - old_active->collect(active_traces); - - // Step 5: Process combined traces - all_traces = standby_traces ∪ active_traces; - processor(all_traces); - - // Step 6: Preserve traces for next cycle - old_scratch->clear(); - for (trace : preserved_traces) { - old_scratch->putWithExistingIdLockFree(trace); - } -} -``` - -## Memory Safety Mechanisms - -The profiler uses thread-local reference counting (RefCountGuard) as the primary memory reclamation mechanism. This provides lock-free, signal-handler-safe protection against use-after-free bugs during concurrent table access and rotation. - -> **Historical Note**: Earlier versions used hazard pointers with bitmap optimization for slot scanning. However, the bitmap-pointer split-update approach had a race condition: the bitmap bit and pointer store were separate atomic operations, creating a window where the scanner could observe an inconsistent state. RefCountGuard eliminates this issue via the pointer-first protocol, where the count field acts as a single atomic activation barrier. Benchmarking confirmed equivalent performance (within 0.25% noise) while providing provably correct memory reclamation. The hazard pointer implementation was removed in favor of the simpler, more correct RefCountGuard approach. - -### ABA Protection - -Generation counters prevent the ABA problem during concurrent access: - -```cpp -// Each storage operation includes generation check -u64 generation = _generation_counter.load(); -CallTraceHashTable* table = _active_storage.load(); - -if (_generation_counter.load() != generation) { - // Storage was rotated, retry or abort -} -``` - -### Thread-Local Reference Counting (RefCountGuard) - -**Production Implementation**: The profiler uses thread-local reference counting via `RefCountGuard` as the primary memory reclamation mechanism. This approach provides superior correctness guarantees compared to hazard pointers while maintaining equivalent performance. - -#### Architecture - -RefCountGuard implements a **cache-aligned, thread-local reference counting scheme** where each thread has a dedicated slot containing: -- Reference count (4 bytes): Number of active guards protecting the table -- Active table pointer (8 bytes): Which table is currently being protected -- Padding (52 bytes): Ensures full cache line alignment (64 bytes total) - -**Memory Layout:** -``` -┌──────────────────────────────────────────────────────────────────┐ -│ RefCountSlot (64 bytes) │ -├──────────────┬──────────────┬─────────────────────────────────────┤ -│ count (4B) │ padding (4B) │ active_table* (8B) │ padding (48B) │ -│ volatile │ │ │ │ -└──────────────┴──────────────┴────────────────────┴───────────────┘ -``` - -**Global Array:** -```cpp -static RefCountSlot refcount_slots[8192]; // 8192 × 64 bytes = 512 KB -static int slot_owners[8192]; // Thread ID ownership tracking -``` - -#### The Pointer-First Protocol - -The correctness of RefCountGuard relies on a **strict ordering protocol** that eliminates race conditions: - -**Constructor (Activation):** -```cpp -RefCountGuard::RefCountGuard(CallTraceHashTable* resource) { - _my_slot = getThreadRefCountSlot(); // Get thread-local slot - - // CRITICAL ORDERING: Store pointer FIRST, then increment count - // Step 1: Store pointer with release semantics - __atomic_store_n(&refcount_slots[_my_slot].active_table, - resource, __ATOMIC_RELEASE); - - // Step 2: Increment count with release semantics - __atomic_fetch_add(&refcount_slots[_my_slot].count, - 1, __ATOMIC_RELEASE); -} -``` - -**Destructor (Deactivation):** -```cpp -RefCountGuard::~RefCountGuard() { - // CRITICAL ORDERING: Decrement count FIRST, then clear pointer - // Step 1: Decrement count with release semantics - __atomic_fetch_sub(&refcount_slots[_my_slot].count, - 1, __ATOMIC_RELEASE); - - // Step 2: Clear pointer with release semantics - __atomic_store_n(&refcount_slots[_my_slot].active_table, - nullptr, __ATOMIC_RELEASE); -} -``` - -**Scanner (Reclamation Check):** -```cpp -void RefCountGuard::waitForRefCountToClear(CallTraceHashTable* table) { - for (int i = 0; i < MAX_THREADS; ++i) { - // CRITICAL: Check count FIRST (pointer-first protocol) - uint32_t count = __atomic_load_n(&refcount_slots[i].count, - __ATOMIC_ACQUIRE); - if (count == 0) continue; // Slot inactive, skip - - // Count > 0: slot is active, check which table it protects - CallTraceHashTable* table = - __atomic_load_n(&refcount_slots[i].active_table, - __ATOMIC_ACQUIRE); - if (table == table_to_delete) { - return false; // Still protected, cannot reclaim - } - } - return true; // Safe to reclaim -} -``` - -#### Why This Protocol Is Correct - -The pointer-first protocol **provably eliminates race conditions** through careful ordering: - -**Scenario 1: Scanner observes during activation (between steps 1 and 2)** -``` -Thread T1 (Constructor): - Step 1: store pointer → refcount_slots[S].active_table = P ✓ DONE - [PREEMPTION - Scanner runs here] - Step 2: increment count → refcount_slots[S].count = 1 PENDING - -Scanner Thread: - Load count = __atomic_load(&refcount_slots[S].count) - → Observes count = 0 (step 2 not yet executed) - → Treats slot S as INACTIVE - → Skips this slot - -Result: SAFE - - Scanner skips the slot (treats as inactive) - - Thread T1 hasn't "activated" protection yet - - No false sense of protection -``` - -**Scenario 2: Scanner observes after activation (after step 2)** -``` -Thread T1 (Constructor): - Step 1: store pointer → refcount_slots[S].active_table = P ✓ DONE - Step 2: increment count → refcount_slots[S].count = 1 ✓ DONE - -Scanner Thread: - Load count = __atomic_load(&refcount_slots[S].count) - → Observes count = 1 (step 2 completed) - → Treats slot S as ACTIVE - → Loads pointer = __atomic_load(&refcount_slots[S].active_table) - → Observes pointer P (step 1 happened-before step 2 via release semantics) - -Result: SAFE - - Scanner sees active slot - - Pointer is guaranteed visible due to release-acquire ordering - - Table P is correctly protected -``` - -**Scenario 3: Scanner observes during deactivation (between steps 1 and 2)** -``` -Thread T1 (Destructor): - Step 1: decrement count → refcount_slots[S].count = 0 ✓ DONE - [PREEMPTION - Scanner runs here] - Step 2: clear pointer → refcount_slots[S].active_table = nullptr PENDING - -Scanner Thread: - Load count = __atomic_load(&refcount_slots[S].count) - → Observes count = 0 (step 1 completed) - → Treats slot S as INACTIVE - → Skips this slot - -Result: SAFE - - Scanner treats slot as inactive (count = 0) - - Doesn't load the pointer at all - - Safe to reclaim (no protection claimed) -``` - -**Key Invariant**: The count field acts as a **single atomic activation barrier**. The scanner checks count first: -- `count == 0` → slot inactive (safe to skip, regardless of pointer value) -- `count > 0` → slot active (pointer is guaranteed visible via release-acquire) - -There is **no window** where the scanner can observe inconsistent state that leads to **use-after-free** bugs. - -#### Trace Drop Window and Revalidation - -There is a narrow window during guard construction where traces may be dropped, but use-after-free is **impossible**: - -**The Scenario:** -``` -Thread T1 (signal handler): - active = _active_storage.load(); // Step 1: Load TableA - RefCountGuard guard(active); // Step 2: Constructor starts - store pointer → slot[S].active_table = TableA; - ⚠️ PREEMPTION - count still 0! - -Thread T2 (scanner): - old = exchange(_active_storage, nullptr); // Nullifies storage - waitForRefCountToClear(TableA); // Scans all slots - for slot[S]: count=0 → skip // Sees inactive - return; // All clear! - delete TableA; // Table deleted - -Thread T1 resumes: - increment count → slot[S].count = 1; // Too late! - - // ⚡ CRITICAL REVALIDATION CHECK: - original_active = _active_storage.load(); // Reads nullptr! - if (original_active != active) { // TRUE (nullptr != TableA) - return DROPPED_TRACE_ID; // ✅ SAFE - never touches table - } - // active->put() is NEVER REACHED ✅ -``` - -**Why This Is Safe:** - -1. **Revalidation Always Happens**: After guard construction completes, we **always** re-check `_active_storage` before using the table pointer -2. **Memory Ordering Guarantees Visibility**: ACQUIRE/RELEASE semantics ensure the revalidation sees the scanner's nullification -3. **No Use-After-Free**: We return `DROPPED_TRACE_ID` and never dereference the deleted table -4. **Acceptable Tradeoff**: Traces arriving during this tiny window (~10-100ns) are dropped, which is acceptable under contention - -**Key Insight**: The scanner can complete on its **first iteration** if all slots have `count=0`. This is by design - we trade this narrow trace-drop window for the simplicity of a two-state protocol (inactive/active) rather than a more complex three-state protocol (inactive/pending/active). - -#### Why Hazard Pointers Had a Race Condition - -The original hazard pointer implementation had a **bitmap-pointer atomicity gap**: - -```cpp -// Hazard Pointer Constructor (FLAWED) -HazardPointer::HazardPointer(CallTraceHashTable* resource) { - // Step 1: Store pointer - global_hazard_list[_my_slot].pointer = resource; - - // Step 2: Set bitmap bit (SEPARATE ATOMIC OPERATION) - set_bitmap_bit(_my_slot); -} - -// Scanner (VULNERABLE) -void waitForHazardPointersToClear(CallTraceHashTable* table) { - // Step 1: Load bitmap - uint64_t bitmap_word = occupied_bitmap[word_index]; - if (bitmap_word == 0) return; // No occupied slots - - // Step 2: Check pointers in occupied slots - for (each bit set in bitmap_word) { - void* ptr = global_hazard_list[slot].pointer; - if (ptr == table) return false; // Still protected - } -} -``` - -**Race Condition Window:** -``` -Thread T1 (Hazard Constructor): - Step 1: store pointer → global_hazard_list[S].pointer = P ✓ DONE - [PREEMPTION - Scanner runs here] - Step 2: set bitmap bit → occupied_bitmap[W] |= (1 << B) PENDING - -Scanner Thread: - Load bitmap = occupied_bitmap[W] - → Observes bit B is NOT SET (step 2 not executed) - → Skips slot S entirely (not in bitmap) - → Concludes table P is not protected - -Result: UNSAFE - - Scanner falsely believes no thread is protecting table P - - May delete/reuse table P - - Thread T1 will access freed memory after step 2 - - Use-after-free bug -``` - -**Why the bitmap approach fails**: The bitmap and pointer are **two separate memory locations** that cannot be updated atomically together. A thread preemption or cache coherency delay between the two operations creates a window where: -1. Pointer is stored (protection intended) -2. Bitmap bit not yet set (protection not visible to scanner) -3. Scanner sees unset bitmap bit → incorrectly assumes no protection → deletes table -4. Thread resumes and uses freed memory → crash - -**RefCountGuard fixes this** by making the count field serve as **both** the "occupied" indicator and the activation barrier, eliminating the two-location race. - -#### Slot Allocation - -RefCountGuard reuses the proven slot allocation strategy from hazard pointers: - -```cpp -int RefCountGuard::getThreadRefCountSlot() { - int tid = OS::threadId(); // Signal-safe cached thread ID - size_t hash = static_cast(tid) * 0x9E3779B97F4A7C15ULL; - int base_slot = (hash >> 51) % MAX_THREADS; // Upper 13 bits - - // Semi-random prime step probing - int step_index = (hash >> 4) % 16; - int prime_step = PRIME_STEPS[step_index]; // e.g., 1009, 1013, etc. - - for (int i = 0; i < MAX_PROBE_DISTANCE; i++) { - int slot = (base_slot + i * prime_step) % MAX_THREADS; - - int expected = 0; - if (__atomic_compare_exchange_n(&slot_owners[slot], &expected, tid, - false, __ATOMIC_ACQ_REL, __ATOMIC_ACQUIRE)) { - return slot; // Successfully claimed - } - - if (__atomic_load_n(&slot_owners[slot], __ATOMIC_ACQUIRE) == tid) { - return slot; // Already owned (reentrant) - } - } - - return -1; // Slot exhaustion - graceful degradation -} -``` - -#### Performance Characteristics - -**Memory Cost:** -- 512 KB for refcount slots (8192 × 64 bytes) -- 32 KB for slot owners (8192 × 4 bytes) -- **Total: 544 KB** (8 KB less than hazard pointers due to no bitmap) - -**Hot Path Cost:** -- Constructor: 2 atomic stores (pointer + count increment) -- Destructor: 2 atomic stores (count decrement + pointer clear) -- No bitmap operations required - -**Benchmark Results (vs Hazard Pointers):** - -| Workload | HazardPointer | RefCountGuard | Difference | -|----------|---------------|---------------|------------| -| 1 thread baseline | 6,139.0 ops/s | 6,134.7 ops/s | -0.07% | -| 8 threads baseline | 49,039.2 ops/s | 49,034.1 ops/s | -0.01% | -| 32 threads baseline | 95,902.9 ops/s | 95,690.1 ops/s | -0.22% | -| 10 threads churn | 176.2 ops/s | 175.9 ops/s | -0.16% | -| 50 threads churn | 46.6 ops/s | 46.5 ops/s | -0.09% | - -**Statistical Analysis:** -- All differences fall within measurement noise (< 0.25%) -- Confidence intervals overlap significantly (91% for single-thread, 79% for 8-thread) -- No meaningful performance difference across all tested scenarios - -**Conclusion**: RefCountGuard provides **equivalent performance** to hazard pointers while offering **provably correct** memory reclamation through the pointer-first protocol. - -#### Signal Handler Safety - -RefCountGuard is fully signal-handler-safe: -- **No malloc/free**: Uses pre-allocated global array -- **No blocking**: Only uses GCC atomic builtins -- **Bounded execution**: Maximum 32 probing attempts for slot allocation -- **Reentrant-safe**: Thread ID verification prevents slot conflicts -- **OS integration**: Uses `OS::threadId()` which caches the thread ID - -#### Usage in Signal Handlers - -```cpp -u64 CallTraceStorage::put(int num_frames, ASGCT_CallFrame* frames, - bool truncated, u64 weight) { - // Protect active table with thread-local reference counting - RefCountGuard guard(_active_storage); - if (!guard.isActive()) { - return DROPPED_TRACE_ID; // Slot exhaustion - graceful degradation - } - - // Safe to use _active_storage - guard prevents reclamation - CallTraceHashTable* table = _active_storage; - return table->put(num_frames, frames, truncated, weight); - - // Guard destructor automatically releases protection -} -``` - -#### Scanner Integration - -The JFR processing thread (scanner) waits for all reference counts to clear before reclaiming a table: - -```cpp -void CallTraceStorage::processTraces(...) { - // ... rotation logic ... - - // Wait for all signal handlers to finish with old active table - RefCountGuard::waitForRefCountToClear(old_active_table); - - // Safe to clear and reuse the table - old_active_table->clear(); -} -``` - -**Scanner Performance:** -- Linear scan of 8192 slots: ~10-20 microseconds on modern CPUs -- Early exit optimization: stops on first active reference -- Amortized cost: negligible compared to JFR serialization (milliseconds) - -#### Comparison Summary - -| Aspect | HazardPointer | RefCountGuard | Winner | -|--------|---------------|---------------|--------| -| **Correctness** | Race condition in bitmap-pointer gap | Provably race-free (pointer-first protocol) | ✅ RefCountGuard | -| **Performance** | ~6,139 ops/s (1T), ~96K ops/s (32T) | ~6,135 ops/s (1T), ~96K ops/s (32T) | ⚖️ Tie | -| **Memory** | 520 KB (array + bitmap) | 512 KB (array only) | ✅ RefCountGuard | -| **Complexity** | Bitmap + pointer synchronization | Single count field as barrier | ✅ RefCountGuard | -| **Signal Safety** | Yes | Yes | ⚖️ Tie | -| **Graceful Degradation** | Yes | Yes | ⚖️ Tie | - -**Production Status**: RefCountGuard is the **production implementation** due to superior correctness properties with zero performance cost. - -## Thread-Local Collections - -Each thread maintains pre-allocated collections to avoid malloc/free in hot paths: - -``` -Thread A Thread B Thread N -──────── ──────── ──────── -ThreadLocalCollections ThreadLocalCollections ThreadLocalCollections -├─ traces_buffer ├─ traces_buffer ├─ traces_buffer -├─ standby_traces ├─ standby_traces ├─ standby_traces -├─ active_traces ├─ active_traces ├─ active_traces -├─ preserve_set ├─ preserve_set ├─ preserve_set -└─ traces_to_preserve └─ traces_to_preserve └─ traces_to_preserve -``` - -## Liveness Preservation - -The system supports pluggable liveness checkers to determine which traces to preserve: - -```cpp -// Liveness checker interface -typedef std::function&)> LivenessChecker; - -// Example: JFR constant pool preservation -registerLivenessChecker([](std::unordered_set& preserve_set) { - // Add trace IDs that appear in active JFR recordings - preserve_set.insert(active_jfr_traces.begin(), active_jfr_traces.end()); -}); -``` - -## 64-Bit Trace ID Architecture - -The system uses a sophisticated 64-bit trace ID scheme that combines collision avoidance with instance tracking to ensure globally unique, stable trace identifiers across buffer rotations. - -### Trace ID Structure - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ 64-bit Trace ID │ -├──────────────────────────────┬──────────────────────────────────────┤ -│ Upper 32 bits │ Lower 32 bits │ -│ Instance Epoch ID │ Hash Table Slot Index │ -│ │ │ -│ Unique per active rotation │ Position in hash table │ -│ Prevents collision across │ (0 to capacity-1) │ -│ buffer swaps │ │ -└──────────────────────────────┴──────────────────────────────────────┘ -``` - -### Instance Epoch ID Generation - -Each time a `CallTraceHashTable` transitions from STANDBY to ACTIVE during buffer rotation, it receives a new instance epoch ID: - -```cpp -// During rotation - Step 2 -current_standby->clear(); -u64 new_instance_id = getNextInstanceId(); // Atomic increment -current_standby->setInstanceId(new_instance_id); - -// Later during trace creation -u64 trace_id = (instance_id << 32) | slot_index; -``` - -### Collision Prevention Across Rotations - -The instance epoch prevents trace ID collisions when the same hash table slot is reused across different active periods: - -``` -Timeline Example: -───────────────────────────────────────────────────────────────────── - -Rotation 1: Instance ID = 0x00000001 -┌─────────────────┐ -│ ACTIVE Table A │ Slot 100 → Trace ID: 0x0000000100000064 -│ Instance: 001 │ Slot 200 → Trace ID: 0x00000001000000C8 -└─────────────────┘ - -Rotation 2: Instance ID = 0x00000002 -┌─────────────────┐ -│ ACTIVE Table A │ Slot 100 → Trace ID: 0x0000000200000064 -│ Instance: 002 │ Slot 200 → Trace ID: 0x00000002000000C8 -│ (same table, │ -│ different ID) │ -└─────────────────┘ -``` - -### JFR Constant Pool Stability - -The trace ID scheme provides crucial benefits for JFR serialization: - -1. **Stable References**: Trace IDs remain consistent during the active period -2. **Unique Across Cycles**: Even if the same slot is reused, the trace ID differs -3. **Collision Avoidance**: 32-bit instance space prevents ID conflicts -4. **Liveness Tracking**: Preserved traces maintain their original IDs - -### Implementation Details - -```cpp -class CallTraceHashTable { - std::atomic _instance_id; // Set when becoming active - - u64 put(int num_frames, ASGCT_CallFrame* frames, bool truncated, u64 weight) { - // ... hash table logic ... - - // Generate unique trace ID - u64 instance_id = _instance_id.load(std::memory_order_acquire); - u64 trace_id = (instance_id << 32) | slot; - - CallTrace* trace = storeCallTrace(num_frames, frames, truncated, trace_id); - return trace->trace_id; - } -}; -``` - -### Instance ID Generation - -```cpp -class CallTraceStorage { - static std::atomic _next_instance_id; // Global counter - - static u64 getNextInstanceId() { - return _next_instance_id.fetch_add(1, std::memory_order_relaxed); - } - - void processTraces() { - // During rotation - assign new instance ID - u64 new_instance_id = getNextInstanceId(); - current_standby->setInstanceId(new_instance_id); - - // Atomic swap: standby becomes new active with fresh instance ID - _active_storage.exchange(current_standby, std::memory_order_acq_rel); - } -}; -``` - -### Reserved ID Space - -The system reserves trace IDs with upper 32 bits = 0 for special purposes: - -```cpp -// Reserved for dropped samples (contention/allocation failures) -static const u64 DROPPED_TRACE_ID = 1ULL; - -// Real trace IDs always have instance_id >= 1 -// Format: (instance_id << 32) | slot where instance_id starts from 1 -// This guarantees no collision with reserved IDs -``` - -### Benefits of This Architecture - -1. **Collision Immunity**: Same slot across rotations generates different trace IDs -2. **JFR Compatibility**: 64-bit IDs work seamlessly with JFR constant pool indices -3. **Liveness Support**: Preserved traces maintain stable IDs across collection cycles -4. **Debug Capability**: Instance ID in trace ID aids in debugging buffer rotation issues -5. **Scalability**: 32-bit instance space supports ~4 billion rotations before wraparound - -This trace ID design ensures that each call trace has a globally unique, stable identifier that survives the complex buffer rotation lifecycle while providing essential metadata about its origin and timing. - -## Performance Characteristics - -### Lock-Free Operations -- **put()**: O(1) average, lock-free with RefCountGuard protection -- **processTraces()**: Lock-free table swapping, O(n) collection where n = trace count - -### Memory Efficiency -- **Zero-Copy Rotation**: Only atomic pointer swaps, no data copying -- **Pre-allocated Collections**: Thread-local collections prevent malloc/free cycles -- **Trace Deduplication**: Hash tables prevent duplicate trace storage - -### Concurrency Benefits -- **Signal Handler Safe**: No blocking operations in signal context -- **Multi-threaded Collection**: Multiple threads can process traces concurrently -- **Contention-Free**: Atomic operations eliminate lock contention - -## Performance Benchmarks - -### Benchmark Suite - -Located in `ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/`: - -**Note**: These benchmarks test the **complete profiling engine** (signal handlers, stack walking, CallTraceStorage, JFR processing), not just isolated CallTraceStorage operations. - -1. **ProfilerThroughputBaselineBenchmark** - Baseline performance with stable threads -2. **ProfilerThroughputThreadChurnBenchmark** - Short-lived thread churn overhead -3. **ProfilerThroughputSlotExhaustionBenchmark** - High concurrency stress tests -4. **ProfilerThroughputQuickBenchmark** - Fast smoke test (~2 minutes) - -### Running Benchmarks - -**Prerequisites**: All benchmarks require the WhiteboxProfiler to be enabled. - -**Quick smoke test (~2 minutes):** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ProfilerThroughputQuickBenchmark -``` - -**Full baseline (~12 minutes):** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ProfilerThroughputBaselineBenchmark -``` - -**Thread churn benchmarks (~20-30 minutes):** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ProfilerThroughputThreadChurnBenchmark -``` - -**Slot exhaustion benchmarks (~15-20 minutes):** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - ProfilerThroughputSlotExhaustionBenchmark -``` - -**Saving results to JSON:** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - -Pjmh.resultFormat=json \ - -Pjmh.resultFile=build/calltrace-results.json \ - ProfilerThroughputQuickBenchmark -``` - -**Faster iterations for development:** -```bash -./gradlew :ddprof-stresstest:jmh \ - -Pjmh.prof='com.datadoghq.profiler.stresstest.WhiteboxProfiler' \ - -Pjmh.fork=1 -Pjmh.wi=1 -Pjmh.i=2 \ - ProfilerThroughputQuickBenchmark.baseline01Thread -``` - -**Common JMH options:** -- `-Pjmh.fork=N`: Number of JVM forks (default: 3) -- `-Pjmh.wi=N`: Warmup iterations (default: 3) -- `-Pjmh.i=N`: Measurement iterations (default: 3) -- `-Pjmh.wt=N`: Warmup time in seconds (default: 1) -- `-Pjmh.t=N`: Measurement time in seconds (default: 3) -- `-Pjmh.resultFormat=json|csv|text`: Output format -- `-Pjmh.resultFile=path`: Output file path - -### Benchmark Results Summary - -**Platform**: macOS arm64 (Apple Silicon), OpenJDK 21.0.5 - -#### Baseline Scaling Analysis - -| Threads | Score (ops/s) | Efficiency | Assessment | -|---------|---------------|------------|------------| -| 1 | 7,766 | 100% | ✅ Baseline | -| 2 | 15,104 | 97% | ✅ Excellent | -| 4 | 30,311 | 98% | ✅ Excellent | -| 8 | 56,870 | 92% | ✅ Good | -| 16 | 59,900 | 48% | ⚠️ Warning | -| 32 | 60,647 | 24% | ⚠️ Warning | -| 64 | 65,077 | 13% | 🔴 Critical | - -**Key Findings:** -- Near-linear scaling up to 8 threads (92-98% efficiency) -- Performance plateaus at ~60k ops/s beyond 8 threads -- Cache line contention on `slot_owners` and `occupied_bitmap` arrays - -#### Thread Churn Performance - -| Configuration | Score (batches/s) | Threads/sec | Per-Thread Overhead | Assessment | -|---------------|-------------------|-------------|---------------------|------------| -| 10 threads × 5ms | 146 ± 21 | **1,460** | **4.86 ms** | 🔴 Critical | -| 50 threads × 5ms | 33 ± 4 | **1,650** | **4.55 ms** | 🔴 Critical | - -**Key Finding**: Thread creation/destruction overhead is ~4.5-4.9ms per thread, driven by: -1. RefCountGuard lifecycle atomic operations (pointer stores and count updates) -2. Prime probing for slot allocation -3. OS thread creation/destruction overhead - -#### Slot Exhaustion - Wave Pattern - -| Thread Count | Time/Thread | Assessment | -|--------------|-------------|------------| -| 500 | 5.3ms | 🔴 Critical | -| 1000 | 5.2ms | 🔴 Critical | -| 2000 | 3.4ms | 🔴 Critical | -| 4000 | 1.3ms | ⚠️ Better | - -**Key Finding**: Per-thread allocation time improves at higher thread counts (better amortization), indicating prime probing is **not** exhibiting quadratic behavior. - -### Performance Targets - -| Metric | Target | Actual | Status | -|--------|--------|--------|--------| -| 8-thread scaling | >80% | **92%** | ✅ Pass | -| 64-thread scaling | >15% | **13%** | ⚠️ Close | -| Thread creation rate | >10k/s | **1.5k/s** | 🔴 Fail | -| Burst allocation | <200μs | **800-5000μs** | 🔴 Fail | - -## Optimization Recommendations - -### High Priority: Cache Line Alignment - -**Impact**: Fixes 16+ thread scaling issue -**Difficulty**: Low -**Risk**: Low - -```cpp -// Before -int slot_owners[MAX_THREADS]; - -// After -alignas(64) int slot_owners[MAX_THREADS]; - -// Pad bitmap words -struct alignas(64) BitmapWord { - uint64_t value; - char padding[56]; -}; -BitmapWord occupied_bitmap[BITMAP_WORDS]; -``` - -**Expected Improvement:** -- 64-thread efficiency: 13% → 40-50% -- Peak throughput: 65k → 150-200k ops/s - -### Medium Priority: Reduce Atomic Operations - -**Impact**: Reduces per-thread overhead -**Difficulty**: Medium -**Risk**: Medium - -Current: 5 atomic ops per put() - -Options: -1. Batch bitmap updates -2. Use relaxed atomics where safe -3. Remove redundant slot ownership checks - -**Expected Improvement:** -- Per-thread overhead: 4.5ms → 2-3ms -- Thread creation rate: 1.5k/s → 3-5k/s - -### Low Priority: Slot Allocation Hints - -**Impact**: Improves slot reuse locality -**Difficulty**: High -**Risk**: Medium - -Maintain per-CPU "last freed slot" hint for better cache locality. - -## Use Case Guidance - -**Good fit:** -- Applications with 1-8 long-lived worker threads ✅ -- Stable thread pools ✅ -- Server applications with persistent connections ✅ - -**Poor fit:** -- Applications creating >100 threads/second 🔴 -- Thread-per-request architectures with <10ms request times 🔴 -- Virtual thread / fiber-based applications 🔴 - -## Usage Example - -```cpp -// Setup -CallTraceStorage storage; -storage.registerLivenessChecker([](auto& preserve_set) { - // Add traces to preserve -}); - -// Signal handler (lock-free) -u64 trace_id = storage.put(num_frames, frames, truncated, weight); - -// Background processing -storage.processTraces([](const std::unordered_set& traces) { - // Serialize to JFR format - for (CallTrace* trace : traces) { - writeToJFR(trace); - } -}); -``` - -## Key Architectural Benefits - -1. **Scalability**: Lock-free design scales linearly with thread count (up to core count) -2. **Reliability**: RefCountGuard prevents memory safety issues through provably correct reclamation -3. **Flexibility**: Pluggable liveness checkers support different use cases -4. **Performance**: Zero-copy operations minimize overhead -5. **Safety**: Signal-handler safe operations prevent deadlocks - -This architecture enables high-performance, concurrent profiling data collection suitable for production environments with minimal impact on application performance. diff --git a/doc/architecture/NativeMemoryProfiling.md b/doc/architecture/NativeMemoryProfiling.md deleted file mode 100644 index b49daefd3..000000000 --- a/doc/architecture/NativeMemoryProfiling.md +++ /dev/null @@ -1,336 +0,0 @@ -# Native Memory Allocation Profiling - -## Overview - -The native memory profiler tracks heap allocations made through the C standard -library (`malloc`, `calloc`, `realloc`, `posix_memalign`, `aligned_alloc`). It -instruments these functions at the GOT (Global Offset Table) level so that every -intercepted call is accounted for without modifying application source code or -requiring a custom allocator. The `free` function is also hooked (to forward calls -correctly through the GOT) but free events are not recorded. - -Sampled allocation events carry a full Java + native stack trace and are emitted as -`profiler.Malloc` JFR events. - -The feature is activated by passing `nativemem=` to the profiler, where -`` is the byte-sampling interval (e.g. `nativemem=524288` samples roughly -one event per 512 KiB allocated). Passing `nativemem=0` records every allocation. - ---- - -## Component Map - -``` - Application code - │ malloc() / calloc() / realloc() / free() / … - ▼ - ┌─────────────┐ GOT patch ┌──────────────────────────┐ - │ libc / musl│ ◄────────── │ malloc_hook / free_hook │ mallocTracer.cpp - └─────────────┘ │ calloc_hook / … │ - └────────────┬─────────────┘ - │ recordMalloc - ▼ - ┌──────────────────────────┐ - │ MallocTracer:: │ mallocTracer.cpp/h - │ shouldSample() │ - │ recordMalloc() ──────► │ profiler.cpp - └────────────┬─────────────┘ - │ walkVM (CSTACK_VM) - ▼ - ┌──────────────────────────┐ - │ JFR buffer │ flightRecorder.cpp - │ profiler.Malloc │ - └──────────────────────────┘ -``` - ---- - -## GOT Patching - -The profiler redirects allocator calls by writing hook function addresses directly -into the importing library's GOT. This is cheaper than `LD_PRELOAD` (no process -restart) and works for libraries loaded at any time. - -### Import IDs - -`codeCache.h` defines an `ImportId` enum with one entry per hooked symbol: - -``` -im_malloc, im_calloc, im_realloc, im_free, im_posix_memalign, im_aligned_alloc -``` - -`CodeCache::patchImport(ImportId, void*)` walks the library's PLT/GOT and overwrites -the matching entry. - -### Hook signatures - -Each hook calls the saved original function first, then records the event: - -| Hook | Calls | Records | -|------|-------|---------| -| `malloc_hook(size)` | `_orig_malloc(size)` | `recordMalloc(ret, size)` if `ret != NULL && size != 0` | -| `calloc_hook(num, size)` | `_orig_calloc(num, size)` | `recordMalloc(ret, total)` if `ret != NULL && num != 0 && size != 0` (total = num×size, clamped to `SIZE_MAX` on overflow) | -| `realloc_hook(addr, size)` | `_orig_realloc(addr, size)` | `recordMalloc(ret, size)` if `ret != NULL && size > 0` | -| `free_hook(addr)` | `_orig_free(addr)` | — (forwards only) | -| `posix_memalign_hook(…)` | `_orig_posix_memalign(…)` | `recordMalloc(*memptr, size)` if `ret == 0 && memptr != NULL && *memptr != NULL && size != 0` | -| `aligned_alloc_hook(align, size)` | `_orig_aligned_alloc(align, size)` | `recordMalloc(ret, size)` if `ret != NULL && size != 0` | - ---- - -## Initialization Sequence - -`MallocTracer::start()` (called once per profiler session) runs: - -1. Resets per-session counters (`_interval`, `_bytes_until_sample`, `_sample_count`, - `_last_config_update_ts`). - -2. On the **first call only** (`!_initialized`), calls `initialize()`: - - a. **`resolveMallocSymbols()`** — calls each intercepted function at least once so - the profiler library's own PLT stubs are resolved by the dynamic linker. This - ensures that subsequent `SAVE_IMPORT` reads get the real libc function pointers - rather than the PLT resolver. - - b. **`SAVE_IMPORT(func)`** — reads the resolved GOT entry for each symbol from the - profiler library's own import table and stores it in the corresponding - `_orig_` static pointer. - - c. **`detectNestedMalloc()`** — probes whether the platform's `calloc` - implementation calls `malloc` internally (as musl does), and whether - `posix_memalign` calls `aligned_alloc` internally. If either is detected, the - corresponding hook is replaced with a dummy variant (`calloc_hook_dummy` or - `posix_memalign_hook_dummy`) that forwards to the original without recording, - preventing double-accounting. The dummy hooks preserve the caller frame pointer - so that the actual call site is not obscured. - - d. **`lib->mark(...)`** — marks the profiler's own hook functions in the code cache - so the stack walker can identify them as profiler frames. - - Then sets `_initialized = true`. - -3. **`patchLibraries()`** — iterates over all currently loaded native libraries and - writes the hook addresses into each library's GOT, under `_patch_lock`. - `_patched_libs` is a monotonic counter so that already-patched libraries are - skipped on subsequent calls. - -4. Sets `_running = true` to enable recording. - -`patchLibraries()` is called again on every `start()` to pick up any libraries -loaded between profiler sessions. - ---- - -## Dynamic Library Handling - -When the application calls `dlopen`, the profiler's `dlopen_hook` (installed as a -GOT hook for `dlopen`) calls `MallocTracer::installHooks()` after the library is -loaded: - -```cpp -// profiler.cpp -void* Profiler::dlopen_hook(const char* filename, int flags) { - void* result = dlopen(filename, flags); - if (result != NULL) { - Libraries::instance()->updateSymbols(false); - MallocTracer::installHooks(); - } - return result; -} -``` - -`installHooks()` calls `patchLibraries()` only if `_running` is `true`, so newly -loaded libraries are automatically hooked without requiring a profiler restart. - ---- - -## Sampling - -Allocation recording uses Poisson-interval sampling via `MallocTracer::shouldSample()`: - -```cpp -// mallocTracer.cpp — lock-free CAS loop with Poisson jitter -static bool shouldSample(size_t size) { - if (_interval <= 1) return true; // nativemem=0 or nativemem=1: record every allocation - while (true) { - u64 prev = _bytes_until_sample; - if (size < prev) { - if (__sync_bool_compare_and_swap(&_bytes_until_sample, prev, prev - size)) - return false; - } else { - u64 next = nextPoissonInterval(); - if (__sync_bool_compare_and_swap(&_bytes_until_sample, prev, next)) - return true; - } - } -} -``` - -`_bytes_until_sample` is a shared volatile counter decremented by each allocation's -size. When exhausted, a new Poisson-distributed interval is generated via -`nextPoissonInterval()` (using `-interval * ln(uniform_random)` where the random -value is derived from TSC ticks via XOR-shift), providing random jitter that avoids -synchronization artifacts. Multiple threads compete via CAS so no mutex is needed. - -A PID controller (`updateConfiguration()`) periodically adjusts `_interval` to -maintain approximately `TARGET_SAMPLES_PER_WINDOW` (100) samples per second. - ---- - -## Stack Trace Capture - -### Why `CSTACK_VM` is needed - -The malloc hooks execute on the calling thread with no signal context (`ucontext == -NULL`). Two distinct levels of stack capture are possible: - -- **Java-only stacks** (`CSTACK_DEFAULT`, `CSTACK_FP`, `CSTACK_DWARF`): Java frames - are still available via ASGCT / `JavaFrameAnchor`. When `ucontext == NULL`, the - profiler falls through to ASGCT so these modes do produce Java-level traces for - malloc events. - -- **Interleaved native + Java stacks** (`CSTACK_VM` only): Native frame unwinding - via frame pointers or DWARF requires a signal context as the starting point. - `CSTACK_VM` avoids this by seeding the unwind from `callerPC()` (no signal context - needed) and transitioning to Java frames via HotSpot's `JavaFrameAnchor`. - -`CSTACK_VM` starts from `callerPC()` (which expands to `__builtin_return_address(0)` -on x86/x86_64/aarch64) for the initial frame and uses HotSpot's `JavaFrameAnchor` -(lastJavaPC / lastJavaSP / lastJavaFP) to transition from native to Java frames. -This works correctly from inside a malloc hook because the anchor is set whenever -the JVM has transitioned from Java to native. - -### Default stack mode - -`CSTACK_DEFAULT` is the initial default (`arguments.h`). At profiler start, -`profiler.cpp` promotes it to `CSTACK_VM` when VMStructs are available **and the OS -is Linux**. If neither condition is met, it falls back to `CSTACK_DWARF` (if -supported) or `CSTACK_NO`: - -```cpp -if (_cstack == CSTACK_DEFAULT) { - if (VMStructs::hasStackStructs() && OS::isLinux()) { - _cstack = CSTACK_VM; - } else if (DWARF_SUPPORTED) { - _cstack = CSTACK_DWARF; - } -} -``` - -If `CSTACK_VM` is explicitly requested but `VMStructs` are not available, the -profiler resets to `CSTACK_DWARF` (if supported) or `CSTACK_NO` and logs an error: - -```cpp -} else if (_cstack == CSTACK_VM) { - if (!VMStructs::hasStackStructs()) { - _cstack = DWARF_SUPPORTED ? CSTACK_DWARF : CSTACK_NO; - Log::error("VMStructs stack walking is not supported on this JVM/platform, defaulting to the default native call stack unwinding mode."); - } -} -``` - -### Code path for malloc stack walking - -`recordSample` in `profiler.cpp` calls `getNativeTrace()` first. For -`_cstack >= CSTACK_VM`, `getNativeTrace` returns 0 immediately (native frames are -not collected via `walkFP`/`walkDwarf`). Then `JVMSupport::walkJavaStack()` is -called, which dispatches to `HotspotSupport::walkJavaStack()`: - -```cpp -// hotspot/hotspotSupport.cpp — walkJavaStack for malloc events -} else if (request.event_type == BCI_CPU || request.event_type == BCI_WALL || request.event_type == BCI_NATIVE_MALLOC) { - if (cstack >= CSTACK_VM) { - java_frames = walkVM(ucontext, frames, max_depth, features, - eventTypeFromBCI(request.event_type), - lock_index, truncated); - } - // ... -} -``` - -`HotspotSupport::walkVM` is the sole source of both native and Java frames for -malloc events. When called with `ucontext == NULL` (as it is for malloc hooks), -it seeds the unwind with `callerPC()` / `callerSP()` / `callerFP()`. - ---- - -## JFR Event Format - -A single event type is defined in `jfrMetadata.cpp` under the -`Java Virtual Machine / Native Memory` category: - -### `profiler.Malloc` (`T_MALLOC`) - -| Field | Type | Description | -|-------|------|-------------| -| `startTime` | `long` (ticks) | TSC timestamp of the allocation | -| `eventThread` | thread ref | Thread that performed the allocation | -| `stackTrace` | stack trace ref | Call stack at the allocation site | -| `address` | `long` (address) | Returned pointer value | -| `size` | `long` (bytes) | Requested allocation size | -| `weight` | `float` | Statistical sample weight based on Poisson sampling probability | -| `spanId` | `long` | Span ID from current context (optional, from context attributes) | -| `localRootSpanId` | `long` | Local root span ID from current context (optional, from context attributes) | - -Events are written by `Recording::recordMallocSample()` in `flightRecorder.cpp`: - -```cpp -buf->putVar64(T_MALLOC); -buf->putVar64(event->_start_time); -buf->putVar32(tid); -buf->putVar64(call_trace_id); -buf->putVar64(event->_address); -buf->putVar64(event->_size); -buf->putFloat(event->_weight); -writeCurrentContext(buf); -``` - ---- - -## Concurrency and Thread Safety - -| Concern | Mechanism | -|---------|-----------| -| GOT patching across threads | `_patch_lock` (Mutex) in `patchLibraries()` | -| Library unload during patching | `UnloadProtection` handle per library | -| Allocation byte counter | Lock-free CAS loop in `shouldSample` | -| JFR buffer writes | Per-lock-index try-lock with 3 attempts; events dropped on contention | -| Hook enable / disable | `volatile bool _running` — checked before every recording call | -| `_initialized` write ordering | Serialized by the profiler's outer state lock (caller responsibility) | - ---- - -## Known Limitations and Design Trade-offs - -**No reentrancy guard.** As documented in `mallocTracer.cpp`: - -> To avoid complexity in hooking and tracking reentrancy, a TLS-based approach is -> not used. Reentrant allocation calls would result in double-accounting. - -When `recordMalloc` calls into the profiler (stack walking, JFR buffer writes), any -allocations made by the profiler itself will re-enter the hooks. Infinite recursion -is prevented because the hook functions call `_orig_malloc` (a saved direct function -pointer) instead of going through the GOT, but profiler-internal allocations may be -double-counted as application allocations. -Leak detection is unaffected: the same address being recorded multiple times is -handled correctly by the tracking logic. - -**Hooks are never uninstalled.** `stop()` only sets `_running = false`. The GOT -entries remain patched for the lifetime of the process. After stopping, every -malloc/free incurs the overhead of one function-pointer indirection plus a volatile -bool read, which is negligible in practice. Uninstalling hooks safely would require -iterating all libraries again under `_patch_lock`, which is deferred. - -**`nativemem=0` records every allocation.** When `_interval == 0`, -`shouldSample` returns `true` on every call (the `interval <= 1` fast path). This -is intentional for 100% sampling but can produce very high event volumes. - -**No free event tracking.** Free calls are hooked (to forward through the GOT -correctly) but not recorded. Sampled mallocs mean most frees would match nothing, -and the immense event volume with no stack traces provides no actionable insight. - -**HotSpot / Linux only for interleaved native+Java stack traces.** `CSTACK_VM` -requires `VMStructs::hasStackStructs() && OS::isLinux()`, which is only true on -HotSpot JVMs on Linux. On other platforms the profiler falls back to `CSTACK_DWARF` -(if supported) or `CSTACK_DEFAULT`. Native frames are still captured via FP/DWARF -unwinding and Java frames via ASGCT, but they are not interleaved through -`JavaFrameAnchor` as they are with `CSTACK_VM`. diff --git a/doc/architecture/RefCountGuard.md b/doc/architecture/RefCountGuard.md deleted file mode 100644 index 29352ee0e..000000000 --- a/doc/architecture/RefCountGuard.md +++ /dev/null @@ -1,212 +0,0 @@ -# RefCountGuard Protocol - -`RefCountGuard` is a generic, lock-free, RAII reference-counting primitive used -to safely reclaim heap-allocated resources that may be accessed concurrently -from signal handlers. Used by `StringDictionary` (buffer rotation) and -`CallTraceHashTable` (table rotation). - -The protocol has three layers: -1. **Slot acquisition** — find a per-thread slot via prime-probing hash. -2. **Activation** — publish the protected pointer; increment the reference count. -3. **Drain** — `waitForRefCountToClear(p)` blocks until no slot references `p`. - -Reentrant signal delivery is handled by parking displaced pointers in a -fixed-size `outer_stack[OUTER_STACK_DEPTH]` array on the slot, so the drain -scanner can see every resource currently in use, not just the innermost one. - ---- - -## Slot layout (one cache line) - -```mermaid -flowchart LR - subgraph slot ["RefCountSlot — one cache line, 64 bytes"] - c["count: uint32_t (4B)"] - g["4-byte gap (alignof void*)"] - a["active_ptr: void* (8B)"] - o0["outer_stack[0] (8B)"] - o1["outer_stack[1] (8B)"] - o2["outer_stack[2] (8B)"] - p["padding[24]"] - c --- g --- a --- o0 --- o1 --- o2 --- p - end -``` - -`active_ptr` is `alignas(alignof(void*))`, which forces a 4-byte gap after the -`uint32_t count` field on 64-bit targets. The trailing `padding[]` is sized -by `DEFAULT_CACHE_LINE_SIZE - alignof(void*) - (1 + OUTER_STACK_DEPTH) * sizeof(void*)` -so the layout fits exactly one cache line; the `RefCountSlot` default ctor's -`static_assert(sizeof(RefCountSlot) == DEFAULT_CACHE_LINE_SIZE, ...)` catches -any drift at compile time. - -`active_ptr` is the resource the *innermost* guard is protecting. -`outer_stack[i]` is the resource that was displaced when reentrant level i+1 -fired (level 1 displaces to slot 0, level 2 to slot 1, ...). - ---- - -## Construction — non-reentrant (root) case - -```mermaid -sequenceDiagram - participant Caller - participant Slot as "RefCountSlot" - participant Scanner as "waitForRefCountToClear scanner" - Caller->>Slot: "store active_ptr := resource (release)" - Caller->>Slot: "count += 1 (release)" - Note right of Scanner: "Scanner sees count > 0 and active_ptr == resource" -``` - -`active_ptr` is stored **before** `count++` so the scanner never sees a stale -pointer for a live slot. - -### Slot exhaustion - -`getThreadRefCountSlot()` walks at most `MAX_PROBE_DISTANCE = 32` probe steps; -if none are free and none belong to the current tid with a live outer guard, -it returns `-1` and the ctor sets `_active = false`. An inactive guard offers -**no protection** — the resource is invisible to the drain. Callers that -require strict protection must check `isActive()`. With `MAX_THREADS = 8192` -prime-probed by tid, exhaustion is effectively unreachable under normal -operation. - ---- - -## Construction — reentrant case - -A signal handler fires while an outer guard is still live on the same thread. -The slot probe returns `slot + MAX_THREADS` to flag reentrancy — but **only** -when the matched slot still has `count > 0`. If the outer guard has already -decremented `count` to 0 (the brief window between `count--` and -`slot_owners[i] := 0` in the non-reentrant dtor), the probe falls through to -search for a fresh slot instead, so the new ctor cannot publish an `active_ptr` -that the outer dtor is about to overwrite with null. - -```mermaid -sequenceDiagram - participant Inner as "Inner guard ctor" - participant Slot as "RefCountSlot" - Inner->>Slot: "load _saved_ptr := active_ptr (acquire)" - Inner->>Slot: "prev_count := count.fetch_add(1, release)" - Note right of Inner: "prev_count is the reentrancy depth this guard occupies" - alt "prev_count - 1 fits in OUTER_STACK_DEPTH" - Inner->>Slot: "store outer_stack[prev_count - 1] := _saved_ptr (release)" - else "depth exceeds OUTER_STACK_DEPTH" - Inner->>Slot: "Log warn once per process; saved pointer invisible to scanner" - end - Inner->>Slot: "store active_ptr := resource (release)" -``` - -After step 3 (or step 4 on overflow), the slot contains: -- `count == prev_count + 1` -- `active_ptr == resource` (inner resource) -- `outer_stack[0..min(prev_count, OUTER_STACK_DEPTH)-1]` filled; deeper levels - live only in the per-guard `_saved_ptr` and trigger the one-time overflow - warning latched by `s_outer_stack_overflow_warned`. - -The scanner walks `active_ptr` plus every entry of `outer_stack[]` and reports -the slot as matching if any of them equals the resource being drained. The -target must be non-null: `nullptr` is the sentinel for "unused" outer-stack -slot, so `waitForRefCountToClear(nullptr)` is not a supported call. - ---- - -## Destruction — reentrant case - -Reverse of construction. Restoring `active_ptr` *before* clearing -`outer_stack[_outer_slot]` *before* `count--` keeps the invariant -**"scanner sees the outer resource while count > 0"**. - -```mermaid -sequenceDiagram - participant Inner as "Inner guard dtor" - participant Slot as "RefCountSlot" - Inner->>Slot: "store active_ptr := _saved_ptr (release)" - opt "this guard parked a slot" - Inner->>Slot: "store outer_stack[_outer_slot] := nullptr (release)" - end - Inner->>Slot: "count -= 1 (release)" -``` - -Destruction — non-reentrant (root) case: the order is inverted — -`count--` first, then `active_ptr := nullptr`, then `slot_owners[slot] := 0`. -Scanner skips a slot whose `count == 0`, so a null `active_ptr` during the -deactivation window is never observed by a live drain. - ---- - -## Worked example — depth-3 nesting - -L0 = JNI lookup on resource `R0`. -L1 = signal handler on the same thread, lookup on resource `R1`. -L2 = nested signal (e.g. SIGSEGV crash handler during L1), lookup on `R2`. - -```mermaid -sequenceDiagram - participant L0 as "L0 (JNI)" - participant L1 as "L1 (signal)" - participant L2 as "L2 (nested signal)" - participant Slot - L0->>Slot: "store active_ptr := R0; count := 1" - Note right of Slot: "active=R0, outer_stack=[null,null,null], count=1" - L1->>Slot: "saved := R0; count := 2; outer_stack[0] := R0; active := R1" - Note right of Slot: "active=R1, outer_stack=[R0,null,null], count=2" - L2->>Slot: "saved := R1; count := 3; outer_stack[1] := R1; active := R2" - Note right of Slot: "active=R2, outer_stack=[R0,R1,null], count=3" - Note over Slot: "Scanner waiting for R1 sees outer_stack[1] = R1 and stays blocked" - L2->>Slot: "active := R1; outer_stack[1] := null; count := 2" - Note right of Slot: "active=R1, outer_stack=[R0,null,null], count=2" - L1->>Slot: "active := R0; outer_stack[0] := null; count := 1" - L0->>Slot: "count := 0; active := null; slot_owners := 0" -``` - -The pre-fix protocol (`outer_ptr` single pointer with `_set_outer_ptr` -discriminator) wrote `R0` to `outer_ptr` at L1 and refused to overwrite it -at L2, so `R1` was held only on L2's stack frame and invisible to the -scanner. A `waitForRefCountToClear(R1)` running during L2 would return -prematurely. With `outer_stack[OUTER_STACK_DEPTH]` every displaced resource -gets its own slot. - ---- - -## Drain — `waitForRefCountToClear(p)` - -```mermaid -flowchart TD - start[start] --> spin{"spun < SPIN_ITERATIONS?"} - spin -->|"yes"| scan["for i in 0..MAX_THREADS: if count[i] > 0 and slot references p, mark not-clear"] - scan --> chk{"any slot references p?"} - chk -->|"no"| done[return] - chk -->|"yes"| pause["spinPause; spun++"] - pause --> spin - spin -->|"no"| sleep_loop["nanosleep 100us; same scan, up to ~500ms"] - sleep_loop --> timeout{"timed out?"} - timeout -->|"no"| done - timeout -->|"yes"| warn["Counters DICTIONARY_DRAIN_TIMEOUTS; Log warn; abort under DEBUG"] - warn --> done -``` - -A slot is considered to reference `p` if `active_ptr == p` or any entry of -`outer_stack[]` equals `p`. Unused outer-stack entries are `nullptr` and -therefore never match a (non-null) drain target. - ---- - -## Invariants - -| Invariant | Enforced by | -|-----------|-------------| -| "Scanner never sees a stale `active_ptr` for a live slot" | `active_ptr` stored before `count++` (root); `active_ptr` restored before `count--` (reentrant) | -| "Every displaced resource on a reentrant chain is visible to the scanner up to OUTER_STACK_DEPTH" | `outer_stack[prev_count-1]` write in ctor; conditional clear in dtor | -| "No deadlock between drain and signal handler" | All scans are bounded; timeout is observable via counter and DEBUG abort | -| "Slot can be reclaimed after drain returns" | `slot_owners[i] = 0` is written only by the non-reentrant teardown path — destructor or move-assignment overwriting an active guard — and only after `count` has been decremented to 0 | -| "Nesting beyond OUTER_STACK_DEPTH is observable" | One-time `Log::warn` latched via `s_outer_stack_overflow_warned` | - ---- - -## Files - -- `ddprof-lib/src/main/cpp/refCountGuard.h` — class declaration, `RefCountSlot` layout, `OUTER_STACK_DEPTH`. -- `ddprof-lib/src/main/cpp/refCountGuard.cpp` — implementation, drain loop, overflow warning. -- `ddprof-lib/src/main/cpp/stringDictionary.h` — primary user; see [StringDictionary](StringDictionary.md). -- `ddprof-lib/src/main/cpp/callTraceHashTable.h` — secondary user. diff --git a/doc/architecture/SigsegvPatching.md b/doc/architecture/SigsegvPatching.md deleted file mode 100644 index dc05f34d9..000000000 --- a/doc/architecture/SigsegvPatching.md +++ /dev/null @@ -1,118 +0,0 @@ -# SIGSEGV Handler Protection via sigaction Interposition - -## Problem - -Some native libraries install SIGSEGV/SIGBUS signal handlers that violate POSIX async-signal-safety requirements. A notable example is **wasmtime**, whose signal handler calls `__tls_get_addr` for TLS access, which in turn may call `malloc()`. - -When the profiler uses `safefetch` (safe memory access via intentional SIGSEGV), the following deadlock can occur: - -1. Application code holds malloc's internal lock -2. Profiler's signal handler runs and calls `safefetch` -3. `safefetch` triggers SIGSEGV -4. Wasmtime's handler (installed on top of ours) runs first -5. Wasmtime's handler calls `__tls_get_addr` → `malloc()` -6. `malloc()` tries to acquire its lock → **deadlock** - -## Solution - -The profiler intercepts `sigaction()` calls via GOT (Global Offset Table) patching. When libraries try to install signal handlers, we: - -1. Store their handler as a "chain target" -2. Keep our handler installed -3. Call their handler from within ours (after our logic completes) - -This ensures: -- Our handler always runs first -- We control when/if the chained handler is invoked -- Problematic handlers never become the "top" handler - -## Implementation - -### Components - -1. **`os_linux.cpp`** - Signal handler protection logic: - - `protectSignalHandlers()` - Registers our handlers for protection - - `sigaction_hook()` - Intercepts sigaction calls, stores chain targets - - `getSegvChainTarget()` / `getBusChainTarget()` - Returns current chain target - -2. **`libraryPatcher_linux.cpp`** - GOT patching: - - `patch_sigaction_in_library()` - Patches sigaction GOT entry in a library - - `patch_sigaction()` - Iterates all libraries and patches them - -3. **`codeCache.cpp`** - Import tracking: - - Tracks `sigaction` imports in loaded libraries (via `im_sigaction`) - -4. **`profiler.cpp`** - Integration: - - `setupSignalHandlers()` - Installs handlers and patches already-loaded libs - - `dlopen_hook()` - Patches newly loaded libraries - - `switchLibraryTrap()` - Enables/disables dlopen hook - -### Initialization Flow - -``` -JVM Initialization - └── VM::ready() - └── Profiler::setupSignalHandlers() - ├── Install SIGSEGV/SIGBUS handlers - ├── OS::protectSignalHandlers() - Mark handlers as protected - └── LibraryPatcher::patch_sigaction() - Patch already-loaded libs - -Profiling Start - └── Profiler::start() - └── switchLibraryTrap(true) - Enable dlopen hook - -Library Load (via dlopen) - └── dlopen_hook() - └── LibraryPatcher::patch_sigaction() - Patch newly loaded libs -``` - -### Signal Handler Chain - -``` -SIGSEGV occurs - └── Profiler::segvHandler() - ├── Handle profiler-related faults (safefetch, etc.) - └── If not handled: call OS::getSegvChainTarget() - └── Invoke chained handler (e.g., wasmtime's, ASAN's) -``` - -## Scope - -All native libraries are patched, including: -- Application libraries (e.g., wasmtime) -- Sanitizer runtime libraries (libasan, libtsan, libubsan) - -This provides defense-in-depth against any library that might install a SIGSEGV/SIGBUS handler. Sanitizer libraries are intentionally patched so our handler can intercept recoverable SIGSEGVs (e.g., from `safefetch`) while still chaining to the sanitizer's handler for unexpected crashes. - -**Exclusions:** -- The profiler's own library -- Only SA_SIGINFO handlers (3-arg form) are intercepted for safe chaining - -## Counters - -Two counters track sigaction patching activity: -- `SIGACTION_PATCHED_LIBS` - Number of libraries where sigaction GOT was patched -- `SIGACTION_INTERCEPTED` - Number of sigaction calls intercepted (handler installations prevented) - -## Why GOT Patching? - -Alternative approaches considered: - -1. **LD_PRELOAD** - Requires modifying JVM launch, not always possible -2. **Rebinding after load** - Libraries install handlers lazily, timing is unreliable -3. **Disabling safefetch** - Would disable core profiler functionality - -GOT patching allows us to intercept function calls from specific libraries without affecting the rest of the process. - -## Thread Safety - -- `_segv_chain_target` / `_bus_chain_target` use atomic operations -- `LibraryPatcher::patch_sigaction()` uses a spinlock -- Signal handlers are async-signal-safe (no allocations) - -## Limitations - -1. Only works on Linux (uses ELF GOT patching) -2. Requires the library to call `sigaction()` via PLT (not inline) -3. Library must be dynamically linked -4. Only SA_SIGINFO (3-arg) handlers are chained; 1-arg handlers pass through diff --git a/doc/architecture/StringDictionary.md b/doc/architecture/StringDictionary.md deleted file mode 100644 index e72035640..000000000 --- a/doc/architecture/StringDictionary.md +++ /dev/null @@ -1,264 +0,0 @@ -# StringDictionary Concurrency Model - -## Overview - -`StringDictionary` is a triple-buffered, lock-free string-to-integer dictionary used to -assign stable JFR constant-pool IDs to class names, endpoint labels, and context values. -Three instances live in `Profiler`: `_class_map`, `_string_label_map`, and -`_context_value_map`. - -Its concurrency model has two orthogonal mechanisms that address two distinct problems: - -| Mechanism | Problem solved | -|-----------|---------------| -| `_accepting` + `RefCountGuard` | Buffer reset safety: no reader is mid-table when `clearAll()` zeroes the root slots | -| `SignalBlocker` | Rotation window safety: no profiling signal fires on the dump thread between Phase 1 and Phase 2 of `rotate()` | - -These are independent. Neither implies the other. - -### Key string ownership — the arena - -Each `StringDictionaryBuffer` owns a `StringArena`: a lock-free bump allocator backed by a -single `malloc`'d block. All key strings are allocated from this arena instead of -individual `malloc` calls. Overflow `SBTable` chain nodes remain heap-allocated. - -Consequences: - -- **`clear()` is O(number-of-overflow-nodes)** rather than O(number-of-entries). The arena - is reset with a single atomic store; no per-key `free()` is needed. -- **The TOCTOU gap between the `_accepting` acquire-load and `RefCountGuard::count++` is - benign.** Even if `clearAll()`'s drain misses a racing caller on a weakly-ordered CPU, - that caller reads from the memset-zeroed root table and returns 0 — the arena memory - is still valid, just logically reclaimed. No UAF is possible. The seq_cst recheck - that previously closed this window has therefore been removed from the hot path. -- **Arena capacity is sized per dictionary** (configured in `Profiler`): - `_class_map` 4 MB per buffer (class names accumulate across rotations); - `_string_label_map` and `_context_value_map` 512 KB per buffer (bounded by `size_limit`). - On exhaustion `insert_with_id` returns 0, the same behaviour as a `malloc` failure. - ---- - -## Buffer Roles - -The three backing buffers (`_a`, `_b`, `_c`) cycle through three roles: - -``` - ┌───────────┐ ┌───────────┐ ┌───────────┐ - │ ACTIVE │ │ DUMP │ │ SCRATCH │ - │ │ │ │ │ │ -Writes → │ new IDs │ │ stable │ │ two │ - │ from all │ │ snapshot │ │ rotations │ - │ callers │ │ for JFR │ │ behind │ - └───────────┘ └───────────┘ └───────────┘ - ↑ rotate() ↑ ↑ - becomes DUMP becomes SCRATCH becomes ACTIVE -``` - -`_rot.active()` — current write target -`_rot.dumpBuffer()` — the buffer handed to `writeCpool()` after `rotate()` -`_rot.clearTarget()` — the scratch buffer (two rotations behind) - ---- - -## Caller Map - -Different callers reach the dictionary under different locking contexts: - -``` - ┌────────────────────────────────────────┐ - │ StringDictionary │ - │ │ - Signal handler ───────┼──→ bounded_lookup(key, len) │ - (SIGPROF/SIGVTALRM) │ read-only, signal-safe │ - │ │ - JNI: recordTrace0 ────┼──→ bounded_lookup(key, len, limit) │ - JNI: registerConst0 ──┼──→ bounded_lookup(key, len, limit) │ - (no shard lock held) │ insert-capable, NOT signal-safe │ - │ │ - JNI: lookupClass ─────┼──→ lookup(key, len) │ - (no shard lock held) │ insert + malloc, NOT signal-safe │ - │ │ - Dump thread ──────────┼──→ lookupDuringDump(key, len) │ - (inside jfr_op, │ reads dump then active; may insert │ - lockAll() held) │ NOT signal-safe, single-threaded │ - │ │ - Profiler::start() ────┼──→ clearAll() │ - (no lock held) │ resets all three buffers │ - └────────────────────────────────────────┘ -``` - -**Key point**: `lockAll()` gates `CallTraceStorage` writers, not dictionary writers. -`recordTrace0` and `registerConstant0` reach `bounded_lookup` before any shard lock -(`_locks[]`) is acquired. `lookupDuringDump` runs inside `jfr_op()` which is called -while `lockAll()` is held, but only because the dump as a whole needs that exclusion — -the dictionary itself does not require it. - ---- - -## RefCountGuard Protocol - -Every `lookup` / `bounded_lookup` call that intends to read or write the active buffer -wraps the access in a `RefCountGuard`: - -``` -Caller clearAll() / rotate() -────── ───────────────────── - -1. load _accepting (acquire) - → false? return 0 - -2. active = _rot.active() - -3. RefCountGuard guard(active) - → store active_ptr (release) - → count++ (release) ← scanner sees this - -4. guard.isActive()? → probe - slot failure? return 0 - -5. _rot.active() == active? - → changed? continue loop - -6. ... read/write buffer ... - -7. ~RefCountGuard() - → count-- (release) - → clear active_ptr (release) - - waitForAllRefCountsToClear(): - scan all slots; block until - every count == 0 or active_ptr - does not match target buffer - - → then arena.reset() + memset root table -``` - -### Why there is no seq_cst recheck - -On weakly-ordered CPUs (ARM64) there is a TOCTOU window between step 1 and step 3: - -``` -Thread A (caller) Thread B (clearAll) -───────────────── ─────────────────── -load _accepting → true (not yet) -active = _rot.active() - _accepting.store(false, seq_cst) - waitForAllRefCountsToClear() - → sees count = 0 for A (count++ not yet visible) - → returns - arena.reset() + memset root table -RefCountGuard guard(...) -count++ (release) -active->lookup(...) - → reads zeroed root table slot → null key → returns 0 -``` - -With the arena, Thread A's read after the memset lands on the zeroed root table and -returns 0 — no UAF because the arena memory is still physically valid. A seq_cst -recheck after step 3 would close the window more tightly (guaranteeing the drain sees -the guard), but at the cost of a barrier on every signal-handler lookup. Since -`clearAll()` is called only at profiler restart (virtually never in production), the -benign-miss trade-off is correct: the recheck was removed. - ---- - -## clearAll() Protocol - -``` -clearAll() - 1. _accepting.store(false, seq_cst) - ↳ subsequent lookup() / bounded_lookup() callers fail their - _accepting acquire-load and return 0 immediately - 2. RefCountGuard::waitForAllRefCountsToClear() - ↳ drains every caller already past the _accepting load. The - drain is best-effort: a racing caller may slip through, but - it then reads from the memset-zeroed root table (step 3) and - returns 0 — no UAF because the arena memory is still valid - (see "Why there is no seq_cst recheck" above). - 3. _a.clear(); _b.clear(); _c.clear() - ↳ freeTable() on each buffer — safe because no guard is live - 4. _rot.reset() - 5. _next_id.store(1, relaxed) - 6. reset counters - 7. _accepting.store(true, release) - ↳ callers can create new guards again -``` - -`clearAll()` is self-contained: no external lock is required. `Profiler::start()` -calls it without `lockAll()`. - ---- - -## rotate() Protocol - -`rotate()` is called inside `rotateDictsAndRun()` under a `SignalBlocker` but -**before** `lockAll()`. It is safe without an external lock. - -``` -rotate() [SignalBlocker active, no external mutex] - -Phase 1: - old_active = _rot.active() - _rot.clearTarget()->copyFrom(*old_active) - ↳ pre-populate the future active buffer with all current entries - _rot.rotate() - ↳ old_active becomes the dump buffer; clearTarget becomes new active - -Drain: - RefCountGuard::waitForRefCountToClear(old_active) - ↳ wait for any JNI thread still holding a guard on old_active - (signal handlers on THIS thread cannot fire: SignalBlocker) - -Phase 2: - _rot.active()->copyFrom(*old_active) - ↳ copy any entries inserted into old_active between Phase 1 and the drain - (late inserts from other threads are captured here) -``` - -`SignalBlocker` is needed to bound the Phase 1→Phase 2 window: without it, a -profiling signal on the dump thread could keep inserting into `old_active` and -defer `waitForRefCountToClear` indefinitely. It does not provide protection for -JNI threads — those are handled by the RefCountGuard drain. - ---- - -## rotateDictsAndRun() Decomposition - -``` -rotateDictsAndRun(jfr_op): - - SignalBlocker blocker ← blocks SIGPROF/SIGVTALRM on THIS thread - - _class_map.rotate() ┐ - _string_label_map.rotate() ├ self-contained; no lockAll() needed - _context_value_map.rotate() ┘ - - lockAll() ← gates CallTraceStorage writers - jfr_op() ← writeCpool() reads dump buffers; - lookupDuringDump() may insert - unlockAll() - - _class_map.clearStandby() ┐ - _string_label_map.clearStandby()├ clears scratch; resets per-dump counters - _context_value_map.clearStandby()┘ -``` - -`rotate()` and `lockAll()` are deliberately separated: - -- `rotate()` needs `SignalBlocker` (to bound the drain window on this thread) but - not `lockAll()`. -- `jfr_op()` needs `lockAll()` (to exclude concurrent `CallTraceStorage` writes) - but rotation is already complete before it runs. - ---- - -## Invariants Summary - -| Invariant | Enforced by | -|-----------|-------------| -| No UAF during `clearAll()` reset | Arena keeps key memory valid; drain ensures no reader is mid-table when memset runs | -| No entry lost during `rotate()` | Two-phase copy + `waitForRefCountToClear(old_active)` drains late JNI insertors | -| No profiling signal inserts into `old_active` between Phase 1 and 2 (dump thread) | `SignalBlocker` in `rotateDictsAndRun()` | -| `writeCpool()` sees a stable dump snapshot | `rotate()` completes (including drain) before `jfr_op()` starts | -| `CallTraceStorage` writers excluded during dump | `lockAll()` around `jfr_op()` | -| Dictionary writers NOT excluded during dump | By design: `rotate()`'s two-phase copy absorbs concurrent inserts | diff --git a/doc/architecture/TLSContext.md b/doc/architecture/TLSContext.md deleted file mode 100644 index 0a17829dc..000000000 --- a/doc/architecture/TLSContext.md +++ /dev/null @@ -1,511 +0,0 @@ -# Thread-Local Context Architecture - -## Overview - -The Thread-Local Context (TLS Context) system provides a high-performance, -signal-handler-safe mechanism for capturing distributed tracing context -(trace IDs, span IDs, and custom attributes) during -profiling events. It enables the profiler to correlate performance samples -with active traces. - -The system uses OTEL profiling signal conventions -([OTEP #4947](https://github.com/open-telemetry/oteps/pull/4947)) as its -sole context storage format. Java code writes tracing context into -thread-local `DirectByteBuffer`s mapped to native structs. Two consumer -paths read this context concurrently: - -1. **DD signal handler (SIGPROF)** — reads integer tag encodings and - root-span ID from the sidecar buffer, and span ID from the OTEL record - (ignores trace ID) -2. **External OTEP-compliant profilers** — discover the - `otel_thread_ctx_v1` TLS symbol via ELF dynsym and read - the `OtelThreadContextRecord` directly. - -All writes from Java are zero-JNI on the hot path (cache-hit case), -using `DirectByteBuffer` with explicit memory ordering. A detach/attach -publication protocol ensures readers see either a complete old record or -a complete new record, never a torn intermediate state. - -For benchmark data, see -[ThreadContext Benchmark Report](../performance/reports/thread-context-benchmark-2026-03-21.md). - -## Core Design Principles - -1. **Zero-Copy Shared Memory** — Java writes to `DirectByteBuffer`s - mapped to native `ProfiledThread` fields; no data copying between - Java and native heaps. -2. **Signal Handler Safety** — all signal-handler reads use lock-free - atomic loads with acquire semantics; no allocation, no locks, no - syscalls. -3. **Detach/Attach Publication Protocol** — the `valid` flag is cleared - before mutation and set after, with `storeFence` barriers between - steps. The TLS pointer is set permanently at thread init. -4. **Two-Phase Attribute Registration** — string attribute values are - registered in the native Dictionary once via JNI; subsequent uses - are zero-JNI ByteBuffer writes from a per-thread encoding cache. -5. **Platform Independence** — correct on both strong (x86/TSO) and - weak (ARM) memory models via explicit `storeFence` / volatile write - barriers. -6. **Low Overhead** — typical span lifecycle write ~30 ns, sidecar - encoding read ~2 ns (no syscalls, no locks). - -## Architecture - -### High-Level Data Flow - -``` -┌─────────────────────────────────────────────────────────────────────┐ -│ Application Thread │ -├─────────────────────────────────────────────────────────────────────┤ -│ │ -│ Tracer calls ThreadContext.put(lrs, spanId, trHi, trLo) │ -│ │ │ -│ ▼ │ -│ ┌───────────────────────────────────────────────────────────────┐ │ -│ │ setContextDirect() │ │ -│ │ 1. detach() — valid ← 0, storeFence │ │ -│ │ 2. ctxBuffer.putLong(traceIdOffset, reverseBytes(trHi)) │ │ -│ │ ctxBuffer.putLong(traceIdOffset+8, reverseBytes(trLo)) │ │ -│ │ ctxBuffer.putLong(spanIdOffset, reverseBytes(spanId)) │ │ -│ │ 3. tag_encodings[0..9] ← 0 │ │ -│ │ attrs_data_size ← LRS_ENTRY_SIZE (keeps fixed LRS at [0]) │ │ -│ │ 4. ctxBuffer.putLong(lrsOffset, lrs) │ │ -│ │ writeLrsHex(lrs) — update fixed LRS entry in attrs_data │ │ -│ │ 5. attach() — storeFence, valid ← 1 │ │ -│ └───────────────────────────────────────────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌──────────────────────────────────────────────────────────────┐ │ -│ │ Unified ctxBuffer (688B, single DirectByteBuffer) │ │ -│ │ ┌──────────────────────┐ ┌───────────────────────────┐ │ │ -│ │ │ OtelThreadContextRec │ │ tag_encodings[10] (u32) │ │ │ -│ │ │ trace_id[16] (BE) │ │ local_root_span_id (u64) │ │ │ -│ │ │ span_id[8] (BE) │ └───────────────────────────┘ │ │ -│ │ │ valid (u8)│ offsets 640..688 in ctxBuffer │ │ -│ │ │ reserved (u8)│ │ │ -│ │ │ attrs_data_size(u16)│ ┌──────────────────────────────┐ │ │ -│ │ │ attrs_data[612] │ │ TLS pointer (8B) │ │ │ -│ │ └──────────────────────┘ │ otel_thread_ctx_v1 │ │ │ -│ │ offsets 0..640 │ (thread_local, DLLEXPORT) │ │ │ -│ └──────────────────────────────────────────────────────────────┘ │ -│ ▲ ▲ │ -│ │ │ │ -│ DD signal handler External OTEP │ -│ reads span_id profiler reads │ -│ from record full record via │ -│ TLS symbol │ -└─────────────────────────────────────────────────────────────────────┘ -``` - -### Component Architecture - -``` -┌──────────────────────────────────────────────────────────────────────┐ -│ Java Layer │ -├──────────────────────────────────────────────────────────────────────┤ -│ │ -│ JavaProfiler │ -│ ├─ ThreadLocal tlsContextStorage │ -│ ├─ initializeContextTLS0(long[] metadata) → ByteBuffer (688B) │ -│ └─ registerConstant0(String value) → int encoding │ -│ │ -│ ThreadContext (per thread) │ -│ ├─ ctxBuffer (688B DirectByteBuffer — record + sidecar contiguous)│ -│ ├─ put(lrs, spanId, trHi, trLo) → setContextDirect() │ -│ ├─ setContextAttribute(keyIdx, value) → setContextAttributeDirect │ -│ ├─ snapshot(byte[], int) / restore(byte[], int) ← nested scopes │ -│ └─ Per-thread caches: │ -│ └─ attrCache[CACHE_SIZE]: String → {int encoding, byte[] utf8}│ -│ │ -│ BufferWriter (memory ordering abstraction) │ -│ ├─ BufferWriter8 (Java 8: Unsafe) │ -│ │ ├─ putOrderedLong / putOrderedInt │ -│ │ └─ storeFence → Unsafe.storeFence() │ -│ └─ BufferWriter9 (Java 9+: VarHandle) │ -│ ├─ setRelease │ -│ └─ storeFence → VarHandle.storeStoreFence() │ -└──────────────────────────────────────────────────────────────────────┘ - -┌─────────────────────────────────────────────────────────────────────┐ -│ Native Layer │ -├─────────────────────────────────────────────────────────────────────┤ -│ │ -│ ProfiledThread (per thread, heap-allocated) │ -│ ├─ OtelThreadContextRecord _otel_ctx_record │ -│ ├─ alignas(8) u32 _otel_tag_encodings[DD_TAGS_CAPACITY] │ -│ ├─ u64 _otel_local_root_span_id │ -│ └─ bool _otel_ctx_initialized │ -│ │ -│ otel_thread_ctx_v1 (thread_local, DLLEXPORT) │ -│ └─ OTEP #4947 TLS pointer for external profiler discovery │ -│ │ -│ Recording::writeCurrentContext(Buffer*) (signal handler) │ -│ ├─ ContextApi::get(spanId, rootSpanId) │ -│ │ └─ acquire load of valid flag, big-endian decode of span_id │ -│ └─ thrd->getOtelTagEncoding(i) for each attribute │ -└─────────────────────────────────────────────────────────────────────┘ -``` - -## Memory Layout - -### OtelThreadContextRecord - -The OTEP #4947 record is a packed struct embedded in each `ProfiledThread`: - -``` -Offset Size Field Description -────────────────────────────────────────────────────────────────── -0x00 16 trace_id 128-bit W3C trace ID (big-endian) -0x10 8 span_id 64-bit span ID (big-endian) -0x18 1 valid 1 = record is consistent, 0 = in-progress -0x19 1 _reserved Reserved (must be 0) -0x1A 2 attrs_data_size Size of attrs_data in bytes (LE uint16) -0x1C 612 attrs_data TLV-encoded key/value attribute entries -────────────────────────────────────────────────────────────────── -Total: 640 bytes (OTEL_MAX_RECORD_SIZE) -``` - -### Sidecar Buffer - -The sidecar is a contiguous, 8-byte-aligned region in `ProfiledThread` -that the DD signal handler reads directly: - -``` -Offset Size Field Description -────────────────────────────────────────────────────────────────── -0x00 40 _otel_tag_encodings[10] Dictionary encoding per attribute (u32) -0x28 8 _otel_local_root_span_id Local root span ID (u64) -────────────────────────────────────────────────────────────────── -Total: 48 bytes -``` - -The sidecar fields are exposed to Java as a single `DirectByteBuffer`. -Tag encodings are integer IDs from the profiler's `Dictionary` constant -pool — the signal handler writes them directly into JFR events without -any string lookup. - -### attrs_data TLV Encoding - -Each entry in `attrs_data` is encoded as: - -``` -┌──────────────┬──────────────┬───────────────────────┐ -│ key_index(1) │ value_len(1) │ value_utf8[value_len] │ -└──────────────┴──────────────┴───────────────────────┘ -``` - -- `key_index` 0 is reserved for the local root span ID (16-char zero-padded lowercase hex string, always fixed at attrs_data[0..17]). -- `key_index` 1..N correspond to user-registered attributes offset by 1. - -## The Detach/Attach Publication Protocol - -### Problem - -Two concurrent readers may observe the record at any point during a -Java-side mutation: - -1. **SIGPROF signal handler** — interrupts the writing thread - mid-sequence, runs on the same thread. -2. **External OTEP profiler** — reads from a different thread via the - `otel_thread_ctx_v1` TLS pointer. - -Both must see either a complete old state or a complete new state, never -a partially-written record. - -### Protocol - -``` -Java writer timeline: -────────────────────────────────────────────────────────────────── -Time 0: detach() - ctxBuffer.put(validOffset, 0) ← mark invalid - storeFence() ← drain store buffer - -Time 1: Mutate record fields - ctxBuffer.putLong(traceIdOffset, ...) - ctxBuffer.putLong(spanIdOffset, ...) - tag_encodings[0..9] ← 0 ← zero tag encodings (offsets 640..680) - attrs_data_size ← LRS_ENTRY_SIZE ← keep only fixed LRS entry at attrs_data[0] - ctxBuffer.putLong(lrsOffset, lrs) ← update LRS at offset 680 - writeLrsHex(lrs) ← update LRS hex entry in attrs_data - - ⚡ SIGPROF may arrive here — handler sees valid=0, skips record - -Time 2: attach() - storeFence() ← ensure writes visible - ctxBuffer.put(validOffset, 1) ← mark valid -────────────────────────────────────────────────────────────────── -``` - -### Reader: DD Signal Handler - -```cpp -// flightRecorder.cpp — Recording::writeCurrentContext() -void Recording::writeCurrentContext(Buffer *buf) { - u64 spanId = 0, rootSpanId = 0; - bool hasContext = ContextApi::get(spanId, rootSpanId); // acquire-loads valid flag - buf->putVar64(spanId); - buf->putVar64(rootSpanId); - - size_t numAttrs = Profiler::instance()->numContextAttributes(); - ProfiledThread* thrd = hasContext ? ProfiledThread::currentSignalSafe() : nullptr; - for (size_t i = 0; i < numAttrs; i++) { - buf->putVar32(thrd != nullptr ? thrd->getOtelTagEncoding(i) : 0); - } -} -``` - -`ContextApi::get()` performs (context_api.cpp): - -```cpp -OtelThreadContextRecord* record = thrd->getOtelContextRecord(); -if (__atomic_load_n(&record->valid, __ATOMIC_ACQUIRE) != 1) { - return false; // record is being mutated — emit zeros -} -u64 val = 0; -for (int i = 0; i < 8; i++) { val = (val << 8) | record->span_id[i]; } -span_id = val; -``` - -The acquire fence pairs with the writer's `storeFence` + `valid=1` -sequence, ensuring all record field writes are visible if `valid` reads -as 1. - -### Reader: External OTEP Profiler - -External profilers follow the OTEP #4947 protocol: - -1. Discover `otel_thread_ctx_v1` via ELF `dlsym`. -2. Read the `OtelThreadContextRecord*` pointer. The pointer is set - permanently at thread init; detach/attach never modify it. It is nulled - on thread exit to prevent use-after-recycle — check for null before - dereferencing. -3. Check `valid == 1`. If not, the record is being updated — skip. -4. Read `trace_id`, `span_id`, `attrs_data` from the record. - -## Memory Ordering - -### Why Barriers Are Needed - -Both consumers need to see field writes ordered before `valid=1`, but for -different reasons: - -- The **signal handler** runs on the same thread as the writer. The CPU presents - its own stores in program order, so CPU store-buffer reordering is not a - concern. The JIT compiler can still reorder stores arbitrarily, so a compiler - barrier is required. -- The **external OTEP profiler** (e.g. eBPF using scheduler events) attaches a - `sched_switch` tracepoint that fires on the same CPU that was executing the - thread. The Linux scheduler acquires `rq_lock` before the tracepoint fires, - which includes a full hardware memory barrier (`smp_mb__before_spinlock` on - ARM). By the time the eBPF probe runs, all prior user-space stores from that - thread are globally visible — including any stores ordered by `DMB ISHST`. - -In both cases `storeFence` serves as a compiler barrier that prevents the JIT -from sinking record field writes past the `valid=1` store. On ARM it also emits -`DMB ISHST`, which is required to order field writes before `valid=1` at the -hardware level — this is not a mere side effect. - -### Barrier Taxonomy - -| Operation | Java 8 | Java 9+ | ARM | x86 | -|-----------|--------|---------|-----|-----| -| `storeFence` | `Unsafe.storeFence` | `VarHandle.storeStoreFence` | DMB ISHST (~2 ns) | compiler barrier (free) | - -On x86, `storeFence` is a compiler-only barrier (TSO guarantees hardware -store ordering for free; Java 9+ `VarHandle.storeStoreFence` emits no -hardware instruction on x86). On ARM it compiles to `DMB ISHST` (~2 ns). - -### Why storeFence, Not fullFence - -The detach/attach protocol only requires store-store ordering — all -operations on the hot path are writes. There are no load-dependent -ordering requirements on the writer side. `storeFence` (~2 ns on ARM) -is sufficient; a full fence (~50 ns on ARM) would be wasteful. - -## Initialization - -### Per-Thread TLS Setup - -When a thread first accesses its `ThreadContext` via the `ThreadLocal`: - -```java -// JavaProfiler.initializeThreadContext() -long[] metadata = new long[6]; -ByteBuffer buffer = initializeContextTLS0(metadata); -return new ThreadContext(buffer, metadata); -``` - -The native `initializeContextTLS0` (in `javaApi.cpp`): - -1. Gets the calling thread's `ProfiledThread` (creates one if needed). -2. Sets `otel_thread_ctx_v1` permanently to the thread's - `OtelThreadContextRecord` (triggering TLS slot init on musl). -3. Fills the `metadata` array with absolute offsets into the unified - buffer (computed via `offsetof` for record fields; `OTEL_MAX_RECORD_SIZE - + DD_TAGS_CAPACITY*sizeof(u32) = 680` for the LRS offset), so Java code - writes to the correct positions regardless of struct packing changes. -4. Creates a single `DirectByteBuffer` spanning the contiguous 688-byte - region: `_otel_ctx_record` (640 B) followed immediately by - `_otel_tag_encodings` (40 B) and `_otel_local_root_span_id` (8 B). - Contiguity is enforced by `alignas(8)` on `_otel_ctx_record` plus - `sizeof(OtelThreadContextRecord)` being a multiple of 8. -5. Returns the single buffer. - -This is the only JNI call in the initialization path. After this, all -hot-path operations are pure Java ByteBuffer writes into offset regions -of the one buffer. - -### Signal-Safe TLS Access - -Signal handlers cannot call `initializeContextTLS0` (it may allocate). The -read path uses a pre-initialized pointer: - -```cpp -// ProfiledThread::currentSignalSafe() — no allocation, no TLS lazy init -ProfiledThread* thrd = ProfiledThread::currentSignalSafe(); -if (thrd == nullptr || !thrd->isContextInitialized()) { - return false; // emit zeros -} -``` - -## Two-Phase Attribute Registration - -String attributes are set via `ThreadContext.setContextAttribute(keyIndex, value)`. -The hot path avoids JNI by splitting the work into two phases: - -### Phase 1: Registration (cache miss) - -On the first call with a new string value: - -```java -encoding = registerConstant0(value); // JNI → Dictionary lookup -utf8 = value.getBytes(UTF_8); // one allocation, cached -attrCacheEncodings[slot] = encoding; -attrCacheBytes[slot] = utf8; -attrCacheKeys[slot] = value; // cache is per-thread; no fence needed -``` - -`registerConstant0` crosses JNI once to register the value in the -native `Dictionary` and returns an integer encoding. - -### Phase 2: Cached Write (cache hit, zero JNI) - -On subsequent calls with the same string: - -```java -if (value.equals(attrCacheKeys[slot])) { - encoding = attrCacheEncodings[slot]; // int read - utf8 = attrCacheBytes[slot]; // ref read -} -// Both sidecar and OTEP attrs_data are written inside the detach/attach window -// so a signal handler never sees a new sidecar encoding alongside old attrs_data. -detach(); -ctxBuffer.putInt(TAG_ENCODINGS_OFFSET + keyIndex * 4, encoding); -replaceOtepAttribute(otepKeyIndex, utf8); -attach(); -``` - -The cache is a 256-slot direct-mapped structure keyed by -`value.hashCode() & 0xFF`. Collisions evict the old entry (benign — -causes a redundant `registerConstant0` call). In production web -applications with 5–50 unique attribute values, the hit rate is -effectively 100%. - -## Signal Handler Read Path - -`Recording::writeCurrentContext()` executes in the SIGPROF handler and -reads context in bounded time (~15 ns) with no allocation: - -1. `ContextApi::get(spanId, rootSpanId)`: - - `ProfiledThread::currentSignalSafe()` — cached pointer, no TLS - lazy init. - - `__atomic_load_n(&record->valid, __ATOMIC_ACQUIRE)` — if 0, emit - zeros (record is being mutated). - - Read `span_id` from `OtelThreadContextRecord`. - - Read `_otel_local_root_span_id` from sidecar. -2. For each registered attribute: - - `thrd->getOtelTagEncoding(i)` — direct u32 read from sidecar. - - Only read when `hasContext` is true; emits 0 otherwise, so tag - encodings are never emitted alongside a zero span ID. - -No dictionary lookup, no string comparison, no allocation. The -encodings written to JFR events are resolved later during JFR parsing. - -## Performance - -### Write Path Costs (arm64, Java 25) - -| Operation | ns/op | Path | -|-----------|------:|------| -| `clearContext` | 5.0 | detach + zero fields | -| `setContextFull` | 11.1 | detach + 3 putLong + attach | -| `setAttrCacheHit` | 10.7 | cache lookup + sidecar write + detach/attach | -| `spanLifecycle` | 30.4 | `setContextFull` + `setAttrCacheHit` | - -### Multi-Threaded Scaling - -| Benchmark | 1 thread | 2 threads | 4 threads | -|-----------|----------|-----------|-----------| -| `setContextFull` | 11.1 ns | 11.1 ns | 11.7 ns | -| `spanLifecycle` | 30.4 ns | 30.7 ns | 32.2 ns | - -No false sharing: each thread's `OtelThreadContextRecord` and sidecar -are embedded in its own heap-allocated `ProfiledThread`. - -### Instrumentation Budget - -At ~35 ns per span (`spanLifecycle` 30.4 ns + `clearContext` 5.0 ns), a single thread -can sustain ~28 million span transitions per second. For a web -application at 100K requests/second, this is <0.004% of CPU time. - -Full benchmark data and analysis: -[thread-context-benchmark-2026-03-21.md](../performance/reports/thread-context-benchmark-2026-03-21.md) - -## Testing - -### Integration Tests (Java) - -`ddprof-test/src/test/java/com/datadoghq/profiler/context/OtelContextStorageModeTest.java`: - -- **testOtelStorageModeContext** — context round-trips correctly with JFR running. -- **testOtelModeCustomAttributes** — verifies attribute TLV encoding in - `attrs_data` via `setContextAttribute`. -- **testOtelModeAttributeOverflow** — overflow of `attrs_data` is handled - gracefully (returns false, no crash). -- **testSequentialContextUpdates** — repeated writes with varying values, - `Long.MAX_VALUE` round-trip, and `clearContext` resetting both IDs to zero. -- **testThreadIsolation** — concurrent writes from multiple threads, - validating thread-local isolation. -- **testSpanTransitionClearsAttributes** — direct span-to-span transition - without `clearContext` does not leak custom attributes from the previous span. - -`ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/ContextWallClockTest.java`: - -- **test** — validates context propagation through wall-clock profiling - samples and JFR event correlation across cstack modes. - -`ddprof-test/src/test/java/com/datadoghq/profiler/context/TagContextTest.java`: - -- **test** — validates integer tag/attribute context propagation through - profiling samples. - -### JMH Benchmarks - -`ddprof-stresstest/src/jmh/java/com/datadoghq/profiler/stresstest/scenarios/throughput/ThreadContextBenchmark.java`: - -- Single-threaded: `setContextFull`, `setAttrCacheHit`, `spanLifecycle`, - `clearContext`, `getContext`. -- Multi-threaded: `setContextFull_2t/4t`, `spanLifecycle_2t/4t` — - `@Threads(2)` and `@Threads(4)` variants to verify linear scaling - (absence of false sharing). - -## OTEP References - -- [OTEP #4947 — Profiling Signal Conventions](https://github.com/open-telemetry/oteps/pull/4947): - Defines the `otel_thread_ctx_v1` TLS symbol, the - `OtelThreadContextRecord` struct layout, and the publication protocol - (valid flag + TLS pointer atomics). -- [OpenTelemetry Profiling SIG](https://github.com/open-telemetry/opentelemetry-specification/tree/main/specification/profiles): - Broader context for profiling signal integration in the OTel - ecosystem. diff --git a/doc/build/BuildSystemGuide.md b/doc/build/BuildSystemGuide.md deleted file mode 100644 index ffbc373d8..000000000 --- a/doc/build/BuildSystemGuide.md +++ /dev/null @@ -1,466 +0,0 @@ -# Build System Maintenance Guide - -This document provides detailed guidance for maintaining and extending the Gradle build system. - -## Convention Plugins Overview - -All custom build logic lives in `build-logic/conventions/`. The plugins are: - -| Plugin ID | Class | Purpose | -|-----------|-------|---------| -| `com.datadoghq.native-build` | `NativeBuildPlugin` | Multi-config C++ compilation | -| `com.datadoghq.native-root` | `RootProjectPlugin` | Root-level config discovery | -| `com.datadoghq.gtest` | `GtestPlugin` | Google Test integration | -| `com.datadoghq.simple-native-lib` | `SimpleNativeLibPlugin` | Simple single-library builds | -| `com.datadoghq.profiler-test` | `ProfilerTestPlugin` | Multi-config Java test generation | -| `com.datadoghq.java-conventions` | `JavaConventionsPlugin` | Java 8 compilation settings | -| `com.datadoghq.versioned-sources` | `VersionedSourcesPlugin` | Java version-specific code | -| `com.datadoghq.fuzz-targets` | `FuzzTargetsPlugin` | libFuzzer integration | -| `com.datadoghq.scanbuild` | `ScanBuildPlugin` | Clang static analysis | -| `com.datadoghq.spotless-convention` | `SpotlessConventionPlugin` | Code formatting | - -## Key Files and Their Purposes - -``` -build-logic/conventions/src/main/kotlin/com/datadoghq/ -├── native/ -│ ├── NativeBuildPlugin.kt # Creates compile/link tasks per config -│ ├── NativeBuildExtension.kt # DSL: nativeBuild { ... } -│ ├── RootProjectPlugin.kt # Provides configs to all subprojects -│ ├── SimpleNativeLibPlugin.kt # Simplified single-lib builds -│ ├── config/ -│ │ └── ConfigurationPresets.kt # Defines release/debug/asan/tsan/fuzzer -│ ├── model/ -│ │ ├── BuildConfiguration.kt # Config model (platform, arch, flags) -│ │ ├── Platform.kt # LINUX, MACOS enum -│ │ └── Architecture.kt # X64, ARM64, etc. -│ ├── tasks/ -│ │ ├── NativeCompileTask.kt # C++ compilation task -│ │ ├── NativeLinkTask.kt # Shared library linking -│ │ └── NativeLinkExecutableTask.kt # Executable linking -│ ├── util/ -│ │ └── PlatformUtils.kt # Platform detection, compiler finding -│ ├── gtest/ -│ │ ├── GtestPlugin.kt # Google Test task generation -│ │ ├── GtestExtension.kt # DSL: gtest { ... } -│ │ └── GtestTaskBuilder.kt # Per-test task creation -│ └── fuzz/ -│ └── FuzzTargetsPlugin.kt # Fuzz testing support -├── profiler/ -│ ├── ProfilerTestPlugin.kt # Multi-config test generation -│ ├── JavaConventionsPlugin.kt # Java 8 --release flag -│ └── SpotlessConventionPlugin.kt # Code formatting -└── java/versionedsources/ - └── VersionedSourcesPlugin.kt # Java 9+ version-specific code -``` - -## How Build Configurations Work - -Build configurations (release, debug, asan, tsan, fuzzer) are defined in `ConfigurationPresets.kt`: - -1. **Registration**: Configs are registered in `NativeBuildExtension.buildConfigurations` -2. **Activation**: Each config checks if it's available on current platform - - release/debug: Always active - - asan: Active if `libasan.so` found via `gcc -print-file-name=libasan.so` - - tsan: Active if `libtsan.so` found - - fuzzer: Active if clang supports `-fsanitize=fuzzer` -3. **Discovery**: Subprojects query active configs via `nativeBuild.buildConfigurations.names` - -### Configuration Flow Diagram - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ settings.gradle.kts │ -│ includeBuild("build-logic") ← loads convention plugins │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ build.gradle.kts (root) │ -│ plugins { id("com.datadoghq.native-root") } │ -│ ↓ │ -│ RootProjectPlugin creates nativeBuild extension │ -│ ↓ │ -│ afterEvaluate: ConfigurationPresets.setupStandardConfigurations│ -│ → Registers: release, debug, asan, tsan, fuzzer │ -│ → Checks availability via PlatformUtils │ -└─────────────────────────────────────────────────────────────────┘ - │ - ▼ -┌─────────────────────────────────────────────────────────────────┐ -│ ddprof-lib/build.gradle.kts │ -│ plugins { id("com.datadoghq.native-build") } │ -│ ↓ │ -│ afterEvaluate: queries nativeBuild.buildConfigurations.names │ -│ → Creates tasks: compileRelease, linkRelease, assembleRelease │ -│ → Creates tasks: compileDebug, linkDebug, assembleDebug │ -│ → (only for active configs on current platform) │ -└─────────────────────────────────────────────────────────────────┘ -``` - -## Adding a New Build Configuration - -To add a new configuration (e.g., "coverage"): - -### Step 1: Add to ConfigurationPresets.kt - -```kotlin -// In setupStandardConfigurations() -extension.buildConfigurations.apply { - // ... existing configs ... - register("coverage") { - configureCoverage(this, currentPlatform, currentArch, version, rootDir) - } -} - -// Add configuration function -fun configureCoverage( - config: BuildConfiguration, - platform: Platform, - architecture: Architecture, - version: String, - rootDir: File -) { - config.platform.set(platform) - config.architecture.set(architecture) - config.active.set(PlatformUtils.hasGcov()) // Add detection in PlatformUtils - - when (platform) { - Platform.LINUX -> { - config.compilerArgs.set( - listOf("-O0", "-g", "--coverage", "-fprofile-arcs", "-ftest-coverage") + - commonLinuxCompilerArgs(version) - ) - config.linkerArgs.set(commonLinuxLinkerArgs() + listOf("--coverage")) - } - Platform.MACOS -> { - config.compilerArgs.set( - listOf("-O0", "-g", "--coverage") + commonMacosCompilerArgs(version) - ) - config.linkerArgs.set(listOf("--coverage")) - } - } -} -``` - -### Step 2: Add Detection to PlatformUtils.kt (if needed) - -```kotlin -fun hasGcov(): Boolean { - return isCompilerAvailable("gcov") -} -``` - -### Step 3: No Changes Needed in Build Scripts - -Configurations are discovered dynamically - all subprojects automatically pick up the new config. - -## Modifying Compiler/Linker Flags - -Flags are centralized in `ConfigurationPresets.kt`: - -| Function | Purpose | -|----------|---------| -| `commonLinuxCompilerArgs()` | Shared C++ flags for all Linux configs | -| `commonLinuxLinkerArgs()` | Shared linker flags for Linux | -| `commonMacosCompilerArgs()` | Shared C++ flags for macOS | -| `configureRelease()` | Release-specific flags | -| `configureDebug()` | Debug-specific flags | -| `configureAsan()` | AddressSanitizer flags | -| `configureTsan()` | ThreadSanitizer flags | -| `configureFuzzer()` | libFuzzer flags | - -### Example: Adding a Flag to All Configs - -```kotlin -private fun commonLinuxCompilerArgs(version: String): List { - return mutableListOf( - "-fPIC", - "-fno-omit-frame-pointer", - "-momit-leaf-frame-pointer", - "-fvisibility=hidden", - "-fdata-sections", - "-ffunction-sections", - "-std=c++17", - // Add new flag here: - "-fstack-protector-strong", - "-DPROFILER_VERSION=\"$version\"", - "-DCOUNTERS" - ) -} -``` - -### Example: Adding a Flag to One Config - -```kotlin -fun configureRelease(...) { - when (platform) { - Platform.LINUX -> { - config.compilerArgs.set( - listOf("-O3", "-DNDEBUG", "-g", "-flto") + // Added -flto - commonLinuxCompilerArgs(version) - ) - } - } -} -``` - -## Creating a New Convention Plugin - -### Step 1: Create the Plugin Class - -Create in `build-logic/conventions/src/main/kotlin/com/datadoghq/example/`: - -```kotlin -package com.datadoghq.example - -import org.gradle.api.Plugin -import org.gradle.api.Project -import org.gradle.api.provider.Property -import javax.inject.Inject - -class MyPlugin : Plugin { - override fun apply(project: Project) { - // Create extension for DSL configuration - val extension = project.extensions.create( - "myPlugin", - MyExtension::class.java, - project - ) - - // Create configurations eagerly (so build scripts can reference them) - val myConfig = project.configurations.create("myConfiguration") - - // Configure tasks in afterEvaluate (when extension values are set) - project.afterEvaluate { - if (extension.enabled.get()) { - createTasks(project, extension) - } - } - } - - private fun createTasks(project: Project, extension: MyExtension) { - project.tasks.register("myTask") { - group = "my-group" - description = "Does something useful" - doLast { - println("Setting: ${extension.someSetting.get()}") - } - } - } -} - -abstract class MyExtension @Inject constructor(project: Project) { - val enabled: Property = project.objects.property(Boolean::class.java) - val someSetting: Property = project.objects.property(String::class.java) - - init { - enabled.convention(true) - someSetting.convention("default-value") - } -} -``` - -### Step 2: Register in build.gradle.kts - -Add to `build-logic/conventions/build.gradle.kts`: - -```kotlin -gradlePlugin { - plugins { - // ... existing plugins ... - create("myPlugin") { - id = "com.datadoghq.my-plugin" - implementationClass = "com.datadoghq.example.MyPlugin" - } - } -} -``` - -### Step 3: Use in Subproject - -```kotlin -plugins { - id("com.datadoghq.my-plugin") -} - -myPlugin { - enabled.set(true) - someSetting.set("custom-value") -} -``` - -## Best Practices for Plugin Development - -### Prefer Lazy Configuration - -```kotlin -// GOOD - lazy task registration -project.tasks.register("myTask", MyTask::class.java) { - inputFile.set(extension.someFile) // Evaluated when task executes -} - -// AVOID - eager task creation -project.tasks.create("myTask", MyTask::class.java) { - inputFile.set(extension.someFile) // Evaluated immediately -} -``` - -### Create Configurations Eagerly, Populate Lazily - -```kotlin -override fun apply(project: Project) { - // Create configurations immediately (so build scripts can use them) - val myConfig = project.configurations.create("myConfiguration").apply { - isCanBeConsumed = true - isCanBeResolved = true - } - - // Populate dependencies in afterEvaluate - project.afterEvaluate { - myConfig.dependencies.add( - project.dependencies.create("com.example:lib:1.0") - ) - } -} -``` - -### Use Lazy Task References - -```kotlin -// GOOD - lazy reference, doesn't force task creation -val linkTask = tasks.named("linkRelease") -myTask.dependsOn(linkTask) - -// AVOID - forces immediate task creation/lookup -val linkTask = tasks.getByName("linkRelease") -``` - -### Check Task Existence Safely - -```kotlin -// GOOD - safe check -if (tasks.names.contains("linkRelease")) { - dependsOn("linkRelease") -} - -// AVOID - throws exception if not found -dependsOn(tasks.getByName("linkRelease")) -``` - -## Testing Build Changes - -After modifying build-logic: - -```bash -# Kill Gradle daemon to pick up plugin changes -./gradlew --stop - -# Verify configuration loads without errors -./.claude/commands/build-and-summarize tasks --group=build -q - -# Verify Java compilation -./.claude/commands/build-and-summarize :ddprof-lib:compileJava :ddprof-test:compileJava -Pskip-native - -# Verify native compilation (on Linux) -./.claude/commands/build-and-summarize assembleRelease - -# Verify tests can be configured -./.claude/commands/build-and-summarize :ddprof-test:testRelease -Pskip-native - -# Verify all tasks are generated correctly -./.claude/commands/build-and-summarize :ddprof-lib:tasks --all -``` - -## Gradle Properties Reference - -All configurable properties (see `gradle.properties.template` for full documentation): - -| Property | Default | Description | -|----------|---------|-------------| -| `skip-tests` | false | Skip Java test execution | -| `skip-native` | false | Skip C++ compilation | -| `skip-gtest` | false | Skip Google Test execution | -| `skip-fuzz` | false | Skip fuzz testing | -| `native.forceCompiler` | auto | Force specific C++ compiler path | -| `ddprof_version` | from build.gradle.kts | Override project version | -| `with-libs` | - | Path to pre-built native libraries | -| `keepJFRs` | false | Keep JFR recordings after tests | -| `validatorArgs` | - | Arguments for UnwindingValidator | -| `CI` | from env | CI environment flag | -| `forceLocal` | false | Use local Nexus for testing | - -## Troubleshooting - -### "Configuration not found" - -Configurations are created in `afterEvaluate`. Ensure you're accessing them in `afterEvaluate` or later: - -```kotlin -// WRONG - runs during configuration phase -val cfg = configurations.getByName("testReleaseImplementation") // May not exist yet - -// RIGHT - runs after all afterEvaluate blocks -project.afterEvaluate { - val cfg = configurations.getByName("testReleaseImplementation") -} - -// ALSO RIGHT - use gradle.projectsEvaluated for cross-project -gradle.projectsEvaluated { - val cfg = configurations.getByName("testReleaseImplementation") -} -``` - -### "Task not found" - -Tasks are created dynamically based on active configs. Verify: - -1. The config is active: Check `PlatformUtils` detection methods -2. The plugin is applied: Check `plugins { }` block -3. You're in the right phase: Use `afterEvaluate` or `gradle.projectsEvaluated` - -### Plugin Changes Not Taking Effect - -The Gradle daemon caches compiled plugins: - -```bash -./gradlew --stop # Kill daemon -./gradlew tasks --no-daemon # Run without daemon -``` - -### Circular Dependencies - -Avoid cross-project `afterEvaluate` dependencies. Use `gradle.projectsEvaluated` instead: - -```kotlin -// WRONG - may cause ordering issues -project.afterEvaluate { - val otherProject = rootProject.findProject(":other") - otherProject?.afterEvaluate { ... } // Nested afterEvaluate -} - -// RIGHT - runs after ALL projects are evaluated -gradle.projectsEvaluated { - val otherProject = rootProject.findProject(":other") - // Safe to access other project's tasks/configs -} -``` - -### Debug Gradle Configuration - -```bash -# Show task dependencies -./gradlew :ddprof-lib:assembleRelease --dry-run - -# Show dependency resolution -./gradlew :ddprof-test:dependencies --configuration testReleaseImplementation - -# Enable debug logging -./gradlew build -d 2>&1 | head -500 -``` - -## See Also - -- [GRADLE-TASKS.md](GRADLE-TASKS.md) - Task reference and workflows -- [build-logic/README.md](../build-logic/README.md) - Plugin implementation details -- [gradle.properties.template](../gradle.properties.template) - All configurable properties diff --git a/doc/build/GradleTasks.md b/doc/build/GradleTasks.md deleted file mode 100644 index 1470b2745..000000000 --- a/doc/build/GradleTasks.md +++ /dev/null @@ -1,207 +0,0 @@ -# Gradle Tasks Reference - -This document describes the custom Gradle tasks available in the java-profiler project. - -## Quick Reference - -| Task | Description | -|------|-------------| -| `./gradlew assembleRelease` | Build release native library and JAR | -| `./gradlew testRelease` | Run Java tests with release library | -| `./gradlew gtestDebug` | Run C++ unit tests (debug config) | -| `./gradlew spotlessApply` | Auto-format all source files | - -## Build Tasks - -### Native Library Compilation - -Tasks are generated for each active build configuration (release, debug, asan, tsan, fuzzer). -Configuration availability depends on platform and compiler capabilities: -- **release/debug**: Always available -- **asan**: Available on Linux when libasan is found -- **tsan**: Available on Linux when libtsan is found -- **fuzzer**: Available when clang with libFuzzer support is detected - -``` -compile{Config} # Compile C++ sources -link{Config} # Link shared library -assemble{Config} # Full build for configuration -assemble{Config}Jar # Build JAR with native library -assembleAll # Build all active configurations -``` - -**Examples:** -```bash -./gradlew assembleRelease # Build release library -./gradlew assembleDebug # Build debug library -./gradlew assembleAll # Build all active configs -``` - -### JAR Tasks - -``` -jar # Standard JAR (requires external libs) -assembleReleaseJar # JAR with release native library -assembleDebugJar # JAR with debug native library -sourcesJar # Sources JAR for publishing -javadocJar # Javadoc JAR for publishing -``` - -## Test Tasks - -### Java Tests (ddprof-test) - -``` -test{Config} # Run Java tests with specified config -testRelease # Java tests with release library -testDebug # Java tests with debug library -testAsan # Java tests with ASAN library (Linux) -testTsan # Java tests with TSAN library (Linux) -``` - -**Examples:** -```bash -./gradlew :ddprof-test:testRelease -./gradlew :ddprof-test:testDebug -Ptests=ProfilerTest -``` - -**Note**: Use `-Ptests` (not `--tests`) with config-specific test tasks. The `-Ptests` property works uniformly across all platforms. On glibc/macOS, config-specific tasks use Gradle's Test task type. On musl systems, they use Exec task type to bypass toolchain probing issues. - -### C++ Unit Tests (Google Test) - -``` -gtest{Config}_{TestName} # Run specific test (e.g., gtestDebug_SymbolAnalyzerTest) -gtest{Config} # Run all tests for config -gtest # Run all tests for all configs -``` - -**Examples:** -```bash -./gradlew :ddprof-lib:gtestDebug # All debug tests -./gradlew :ddprof-lib:gtest # All tests, all configs -``` - -### Stress Tests (JMH Benchmarks) - -``` -jmh # Run JMH benchmarks -jmhJar # Build executable JMH JAR -runStressTests # Run stress tests with profiler -``` - -## Application Tasks - -### Unwinding Validator - -``` -runUnwindingValidator{Config} # Run validator with config -runUnwindingValidator # Run validator (release or debug) -unwindingReport{Config} # Generate markdown report -unwindingReport # Generate report (release or debug) -``` - -**Examples:** -```bash -./gradlew :ddprof-test:runUnwindingValidatorRelease -./gradlew :ddprof-test:runUnwindingValidatorRelease -PvalidatorArgs="--verbose" -``` - -## Code Quality Tasks - -### Formatting (Spotless) - -``` -spotlessCheck # Check formatting violations -spotlessApply # Auto-fix formatting issues -``` - -### Static Analysis - -``` -scanbuild{Config} # Run Clang Static Analyzer -``` - -## Fuzz Testing - -``` -fuzz_{TargetName} # Run specific fuzz target -compileFuzz_{TargetName} # Compile fuzz target -linkFuzz_{TargetName} # Link fuzz target -``` - -**Example:** -```bash -./gradlew :ddprof-lib:fuzz:fuzz_symbolAnalyzer -``` - -## Task Dependency Graph - -``` - ┌─────────────────┐ - │ assembleAll │ - └────────┬────────┘ - │ - ┌───────────────────┼───────────────────┐ - ▼ ▼ ▼ -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ assembleRelease │ │ assembleDebug │ │ assembleAsan │ ... -└────────┬────────┘ └────────┬────────┘ └────────┬────────┘ - │ │ │ - ▼ ▼ ▼ -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ linkRelease │ │ linkDebug │ │ linkAsan │ -└────────┬────────┘ └────────┬────────┘ └────────┬────────┘ - │ │ │ - ▼ ▼ ▼ -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ compileRelease │ │ compileDebug │ │ compileAsan │ -└─────────────────┘ └─────────────────┘ └─────────────────┘ - - - ┌─────────────────┐ - │ gtest │ - └────────┬────────┘ - │ - ┌───────────────────┼───────────────────┐ - ▼ ▼ ▼ -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ gtestRelease │ │ gtestDebug │ │ gtestAsan │ -└────────┬────────┘ └────────┬────────┘ └────────┬────────┘ - │ │ │ - (per-test tasks) (per-test tasks) (per-test tasks) -``` - -## Common Workflows - -### Full Build and Test -```bash -./gradlew assembleRelease :ddprof-test:testRelease :ddprof-lib:gtestRelease -``` - -### Quick Development Cycle -```bash -./gradlew assembleDebug :ddprof-test:testDebug -Ptests=MyTest -``` - -### Pre-commit Checks -```bash -./gradlew spotlessCheck :ddprof-lib:gtestDebug :ddprof-test:testDebug -``` - -### CI Pipeline -```bash -./gradlew -PCI assembleAll :ddprof-test:testRelease :ddprof-lib:gtest spotlessCheck -``` - -## Gradle Properties - -See `gradle.properties.template` for all available properties: - -| Property | Description | -|----------|-------------| -| `skip-tests` | Skip Java test execution | -| `skip-native` | Skip native C++ compilation | -| `skip-gtest` | Skip Google Test execution | -| `native.forceCompiler` | Force specific C++ compiler | -| `ddprof_version` | Override project version | -| `CI` | Enable CI-specific behavior | diff --git a/doc/build/NativeBuildPlugin.md b/doc/build/NativeBuildPlugin.md deleted file mode 100644 index 965d4f230..000000000 --- a/doc/build/NativeBuildPlugin.md +++ /dev/null @@ -1,410 +0,0 @@ -# Native Build Plugin Architecture - -This document describes the architecture of the Kotlin-based native build plugin (`com.datadoghq.native-build`) used for C++ compilation in the Datadog Java Profiler project. - -## Overview - -The native build plugin replaces Gradle's built-in `cpp-library` and `cpp-application` plugins with a custom, type-safe solution that directly invokes compilers without version string parsing. This design avoids known issues with Gradle's native plugins while providing a clean DSL for configuration. - -## Why Custom Build Tasks? - -Gradle's native plugins have several problems: - -1. **Version Parsing Failures**: The plugins parse compiler version strings which breaks with newer gcc/clang versions -2. **JNI Header Detection Issues**: Problems with non-standard JAVA_HOME layouts -3. **Unresponsive Maintainers**: Plugin maintainers are unresponsive to fixes -4. **Undocumented Internals**: The plugins use internals that change between Gradle versions - -**Solution**: Direct compiler invocation without version parsing. The tasks simply find `clang++` or `g++` on PATH and invoke them with configured flags. - -## Component Architecture - -``` -build-logic/ -└── conventions/ - └── src/main/kotlin/com/datadoghq/native/ - ├── NativeBuildPlugin.kt # Main native build plugin - ├── NativeBuildExtension.kt # DSL extension for configuration - ├── config/ - │ └── ConfigurationPresets.kt # Standard build configurations - ├── gtest/ - │ ├── GtestPlugin.kt # Google Test integration plugin - │ └── GtestExtension.kt # DSL extension for gtest config - ├── model/ - │ ├── Architecture.kt # x64, arm64 enum - │ ├── Platform.kt # linux, macos enum - │ ├── BuildConfiguration.kt # Configuration model - │ ├── LogLevel.kt # QUIET, NORMAL, VERBOSE, DEBUG - │ ├── ErrorHandlingMode.kt # FAIL_FAST, COLLECT_ALL - │ └── SourceSet.kt # Per-directory compiler flags - ├── tasks/ - │ ├── NativeCompileTask.kt # C++ compilation task - │ ├── NativeLinkTask.kt # Library linking task - │ └── NativeLinkExecutableTask.kt # Executable linking task - └── util/ - └── PlatformUtils.kt # Platform detection utilities -``` - -## Plugin Lifecycle - -### 1. Plugin Application - -When `com.datadoghq.native-build` is applied to a project: - -```kotlin -plugins { - id("com.datadoghq.native-build") -} -``` - -The plugin: -1. Creates the `nativeBuild` extension for DSL configuration -2. Registers an `afterEvaluate` hook for task generation - -### 2. Configuration Phase - -During project evaluation, users configure the build: - -```kotlin -nativeBuild { - version.set(project.version.toString()) - cppSourceDirs.set(listOf("src/main/cpp")) - includeDirectories.set(listOf("src/main/cpp")) -} -``` - -### 3. Task Generation (afterEvaluate) - -After project evaluation, the plugin: - -1. **Detects Current Platform**: Uses `PlatformUtils.currentPlatform` and `PlatformUtils.currentArchitecture` - -2. **Detects Compiler**: Runs the compiler detection algorithm (see below) - -3. **Creates Standard Configurations**: If no configurations are explicitly defined, creates release, debug, asan, tsan, and fuzzer configurations - -4. **Filters Active Configurations**: Only configurations matching the current platform/architecture are processed - -5. **Generates Tasks**: For each active configuration, creates: - - `compile{Config}` - Compiles C++ sources - - `link{Config}` - Links shared library - - `assemble{Config}` - Aggregates the above - -6. **Creates Aggregation Tasks**: `assembleAll` depends on all individual assemble tasks - -## Compiler Detection - -The compiler detection algorithm prioritizes explicit overrides, then auto-detection: - -``` -┌─────────────────────────────────────────┐ -│ Check -Pnative.forceCompiler property │ -└─────────────────┬───────────────────────┘ - │ - ┌─────────▼─────────┐ - │ Property defined? │ - └─────────┬─────────┘ - Yes │ No - ┌─────────▼─────────┐ ┌─────────────────────┐ - │ Validate compiler │ │ Try clang++ │ - │ with --version │ │ (preferred) │ - └─────────┬─────────┘ └──────────┬──────────┘ - │ │ - ┌─────────▼─────────┐ ┌──────────▼──────────┐ - │ Available? │ │ Available? │ - └─────────┬─────────┘ └──────────┬──────────┘ - Yes │ No Yes │ No - ▼ │ ▼ │ - Return │ Return │ - ▼ ▼ - GradleException Try g++ → c++ - │ - ┌─────▼─────┐ - │ None found│ - └─────┬─────┘ - ▼ - GradleException -``` - -**Usage:** -```bash -# Auto-detect (default) -./gradlew build - -# Force specific compiler -./gradlew build -Pnative.forceCompiler=clang++ -./gradlew build -Pnative.forceCompiler=/usr/bin/g++-13 -``` - -## Build Configurations - -### Standard Configurations - -| Config | Active When | Optimization | Debug | Sanitizers | -|---------|--------------------------------------|--------------|-------|------------| -| release | Always | `-O3` | `-g` | None | -| debug | Always | `-O0` | `-g` | None | -| asan | `libasan` found + not musl | None | `-g` | ASan, UBSan, LSan | -| tsan | `libtsan` found + not musl | None | `-g` | TSan | -| fuzzer | clang++ with libFuzzer + not musl | None | `-g` | ASan, UBSan | - -### Configuration Model - -Each `BuildConfiguration` contains: - -```kotlin -abstract class BuildConfiguration { - val platform: Property // LINUX or MACOS - val architecture: Property // X64 or ARM64 - val compilerArgs: ListProperty // Compiler flags - val linkerArgs: ListProperty // Linker flags - val testEnvironment: MapProperty // Test env vars - val active: Property // Whether to build -} -``` - -### Platform-Specific Flags - -**Common Linux Flags:** -``` --fPIC -fno-omit-frame-pointer -momit-leaf-frame-pointer --fvisibility=hidden -fdata-sections -ffunction-sections -std=c++17 -``` - -**Common macOS Additions:** -``` --D_XOPEN_SOURCE -D_DARWIN_C_SOURCE -``` - -**Release Linker Flags (Linux):** -``` --Wl,-z,nodelete -static-libstdc++ -static-libgcc --Wl,--exclude-libs,ALL -Wl,--gc-sections -``` - -## Task Architecture - -### NativeCompileTask - -Compiles C++ source files in parallel: - -``` -┌──────────────────────────────────────────────────────┐ -│ NativeCompileTask │ -├──────────────────────────────────────────────────────┤ -│ Inputs: │ -│ - compiler: String (e.g., "clang++") │ -│ - compilerArgs: List │ -│ - sources: FileCollection │ -│ - includes: FileCollection │ -│ - sourceSets: NamedDomainObjectContainer│ -│ │ -│ Outputs: │ -│ - objectFileDir: Directory │ -│ │ -│ Features: │ -│ - Parallel compilation (configurable jobs) │ -│ - Per-source-set compiler flags │ -│ - FAIL_FAST or COLLECT_ALL error modes │ -│ - Configurable logging verbosity │ -│ - Convenience methods: define(), standard() │ -└──────────────────────────────────────────────────────┘ -``` - -**Source Sets Support:** - -Source sets allow different parts of the codebase to have different compilation flags: - -```kotlin -tasks.register("compile", NativeCompileTask::class) { - compilerArgs.set(listOf("-std=c++17", "-O3")) // Base flags - - sourceSets { - create("main") { - sources.from(fileTree("src/main/cpp")) - compilerArgs.add("-fPIC") - } - create("legacy") { - sources.from(fileTree("src/legacy")) - compilerArgs.addAll("-Wno-deprecated", "-std=c++11") - excludes.add("**/broken/*.cpp") - } - } -} -``` - -### NativeLinkTask - -Links object files into shared libraries: - -``` -┌──────────────────────────────────────────────────────┐ -│ NativeLinkTask │ -├──────────────────────────────────────────────────────┤ -│ Inputs: │ -│ - linker: String │ -│ - linkerArgs: List │ -│ - objectFiles: FileCollection │ -│ - exportSymbols: List │ -│ - hideSymbols: List │ -│ │ -│ Outputs: │ -│ - outputFile: RegularFile │ -│ - debugSymbolsDir: Directory (optional) │ -│ │ -│ Features: │ -│ - Symbol visibility control (version scripts) │ -│ - Debug symbol extraction (release builds) │ -│ - Platform-specific linking │ -│ - macOS wildcard warning │ -└──────────────────────────────────────────────────────┘ -``` - -**Symbol Visibility:** - -The task generates platform-specific symbol export files: - -- **Linux**: Version script (`.ver`) with wildcard support (`Java_*`) -- **macOS**: Exported symbols list (`.exp`) - **no wildcard support** - -```kotlin -tasks.register("link", NativeLinkTask::class) { - exportSymbols.set(listOf("Java_*", "JNI_OnLoad", "JNI_OnUnload")) - hideSymbols.set(listOf("*_internal*")) -} -``` - -**Note:** On macOS, the task warns when wildcards are used since they're not supported. - -## Task Dependencies - -``` -compile{Config} - │ - ▼ - link{Config} - │ - ├──────────────────┐ - │ │ - ▼ ▼ -extractDebugLib stripLib{Config} - (release only) (release only) - │ │ - └────────┬─────────┘ - │ - ▼ - assemble{Config} - │ - ▼ - assembleAll -``` - -## Debug Symbol Extraction - -Release builds automatically extract debug symbols for optimal deployment: - -### Linux Workflow -```bash -objcopy --only-keep-debug library.so library.so.debug -objcopy --add-gnu-debuglink=library.so.debug library.so -strip --strip-debug library.so -``` - -### macOS Workflow -```bash -dsymutil library.dylib -o library.dylib.dSYM -strip -S library.dylib -``` - -### Size Reduction -- Original with debug: ~6.1 MB -- Stripped library: ~1.2 MB (80% reduction) -- Debug symbols: ~6.1 MB (separate file) - -## Platform Utilities - -`PlatformUtils` provides platform detection and tool location: - -| Function | Description | -|----------|-------------| -| `currentPlatform` | Detects LINUX or MACOS | -| `currentArchitecture` | Detects X64 or ARM64 | -| `isMusl()` | Detects musl libc (Alpine Linux) | -| `javaHome()` | Finds JAVA_HOME | -| `jniIncludePaths()` | Returns JNI header paths | -| `isCompilerAvailable(compiler)` | Tests compiler with `--version` | -| `locateLibasan(compiler)` | Finds ASan library path | -| `locateLibtsan(compiler)` | Finds TSan library path | -| `hasFuzzer()` | Tests libFuzzer support | -| `sharedLibExtension()` | Returns "so" or "dylib" | - -## Plugin Components - -The `build-logic` directory contains all native build plugins: - -| Component | Plugin ID | Purpose | -|-----------|-----------|---------| -| `NativeBuildPlugin` | `com.datadoghq.native-build` | C++ compilation and linking | -| `GtestPlugin` | `com.datadoghq.gtest` | Google Test integration | -| `NativeCompileTask` | - | Parallel C++ compilation task | -| `NativeLinkTask` | - | Shared library linking task | -| `NativeLinkExecutableTask` | - | Executable linking task (for gtest) | -| `PlatformUtils` | - | Platform detection and compiler location | - -## GtestPlugin Integration - -The `GtestPlugin` consumes configurations from `NativeBuildPlugin`: - -```kotlin -plugins { - id("com.datadoghq.native-build") - id("com.datadoghq.gtest") -} - -gtest { - testSourceDir.set(layout.projectDirectory.dir("src/test/cpp")) - mainSourceDir.set(layout.projectDirectory.dir("src/main/cpp")) - includes.from("src/main/cpp", "$javaHome/include") -} -``` - -For each test file, GtestPlugin creates: -- `compileGtest{Config}_{TestName}` - Compile sources with test -- `linkGtest{Config}_{TestName}` - Link test executable -- `gtest{Config}_{TestName}` - Execute the test - -See `build-logic/README.md` for full GtestPlugin documentation. - -## Error Handling Modes - -### FAIL_FAST (Default) -- Stops compilation on first error -- Uses sequential stream processing -- Provides immediate feedback - -### COLLECT_ALL -- Compiles all files regardless of errors -- Uses parallel stream processing -- Reports all errors at end -- Configurable max errors to show - -## Logging Levels - -| Level | Description | -|-------|-------------| -| QUIET | Minimal output | -| NORMAL | Standard progress (default) | -| VERBOSE | Progress per N files | -| DEBUG | Full command lines | - -## Future Considerations - -1. **Windows Support**: Add MSVC/MinGW compiler support if needed -2. **Fuzzer Compiler Detection**: Currently hardcodes clang++ -3. **Per-Configuration Compiler**: Allow different compilers per configuration -4. **Incremental Compilation**: Track source dependencies for partial rebuilds - -## Related Documentation - -- `build-logic/README.md` - Native build and GtestPlugin usage documentation -- `CLAUDE.md` - Build commands reference diff --git a/doc/build/TestingGuide.md b/doc/build/TestingGuide.md deleted file mode 100644 index a7073598c..000000000 --- a/doc/build/TestingGuide.md +++ /dev/null @@ -1,303 +0,0 @@ -# Testing Guide - -This document describes the complete test strategy for the java-profiler project: -what runs where, what each tier is designed to catch, and how to run each tier -locally. - -## Overview - -Tests are split across four tiers based on what they detect and what infrastructure -they require: - -| Tier | System | When | Sanitizers | Purpose | -|------|--------|------|-----------|---------| -| **C++ unit tests** | GitLab | Every branch push | ASan + TSan | Data races and memory errors in native internals | -| **Java functional tests** | GitHub Actions | Nightly | ASan | Correctness + memory errors in JVMTI paths | -| **dd-trace integration** | GitLab | Every branch push | None | Compatibility with the tracer agent | -| **Chaos / reliability** | GitLab | Nightly scheduled | None | Long-duration stability and probabilistic crash detection | - ---- - -## Tier 1 — C++ Unit Tests (Every Branch Push) - -**Pipeline:** `.gitlab/sanitizer-tests/.gitlab-ci.yml`, `build` stage - -**Jobs:** `gtest-asan-amd64`, `gtest-tsan-amd64`, `gtest-asan-arm64`, `gtest-tsan-arm64` - -**Gradle tasks:** `:ddprof-lib:gtestAsan`, `:ddprof-lib:gtestTsan` - -**Runs on:** amd64 and aarch64, using the standard `BUILD_IMAGE_X64` / `BUILD_IMAGE_ARM64` -images, on every branch push (same trigger as the dd-trace integration tests) - -The C++ gtest suite in `ddprof-lib/src/test/cpp/` exercises profiler internals -directly, without a JVM. This makes both ASan and TSan effective: - -- **ASan** (`-fsanitize=address,undefined`) detects buffer overflows, use-after-free, - and pointer arithmetic errors in the signal handler path and native data structures. -- **TSan** (`-fsanitize=thread`) detects data races between signal handlers, profiling - threads, and class-unload callbacks — exactly the class of bug most likely to - produce intermittent JVM crashes in the field. - -TSan is only viable at this tier. The JVM binary contains intentional unsynchronized -patterns (lock-free GC internals, biased locking) that produce too many false -positives in the Java functional tier. `gradle/sanitizers/tsan.supp` captures -suppressions from earlier attempts; it exists for the benefit of any future JVM-level -TSan runs, but is not applied here since these tests never load a JVM. - -**Why GitLab and not GitHub Actions:** TSan requires `vm.mmap_rnd_bits ≤ 28` and its -re-exec fallback (`personality(ADDR_NO_RANDOMIZE)`) to handle ASLR conflicts. GitHub -Actions' ubuntu-latest runners have `vm.mmap_rnd_bits=32` and their seccomp profile -blocks the `personality` syscall. The Datadog GitLab runners have stable kernel -settings tuned for benchmark workloads. - -**Key test files:** - -| File | Covers | -|------|--------| -| `dictionary_concurrent_ut.cpp` | Concurrent read/write/clear of `Dictionary` — the `std::_Rb_tree_increment` race path | -| `thread_teardown_safety_ut.cpp` | Signal delivery during `ProfiledThread` TLS clear and delete | -| `profiler_null_calltrace_buffer_ut.cpp` | Null calltrace buffer guard in the JVMTI sample path (PROF-14679) | -| `stress_callTraceStorage.cpp` | `CallTraceStorage` under concurrent write pressure | -| `test_callTraceStorage.cpp` | `CallTraceStorage` buffer swap correctness | -| `sigaction_interception_ut.cpp` | `sigaction` interception correctness and re-entrancy | -| `signalOrigin_ut.cpp` | Signal origin detection and classification | -| `spinlock_bounded_ut.cpp` | `SpinLock` / `BoundedOptionalSharedLockGuard` under contention | - -**Local run:** -```bash -# Individual sanitizer configs -./gradlew :ddprof-lib:gtestAsan -./gradlew :ddprof-lib:gtestTsan - -# All configs (debug, release, asan, tsan) -./gradlew :ddprof-lib:gtest - -# Specific test -./gradlew :ddprof-lib:gtestAsan_dictionary_concurrent_ut -``` - -Prerequisites on Ubuntu: -```bash -sudo apt-get install -y libgtest-dev libgmock-dev cmake g++ clang -``` - -The sanitizer runtimes are bundled with `g++` and `clang` on modern Ubuntu — no -separate `libasan` or `libtsan` package is needed. - -On TSan failure the report is written to stderr and appears directly in the GitLab -job log. - ---- - -## Tier 2 — Java Functional Tests (Nightly) - -**Workflow:** `.github/workflows/nightly.yml` → `test_workflow.yml` - -**Gradle task:** `:ddprof-test:testAsan -Pskip-gtest` - -**Runs on:** amd64 and aarch64 × glibc and musl × -HotSpot / OpenJ9 / GraalVM / IBM / Liberica across JDK 8–25 - -**Triggers:** nightly at 03:00 UTC; also `workflow_dispatch` for manual runs - -The Java functional tests run the profiler as a JVMTI agent attached to a real JVM -and assert correctness: allocation profiling reports the right classes, CPU samples -land on the right frames, class unloading is handled cleanly, wall-clock profiling -produces expected output. - -ASan is applied here even though the JVM is not instrumented, because -`libjavaProfiler.so` is instrumented and ASan intercepts memory errors in JVMTI -callback paths — actual `GetStackTrace` calls, real `SampledObjectAlloc` events, real -class load/unload sequences — that cannot be fully replicated in C++ unit tests. - -TSan is not run against the Java functional tests (see Tier 1 rationale above). - -C++ gtests are skipped (`-Pskip-gtest`) because they already run on every PR in -Tier 1. - -**Test configurations triggered by PR labels** (optional, in addition to the always-on -debug build): - -| Label | Effect | -|-------|--------| -| `test:release` | Run Java functional tests with release library | -| `test:asan` | Run Java functional tests with ASan library on the PR | -| `test:tsan` | Run Java functional tests with TSan library on the PR (expect JVM false positives) | - -**Local run:** -```bash -# Match the nightly configuration -./gradlew :ddprof-test:testAsan -Pskip-gtest - -# Run against a specific JDK and libc via Docker (matches CI exactly) -./utils/run-docker-tests.sh --config=asan --jdk=21 --libc=glibc - -# Run a single test -./gradlew :ddprof-test:testAsan -Ptests=AllocationProfilerTest -Pskip-gtest -``` - -On failure the workflow reports affected scenarios to Slack and uploads test reports -as artifacts. - ---- - -## Tier 3 — dd-trace Integration Tests (GitLab, Every Push) - -**Pipeline:** `.gitlab/dd-trace-integration/.gitlab-ci.yml` - -**Runs on:** amd64 and aarch64 × glibc and musl × HotSpot + OpenJ9, JDK 8–25 - -**Triggers:** every branch push; skipped when `CI_PIPELINE_SOURCE` is -`merge_request_event` (GitLab merge-request pipeline) or when JDK integration -variables are set (`JDK_VERSION`, `DEBUG_LEVEL`, `HASH`, `DOWNSTREAM`) - -This tier patches the latest `dd-java-agent.jar` snapshot with the locally built -`ddprof.jar` and runs integration tests against the combined agent. The patch -replaces the bundled (relocated) profiler classes inside the agent with the version -under test, keeping the classloader/relocation path identical to production. - -It tests end-to-end agent startup, profiling data collection, and tracer/profiler -co-existence across the full JDK × libc matrix. Failures are posted as PR comments -and published to GitHub Pages as a compatibility matrix. - -No sanitizers are applied here. The goal is compatibility verification, not crash -or race detection. - -**Manual trigger:** The `DD_TRACE_VERSION` variable can be set to test against a -specific dd-java-agent snapshot version rather than auto-detecting the latest. - ---- - -## Tier 4 — Chaos and Reliability (GitLab, Nightly Scheduled) - -**Pipeline:** `.gitlab/reliability/.gitlab-ci.yml` - -**Runs on:** amd64 and aarch64, nightly via GitLab pipeline schedule - -This tier runs long-duration workloads designed to provoke probabilistic crashes and -stability regressions that bounded-time unit tests cannot reliably trigger. - -### Reliability variants (`jit` and `memory`) - -Runs `renaissance.jar akka-uct` repeatedly under the profiler for up to 6 hours. -Tests `profiler` and `profiler+tracer` configurations against `gmalloc`, `jemalloc`, -and `tcmalloc` allocators. Detects crashes that require sustained JIT compilation -churn and GC pressure to manifest. - -The `memory` variant additionally monitors RSS over time (via `memwatch.log`) and -runs `memory_trend_check.py` to detect upward memory trends. - -### Chaos variant - -Patches the latest `dd-java-agent.jar` with the locally built `ddprof.jar` (same -patch mechanism as Tier 3) and runs the `ddprof-stresstest` chaos harness under -continuous antagonist load: - -| Antagonist | What it stresses | -|-----------|-----------------| -| `thread-churn` | 64 short-lived threads racing signal delivery, `RefCountGuard` slot allocation | -| `classloader-churn` | Rapid class definition and GC, `StringDictionary` insert/collect/clear races | -| `alloc-storm` | Continuous allocation pressure against the allocation profiler | -| `vthread-churn` | Virtual thread mount/unmount lifecycle against wall-clock profiling | -| `trace-context` | Trace context propagation under concurrent profiling (requires `profiler+tracer`) | - -Failure criterion: a non-zero exit code (JVM crash), captured as an `hs_err.log` -artifact. Crashes are also reported to Slack. - -No sanitizers are used. Tier 4 catches races that require hours at production-scale -concurrency to trigger with meaningful probability. - -### JDK integration tests - -`.gitlab/jdk-integration/.gitlab-ci.yml` handles upstream testing against custom JDK -builds. It is triggered externally (from the `async-profiler-build` pipeline) with -specific `JDK_VERSION`, `DEBUG_LEVEL`, and `HASH` parameters and runs `testDebug` -against that JDK build. This is used to validate compatibility with unreleased JDK -versions. - ---- - -## Why the Split - -| Bug class | Caught by | -|-----------|-----------| -| Data race in native data structures (signal handler vs. mutator) | Tier 1 — TSan gtest | -| Memory corruption in signal handler path | Tier 1 — ASan gtest | -| Memory error in JVMTI callback path | Tier 2 — ASan Java functional | -| Correctness regression (wrong profiling output) | Tier 2 — Java functional | -| Tracer / profiler incompatibility | Tier 3 — dd-trace integration | -| Probabilistic crash under sustained load | Tier 4 — chaos / reliability | -| JDK-version-specific crash | Tier 4 — JDK integration | - -**Tier 1** provides the fastest feedback (every PR, minutes). TSan without a JVM is -definitive for the class of race that has caused the most production crashes: signal -handlers accessing shared data structures concurrently with writers on other threads. - -**Tier 2** covers correctness and integration with real JVM behaviour. Some paths -(actual `GetStackTrace` interleaving with class unload, real `SampledObjectAlloc` -callback ordering) are impractical to replicate in C++ unit tests. - -**Tier 3** catches regressions in the tracer/profiler integration boundary that would -otherwise only surface after a combined dd-trace-java release. - -**Tier 4** provides long-duration soak coverage at realistic concurrency levels, -catching races with per-second probability too low for any bounded CI window. - ---- - -## Local Development - -### Quick feedback cycle - -```bash -# C++ unit tests — debug build, fast -./gradlew :ddprof-lib:gtestDebug - -# Java functional tests — debug build -./gradlew :ddprof-test:testDebug - -# Single test -./gradlew :ddprof-test:testDebug -Ptests=WallclockDumpSmokeTest -``` - -### Sanitizer builds - -```bash -# C++ ASan + TSan (no JVM needed) -./gradlew :ddprof-lib:gtestAsan -./gradlew :ddprof-lib:gtestTsan - -# Java functional tests under ASan (JVM required) -./gradlew :ddprof-test:testAsan -Pskip-gtest -``` - -### Using Docker to match CI exactly - -```bash -# Matches the nightly configuration -./utils/run-docker-tests.sh --config=asan --jdk=21 --libc=glibc - -# Debug build against a specific JDK -./utils/run-docker-tests.sh --config=debug --jdk=17-j9 --libc=glibc - -# Musl build -./utils/run-docker-tests.sh --config=debug --jdk=21-librca --libc=musl - -# With C++ gtests enabled (disabled by default in run-docker-tests.sh) -./utils/run-docker-tests.sh --config=asan --jdk=21 --libc=glibc --gtest -``` - -### Running the chaos harness locally - -```bash -# Build the chaos jar (auto-detected by chaos_check.sh when present) -./gradlew :ddprof-stresstest:chaosJar - -# Run the chaos check (uses the local build artifact; downloads dd-java-agent.jar) -.gitlab/reliability/chaos_check.sh 300 profiler+tracer gmalloc -``` - -`chaos_check.sh` looks for `ddprof-lib/build/libs/ddprof-*.jar` first and only -falls back to downloading from Maven snapshots if not found (requiring -`CURRENT_VERSION` to be set in that case). Build the jar locally to skip the -Maven download. diff --git a/doc/octo-sts-pr-comment-policy.yaml b/doc/octo-sts-pr-comment-policy.yaml deleted file mode 100644 index b0dd15e5e..000000000 --- a/doc/octo-sts-pr-comment-policy.yaml +++ /dev/null @@ -1,19 +0,0 @@ -# Octo-STS Policy for PR Comments -# -# This file should be placed in the java-profiler repository at: -# .github/chainguard/async-profiler-build.pr-comment.sts.yaml -# -# It allows the java-profiler-build CI to post comments on PRs -# with integration test results. -# -# Reference: https://github.com/chainguard-dev/octo-sts - -issuer: https://gitlab.ddbuild.io - -subject_pattern: project_path:DataDog/apm-reliability/async-profiler-build:* - -permissions: - # Required for posting PR comments (uses Issues API) - issues: write - # Required for looking up PRs by branch - pull_requests: read diff --git a/doc/octo-sts-update-images-policy.yaml b/doc/octo-sts-update-images-policy.yaml deleted file mode 100644 index 458479704..000000000 --- a/doc/octo-sts-update-images-policy.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# Octo-STS Trust Policy for Image Update PRs -# -# This is a DOCUMENTATION COPY of the policy that should be created in the -# GitHub repository: DataDog/async-profiler-build -# -# Location in GitHub repo: .github/chainguard/update-images.sts.yaml -# -# PURPOSE: -# This policy allows the GitLab CI check-image-updates job to: -# - Push branches to the repository -# - Create pull requests for image updates -# -# BOOTSTRAP: -# On first run, the create-image-update-pr.sh script will automatically -# include this policy file in the PR if it doesn't exist. After merging -# that first PR, subsequent runs will use Octo-STS for authentication. -# -# For the bootstrap run, set GITHUB_TOKEN as a CI variable with a -# personal access token that has 'repo' scope. -# -# DOCUMENTATION: -# - Octo-STS: https://edu.chainguard.dev/open-source/octo-sts/ -# - GitLab OIDC: https://docs.gitlab.com/ee/ci/secrets/id_token_authentication.html - -# GitLab OIDC issuer -issuer: https://gitlab.ddbuild.io - -# Match GitLab CI jobs from the async-profiler-build project -# The scheduled job runs on main, but we allow any branch for manual triggers -subject_pattern: project_path:DataDog/apm-reliability/async-profiler-build:ref_type:branch:ref:.* - -# GitHub API permissions -permissions: - # Required to push branches - contents: write - # Required to create PRs - pull_requests: write diff --git a/doc/performance/reports/thread-context-benchmark-2026-03-21.md b/doc/performance/reports/thread-context-benchmark-2026-03-21.md deleted file mode 100644 index 6af412539..000000000 --- a/doc/performance/reports/thread-context-benchmark-2026-03-21.md +++ /dev/null @@ -1,314 +0,0 @@ -# ThreadContext Benchmark Report — 2026-03-21 (updated 2026-04-07) - -## Objective - -Verify that `ThreadContext` operations scale linearly across threads and -that no false sharing exists between per-thread `Context` structs -allocated on the native heap via `ProfiledThread`. - -## Environment - -### 2026-04-07 (current) - -| Property | Value | -|----------------|---------------------------------| -| OS | macOS Darwin 25.3.0 (arm64) | -| JDK | Temurin 25.0.2 | -| JMH | 1.37 | -| Forks | 1 | -| Warmup | 2 iterations x 1 s | -| Measurement | 3 iterations x 2 s | -| Mode | Average time (ns/op) | - -### 2026-03-21 (baseline) - -| Property | Value | -|----------------|---------------------------------| -| OS | macOS Darwin 25.3.0 (arm64) | -| JDK | Zulu 25.0.2 | -| JMH | 1.37 | -| Forks | 2 | -| Warmup | 3 iterations x 1 s | -| Measurement | 5 iterations x 2 s | -| Mode | Average time (ns/op) | - -## Benchmark Design - -The benchmark class `ThreadContextBenchmark` uses two JMH `@State` scopes: - -- **`ProfilerState` (`Scope.Benchmark`)** — starts the profiler once per - trial, shared across all threads. Prevents the "Profiler already - started" race that occurs when multiple threads each try to call - `profiler.execute("start,...")`. - -- **`ThreadState` (`Scope.Thread`)** — each JMH worker thread gets its own - `ThreadContext`, random span IDs, and counter. This mirrors production - behavior where each application thread owns an independent context. - -Multi-threaded variants use JMH's `@Threads` annotation at the method -level. The single-threaded benchmarks run with the default thread count -of 1. - -## Results - -### 2026-04-07 — Current (Temurin 25, 1 fork) - -``` -Benchmark Mode Cnt Score Error Units -ThreadContextBenchmark.clearContext avgt 3 4.279 ± 0.236 ns/op -ThreadContextBenchmark.getSpanId avgt 3 1.046 ± 0.054 ns/op -ThreadContextBenchmark.setAttrCacheHit avgt 3 9.512 ± 0.213 ns/op -ThreadContextBenchmark.setContextFull avgt 3 5.993 ± 0.094 ns/op -ThreadContextBenchmark.setContextFull_2t avgt 3 6.360 ± 0.680 ns/op -ThreadContextBenchmark.setContextFull_4t avgt 3 6.162 ± 1.297 ns/op -ThreadContextBenchmark.spanLifecycle avgt 3 16.345 ± 0.935 ns/op -ThreadContextBenchmark.spanLifecycle_2t avgt 3 14.475 ± 1.104 ns/op -ThreadContextBenchmark.spanLifecycle_4t avgt 3 15.233 ± 0.302 ns/op -``` - -### 2026-03-21 — Baseline (Zulu 25, 2 forks) - -``` -Benchmark Mode Cnt Score Error Units -ThreadContextBenchmark.clearContext avgt 9 5.011 ± 0.039 ns/op -ThreadContextBenchmark.setAttrCacheHit avgt 9 10.707 ± 0.077 ns/op -ThreadContextBenchmark.setContextFull avgt 9 11.104 ± 0.338 ns/op -ThreadContextBenchmark.setContextFull_2t avgt 9 11.081 ± 0.105 ns/op -ThreadContextBenchmark.setContextFull_4t avgt 9 11.741 ± 0.338 ns/op -ThreadContextBenchmark.spanLifecycle avgt 9 30.430 ± 0.129 ns/op -ThreadContextBenchmark.spanLifecycle_2t avgt 9 30.674 ± 0.093 ns/op -ThreadContextBenchmark.spanLifecycle_4t avgt 9 32.203 ± 0.309 ns/op -``` - -### Single-Threaded Breakdown (2026-04-07) - -| Benchmark | ns/op | Error | vs Baseline | Description | -|------------------|--------|---------|-------------|------------------------------------------------| -| `clearContext` | 4.279 | ± 0.236 | -14.6% | Zero-fill all four context fields | -| `getSpanId` | 1.046 | ± 0.054 | N/A (new) | Direct ByteBuffer read of span ID | -| `setContextFull` | 5.993 | ± 0.094 | **-46.0%** | Write localRootSpanId, spanId, traceIdHigh, traceIdLow | -| `setAttrCacheHit`| 9.512 | ± 0.213 | -11.2% | Set string attribute (dictionary cache hit) | -| `spanLifecycle` | 16.345 | ± 0.935 | **-46.3%** | `setContextFull` + `setAttrCacheHit` combined | - -`getSpanId` at **1.046 ns** confirms the direct ByteBuffer read compiles to a -single memory load — effectively free compared to the removed `getContext()` path -(71.6 ns, dominated by `long[]` JNI allocation). - -`setContextFull` improved from 11.1 ns to 6.0 ns (-46%), and `spanLifecycle` from -30.4 ns to 16.3 ns (-46%). This reflects OTEP record write optimizations accumulated -since the baseline (removed redundant `full_fence`, tightened detach/attach window, -direct sidecar writes replacing dictionary lookups for LRS). - -### False Sharing Analysis (2026-04-07) - -| Benchmark | 1 thread | 2 threads | 4 threads | 1t to 2t | 1t to 4t | -|------------------|----------|-----------|-----------|----------|----------| -| `setContextFull` | 5.993 | 6.360 | 6.162 | +6.1% | +2.8% | -| `spanLifecycle` | 16.345 | 14.475 | 15.233 | -11.5%\* | -6.8%\* | - -\* The `spanLifecycle` multi-thread results are below the single-thread score. -With 1 fork and 3 measurement iterations, noise floor is ~1 ns. The inversion is -within measurement noise, not a real effect. Re-run with 2+ forks to confirm. - -**At 4 threads (`setContextFull`):** +2.8% increase. Well below false-sharing -threshold (typically 2-10x). Consistent with L2/L3 cache pressure and MOESI -coherency traffic with no same-cache-line contention. - -### Verdict - -**No false sharing detected.** Consistent with the 2026-03-21 result. Per-thread -`OtelThreadContextRecord` structs embedded in `ProfiledThread` are sufficiently -separated in memory. Throughput scales near-linearly from 1 to 4 threads. - -## JNI vs ByteBuffer Implementation Comparison - -Prior to commit `e7e1c3ce`, the context write path used JNI native -methods for every operation. The current implementation replaced the -per-operation JNI calls with direct `ByteBuffer` writes to native memory, -keeping JNI only for one-time value registration. This section documents -the measured regression in the old JNI path and explains its root cause. - -### Before/After Benchmark Data - -The "JNI" column was captured on Java 25 (Temurin 25.0.2) where -`ThreadContext` routed all writes through JNI native methods. The -"ByteBuffer" column is the current implementation on Java 25 (Zulu -25.0.2) where all hot-path writes go through `DirectByteBuffer`. -Both are standard OpenJDK 25 distributions with no profiling-relevant -divergences; the vendor difference does not materially affect the comparison. - -| Benchmark | JNI (ns/op) | ByteBuffer 2026-03-21 | ByteBuffer 2026-04-07 | JNI→current | -|------------------|------------:|----------------------:|----------------------:|------------:| -| `clearContext` | 6.5 | 5.011 | 4.279 | 1.5x | -| `setContextFull` | 20.0 | 11.104 | 5.993 | 3.3x | -| `setAttrCacheHit`| 114.8 | 10.707 | 9.512 | 12.1x | -| `spanLifecycle` | 146.3 | 30.430 | 16.345 | 9.0x | - -The span lifecycle hot path improved by **9.0x** from JNI (146 ns) to the current -ByteBuffer implementation (16.3 ns). The dominant contributor is `setContextAttribute`, -which improved by **12.1x** (115 ns → 9.5 ns). Additional gains since the 2026-03-21 -baseline reflect OTEP record write optimizations (removed `full_fence`, tightened -detach/attach, direct sidecar writes). - -### Why the JNI Path Was Slow - -The old `setContextAttribute0` JNI method accepted a `java.lang.String`: - -```c -JNICALL Java_com_datadoghq_profiler_ThreadContext_setContextAttribute0( - JNIEnv* env, jclass unused, jint keyIndex, jstring value) -``` - -Every invocation paid three costs that the ByteBuffer path avoids: - -**1. JNI boundary crossing (~15-20 ns)** - -Each JNI call transitions from compiled Java code to native code through -the JNI stub. The JVM must save the Java frame state, set up the native -frame, and pass arguments through the JNI calling convention. On return, -it must restore Java state and check for pending exceptions. Even for -trivial native methods with all-primitive signatures (like -`setContextFull0(long, long, long, long)`), this costs ~10 ns. With -object references in the signature, the cost increases due to handle -table management (see point 2). - -**2. JNI String marshalling (~40-60 ns)** - -The `jstring` parameter triggers JNI local reference tracking. Inside -the native method, `JniString` called `GetStringUTFChars()` to obtain a -modified-UTF-8 copy of the string data. This involves: - -- Acquiring the string's backing array (may pin or copy). -- Converting UTF-16 code units to modified-UTF-8. -- Allocating a temporary native buffer for the result. -- Releasing the string via `ReleaseStringUTFChars()` on scope exit. - -For a typical 15-character HTTP route like `"GET /api/users"`, this -UTF-16-to-UTF-8 conversion and copy dominates the call cost. - -**3. Dictionary lookup on every call (~30-40 ns)** - -After extracting the C string, `ContextApi::setAttribute()` performed a -`Dictionary::bounded_lookup()` to resolve the string to a numeric -encoding. This hash-table lookup happened on every call, even for -repeated values of the same attribute — the JNI path had no caching -layer. - -**Total JNI path cost:** ~15 + ~50 + ~35 = ~100 ns, consistent with -the measured 115 ns (the remainder is `JniString` construction overhead -and mode-checking branches). - -### Why the ByteBuffer Path Is Fast - -The current implementation splits attribute setting into two phases: - -**Phase 1 — One-time registration (cache miss, ~100+ ns, amortized away)** - -On the first call with a new string value, `registerConstant0(String)` -crosses JNI once to register the value in the native `Dictionary` and -returns its integer encoding. The encoding and the string's UTF-8 bytes -are stored in a per-thread 256-slot direct-mapped cache: - -```java -encoding = registerConstant0(value); // one JNI call -utf8 = value.getBytes(StandardCharsets.UTF_8); // one allocation -attrCacheEncodings[slot] = encoding; -attrCacheBytes[slot] = utf8; -attrCacheKeys[slot] = value; // commit -``` - -**Phase 2 — Cached hot path (cache hit, ~11 ns, zero JNI)** - -On subsequent calls with the same string value (the common case — web -applications cycle through a small set of routes), the path is: - -```java -if (value.equals(attrCacheKeys[slot])) { - encoding = attrCacheEncodings[slot]; // int read - utf8 = attrCacheBytes[slot]; // ref read -} -BUFFER_WRITER.writeOrderedInt(sidecarBuffer, keyIndex * 4, encoding); -detach(); -replaceOtepAttribute(otepKeyIndex, utf8); -attach(); -``` - -This performs: -- One `String.equals()` comparison (~3 ns for short strings). -- One ordered int write to the sidecar `DirectByteBuffer` (~2 ns). -- Detach/attach protocol with `storeFence` barriers (~4 ns on ARM). -- A `ByteBuffer.put()` sequence for the OTEP attrs_data (~2 ns). - -No JNI crossing. No string marshalling. No dictionary lookup. - -### Why All-Primitive JNI Was Also Slower - -Even `setContextFull0(long, long, long, long)` — which has no object -parameters — was 1.8x slower than the ByteBuffer path (20 ns vs 11 ns). -The JNI stub overhead alone accounts for ~10 ns. The ByteBuffer path -replaces this with `ByteBuffer.putLong()` calls that the JIT compiles -to direct memory stores, plus two `storeFence` barriers for the -detach/attach protocol. On ARM, `storeFence` compiles to a DMB ISHST -instruction (~2 ns), making the total ByteBuffer overhead ~6 ns for -four long writes plus two fences — well under the JNI stub cost. - -On x86, `storeFence` is a no-op (Total Store Order provides free -store-store ordering), so the ByteBuffer path would be even cheaper. - -### Cache Hit Rate in Production - -The attribute cache is a 256-slot direct-mapped cache keyed by -`String.hashCode() & 0xFF`. In typical web applications: - -- HTTP route attributes cycle through 5-50 unique values. -- Span IDs change every request, but they use the `put()` path (no - string marshalling). -- The cache hit rate for string attributes is effectively 100% in - steady state, since the working set is far smaller than 256 slots. - -This means the amortized cost of `setContextAttribute` in production is -the cache-hit cost of ~11 ns, not the registration cost. - -## Cost Model for Instrumentation Budget - -Based on these results, the per-span instrumentation cost on the context -hot path is: - -| Operation | Cost (2026-03-21) | Cost (2026-04-07) | -|------------------------|------------------:|------------------:| -| Span start (set IDs) | ~11 ns | ~6 ns | -| Set one attribute | ~11 ns | ~10 ns | -| Full span lifecycle | ~30 ns | ~16 ns | -| Span end (clear) | ~5 ns | ~4 ns | -| Read span ID | ~72 ns (alloc) | ~1 ns (direct) | -| **Total per-span** | **~35 ns** | **~20 ns** | - -At 20 ns per span, a single thread can sustain ~50 million span -transitions per second before the context layer becomes a bottleneck. -For a typical web application processing 10K-100K requests/second, this -represents less than 0.002% of available CPU time. - -## Recommendations - -1. **No action needed on false sharing.** The current `ProfiledThread` - allocation strategy (one heap allocation per thread) provides - sufficient cache-line isolation. Confirmed by both the 2026-03-21 and - 2026-04-07 runs. - -2. **`getContext` allocation (resolved).** The original `getContext` method - allocated a `long[]` per call (~71.6 ns). Resolved by replacing `getContext()` - with direct `DirectByteBuffer` reads in `getSpanId()` (1.0 ns) and - `getRootSpanId()`, eliminating the JNI call and array allocation entirely. - -3. **Re-run multi-threaded variants with 2+ forks.** The 2026-04-07 run used - 1 fork with 3 iterations; `setContextFull_4t` has a ±21% error bar and the - `spanLifecycle` multi-thread results show noise-floor inversion. A 2-fork run - would produce publication-quality error bars. - -4. **Higher thread counts.** To stress-test on server hardware, add `@Threads(8)` - and `@Threads(16)` variants and run on a machine with sufficient physical cores. - On a laptop, 4 threads saturates the performance core complex. - -5. **Linux `perf stat` validation.** For definitive cache-miss evidence, run on - Linux with `perf stat -e L1-dcache-load-misses,LLC-load-misses` and compare - miss rates across thread counts. diff --git a/doc/plans/2026-05-29-logs-backend-crash-simulation-design.md b/doc/plans/2026-05-29-logs-backend-crash-simulation-design.md deleted file mode 100644 index 0644f8fbf..000000000 --- a/doc/plans/2026-05-29-logs-backend-crash-simulation-design.md +++ /dev/null @@ -1,186 +0,0 @@ -# Reproducing the logs-backend profiler crashes in java-profiler testing - -**Date:** 2026-05-29 -**Status:** Design (pending review) -**Author:** investigation driven from Datadog crash-tracking telemetry - -## 1. Problem - -Datadog-internal services built from `~/dd/logs-backend` (event-platform / `*-reducer` -workloads) crash far more than our test suite or external customers surface. The goal is -to reproduce the responsible conditions inside java-profiler's own testing so we gain -confidence that fixes hold and regressions are caught. - -## 2. Evidence (crash-tracking telemetry, Org2, 7-day window, all versions) - -Source: `service:instrumentation-telemetry-data-jvm @lib_language:jvm`, crash events. - -### 2.1 Why we weren't seeing them - -The standard triage filter keys on `@tags.crash_datadog:true`. **Most logs-backend crashes -are not tagged `crash_datadog`** (they are `crash_runtime` / `crash_unactionable`), so they -were filtered out. Removing that clause surfaces hundreds of crashes in 7 days -(`event-context-writer`: 110 SIGSEGV + 14 SIGBUS; `event-heartbeat-emitter`: 27 SIGSEGV + -23 SIGBUS; `event-context-provider-skeleton`: 30 SIGSEGV; etc.). - -### 2.2 Uniform platform - -All on **JDK 25.0.3**, `env:staging`, tracer **`1.63.0-snapshot`** (dev build). This is -Datadog dogfooding on bleeding-edge Java 25. - -### 2.3 Attribution split (SIGSEGV/SIGBUS, JFR-crashing services) - -| Tags | Events | Heuristic verdict | -|---|---|---| -| `crash_runtime` + `crash_unactionable` | ~179 | "JVM bug" — **but see §2.5** | -| `crash_profiler` + `crash_datadog` | ~25 | attributed to this profiler | - -The `crash_runtime`/`crash_unactionable` tag records only **where the crashing PC landed** -(JVM code). It is **not** a root-cause determination. - -### 2.4 Crash taxonomy - -**Family 1 — PC in JVM/JDK code (the ~179):** -`JfrJavaThreadIteratorAdapter::next → jfr_emit_event → jdk.jfr.internal.periodic.*` -(JDK's own Flight Recorder iterating the thread list), `SafepointSynchronize::arm_safepoint`, -`ThreadsSMRSupport::free_list/add_thread`, `GlobalCounter::write_synchronize`, ZGC mark/relocate. - -**Family 2 — PC in this profiler's code (the ~25):** -- **A. Thread-lifecycle callbacks** — `Profiler::onThreadEnd+0x65` during `JavaThread::exit`, - `Profiler::onThreadStart`, `Profiler::updateThreadName → ThreadInfo::set`. -- **B. Dump / chunk serialization** — `Profiler::dump → FlightRecorder::dump → - Recording::switchChunk / writeCpool / writeStackTraces / cleanupUnreferencedMethods`, - `CallTraceStorage::processTraces`, `Dictionary::clear`, `std::_Rb_tree_increment` in - `writeCpool`. Frequently triggered via `JavaProfiler.dump0`. -- **C. Virtual threads** — `ExceptionSampleEvent.commit → Continuation.enterSpecial → - VirtualThread.runContinuation`. - -### 2.5 Fault-address forensics — corruption, not clean nulls - -For the Family-1 services (SIGSEGV/SIGBUS): -- **SIGBUS / `BUS_ADRALN`** (~25 events): misaligned access ⇒ the pointer value itself is - garbage read from corrupted/freed memory. Correct codegen never misaligns. -- **Non-null `SEGV_MAPERR`**: garbage 32-bit-range addresses (`0x2d6c1892`, `0x86471d9e`…), - **not** valid Java-25 heap pointers (`0x00007f…`), and they **cluster on identical - low-16-bit suffixes** (`…1d9e` ×15, `…1ea6` ×12, `…1892` ×6) — the same field offset in - the same structure read off a varying garbage base. Textbook **use-after-free / - type-confusion**. -- Only the `SI_KERNEL / 0x0` bucket (~32) looks like a plain null. - -**Conclusion (CONFIRMED):** Family 1 and Family 2 are **one underlying profiler -memory-safety bug** (heap corruption / UAF in the thread-churn × dump interaction on Java 25). -**Disabling the profiler on the affected staging services removes the crashes** — the decisive -control experiment. So the ~179 `crash_runtime`/`unactionable` events are profiler-induced too; -the attribution tag undercounted our true crash volume by roughly 8× (~25 tagged vs ~204 actual). -The crash tag merely records where the corruption happened to surface. `ThreadInfo` itself is -**correctly locked** (every method holds `_ti_lock`, `get()` deep-copies via `shared_ptr`), so -`ThreadInfo::set` is a *victim* site touching an already-poisoned heap, not the source. The -source must be found by sanitizing the interacting components together. - -### 2.6 Common trigger - -Rapid **thread creation/destruction (churn)** intersecting the **recording dump** path, on -**Java 25**. Service names corroborate: heartbeat emitters, periodic writers, reducers spin up -short-lived workers. - -## 3. Testing gaps that allowed the escape - -1. **ASan/TSan run only on isolated C++ gtest binaries** (`buildGtest{Asan,Tsan}`, no JVM in - the process). They cannot — and must not — run against a live JVM (ASan's global - malloc/signal/guard-page interception fights JVM signal handling, JIT, and deliberate - faults; TSan floods on uninstrumented JVM synchronization and its shadow region collides - with JVM mappings, incl. the Kata issue already noted in CI). So a UAF that only appears - under *live JVM + churn + dump* is invisible to the sanitizers. -2. **JVM integration / chaos / reliability run on JDK 21** (`chaos_check.sh` hardcodes - `java 21.0.3-tem`; repo default `JAVA_TEST_VERSION=21`). Every production crash is **Java 25**. -3. **No concurrent `dump()` stress.** The chaos harness only crash-detects under churn; it - never drives the dump path that Family-2 B/C fault in. - -## 4. Design — two layers, mapped onto existing structure - -Detection is split by layer because sanitizers are valid only without a JVM in the process. - -### 4.1 Layer 1 — gtest sanitizer concurrency stress (finds the bug) - -**Where:** `ddprof-lib/src/test/` (ASan+TSan via existing `buildGtestAsan`/`buildGtestTsan` -→ `gtest-asan-amd64`/`gtest-tsan-amd64`). No new infra. Existing neighbors: -`stress_callTraceStorage.cpp`, `thread_teardown_safety_ut.cpp`, `threadIdTable_ut.cpp`, -`dictionary_ut.cpp`. - -**New:** `stress_threadLifecycle_ut.cpp` — a no-JVM concurrency model of the churn × dump -interaction: -- M worker threads loop: `ProfiledThread` register → updateName → engine register - (`_cpu_engine`/`_wall_engine`) → unregister → `ProfiledThread::release()` (incl. buffer - recycling path). -- 1–2 "dump" threads loop: `CallTraceStorage::processTraces(...)` + `Dictionary::clear()` - (models `Recording::switchChunk`/`writeStackTraces`). -- Run for a fixed iteration budget; rely on ASan (UAF at origin) and TSan (data race) as the - oracle, plus existing crash-on-signal. - -Scope note: drive the profiler's *own* data structures directly. Anything requiring genuine -JVMTI inputs (a real `jthread`) stays in Layer 2. - -### 4.2 Layer 2 — chaos antagonist + reliability (proves the real scenario, guards regressions) - -**Where:** `ddprof-stresstest/src/chaos/java/.../chaos/` + `.gitlab/reliability/`. No -sanitizer. Existing: `ThreadChurnAntagonist`, `VirtualThreadChurnAntagonist`, `gmalloc` -guard-allocator already in the matrix. - -**Changes:** -1. **`DumpStormAntagonist`** — drives frequent recording rotation/dump concurrently with - churn. Harness is black-box (patched `dd-java-agent`), so trigger dumps via a short - profiler recording/upload interval rather than calling `dump0` directly. Exercises - `Recording::switchChunk/writeCpool`, `updateJavaThreadNames → ThreadInfo::set`, - `Dictionary::clear`. Register in `Main.create()`. -2. **Add Java 25 to the chaos/reliability matrix** — `chaos_check.sh` currently installs only - `21.0.3-tem`. Add a 25.x cell (keep 21 as control). This alone may reproduce the crash. - -The `gmalloc` allocator already in the matrix provides JVM-safe guard-page detection of -overflows/UAF for Layer 2, complementing crash-on-signal. - -## 5. Out of scope / explicitly not done - -- **Not** chasing Family-1 PCs as JDK bugs — the control experiment (profiler off ⇒ no - crashes) confirms they are profiler-induced, not JDK bugs. -- **No** ASan/TSan against a live JVM (technically invalid — see §3.1). -- **No** source-scan of logs-backend; telemetry already gave the exact code paths. -- **No** new test-only production APIs unless proposed and approved first. - -## 6. Validation plan - -- Layer 1: new stress test fails (ASan/TSan report) on current `main`; passes after the fix. -- Layer 2: chaos+`DumpStorm` on **Java 25** reproduces a crash (or `gmalloc` fault) on current - `main`; clean after the fix; Java 21 cell stays green throughout (control). -- Cross-check: **DONE** — disabling the profiler on the affected staging services removes the - crashes, confirming the single-root-cause (profiler) hypothesis. - -## 7. Open questions - -- Exact black-box mechanism to force frequent dumps in the chaos harness (short recording - interval config vs. a benign agent hook) — to resolve during implementation. -- Whether a Java-25 chaos cell needs a new CI build image / SDKMAN candidate availability. - -## 8. Reproduction results - -### Layer 1 — ASan gtest (JDK 25, aarch64/glibc, 2026-05-29) - -Command: `./utils/run-containers-tests.sh --config=asan --gtest --jdk=25 --mount` - -Results: -- `stress_threadLifecycle_ut` compiled and linked under ASan: **PASS** -- `StressThreadLifecycle.Smoke`: **PASS** (no ASan report) -- `StressThreadLifecycle.ChurnOnly`: **PASS** (no ASan report) -- `StressThreadLifecycle.ChurnDuringDump`: **PASS** (no ASan report) - -**Interpretation:** The profiler's own data structures (`ProfiledThread`, `ThreadFilter`, -`CallTraceStorage`, `Dictionary`) are race-free in isolation at the unit level. The UAF -requires **real JVMTI inputs** — live `jthread` handles, JVM-side native thread lifecycle, -JVMTI callbacks. Layer 2 (chaos antagonists under a real JVM on Java 25) is the decisive -reproducer. - -Pre-existing unrelated failure: `CollapsingSleepTest > testSleep()` (Java integration test, -not touched by this work). - -### Layer 2 — chaos + Java 25 CI cell - -Pending: Task 8 (CI matrix change) not yet run. diff --git a/doc/plans/2026-05-29-logs-backend-crash-simulation-plan.md b/doc/plans/2026-05-29-logs-backend-crash-simulation-plan.md deleted file mode 100644 index 731bf3149..000000000 --- a/doc/plans/2026-05-29-logs-backend-crash-simulation-plan.md +++ /dev/null @@ -1,529 +0,0 @@ -# logs-backend Crash Simulation Implementation Plan - -> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. - -**Goal:** Reproduce, inside java-profiler's own test suite, the confirmed profiler memory-safety bug that crashes logs-backend services (Java 25, thread-churn × recording-dump), so we can find the root cause and guard against regressions. - -**Architecture:** Two layers. **Layer 1** is a no-JVM gtest concurrency stress (`stress_threadLifecycle_ut.cpp`) that drives the profiler's own thread-lifecycle + dump data structures under ASan/TSan to surface the UAF/race at its origin. **Layer 2** adds a `DumpStormAntagonist` and a **Java 25** cell to the existing black-box chaos/reliability suite (JVM-safe detectors only: `gmalloc` + crash-on-signal). - -**Tech Stack:** C++17, GoogleTest (`com.datadoghq.gtest` Gradle plugin, auto-discovers `src/test/cpp/*.cpp`), ASan/TSan via `buildGtestAsan`/`buildGtestTsan`; Java chaos harness (`ddprof-stresstest`), GitLab CI (`.gitlab/reliability`, `.gitlab/sanitizer-tests`), SDKMAN. - -**Spec:** `doc/plans/2026-05-29-logs-backend-crash-simulation-design.md` - -**Branch:** Work on `prof-logs-backend-crash-sim` (do not commit to `main`). - -**Execution environment (IMPORTANT — overrides the bare `./gradlew` lines below):** -This host is macOS/arm64; the test is `#ifdef __linux__` and the sanitizers are Linux-only. -ALL build/run steps execute via the repo's Docker runner on **JDK 25**, which gives faithful -Linux ASan/TSan: -- Debug build + gtests: `./utils/run-containers-tests.sh --config=debug --gtest --jdk=25 --mount` -- ASan: `./utils/run-containers-tests.sh --config=asan --gtest --jdk=25 --mount` -- TSan: `./utils/run-containers-tests.sh --config=tsan --gtest --jdk=25 --mount` -- Interactive iteration: add `--shell` to drop into the container and run the built - `*_stress_threadLifecycle_ut*` binary directly with `--gtest_filter=StressThreadLifecycle.*`. - -The `./gradlew :ddprof-lib:buildGtest*` commands in the tasks are the *in-container* equivalents; -run them through the Docker wrapper above, not directly on the host. - -**Commit policy:** Per project rule, do NOT `git commit` during execution. Author + verify, then -leave changes in the working tree for the user to review and commit. The `git commit` steps in -the tasks are deferred until the user approves. - ---- - -## File Structure - -| File | Status | Responsibility | -|---|---|---| -| `ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp` | Create | Layer-1 concurrency reproducer: churn workers (ProfiledThread + ThreadFilter) vs. dump thread (CallTraceStorage + Dictionary), under ASan/TSan. | -| `ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java` | Create | Layer-2 antagonist: forces frequent recording rotation/dump concurrently with churn. | -| `ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java` | Modify | Register `dump-storm` antagonist in `create()`. | -| `.gitlab/reliability/chaos_check.sh` | Modify | Add a Java 25 install/cell alongside the existing `21.0.3-tem`. | -| `.gitlab/reliability/.gitlab-ci.yml` | Modify | Add `JDK` dimension (21, 25) to the chaos matrix; include `dump-storm` in the antagonist set. | - -**Reference neighbors (read before coding):** `thread_teardown_safety_ut.cpp` (ProfiledThread lifecycle idioms, sigaction guards, PROF-14603), `threadFilter_ut.cpp` (ThreadFilter construction/`init`), `test_callTraceStorage.cpp` + `stress_callTraceStorage.cpp` (CallTraceStorage `put`/`processTraces`/`clear`, `ASGCT_CallFrame`, `gtest_crash_handler.h`), `dictionary_ut.cpp` (Dictionary `lookup`/`clear`). - -**Note on test nature:** This is a *characterization/reproduction* harness, not classic TDD. Success criterion for Phase 1 is: the harness runs **clean when the code is correct**, and we run it under ASan/TSan on current `main` to see whether it surfaces the corruption. If it stays clean on `main`, that is itself a result — it narrows the root cause to a path that requires real JVMTI/JVM state (→ Layer 2). - ---- - -## Phase 1 — Layer 1: gtest concurrency reproducer - -### Task 1: Scaffold the stress test file (compiles, runs, passes) - -**Files:** -- Create: `ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp` - -- [ ] **Step 1: Create the file with includes, crash-handler hook, and one trivial test** - -```cpp -/* - * Copyright 2026, Datadog, Inc. - * SPDX-License-Identifier: Apache-2.0 - * - * Layer-1 reproducer for the logs-backend crash (Java 25, thread-churn × - * recording-dump). Drives the profiler's own thread-lifecycle and dump data - * structures concurrently, with NO JVM in the process, so ASan/TSan can flag - * the use-after-free / race at its origin. - * - * See doc/plans/2026-05-29-logs-backend-crash-simulation-design.md - */ -#include "gtest/gtest.h" - -#ifdef __linux__ - -#include "callTraceStorage.h" -#include "callTraceHashTable.h" -#include "dictionary.h" -#include "threadFilter.h" -#include "thread.h" -#include "arch.h" - -#include -#include -#include -#include -#include "../../main/cpp/gtest_crash_handler.h" - -static constexpr const char STRESS_TEST_NAME[] = "StressThreadLifecycle"; - -TEST(StressThreadLifecycle, Smoke) { - // Sanity: structures construct and tear down without a JVM. - CallTraceStorage storage; - Dictionary dict; - EXPECT_EQ(0u, dict.lookup("smoke") == 0u ? 0u : 0u); // lookup is callable - storage.clear(); - SUCCEED(); -} - -#endif // __linux__ -``` - -- [ ] **Step 2: Build under the normal (non-sanitizer) gtest config** - -Run: `./gradlew :ddprof-lib:buildGtestDebug --no-daemon` -Expected: BUILD SUCCESSFUL; a binary matching `stress_threadLifecycle_ut` appears under `ddprof-lib/build/bin/gtest/`. - -- [ ] **Step 3: Run the smoke test** - -Run: `find ddprof-lib/build/bin/gtest -name '*stress_threadLifecycle_ut*' -type f -executable -exec {} \;` -Expected: `[ PASSED ] 1 test.` - -- [ ] **Step 4: Commit** - -```bash -git add ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp -git commit -m "test: scaffold thread-lifecycle stress reproducer" -``` - ---- - -### Task 2: Churn workers modelling onThreadStart / onThreadEnd - -Models `Profiler::onThreadStart`/`onThreadEnd`: each worker initialises its `ProfiledThread`, registers a filter slot, does work, then unregisters and releases — the exact sequence in `profiler.cpp:75-130`. - -**Files:** -- Modify: `ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp` - -- [ ] **Step 1: Add the churn-worker test (verify ThreadFilter ctor/init against `threadFilter_ut.cpp` first)** - -```cpp -// Shared filter, mirroring Profiler::_thread_filter (one instance, many threads). -static ThreadFilter g_filter; -static std::atomic g_run{false}; - -static void churn_worker() { - while (!g_run.load(std::memory_order_acquire)) { /* spin until start */ } - for (int i = 0; i < 2000 && g_run.load(std::memory_order_relaxed); i++) { - // onThreadStart sequence - ProfiledThread::initCurrentThread(); - ProfiledThread* self = ProfiledThread::current(); - ASSERT_NE(nullptr, self); - ThreadFilter::SlotID slot = g_filter.registerThread(); - self->setFilterSlotId(slot); - g_filter.add(self->tid(), slot); - - // minimal "work" so the thread is live across a scheduling quantum - std::this_thread::yield(); - - // onThreadEnd sequence - g_filter.remove(slot); - g_filter.unregisterThread(slot); - self->setFilterSlotId(-1); - ProfiledThread::release(); - } -} - -TEST(StressThreadLifecycle, ChurnOnly) { - g_filter.init("*"); // enable filtering; confirm arg form via threadFilter_ut.cpp - ASSERT_TRUE(g_filter.enabled()); - g_run.store(true, std::memory_order_release); - - std::vector ts; - for (int t = 0; t < 16; t++) ts.emplace_back(churn_worker); - for (auto& t : ts) t.join(); - - g_run.store(false); - SUCCEED(); -} -``` - -- [ ] **Step 2: Build and run (Debug)** - -Run: `./gradlew :ddprof-lib:buildGtestDebug --no-daemon && find ddprof-lib/build/bin/gtest -name '*stress_threadLifecycle_ut*' -type f -executable -exec {} --gtest_filter=StressThreadLifecycle.ChurnOnly \;` -Expected: `[ PASSED ]`. If it crashes, capture the stack — that may already be the bug. - -- [ ] **Step 3: Commit** - -```bash -git add ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp -git commit -m "test: add thread-churn worker to lifecycle stress" -``` - ---- - -### Task 3: Concurrent dump thread (CallTraceStorage + Dictionary) - -Models `Profiler::dump`: while workers churn, a dump thread repeatedly walks call traces and clears the dictionary — `CallTraceStorage::processTraces` + `Dictionary::clear` from the design's Family-2 B. - -**Files:** -- Modify: `ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp` - -- [ ] **Step 1: Add a shared storage + dump thread, and have workers `put()` traces (mirror `put()` usage in `test_callTraceStorage.cpp`)** - -```cpp -static CallTraceStorage g_storage; -static Dictionary g_dict; - -// Build a small fake stack and record it (signal-handler-style put()). -static void record_trace(int salt) { - ASGCT_CallFrame frames[4]; - std::memset(frames, 0, sizeof(frames)); - for (int i = 0; i < 4; i++) { - frames[i].bci = i + salt; - frames[i].method_id = reinterpret_cast(static_cast(0x1000 + i + salt)); - } - g_storage.put(4, frames, false, 1); - g_dict.lookup("logs-backend-sim"); -} - -static void dump_thread() { - while (g_run.load(std::memory_order_relaxed)) { - g_storage.processTraces([](const std::unordered_set& traces) { - volatile size_t n = 0; - for (CallTrace* t : traces) if (t) n += 1; // touch each, like writeStackTraces - (void)n; - }); - g_dict.clear(); // models Dictionary::clear() during Profiler::dump - g_storage.clear(); // models chunk rotation - } -} -``` - -Then, inside `churn_worker`'s loop, add `record_trace(i)` right after `g_filter.add(...)`. - -- [ ] **Step 2: Add the combined test that runs churn + dump together** - -```cpp -TEST(StressThreadLifecycle, ChurnDuringDump) { - g_filter.init("*"); - ASSERT_TRUE(g_filter.enabled()); - g_run.store(true, std::memory_order_release); - - std::thread dumper(dump_thread); - std::vector ts; - for (int t = 0; t < 16; t++) ts.emplace_back(churn_worker); - for (auto& t : ts) t.join(); - - g_run.store(false); - dumper.join(); - SUCCEED(); -} -``` - -- [ ] **Step 3: Build and run (Debug)** - -Run: `./gradlew :ddprof-lib:buildGtestDebug --no-daemon && find ddprof-lib/build/bin/gtest -name '*stress_threadLifecycle_ut*' -type f -executable -exec {} --gtest_filter=StressThreadLifecycle.ChurnDuringDump \;` -Expected: `[ PASSED ]` in a correct build (or a crash that reproduces the bug). - -- [ ] **Step 4: Commit** - -```bash -git add ddprof-lib/src/test/cpp/stress_threadLifecycle_ut.cpp -git commit -m "test: drive concurrent dump against thread churn" -``` - ---- - -### Task 4: Run under AddressSanitizer on `main` (the reproduction experiment) - -**Files:** none (investigation step — record findings). - -- [ ] **Step 1: Build the ASan gtest binaries** - -Run: `./gradlew :ddprof-lib:buildGtestAsan --no-daemon --parallel` -Expected: BUILD SUCCESSFUL. - -- [ ] **Step 2: Run the new test under ASan, many iterations** - -Run: -```bash -bin=$(find ddprof-lib/build/bin/gtest -type f -executable -name 'asan_*stress_threadLifecycle_ut*') -for i in $(seq 1 50); do "$bin" --gtest_filter='StressThreadLifecycle.*' || { echo "ASAN HIT on iter $i"; break; }; done -``` -Expected outcome is one of: -- ASan reports `heap-use-after-free` / `heap-buffer-overflow` with a profiler stack → **bug reproduced**; record the report verbatim (this is the root-cause pointer). -- All 50 iterations PASS → record "ASan clean at unit level"; proceed to TSan (Task 5). - -- [ ] **Step 3: Record findings** - -Append the ASan output (or "clean") to the design doc under a new "## 8. Reproduction results" section. - -```bash -git add doc/plans/2026-05-29-logs-backend-crash-simulation-design.md -git commit -m "docs: record ASan reproduction results" -``` - ---- - -### Task 5: Run under ThreadSanitizer on `main` - -**Files:** none (investigation step). - -- [ ] **Step 1: Build the TSan gtest binaries** - -Run: `./gradlew :ddprof-lib:buildGtestTsan --no-daemon --parallel` -Expected: BUILD SUCCESSFUL. (TSan must run on a non-Kata runner — locally is fine; in CI it uses the `docker-in-docker:amd64`→EC2 path per `.gitlab/sanitizer-tests/.gitlab-ci.yml`.) - -- [ ] **Step 2: Run the new test under TSan** - -Run: -```bash -bin=$(find ddprof-lib/build/bin/gtest -type f -executable -name 'tsan_*stress_threadLifecycle_ut*') -GTEST_DEATH_TEST_STYLE=threadsafe "$bin" --gtest_filter='StressThreadLifecycle.*' -``` -Expected: either a `data race` report naming a profiler structure (record it) or clean. - -- [ ] **Step 3: Record findings and commit (as in Task 4 Step 3).** - -```bash -git add doc/plans/2026-05-29-logs-backend-crash-simulation-design.md -git commit -m "docs: record TSan reproduction results" -``` - ---- - -## Phase 2 — Layer 2: chaos antagonist + Java 25 - -### Task 6: `DumpStormAntagonist` - -Forces frequent recording rotation/dump concurrently with churn. The harness is black-box (patched `dd-java-agent`), so it cannot call `dump0` directly; it drives dumps by keeping the profiler busy and relying on the agent's short recording interval (configured in Task 8). This antagonist supplies the *concurrent churn-with-allocation* pressure that makes each rotation race thread teardown. - -**Files:** -- Create: `ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java` - -- [ ] **Step 1: Create the antagonist (mirror `ThreadChurnAntagonist` structure exactly)** - -```java -/* - * Copyright 2026, Datadog, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - */ -package com.datadoghq.profiler.chaos; - -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; - -/** - * Spawns short-lived, frequently-renamed threads that each generate distinct - * stack shapes, maximising churn in the profiler's thread-name table and call- - * trace storage right as recording chunks rotate. - * - *

Targets the dump path observed in production crashes: - * {@code Recording::switchChunk/writeCpool}, {@code updateJavaThreadNames -> - * ThreadInfo::set}, {@code Dictionary::clear}. Pair with a short profiler - * recording interval (see chaos_check.sh) so dumps fire continuously. - */ -public final class DumpStormAntagonist implements Antagonist { - - private final int concurrentThreads; - private volatile boolean running; - private Thread driver; - - public DumpStormAntagonist() { - this(96); - } - - public DumpStormAntagonist(int concurrentThreads) { - this.concurrentThreads = concurrentThreads; - } - - @Override - public String name() { - return "dump-storm"; - } - - @Override - public void start() { - running = true; - driver = new Thread(this::loop, "chaos-dump-storm"); - driver.setDaemon(true); - driver.start(); - } - - @Override - public void stopGracefully(Duration timeout) { - running = false; - try { - if (driver != null) driver.join(timeout.toMillis()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - - private void loop() { - long seq = 0; - while (running) { - List batch = new ArrayList<>(concurrentThreads); - for (int i = 0; i < concurrentThreads && running; i++) { - final long id = seq++; - Thread t = new Thread(() -> distinctStack(id, 0)); - // Frequent renames hammer updateThreadName/ThreadInfo::set. - t.setName("dump-storm-" + id); - t.setDaemon(true); - t.start(); - batch.add(t); - } - for (Thread t : batch) { - try { - t.join(500L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - return; - } - } - } - } - - // Recurse to a per-thread depth so each thread yields a unique stack shape, - // forcing new call-trace + symbol entries that the dump path must serialise. - private long distinctStack(long id, int depth) { - if (depth >= (int) (id % 32)) { - long r = id; - for (int i = 0; i < 5000; i++) r = (r * 1103515245L + 12345L) & 0x7fffffffL; - return r; - } - return distinctStack(id, depth + 1) + depth; - } -} -``` - -- [ ] **Step 2: Compile the stresstest module** - -Run: `./gradlew :ddprof-stresstest:compileChaosJava --no-daemon` (confirm the source-set task name via `./gradlew :ddprof-stresstest:tasks --all | grep -i chaos`) -Expected: BUILD SUCCESSFUL. - -- [ ] **Step 3: Commit** - -```bash -git add ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/DumpStormAntagonist.java -git commit -m "test: add dump-storm chaos antagonist" -``` - ---- - -### Task 7: Register the antagonist - -**Files:** -- Modify: `ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java` - -- [ ] **Step 1: Add the `dump-storm` case to `create()`** (after the `trace-context` case, before `default`): - -```java - case "dump-storm": - return new DumpStormAntagonist(); -``` - -- [ ] **Step 2: Compile** - -Run: `./gradlew :ddprof-stresstest:compileChaosJava --no-daemon` -Expected: BUILD SUCCESSFUL. - -- [ ] **Step 3: Smoke-run the harness briefly (no agent needed to prove wiring)** - -Run: `./gradlew :ddprof-stresstest:run -PchaosArgs="--duration 5s --antagonists dump-storm" --no-daemon` (adapt to the module's actual run task; otherwise run the built class directly with `--duration 5s --antagonists dump-storm`). -Expected: logs `antagonist started: dump-storm` then `completed cleanly`. - -- [ ] **Step 4: Commit** - -```bash -git add ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/Main.java -git commit -m "test: register dump-storm antagonist" -``` - ---- - -### Task 8: Add Java 25 + dump-storm to the chaos CI matrix - -**Files:** -- Modify: `.gitlab/reliability/chaos_check.sh` -- Modify: `.gitlab/reliability/.gitlab-ci.yml` - -- [ ] **Step 1: Parameterise the JDK in `chaos_check.sh`** - -Replace the hardcoded install (`.gitlab/reliability/chaos_check.sh:16`): -```bash -timeout 300 sdk install java 21.0.3-tem 1>/dev/null 2>/dev/null -``` -with a parameterised version + a short recording interval so dumps fire continuously: -```bash -JDK_CANDIDATE="${CHAOS_JDK:-21.0.3-tem}" -timeout 300 sdk install java "${JDK_CANDIDATE}" 1>/dev/null 2>/dev/null -sdk use java "${JDK_CANDIDATE}" -# Force frequent recording rotation so the dump path is exercised under churn. -export DD_PROFILING_UPLOAD_PERIOD=5 -``` -(Validate the exact env var name against the dd-java-agent build used by the harness; the intent is "shortest safe upload/recording period".) - -- [ ] **Step 2: Validate YAML/script edits** - -Run: `bash -n .gitlab/reliability/chaos_check.sh && python3 -c "import yaml,sys; yaml.safe_load(open('.gitlab/reliability/.gitlab-ci.yml')); print('yaml ok')"` -Expected: no syntax errors; `yaml ok`. - -- [ ] **Step 3: Add a `CHAOS_JDK` dimension and `dump-storm` to the chaos matrix in `.gitlab/reliability/.gitlab-ci.yml`** - -In `.reliability_chaos_job.parallel.matrix`, add the JDK dimension (keep 21 as control): -```yaml - parallel: - matrix: - - CONFIG: ["profiler", "profiler+tracer"] - ALLOCATOR: ["gmalloc", "jemalloc", "tcmalloc"] - CHAOS_JDK: ["21.0.3-tem", "25.0.3-tem"] -``` -And ensure the run invocation passes the dump-storm antagonist (in `chaos_check.sh`'s harness call, set the antagonist list to include `thread-churn,vthread-churn,dump-storm`). Confirm `25.0.3-tem` (or the closest available 25.x) is a valid SDKMAN candidate; if not, pin the available 25 build. - -- [ ] **Step 4: Re-validate YAML** - -Run: `python3 -c "import yaml; yaml.safe_load(open('.gitlab/reliability/.gitlab-ci.yml')); print('yaml ok')"` -Expected: `yaml ok`. - -- [ ] **Step 5: Commit** - -```bash -git add .gitlab/reliability/chaos_check.sh .gitlab/reliability/.gitlab-ci.yml -git commit -m "ci: run chaos suite on Java 25 with dump-storm antagonist" -``` - ---- - -## Self-Review notes - -- **Spec coverage:** §4.1 Layer 1 → Tasks 1-5; §4.2 Layer 2 `DumpStormAntagonist` → Tasks 6-7, Java 25 matrix → Task 8; §3 gaps (JDK 21, no dump stress, sanitizers gtest-only) all addressed. §6 validation (ASan/TSan on main) → Tasks 4-5. -- **Known unknowns flagged inline (not placeholders):** exact `ThreadFilter::init` arg form (verify vs `threadFilter_ut.cpp`), the chaos module's run/compile task names, the dd-java-agent recording-period env var, and SDKMAN's 25.x candidate string. Each step says what to confirm and against what. -- **Type consistency:** `g_filter`/`g_storage`/`g_dict`/`g_run` used consistently across Tasks 2-3; antagonist `name()` returns `"dump-storm"` matching the `create()` case and the CI antagonist list. diff --git a/doc/plans/SignalOriginValidation.md b/doc/plans/SignalOriginValidation.md deleted file mode 100644 index e664b52c6..000000000 --- a/doc/plans/SignalOriginValidation.md +++ /dev/null @@ -1,238 +0,0 @@ -# Plan: Signal Origin Validation for ddprof Signal Handlers - -## Goal -Make ddprof's signal handlers process only signals that originated from ddprof's own timers / inter-thread kills. Foreign signals are forwarded to the previously-installed handler. - -Context: A Go-cgo shared library (e.g. trivyjni) linked into the JVM can install -`setitimer(ITIMER_PROF)` via `dd-trace-go`'s CPU profiler. That timer is -process-wide and delivers SIGPROF to arbitrary threads — including threads that -were never registered with ddprof's `pthread_create` interceptor. ddprof's -SIGPROF handler currently processes every SIGPROF, which can land on threads -whose dynamic-TLS blocks for libjavaProfiler.so have not been allocated yet. -First-touch access of any `thread_local` on such a thread inside the signal -handler triggers `__tls_get_addr` → `calloc`, which can deadlock against a -malloc lock held by the interrupted thread. - -## Scope - -- SIGPROF (CTimer — default CPU engine, `timer_create`-based) -- SIGVTALRM (WallClock engine, ASGCT variant) -- **Out of scope**: - - SIGSEGV / SIGBUS — synchronous faults use a different discrimination - mechanism (SafeAccess PC range check). - - ITimer (alternate CPU engine, `setitimer(ITIMER_PROF)`-based) — setitimer - signals carry `si_code == SI_KERNEL` with no `sigval` payload, so an - origin cookie cannot be attached. Migrating ITimer to `timer_create` - would duplicate CTimer. ITimer is a fallback for platforms where - `timer_create` is unavailable; deployments vulnerable to the Go - `ITIMER_PROF` deadlock should use CTimer (the default). - -## Design - -### Discriminators by signal source - -| Source | `si_code` | Additional validation | -|---|---|---| -| `timer_create` (CTimer; ITimer post-migration) | `SI_TIMER` (-2) | `siginfo->si_value.sival_ptr == CPU_COOKIE` | -| `rt_tgsigqueueinfo` (WallClock — new) | `SI_QUEUE` (-1) | `siginfo->si_value.sival_ptr == WALLCLOCK_COOKIE` | -| `setitimer` (foreign — Go's CPU profiler) | `SI_KERNEL` / kernel-defined | none → untrusted, must forward | -| `kill` (cross-process/thread) | `SI_USER` (0) | none → forward | -| `raise` / `tgkill` / `pthread_kill` (foreign) | `SI_TKILL` (-6) | none → untrusted, must forward | -| `sigqueue` from another process | `SI_QUEUE` (-1) + foreign cookie | cookie mismatch → forward | - -### Cookie strategy - -Cookies live in `signalCookie.h` and are the addresses of private static tag -variables — unique per shared-library image, forgery-resistant against -in-process attackers who would have to read the DSO's symbols to guess a -valid cookie: - -```cpp -namespace SignalCookie { - namespace detail { - inline char cpu_tag; - inline char wallclock_tag; - } - inline void* cpu() { return &detail::cpu_tag; } - inline void* wallclock() { return &detail::wallclock_tag; } -} -``` - -One cookie for CTimer (the only CPU engine that can carry a cookie), one for -wallclock. Rationale for wallclock needing a cookie: `SI_TKILL` + -`si_pid == getpid()` is **not** sufficient — `si_pid` for `SI_TKILL` is the -sending process's PID, and all threads in our process share the same PID, so -that check accepts any in-process `tgkill` of SIGVTALRM (including from other -libraries). Real isolation requires a payload. - -Earlier iterations used magic numbers (`reinterpret_cast(0xDDDD01)`). -Symbol-address cookies are strictly stronger: they cannot collide with a -legitimate pointer value from third-party code (the tag lives in a fresh -DSO-private BSS slot), and they are resistant to an attacker who only knows -the constants published in open-source headers. - -### Portable per-thread queued signal - -`pthread_sigqueue(3)` is a glibc-only extension; musl does not implement it. -To keep musl support, send via the `rt_tgsigqueueinfo(2)` syscall directly — -it is the kernel primitive both glibc and musl expose via `syscall(...)`: - -```cpp -#include -#include // siginfo_t - -static int tg_sigqueue_thread(pid_t tgid, pid_t tid, int sig, void* cookie) { - siginfo_t si = {}; - si.si_signo = sig; - si.si_code = SI_QUEUE; // must be < 0 to be accepted by kernel - si.si_value.sival_ptr = cookie; - si.si_pid = tgid; // convention: sender's TGID - si.si_uid = getuid(); - return (int)syscall(SYS_rt_tgsigqueueinfo, tgid, tid, sig, &si); -} -``` - -The receiving handler then sees: -- `siginfo->si_code == SI_QUEUE` -- `siginfo->si_value.sival_ptr == WALLCLOCK_COOKIE` - -The kernel rejects any positive `si_code` from userspace (reserved for -kernel-generated signals), so this is safe. - -### Handler gate (common helper) - -```cpp -// os.h -bool OS::shouldProcessSignal(siginfo_t* si, int expected_si_code, void* expected_cookie); -bool OS::sendSignalWithCookie(int thread_id, int signo, void* cookie); // rt_tgsigqueueinfo wrapper -void OS::forwardForeignSignal(int signo, siginfo_t* si, void* uctx); -``` - -Handler skeleton: - -```cpp -void Engine::signalHandler(int signo, siginfo_t* si, void* uctx) { - if (!OS::shouldProcessSignal(si, SI_TIMER, ENGINE_COOKIE)) { - Counters::increment(ENGINE_SIGNAL_FOREIGN); - OS::forwardForeignSignal(signo, si, uctx); - return; - } - Counters::increment(ENGINE_SIGNAL_OWN); - // existing handler body -} -``` - -### Foreign-signal forwarding - -Must preserve correctness for other libraries sharing the signal (e.g. Go's -`runtime.sighandler`). Store the previous `sigaction` at install time and -forward via: - -```cpp -if (prev.sa_flags & SA_SIGINFO) { - prev.sa_sigaction(signo, si, uctx); -} else if (prev.sa_handler != SIG_DFL && prev.sa_handler != SIG_IGN) { - prev.sa_handler(signo); -} -// SIG_DFL: drop (reproducing termination on every foreign signal is worse than ignoring it) -// SIG_IGN: drop (matches kernel ignore semantics) -``` - -`OS::installSignalHandler` already captures this via the `oldact` out-parameter -of `sigaction` — just need to make it retrievable by signal number. - -#### sa_mask chaining — opt-in via env var - -By default, chained handlers run under whatever signal mask the profiler's -handler was given by the kernel (which has `signo` blocked because we install -without `SA_NODEFER`). This is the **fast chain path**: no mask-setup syscalls, -~24 ns pure function cost, ~37 ns end-to-end including signal delivery -(measurements from `signalOrigin_bench.cpp` on Linux x86_64). - -Applying `prev.sa_mask` faithfully — so the chained handler sees the same -masked environment the kernel would have given it under normal delivery — -requires two raw `rt_sigprocmask` syscalls (`SIG_BLOCK` then `SIG_SETMASK`). -On modern Linux this measures ~1 µs per foreign signal, ~30% per-signal -end-to-end overhead. (`pthread_sigmask` is NOT used: it would call libc internals -and is not async-signal-safe on all configurations.) - -Because the realistic foreign-signal sources this plan targets -(Go's `ITIMER_PROF`, `raise()`, most `tgkill` callers) do not rely on -`sa_mask` for correctness, we take the fast path by default and expose an -env-var escape hatch: - -| Env var value | Behavior | -|---|---| -| `DDPROF_FORWARD_APPLY_SIGMASK=1` (or `true` / `on` / `yes`) | Slow path: `rt_sigprocmask` block+restore around the chained call | -| unset / any other value | Fast path (default): no mask syscalls; chained handler runs with only `signo` blocked | - -Enable the slow path when profiling exposes a chained handler that depends -on the kernel's normal masked environment — e.g. a crash handler installed -before the profiler that expects other fatal signals blocked during its run. - -SIGSEGV / SIGBUS forwarding is not affected by this flag — those use the -existing SafeAccess PC-range discrimination, not `forwardForeignSignal`. - -## Implementation steps - -1. **`os.h` / `os_linux.cpp`**: add the three helper functions + previous-action - storage keyed by signal number. -2. **`ctimer_linux.cpp`**: set `sev.sigev_value.sival_ptr = CPU_COOKIE` in the - `timer_create` call; add origin gate at top of `signalHandler` (current - line 145). -3. **`itimer.cpp`**: left unchanged — see scope note. Callers who need the - origin check must configure the profiler to use CTimer (the default). -4. **`wallClock.cpp` + `os_linux.cpp`**: - - Replace the current `OS::sendSignalToThread(tid, SIGVTALRM)` implementation - (uses `tgkill` under the hood) with `OS::sendSignalWithCookie(tid, - SIGVTALRM, WALLCLOCK_COOKIE)` which invokes `rt_tgsigqueueinfo` - directly — portable across glibc and musl. - - Add `SI_QUEUE` + `WALLCLOCK_COOKIE` gate in - `WallClockASGCT::sharedSignalHandler` (current line 52). Forward on mismatch. - (J9WallClock uses JVMTI GetAllStackTracesExtended polling — no signal handler — and is out of scope) -5. **`counters.h`**: add `CPU_SIGNAL_FOREIGN` / `CPU_SIGNAL_OWN` and - `WALLCLOCK_SIGNAL_FOREIGN` / `WALLCLOCK_SIGNAL_OWN` counters. -6. **Feature flag**: gate the whole change behind `DDPROF_SIGNAL_ORIGIN_CHECK` - (default ON) so it can be disabled if a regression emerges. - -## Testing - -1. **Unit tests** (per engine): - - CPU: `timer_create` + `timer_settime` with `CPU_COOKIE` → assert `OWN` - counter increments. - - CPU: `setitimer(ITIMER_PROF)` → assert `FOREIGN` counter increments and - signal is forwarded. - - CPU: `raise(SIGPROF)` (SI_TKILL on glibc/musl, no cookie) → assert `FOREIGN`. - - Wallclock: `rt_tgsigqueueinfo(..., WALLCLOCK_COOKIE)` → assert `OWN`. - - Wallclock: `tgkill(SIGVTALRM)` from another thread (no cookie) → assert - `FOREIGN` and forwarded. - - Wallclock: `rt_tgsigqueueinfo` with a foreign cookie → assert `FOREIGN`. - - Musl build: confirm `rt_tgsigqueueinfo` path compiles and signals are - delivered with correct `si_code` / `si_value`. -2. **Integration**: a test process that installs `setitimer(ITIMER_PROF)` - alongside ddprof; verify ddprof samples are all attributed to registered - threads (none from kernel-selected pre-existing threads). -3. **Regression**: reproduce the trivyjni scenario — load a Go-cgo lib that - calls `profiler.Start(CPUProfile)`, run with ddprof active, confirm no - lockup and no ddprof samples from Go-runtime threads that weren't explicitly - registered. - -## Metrics to watch post-deploy - -- `CPU_SIGNAL_FOREIGN > 0` on services with Go cgo + dd-trace-go CPU profiler - → confirms the fix is discriminating. -- `CPU_SIGNAL_OWN` unchanged vs. pre-fix baseline → confirms no false negatives. -- `WALLCLOCK_SIGNAL_OWN` unchanged vs. pre-fix baseline → confirms the - `rt_tgsigqueueinfo` migration preserves delivery. -- `WALLCLOCK_SIGNAL_FOREIGN` should be 0 under normal conditions; non-zero - indicates another in-process sender or external `sigqueue` to us. -- P99 request latency on trivy-affected services → should lose the lockup tail. - -## What this does NOT fix - -- SIGSEGV on pre-existing threads touching a not-yet-allocated `thread_local` - for the first time (separate audit — need to enumerate profiler-owned - `thread_local`s and either eliminate, make allocation-free, or pre-touch at - registration). -- If Go ever adds a `SIGEV_SIGNAL`-targeted `SIGPROF` with its own sival → - our cookie is non-null and distinct, so it is still filtered. diff --git a/doc/reference/EventTypeSystem.md b/doc/reference/EventTypeSystem.md deleted file mode 100644 index f3b3d49ae..000000000 --- a/doc/reference/EventTypeSystem.md +++ /dev/null @@ -1,234 +0,0 @@ -# Event Type System - -## Overview - -The Datadog Java Profiler uses two distinct type systems for event identification: -- **`EventType`** enum (from upstream async-profiler) -- **`ASGCT_CallFrameType`** (BCI_* constants) - -This document explains the relationship between these systems and how Datadog's implementation differs from upstream async-profiler. - -## Type Definitions - -### EventType Enum - -Defined in `ddprof-lib/src/main/cpp/event.h`: - -```cpp -enum EventType { - PERF_SAMPLE, // 0 - EXECUTION_SAMPLE, // 1 - WALL_CLOCK_SAMPLE, // 2 - MALLOC_SAMPLE, // 3 - INSTRUMENTED_METHOD, // 4 - METHOD_TRACE, // 5 - ALLOC_SAMPLE, // 6 - ALLOC_OUTSIDE_TLAB, // 7 - LIVE_OBJECT, // 8 - LOCK_SAMPLE, // 9 - PARK_SAMPLE, // 10 - PROFILING_WINDOW, // 11 - USER_EVENT, // 12 -}; -``` - -### ASGCT_CallFrameType (BCI_* Constants) - -Defined in `ddprof-lib/src/main/cpp/vmEntry.h`: - -```cpp -enum ASGCT_CallFrameType { - BCI_CPU = 0, // cpu time - BCI_WALL = -10, // wall time - BCI_NATIVE_FRAME = -11, // native function name (char*) - BCI_ALLOC = -12, // name of the allocated class - BCI_ALLOC_OUTSIDE_TLAB = -13, // name of the class allocated outside TLAB - BCI_LIVENESS = -14, // name of the allocated class - BCI_LOCK = -15, // class name of the locked object - BCI_PARK = -16, // class name of the park() blocker - BCI_THREAD_ID = -17, // method_id designates a thread - BCI_ERROR = -18, // method_id is an error string -}; -``` - -## Upstream vs Datadog Implementation - -### Upstream async-profiler - -Uses `EventType` consistently throughout: - -- **Function signature** (cpp/profiler.h:213): - ```cpp - u64 recordSample(void* ucontext, u64 counter, EventType event_type, Event* event); - ``` - -- **Event-to-frame-type conversion** when building call traces: - ```cpp - // Convert EventType to BCI_* frame type - jint frame_type = BCI_ALLOC - (event_type - ALLOC_SAMPLE); - ``` - -- **Type-safe comparisons**: - ```cpp - if (event_type <= MALLOC_SAMPLE) { ... } - if (event_type >= ALLOC_SAMPLE && event_type <= ALLOC_OUTSIDE_TLAB) { ... } - ``` - -### Datadog Implementation - -Uses BCI_* values directly as event identifiers: - -- **Modified function signature** (ddprof-lib/src/main/cpp/profiler.h:247-248): - ```cpp - void recordSample(void *ucontext, u64 weight, int tid, jint event_type, - u64 call_trace_id, Event *event); - ``` - -- **Direct BCI_* comparisons**: - ```cpp - if (event_type == BCI_CPU && _cpu_engine == &perf_events) { ... } - if (event_type == BCI_CPU || event_type == BCI_WALL) { ... } - ``` - -- **Call sites pass BCI_* values**: - ```cpp - Profiler::instance()->recordSample(ucontext, _interval, tid, BCI_CPU, 0, &event); - Profiler::instance()->recordSample(ucontext, last_sample, tid, BCI_WALL, call_trace_id, &event); - ``` - -## Type Conversion - -### The Problem - -Datadog's `StackWalker::walkVM()` (inherited from upstream) expects `EventType` but receives BCI_* values. Previously, this used an unsafe cast: - -```cpp -// Old approach - undefined behavior for negative BCI_* values -static_cast(event_type) -``` - -This cast is technically undefined behavior because: -- BCI_WALL = -10 is not a valid EventType enum value (0-12) -- Casting negative integers to enums with only positive values is undefined in C++ -- It worked by accident because numeric comparisons still functioned - -### The Solution - -A conversion function maps BCI_* values to appropriate EventType values: - -```cpp -inline EventType eventTypeFromBCI(jint bci_type) { - switch (bci_type) { - case BCI_CPU: return EXECUTION_SAMPLE; - case BCI_WALL: return WALL_CLOCK_SAMPLE; - case BCI_ALLOC: return ALLOC_SAMPLE; - case BCI_ALLOC_OUTSIDE_TLAB: return ALLOC_OUTSIDE_TLAB; - case BCI_LIVENESS: return LIVE_OBJECT; - case BCI_LOCK: return LOCK_SAMPLE; - case BCI_PARK: return PARK_SAMPLE; - default: return EXECUTION_SAMPLE; - } -} -``` - -Usage in profiler.cpp: - -```cpp -num_frames += StackWalker::walkVM(ucontext, frames + num_frames, - max_remaining, _features, - eventTypeFromBCI(event_type), &truncated); -``` - -## Type Mapping - -| BCI_* Constant | Value | EventType | Value | Notes | -|----------------|-------|-----------|-------|-------| -| BCI_CPU | 0 | EXECUTION_SAMPLE | 1 | CPU samples (perf/itimer) | -| BCI_WALL | -10 | WALL_CLOCK_SAMPLE | 2 | Wall clock samples | -| BCI_ALLOC | -12 | ALLOC_SAMPLE | 6 | TLAB allocations | -| BCI_ALLOC_OUTSIDE_TLAB | -13 | ALLOC_OUTSIDE_TLAB | 7 | Non-TLAB allocations | -| BCI_LIVENESS | -14 | LIVE_OBJECT | 8 | Live heap objects | -| BCI_LOCK | -15 | LOCK_SAMPLE | 9 | Monitor contention | -| BCI_PARK | -16 | PARK_SAMPLE | 10 | Thread park events | - -## Key Files - -### Type Definitions -- `ddprof-lib/src/main/cpp/event.h` - EventType enum -- `ddprof-lib/src/main/cpp/vmEntry.h` - ASGCT_CallFrameType (BCI_* constants) - -### Conversion Function -- `ddprof-lib/src/main/cpp/profiler.h:61-96` - eventTypeFromBCI() implementation - -### Usage Sites -- `ddprof-lib/src/main/cpp/profiler.cpp:716,719` - StackWalker::walkVM() calls -- `ddprof-lib/src/main/cpp/profiler.cpp:690,717` - BCI_* comparisons -- `ddprof-lib/src/main/cpp/flightRecorder.cpp:1620-1642` - Event recording switch - -### Call Sites (BCI_* sources) -- `ddprof-lib/src/main/cpp/itimer.cpp:55` - BCI_CPU -- `ddprof-lib/src/main/cpp/ctimer_linux.cpp:171` - BCI_CPU -- `ddprof-lib/src/main/cpp/perfEvents_linux.cpp:749` - BCI_CPU -- `ddprof-lib/src/main/cpp/wallClock.cpp:105` - BCI_WALL - -## Why Two Type Systems? - -### Historical Context - -1. **Upstream design**: async-profiler uses EventType for event categorization and converts to BCI_* frame types only when building call traces. - -2. **Datadog divergence**: Datadog's fork uses BCI_* values as the primary event identifiers throughout the codebase. - -### Rationale - -Using BCI_* values directly has advantages: -- **Frame type alignment**: BCI_* values are already frame types, eliminating conversion steps -- **Semantic clarity**: BCI_CPU, BCI_WALL are more descriptive than numeric EventType values -- **Simpler call sites**: Event sources directly specify the frame type they produce - -However, it creates the conversion requirement when calling upstream code that expects EventType. - -## Best Practices - -### When Adding New Event Types - -1. **Choose the identifier**: Use BCI_* constant for the event type identifier -2. **Update conversion function**: Add mapping in eventTypeFromBCI() -3. **Update FlightRecorder**: Add case to recordEvent() switch statement -4. **Add call sites**: Pass BCI_* value to recordSample() - -### Code Review Checklist - -- [ ] All event_type comparisons use symbolic names (BCI_CPU, not 0) -- [ ] No raw static_cast on event_type parameters -- [ ] New BCI_* values have eventTypeFromBCI() mappings -- [ ] FlightRecorder::recordEvent() handles the new event type -- [ ] Call sites pass correct BCI_* constant - -## Future Considerations - -### Option 1: Align with Upstream -Convert back to using EventType throughout Datadog's code: -- **Pros**: Type safety, upstream compatibility -- **Cons**: Large refactoring, frame type conversion overhead - -### Option 2: Maintain Current Design -Keep using BCI_* with conversion function: -- **Pros**: Minimal changes, semantically clear -- **Cons**: Type system mismatch persists - -### Option 3: Diverge Further -Fork StackWalker to accept BCI_* directly: -- **Pros**: Eliminates conversion, full type consistency -- **Cons**: Maintenance burden, harder to merge upstream changes - -## Conclusion - -The eventTypeFromBCI() conversion function provides a clean bridge between Datadog's BCI_*-based event identification and upstream's EventType-based stack walking logic. This approach: - -- Eliminates undefined behavior from raw casts -- Documents the intentional type system divergence -- Maintains compatibility with upstream StackWalker code -- Preserves the semantic benefits of BCI_* identifiers - -When working with event types, always use the conversion function when calling code that expects EventType, and use BCI_* constants directly in Datadog-specific code. diff --git a/doc/reference/ProfilerMemoryRequirements.md b/doc/reference/ProfilerMemoryRequirements.md deleted file mode 100644 index 3d5ef94c2..000000000 --- a/doc/reference/ProfilerMemoryRequirements.md +++ /dev/null @@ -1,678 +0,0 @@ -# Profiler Memory Requirements and Limitations - -**Last Updated:** 2026-01-13 - -## Overview - -The Datadog Java Profiler has inherent memory requirements due to its architecture combining signal-safe stack sampling, JFR event recording, and constant pool management. This document describes all major memory consumers, their costs, limitations, and when the profiler may not be appropriate for certain workloads. - -## Memory Requirements - -### 1. jmethodID Preallocation (Required for AGCT) - -**What it is:** -- Every Java method needs a jmethodID allocated before the profiler can identify it in stack traces -- AGCT operates in signal handlers where lock acquisition is forbidden -- Therefore, jmethodIDs must be preallocated when classes are loaded (ClassPrepare callback) - -**Memory cost:** -``` -Memory = (Number of Classes) × (Average Methods per Class) × (JVM Method Structure Size) - -Typical overhead: -- Normal application: 10K-100K classes × 15 methods × 68 bytes = 10-150 MB -- High class churn: 1M+ classes × 15 methods × 68 bytes = 1GB+ -``` - -**JVM internal allocation breakdown:** -- Method structure: ~40-60 bytes (varies by JDK version) -- ConstMethod metadata: variable (includes bytecode) -- Hash table entries for jmethodID lookup -- Memory allocator overhead - -**Key constraint:** -- Memory persists until class is unloaded (cannot be freed while class exists) -- Classes with long-lived ClassLoaders never free this memory -- **This is NOT a bug** - it's fundamental to AGCT architecture - -### 2. Line Number Tables (Optimized) - -**What it is:** -- Maps bytecode index (BCI) to source line numbers for stack traces -- Only allocated for methods that appear in samples (lazy allocation) - -**Memory cost:** -``` -Memory = (Number of Sampled Methods) × (Average Entries per Method) × (Entry Size) - -Typical overhead: -- Sampled methods: 1K-10K (subset of all methods) -- Entries per method: 5-20 line number mappings -- Entry size: 8 bytes (jvmtiLineNumberEntry) -- Total: 40 KB - 1.6 MB (negligible) -``` - -**Lifecycle:** -- Allocated during JFR flush when method first appears in samples -- Stored in SharedLineNumberTable with shared_ptr reference counting -- Tracked via LINE_NUMBER_TABLES counter (incremented on allocation, decremented on deallocation) -- Deallocated when MethodInfo is destroyed (profiler stop/restart or age-based cleanup) - -### 3. CallTraceStorage and Hash Tables - -**What it is:** -- Triple-buffered hash tables storing unique stack traces with lock-free rotation -- Active storage: Accepts new traces from profiling events -- Standby storage: Background storage for JFR serialization -- Scratch storage: Spare table for rotation - -**Memory cost:** -``` -Per hash table: -- Initial capacity: 65,536 entries -- Entry size: 16 bytes (8-byte key + 8-byte CallTraceSample) -- Initial table: ~1 MB -- Auto-expands at 75% capacity (doubles to 128K, 256K, etc.) -- LinearAllocator chunks: 8 MB per chunk - -Typical overhead: -- 3 hash tables × 1-4 MB = 3-12 MB (depends on active traces) -- Chunk allocations: 8-32 MB (depends on stack depth and diversity) -- Total: 11-44 MB for typical applications -``` - -**Growth patterns:** -- Bounded by unique stack trace count (converges after warmup) -- Applications with stable code paths: 10K-100K unique traces -- Applications with dynamic dispatch: 100K-1M unique traces - -**Lifecycle:** -- Allocated at profiler start -- Grows during warmup as new traces are discovered -- Converges to stable size once all code paths are sampled -- Cleared and reset during profiler stop/restart - -### 4. RefCountSlot Arrays (Thread-Local Reference Counting) - -**What it is:** -- Cache-aligned slots for lock-free memory reclamation of hash tables -- Each slot occupies one cache line (64 bytes) to eliminate false sharing - -**Memory cost:** -``` -Fixed allocation: -- MAX_THREADS = 8192 slots -- Slot size: 64 bytes (cache line aligned) -- Total: 512 KB (fixed, independent of actual thread count) -``` - -**Lifecycle:** -- Allocated once at profiler initialization -- Never grows or shrinks -- Reused throughout profiler lifetime - -### 5. Dictionary Instances (String Deduplication) - -**What it is:** -- Four Dictionary instances for JFR constant pools: - - _class_map: Class name strings - - _string_label_map: Label strings - - _context_value_map: Tracer context attribute values - - _packages (in Lookup): Package name strings -- Multi-level hash table with 128 rows × 3 cells per level - -**Memory cost:** -``` -Per dictionary: -- Initial table: sizeof(DictTable) = ~3 KB -- Key storage: Variable-length strings (malloc'd) -- Additional levels: 3 KB per level (rare, only on collision) - -Typical overhead: -- 4 dictionaries × 3 KB = 12 KB (initial tables) -- String storage: 100 KB - 2 MB (depends on unique strings) -- Context values: Variable (depends on tracer integration) -- Total: 112 KB - 8 MB for typical applications -``` - -**Growth patterns:** -- Grows with unique class/method names encountered -- Class names: Bounded by loaded class count -- String labels: Bounded by profiling event labels -- Context values: Bounded by unique span/trace attribute values -- Package names: Typically small (< 1000 unique packages) - -**Lifecycle:** -- Allocated at profiler start -- Grows during warmup -- Converges once all classes/methods/contexts are sampled -- Cleared during profiler stop/restart - -### 6. Recording Buffers (JFR Event Storage) - -**What it is:** -- Thread-local buffers for JFR event serialization -- CONCURRENCY_LEVEL = 16 buffers to minimize lock contention - -**Memory cost:** -``` -Per buffer: -- RecordingBuffer size: 65,536 bytes + 8,192 overflow guard = 73,728 bytes - -Total allocation: -- 16 buffers × 73,728 bytes = 1,179,648 bytes (~1.1 MB) -``` - -**Lifecycle:** -- Allocated at profiler start -- Fixed size, no growth -- Flushed periodically to JFR file -- Deallocated at profiler stop - -### 7. ThreadIdTable Arrays - -**What it is:** -- Hash tables tracking active thread IDs for JFR thread metadata -- Two dimensions: CONCURRENCY_LEVEL (16) × 2 (double-buffering) - -**Memory cost:** -``` -Per table: -- TABLE_SIZE = 256 entries -- Entry size: 4 bytes (atomic) -- Table size: 1,024 bytes - -Total allocation: -- 16 concurrency levels × 2 tables × 1,024 bytes = 32 KB -``` - -**Lifecycle:** -- Allocated at profiler start -- Cleared during buffer rotation -- Fixed size, no growth - -### 8. MethodMap (MethodInfo Storage) - -**What it is:** -- std::map storing metadata for sampled methods -- Only methods that appear in stack traces are stored (lazy allocation) -- Implements age-based cleanup to prevent unbounded growth in continuous profiling - -**Memory cost:** -``` -Per MethodInfo: -- MethodInfo struct: ~56 bytes -- shared_ptr: 16 bytes -- std::map overhead: ~32 bytes per entry -- Total per method: ~104 bytes - -Typical overhead: -- Sampled methods: 1K-10K -- Total: 104 KB - 1 MB -``` - -**Growth patterns:** -- Grows with sampled method diversity during warmup -- Converges once all hot methods are encountered -- Bounded by unique methods in active code paths -- **With cleanup enabled (default):** Methods unused for 3+ chunks are automatically removed -- **Without cleanup:** Would grow unboundedly in applications with high method churn - -**Lifecycle:** -- Allocated during JFR flush when methods are first sampled -- Age counter incremented for unreferenced methods at each chunk boundary -- Methods with age >= 3 chunks are removed during switchChunk() -- Line number tables deallocated via shared_ptr when MethodInfo is destroyed -- Cleanup can be disabled with `mcleanup=false` (not recommended; default is `mcleanup=true`) - -**Cleanup behavior:** -- Triggered during switchChunk() (typically every 10-60 seconds) -- Mark phase: Reset all _referenced flags before serialization -- Reference phase: Mark methods in active stack traces during writeStackTraces() -- Sweep phase: Increment age for unreferenced methods, remove if age >= 3 -- Conservative strategy: Methods must be unused for 3 consecutive chunks before removal - -### 9. Thread-Local Context Storage (Tracer Integration) - -**What it is:** -- thread_local Context structures for APM tracer integration -- Each thread has a cache-line aligned Context containing span IDs and tags -- Pointer stored in ProfiledThread for signal-safe access - -**Memory cost:** -``` -Per thread Context: -- spanId: 8 bytes -- rootSpanId: 8 bytes -- checksum: 8 bytes -- tags array: 10 × 4 bytes = 40 bytes -- Cache line alignment padding: ~0-8 bytes -- Total per thread: 64 bytes (cache-line aligned) - -Typical overhead: -- 100-500 threads: 6.4 KB - 32 KB -- 1000+ threads: 64 KB+ -``` - -**Growth patterns:** -- Grows with thread count (one Context per thread) -- Bounded by application thread count -- Context values stored in _context_value_map Dictionary (see section 5) - -**Lifecycle:** -- Allocated lazily on first TLS access per thread -- Persists throughout thread lifetime -- Deallocated when thread terminates - -### Summary: Total Memory Overhead - -**Typical Java Application (10K-100K classes, stable code paths):** -``` -Component Memory Overhead -───────────────────────────────────────────────────── -jmethodID preallocation 10-150 MB (JVM internal, NMT Internal category) -Line number tables 40 KB - 1.6 MB -CallTraceStorage hash tables 11-44 MB -RefCountSlot arrays 512 KB (fixed) -Dictionary instances (4x) 112 KB - 8 MB -Recording buffers 1.1 MB (fixed) -ThreadIdTable arrays 32 KB (fixed) -MethodMap 104 KB - 1 MB -Thread-local Contexts 6-64 KB (depends on thread count) -───────────────────────────────────────────────────── -TOTAL (excluding jmethodID): ~14-56 MB -TOTAL (including jmethodID): 24-206 MB -``` - -**High Class Churn Application (1M+ classes):** -``` -Component Memory Overhead -───────────────────────────────────────────────────── -jmethodID preallocation 1+ GB (grows with class count) -Line number tables 1-10 MB -CallTraceStorage hash tables 50-200 MB (more unique traces) -RefCountSlot arrays 512 KB (fixed) -Dictionary instances (4x) 10-50 MB (more unique strings/contexts) -Recording buffers 1.1 MB (fixed) -ThreadIdTable arrays 32 KB (fixed) -MethodMap 1-10 MB -Thread-local Contexts 64-256 KB (high thread count) -───────────────────────────────────────────────────── -TOTAL (excluding jmethodID): ~63-273 MB -TOTAL (including jmethodID): 1+ GB (dominated by jmethodID) -``` - -**Key observations:** -- For normal applications: Total overhead is 24-206 MB (acceptable) -- For high class churn: jmethodID dominates memory usage (1+ GB) -- Most non-jmethodID memory converges after warmup -- Only jmethodID and CallTraceStorage can grow unbounded (jmethodID requires class unloading) -- MethodMap now bounded by age-based cleanup (enabled by default) -- Tracer context overhead is negligible (< 256 KB even with 1000+ threads) - -## Limitations and Restrictions - -### 1. High Class Churn Applications - -**Symptom:** -- Native memory (NMT Internal category) grows continuously -- Growth proportional to class loading rate -- Memory does not decrease even if classes are GC'd (requires ClassLoader unload) - -**Root cause:** -- Application continuously generates new classes without unloading -- Common culprits: - - Groovy scripts evaluated without caching - - Dynamic proxies created per-request - - CGLIB/Javassist code generation without caching - - ClassLoader leaks preventing class unloading - -**Impact:** -- 1M classes = ~1 GB overhead (acceptable in some cases) -- 10M classes = ~10 GB overhead (likely unacceptable) - -**When profiler is NOT appropriate:** -- Applications that generate millions of classes -- Unbounded class growth patterns -- ClassLoader leaks that prevent class unloading - -**Recommendation:** -``` -If NMT Internal category grows beyond acceptable limits: -1. Diagnose class explosion (see diagnosis section below) -2. Fix application-level class leak or caching issue -3. If unfixable: Disable profiler in production - - Profile only in staging with shorter runs - - Use alternative observability (JFR events, metrics, tracing) -``` - -### 2. No Per-Method Memory Control - -**Limitation:** -- Cannot selectively preallocate jmethodIDs for specific methods -- ClassPrepare callback must allocate for ALL methods in the class -- Cannot free jmethodIDs while profiling (required for signal-safe operation) - -**Why:** -- AGCT can encounter any method in any stack trace unpredictably -- Signal handler cannot call JVM to allocate jmethodIDs on-demand (not signal-safe) -- Selective allocation would cause profiler failures (missing method information) - -### 3. Memory Persists Until Class Unload - -**Limitation:** -- jmethodID memory cannot be freed while class is loaded -- Even if method is never sampled, its jmethodID exists -- Only freed when ClassLoader is GC'd and class is unloaded - -**Impact:** -- Long-lived applications with stable classes: Acceptable (one-time cost) -- Applications with high class churn: Unbounded growth - -**No workaround exists:** -- This is fundamental to JVM's jmethodID architecture -- All profiling approaches (AGCT, VM stack walker, etc.) require jmethodIDs -- jmethodIDs are the only reliable way to identify methods - -## When to Use the Profiler - -### ✅ Appropriate Use Cases - -**Stable class count applications:** -- Typical web services, microservices -- Batch processing jobs -- Applications with well-defined class sets -- Expected memory overhead: 24-206 MB total - - jmethodID: 10-150 MB - - Profiler data structures: 14-56 MB - - Tracer context: < 64 KB (negligible) - -**Moderate class churn:** -- Applications loading 100K-1M classes total -- Expected memory overhead: 100 MB - 1 GB total - - jmethodID: Dominant component (70-90% of total) - - Profiler data structures: 63-273 MB - - Tracer context: < 256 KB (negligible) -- Monitor NMT Internal category to ensure convergence - -### ⚠️ Caution Required - -**Dynamic scripting with caching:** -- Groovy, JavaScript engines IF scripts are cached -- Code generation frameworks IF classes are cached -- Monitor NMT Internal category growth closely - -**Microservices with hot reloading:** -- Frequent redeployments cause class reloading -- Acceptable if reloads are infrequent (hourly/daily) -- Problematic if reloads are continuous - -### ❌ NOT Appropriate - -**Unbounded class generation:** -- Groovy scripts evaluated per-request without caching -- Dynamic proxies generated per-request -- Code generation without caching -- Expected memory: Unbounded growth (9+ GB observed in production) -- Root cause: Application generates millions of classes (class explosion bug) - -**Known ClassLoader leaks:** -- Applications that leak ClassLoaders -- Classes never get unloaded -- Memory grows without bound -- Example: 9.1M classes = ~9.2 GB jmethodID overhead alone - -**Extreme class counts:** -- Applications requiring 10M+ classes -- Expected memory: 10+ GB total overhead - - jmethodID: 9-10 GB - - Profiler data structures: 1-2 GB -- **This is unacceptable** - disable profiler and fix application class explosion bug first - -## Diagnosing Class Explosion - -If NMT Internal category shows unbounded growth: - -### Step 1: Enable Class Loading Logging - -```bash -# JDK 9+ --Xlog:class+load=info,class+unload=info:file=/tmp/class-load.log - -# JDK 8 --XX:+TraceClassLoading -XX:+TraceClassUnloading -``` - -### Step 2: Monitor Class Counts - -```bash -# Count loaded classes -jcmd VM.classloader_stats - -# Show class histogram (top 100) -jcmd GC.class_histogram | head -100 - -# Count total methods -jcmd VM.class_hierarchy | grep -c "method" - -# Examine metaspace -jcmd VM.metaspace statistics -``` - -### Step 3: Identify Patterns - -**Look for:** -- Repeated class name patterns (e.g., `Script123`, `$$Lambda$456`, `EnhancerByCGLIB$$abc`) -- ClassLoaders with high class counts that never get GC'd -- Libraries known for code generation (Groovy, CGLIB, Javassist, ASM) -- Method count per class (if unusually high, indicates code complexity) - -**Expected findings for problematic applications:** -- Class names show sequential numbering (leak/no caching) -- ClassLoaders persist with growing class counts (ClassLoader leak) -- Class load rate is constant over time (not converging) - -### Step 4: Fix Root Cause - -**Common fixes:** - -1. **Cache compiled scripts:** - ```java - // BAD: New class per evaluation - new GroovyShell().evaluate(script); - - // GOOD: Cache compiled classes - scriptCache.computeIfAbsent(scriptHash, - k -> new GroovyShell().parse(script)); - ``` - -2. **Reuse dynamic proxies:** - ```java - // BAD: New proxy class per instance - Proxy.newProxyInstance(loader, interfaces, handler); - - // GOOD: Cache proxy classes or use interfaces - ``` - -3. **Configure framework caching:** - - CGLIB: `Enhancer.setUseCache(true)` - - Javassist: Reuse `ClassPool` instances - - Groovy: Configure `CompilerConfiguration` with caching - -4. **Fix ClassLoader leaks:** - - Properly dispose of ClassLoaders - - Use weak references for dynamic class caches - - Monitor ClassLoader lifecycle - -### Step 5: Verify Fix - -After fixing application: -```bash -# Class count should stabilize after warmup -watch -n 10 'jcmd GC.class_histogram | head -5' - -# NMT Internal should plateau -watch -n 10 'jcmd VM.native_memory summary | grep Internal' -``` - -**Expected result:** -- Class count converges to stable number (10K-100K for typical apps) -- Method count stabilizes (150K-1.5M methods for typical apps) -- NMT Internal growth stops after warmup -- Overhead: 10-150 MB (acceptable) - -## Monitoring Recommendations - -### NMT Metrics to Track - -**Note on RSS (Resident Set Size) Measurements:** -RSS measurements are unreliable for tracking profiler memory usage: -- GraalVM JVMCI: Can show negative RSS growth due to aggressive GC shrinking heap -- Zing JDK: Large divergence between NMT and RSS measurements (3.6% vs 82.5%) -- Recommendation: **Use NMT Internal category only** for accurate profiler memory tracking - -```bash -# Enable NMT in production (minimal overhead) --XX:NativeMemoryTracking=summary - -# Collect baseline after warmup -jcmd VM.native_memory baseline - -# Check growth periodically -jcmd VM.native_memory summary.diff -``` - -**Alert thresholds for NMT Internal category:** -- Growth > 500 MB after warmup: Investigate class loading patterns -- Growth > 2 GB: Likely class explosion (check jmethodID allocations) -- Growth > 10 GB: **Unacceptable** - disable profiler immediately -- Continuous growth (not converging): Application bug requiring fix - -**Breakdown analysis:** -- Use `jcmd VM.native_memory detail` to see allocation sites -- GetClassMethods: jmethodID allocations (should converge) -- GetLineNumberTable: Line number tables (should converge) -- Other malloc: Profiler data structures (CallTraceStorage, Dictionary, etc.) - -### Class Count Metrics - -```bash -# Track loaded classes over time -jcmd GC.class_histogram | head -1 - -# Expected pattern: -# - Rapid growth during warmup (first few minutes) -# - Convergence to stable count -# - No growth during steady state -``` - -## References - -### Why jmethodID Preallocation is Required - -**AsyncGetCallTrace (AGCT):** -- Signal-safe stack walking (operates in SIGPROF handler) -- Cannot acquire locks or call most JVM functions -- Must have all jmethodIDs allocated before profiling - -**Detailed explanation:** -- [jmethodIDs in Profiling: A Tale of Nightmares](https://mostlynerdless.de/blog/2023/07/17/jmethodids-in-profiling-a-tale-of-nightmares/) - -**Key quote:** -> "Profilers must ensure every method has an allocated jmethodID before profiling starts. Without preallocation, profilers risk encountering unallocated jmethodIDs in stack traces, making it impossible to identify methods safely." - -### JVM Internals - -**Method structure allocation:** -- [JDK-8062116](https://bugs.openjdk.org/browse/JDK-8062116) - GetClassMethods performance (JDK 8 specific) -- [JDK-8268406](https://www.mail-archive.com/serviceability-dev@openjdk.org/msg22686.html) - jmethodID memory management - -**JVMTI specification:** -- [JVMTI 1.2 Specification](https://docs.oracle.com/javase/8/docs/platform/jvmti/jvmti.html) - -## Implementation Details - -### Code Locations - -**1. jmethodID preallocation:** -- `ddprof-lib/src/main/cpp/vmEntry.cpp:497-531` - `VM::loadMethodIDs()` -- Called from ClassPrepare callback for every loaded class -- Must call `GetClassMethods()` to trigger JVM internal allocation - -**2. Line number table management:** -- `ddprof-lib/src/main/cpp/flightRecorder.cpp:46-63` - `SharedLineNumberTable` destructor -- `ddprof-lib/src/main/cpp/flightRecorder.cpp:287-295` - Lazy allocation in `fillJavaMethodInfo()` -- `ddprof-lib/src/main/cpp/counters.h` - LINE_NUMBER_TABLES counter for tracking live tables -- Properly deallocates via `jvmti->Deallocate()` (fixed in commit 8ffdb30e) -- Counter tracking added in commit 257d982d for observable line number table lifecycle - -**3. CallTraceStorage:** -- `ddprof-lib/src/main/cpp/callTraceStorage.h` - Triple-buffered hash table management -- `ddprof-lib/src/main/cpp/callTraceHashTable.h` - Hash table structure and operations -- `INITIAL_CAPACITY = 65536` entries, expands at 75% capacity -- `CALL_TRACE_CHUNK = 8 MB` per LinearAllocator chunk - -**4. RefCountSlot arrays:** -- `ddprof-lib/src/main/cpp/callTraceStorage.h:43-53` - RefCountSlot structure (64 bytes) -- `MAX_THREADS = 8192` slots, cache-line aligned to eliminate false sharing -- Used for lock-free memory reclamation of hash tables - -**5. Dictionary instances:** -- `ddprof-lib/src/main/cpp/dictionary.h` - Multi-level hash table for string deduplication -- `ROWS = 128`, `CELLS = 3` per row -- Four instances: _class_map, _string_label_map, _context_value_map, _packages - -**6. Recording buffers:** -- `ddprof-lib/src/main/cpp/buffers.h` - RecordingBuffer implementation -- `RECORDING_BUFFER_SIZE = 65536` bytes + `RECORDING_BUFFER_OVERFLOW = 8192` guard -- `CONCURRENCY_LEVEL = 16` buffers for thread-local event storage - -**7. ThreadIdTable:** -- `ddprof-lib/src/main/cpp/threadIdTable.h` - Thread ID tracking for JFR metadata -- `TABLE_SIZE = 256` entries per table -- 16 concurrency levels × 2 tables (double-buffering) = 32 tables total - -**8. MethodMap:** -- `ddprof-lib/src/main/cpp/flightRecorder.h:107-110` - MethodMap (std::map) -- `ddprof-lib/src/main/cpp/flightRecorder.h:68-105` - MethodInfo structure (_referenced, _age fields) -- `ddprof-lib/src/main/cpp/flightRecorder.cpp:601-658` - cleanupUnreferencedMethods() implementation -- `ddprof-lib/src/main/cpp/flightRecorder.cpp:517-563` - switchChunk() calls cleanup after finishChunk() -- `ddprof-lib/src/main/cpp/flightRecorder.cpp:1196-1242` - writeStackTraces() marks referenced methods -- `ddprof-lib/src/main/cpp/arguments.h:191` - _enable_method_cleanup flag (default: true) -- `ddprof-lib/src/main/cpp/arguments.cpp` - mcleanup=true/false parsing -- Stores metadata for sampled methods with lazy allocation -- Age-based cleanup removes methods unused for 3+ consecutive chunks -- Cleanup logs LINE_NUMBER_TABLES counter value via TEST_LOG for observability - -**9. Thread-local Context storage:** -- `ddprof-lib/src/main/cpp/context.h:32-40` - Context structure (cache-line aligned, 64 bytes) -- `ddprof-lib/src/main/cpp/context.h:57` - thread_local context_tls_v1 declaration -- `DD_TAGS_CAPACITY = 10` tags per context -- Context values stored in _context_value_map Dictionary (profiler.h:122) - -### Known Issues Fixed - -**GetLineNumberTable leak (Fixed):** -- SharedLineNumberTable destructor was not properly deallocating JVMTI memory -- Impact: 1.2 GB leak for applications sampling 3.8M methods -- Fix: Added null checks and error handling in destructor (commit 8ffdb30e) -- Tracking: LINE_NUMBER_TABLES counter provides observable lifecycle tracking (commit 257d982d) -- Test: `GetLineNumberTableLeakTest` validates cleanup via TEST_LOG output (RSS unreliable across JVMs) - -**MethodMap unbounded growth (Fixed):** -- Recording._method_map accumulated ALL methods forever in long-running applications -- Impact: 3.8M methods × ~300 bytes = 1.2 GB over days in production -- Root cause: Recording objects live for entire app lifetime, never freed methods -- Fix: Age-based cleanup removes methods unused for 3+ consecutive chunks -- Implementation: Mark-and-sweep during switchChunk(), enabled by default -- Test: `GetLineNumberTableLeakTest.testMethodMapCleanupDuringContinuousProfile()` validates bounded growth -- Feature flag: `mcleanup=true` (default: enabled), `mcleanup=false` to disable - -**Test validation approach:** -- Test uses TEST_LOG output for validation (commits 7ed1e7eb, 257d982d) -- RSS measurements removed due to unreliability across JVMs (commits 33f6c5c0, 4bbfcfe8) -- Counter infrastructure provides observable line number table lifecycle tracking - -**Previous investigation findings:** -- See git history for detailed investigation (commits 8ffdb30e, a9fa649c, 2ab1d263) -- Investigation confirmed jmethodID preallocation is required, not a bug diff --git a/doc/reference/RemoteSymbolication.md b/doc/reference/RemoteSymbolication.md deleted file mode 100644 index 5e9313a93..000000000 --- a/doc/reference/RemoteSymbolication.md +++ /dev/null @@ -1,312 +0,0 @@ -# Remote Symbolication Implementation - -This document describes the implementation of build-id and pc/offset storage in native frames for remote symbolication in the Java profiler. - -## Overview - -The enhancement allows the Java profiler to store raw build-id and PC offset information for native frames instead of resolving symbols locally. This enables remote symbolication services to handle symbol resolution, which is especially useful for: - -- Distributed profiling scenarios where symbol files aren't available locally -- Reduced profiler overhead by deferring symbol resolution -- Better support for stripped binaries -- Centralized symbol management - -## Implementation Summary - -### 1. **Build-ID Extraction** (`symbols_linux_dd.h/cpp`) - -- **SymbolsLinux**: Utility class to extract GNU build-id from ELF files -- Supports both file-based and memory-based extraction -- Handles .note.gnu.build-id section parsing -- Returns hex-encoded build-id strings - -### 2. **Enhanced CodeCache** (`codeCache.h/cpp`) - -Added fields to store build-id information: -- `_build_id`: Hex-encoded build-id string -- `_build_id_len`: Raw build-id length in bytes -- `_load_bias`: Load bias for address calculations -- Methods: `hasBuildId()`, `buildId()`, `setBuildId()`, etc. - -### 3. **Packed Remote Frame Data** (`profiler.h`) - -- **RemoteFramePacker**: Utility struct for packing/unpacking remote symbolication data - - Packs into 64-bit jmethodID: `pc_offset (44 bits) | mark (3 bits) | lib_index (17 bits)` - - PC offset: 44 bits = 16 TB address range - - Mark: 3 bits = 0-7 values (JVM internal frame markers) - - Library index: 17 bits = 131K libraries max -- **RemoteFrameInfo**: Structure for JFR serialization (vmEntry.h): - - `build_id`: Library build-id string - - `pc_offset`: PC offset within library - - `lib_index`: Library table index -- **BCI_NATIVE_FRAME_REMOTE**: Frame encoding (-19) indicates packed remote data - -### 4. **Enhanced Frame Collection** (`profiler.cpp`, `stackWalker.h`) - -Modified frame collection to support dual modes: -- **Traditional mode**: Stores resolved symbol names (existing behavior) -- **Remote mode**: Stores RemoteFrameInfo with build-id and offset - -**Key Functions**: -- `populateRemoteFrame()`: Packs pc_offset, mark, and lib_index into jmethodID field -- `resolveNativeFrameForWalkVM()`: Resolves native frames for walkVM/walkVMX modes - - Performs binarySearch() to get symbol name - - Extracts mark via NativeFunc::read_mark() (O(1)) - - Packs data using RemoteFramePacker::pack() -- `convertNativeTrace()`: Converts raw PCs to frames for walkFP/walkDwarf modes - - Checks marks to terminate at JVM internal frames - - Calls populateRemoteFrame() to pack data - -**Mark Checking**: -- Uses binarySearch() + NativeFunc::read_mark() approach (O(log n) + O(1)) -- Performance identical to traditional symbolication -- Simpler than maintaining separate marked ranges index -- Mark values packed into jmethodID for later unpacking - -**Stack Walker Integration**: -- **walkFP/walkDwarf**: Return raw PCs → `convertNativeTrace()` → `populateRemoteFrame()` -- **walkVM/walkVMX**: Directly call `resolveNativeFrameForWalkVM(pc, lock_index)` during stack walk (patched via gradle/patching.gradle) - -### 5. **JFR Serialization** (`flightRecorder.cpp/h`) - -- **resolveMethod()**: Unpacks remote frame data during JFR serialization - - Uses RemoteFramePacker::unpackPcOffset/Mark/LibIndex() - - Looks up library by index via Libraries::getLibraryByIndex() - - Creates temporary RemoteFrameInfo with build_id and pc_offset -- **fillRemoteFrameInfo()**: Serializes remote frame data to JFR format - - Stores `.` in class name field (e.g., `deadbeef1234567890abcdef.`) - - Stores PC offset in signature field (e.g., `(0x1234)`) - - Uses modifier flag 0x100 (ACC_NATIVE, same as regular native frames) -- **Thread Safety**: Called during JFR serialization with lockAll() held - - Library array is stable (no concurrent dlopen_hook modifications) - - No additional locking needed - -### 6. **Configuration** (`arguments.h/cpp`) - -- **remotesym[=BOOL]**: New profiler argument -- Default: disabled -- Can be enabled with `remotesym=true` or `remotesym=y` - -### 7. **Libraries Integration** (`libraries.h/cpp`, `libraries_linux.cpp`) - -- **updateBuildIds()**: Extracts build-ids for all loaded libraries - - Called during profiler startup when remote symbolication is enabled - - Uses O(1) cache lookup via `_build_id_processed` set - - Mirrors `_parsed_inodes` pattern from symbols_linux.cpp - - Linux-only implementation using ELF parsing -- **getLibraryByIndex()**: Retrieves CodeCache by library index - - Parameter type: uint32_t (matches 17-bit lib_index packing) - - Returns nullptr if index out of bounds - - Used during JFR serialization to unpack remote frames - -### 8. **Upstream Stack Walker Integration** (`gradle/patching.gradle`) - -Patches async-profiler's `stackWalker.h` and `stackWalker.cpp` to integrate remote symbolication: - -**Header Patches (stackWalker.h)**: -- Adds `lock_index` parameter to all three `walkVM` signatures (private implementation, public with features, public with anchor) -- Enables per-strip RemoteFrameInfo pool access during stack walking - -**Implementation Patches (stackWalker.cpp)**: -- Updates all `walkVM` signatures to accept and propagate `lock_index` -- **Critical patch at line 454**: Replaces `profiler->findNativeMethod(pc)` with `profiler->resolveNativeFrameForWalkVM(pc, lock_index)` -- Adds dynamic BCI selection (BCI_NATIVE_FRAME vs BCI_NATIVE_FRAME_REMOTE) -- Adds `fillFrame()` overload for void* method_id to support both symbol names and RemoteFrameInfo pointers -- Handles marked C++ interpreter frames (terminates scan if detected) - -## Usage - -### Enable Remote Symbolication - -```bash -java -agentpath:/libjavaProfiler.so=start,cpu,remotesym=true,file=profile.jfr MyApp -``` - -### Mixed Configuration - -```bash -java -agentpath:/libjavaProfiler.so=start,event=cpu,interval=1000000,remotesym=true,file=profile.jfr MyApp -``` - -## JFR Output Format - -When remote symbolication is enabled, native frames in the JFR output contain: - -- **Class Name**: Build-ID hex string (e.g., `deadbeef1234567890abcdef`) - - Stored via `_classes->lookup(rfi->build_id)` - - Deduplicated in JFR constant pool -- **Method Name**: `` - - Constant string indicating remote symbolication needed - - Stored via `_symbols.lookup("")` -- **Signature**: PC offset in hex (e.g., `0x1a2b`) - - Formatted with `snprintf(buf, size, "0x%lx", pc_offset)` - - Stored via `_symbols.lookup(offset_hex)` - - Note: No parentheses, just hex value -- **Modifier**: `0x100` (ACC_NATIVE) - - Same as regular native frames for consistency -- **Frame Type**: `FRAME_NATIVE_REMOTE` (7) - - Distinguishes from regular native frames (FRAME_NATIVE = 6) - - Allows parsers to identify frames needing remote symbolication - -## Backward Compatibility - -- **Default behavior**: No changes (remote symbolication disabled) -- **Mixed traces**: Supports both local and remote frames in same trace -- **Fallback**: Gracefully falls back to local symbolication when build-id unavailable - -## Memory Management - -- **Build-IDs**: Stored once per CodeCache, shared across frames - - Hex string allocated with malloc (one-time, ~40 bytes per library) - - Freed in CodeCache destructor - - Total overhead: < 2 KB for typical applications -- **Packed Remote Frames**: No separate allocation needed - - Data packed directly into 64-bit jmethodID field - - Zero additional memory overhead per frame - - Eliminates need for signal-safe pool allocation -- **JFR Serialization**: Temporary RemoteFrameInfo created during unpacking - - Stack-allocated, no heap allocation - - Only exists during JFR serialization with lockAll() held - -## Testing - -### Unit Tests -- **remotesymbolication_ut.cpp**: Tests RemoteFrameInfo structure and build-id extraction -- **remoteargs_ut.cpp**: Tests argument parsing for remote symbolication option - -### Test Coverage -- Build-ID extraction from ELF files -- Frame encoding/decoding -- Argument parsing -- Error handling for invalid inputs - -## Platform Support - -- **Linux**: Full support with ELF build-id extraction -- **macOS/Windows**: Framework in place, needs platform-specific implementation - -## Observability and Metrics - -The following counters track remote symbolication usage (added to `counters.h`): - -- **REMOTE_SYMBOLICATION_FRAMES**: Number of frames using remote symbolication - - Incremented in `populateRemoteFrame()` each time a remote frame is created - - Indicates actual usage of the feature -- **REMOTE_SYMBOLICATION_LIBS_WITH_BUILD_ID**: Libraries with extracted build-IDs - - Incremented in `updateBuildIds()` after successful build-ID extraction - - Shows how many libraries are eligible for remote symbolication -- **REMOTE_SYMBOLICATION_BUILD_ID_CACHE_HITS**: Build-ID cache hit rate - - Incremented when `_build_id_processed` cache prevents redundant extraction - - Demonstrates effectiveness of O(1) caching strategy - -These metrics appear in profiler statistics and can be used to monitor: -- Feature adoption rate (frames with remote symbolication vs total native frames) -- Build-ID coverage (libraries with build-IDs vs total libraries) -- Cache efficiency (cache hits vs total updateBuildIds() calls) - -## Performance Considerations - -### Benefits -- **Identical hot-path performance** to traditional symbolication - - Same O(log n) binarySearch for mark checking - - Zero additional overhead for packed representation -- **Reduced memory footprint**: 8 bytes per frame vs storing symbol strings -- **Faster profiling** with deferred full symbolication to post-processing -- **Eliminated duplicate lookups**: Single binarySearch per frame (was 2x before optimization) - -### Costs -- **One-time build-ID extraction** during startup (~ms per library) - - Cached with O(1) lookup to prevent redundant work - - Only extracted for libraries loaded when profiler starts or via dlopen -- **Library index lookup** during JFR serialization - - O(1) array access with lockAll() held - - No contention (serialization is single-threaded) - -## Future Enhancements - -1. **macOS Support**: Implement Mach-O UUID extraction -2. **Caching**: Cache build-ids across profiler sessions -3. **Compression**: Compress build-ids in JFR output -4. **Validation**: Add runtime validation of build-id consistency -5. **Dynamic Pool Sizing**: Adjust RemoteFrameInfo pool size based on workload -6. **Native Frame Modifier Optimization**: Change native frame modifiers from `0x100` to `0x0` - - Current: All native frames use `0x100` (ACC_NATIVE) = 2-byte varint encoding - - Proposed: Use `0x0` (no modifiers) = 1-byte varint encoding - - Benefit: **Save 1 byte per native frame** across all JFR recordings - - Impact: Significant space savings for native-heavy profiles (C++ applications) - - Note: Would require coordination with JFR parsing tools - -## File Structure - -``` -ddprof-lib/src/main/cpp/ -├── symbols_linux_dd.h # Build-ID extraction interface (Linux-specific) -├── symbols_linux_dd.cpp # Build-ID extraction with bounds/alignment checks -├── vmEntry.h # Enhanced with RemoteFrameInfo and BCI constants -├── codeCache.h # Enhanced with build-id fields (cleaned up operator[]) -├── codeCache.cpp # Build-id storage implementation -├── profiler.h # Added resolveNativeFrame/ForWalkVM, RemoteFrameInfo pool -├── profiler.cpp # Remote symbolication logic and pool allocation -├── stackWalker.h # Enhanced with lock_index and truncated parameters -├── stackWalker.cpp # Remote symbolication and truncation detection logic -├── flightRecorder.h # Added fillRemoteFrameInfo declaration -├── flightRecorder.cpp # Remote frame JFR serialization -├── arguments.h # Added _remote_symbolication field -├── arguments.cpp # Remote symbolication argument parsing -├── libraries.h # Added updateBuildIds method -└── libraries.cpp # Build-id extraction for loaded libraries - -ddprof-lib/src/test/cpp/ -├── remotesymbolication_ut.cpp # Unit tests for remote symbolication -└── remoteargs_ut.cpp # Unit tests for argument parsing - -ddprof-test/src/test/java/ -└── RemoteSymbolicationTest.java # Integration tests for all cstack modes -``` - -## Implementation Notes - -### Thread Safety -- **Build-ID extraction**: Protected by `_build_id_lock` mutex in `updateBuildIds()` -- **Build-ID cache**: `_build_id_processed` set provides O(1) duplicate detection -- **JFR serialization**: Called with `lockAll()` held, library array is stable - - No concurrent dlopen_hook modifications possible - - No additional locking needed for `getLibraryByIndex()` - -### Signal Handler Safety -- **Packed representation**: No allocations in signal handlers - - Data packed directly into 64-bit jmethodID field - - Zero memory overhead, eliminates need for signal-safe pools -- **Read-only operations**: binarySearch() and mark checking are signal-safe - - No malloc, no locks (except tryLock which is acceptable) - - Atomic operations where needed - -### Error Handling -- **Graceful fallback**: Falls back to traditional symbolication when: - - Library has no build-ID - - Library index out of bounds during unpacking - - Build-ID extraction fails -- **Defensive programming**: Null checks before dereferencing pointers -- **Logging**: TEST_LOG() for debugging production issues - -### ELF Security -- Bounds checking for program header table (prevents reading beyond mapped region) -- Alignment verification for program header offset (prevents misaligned pointer access) -- Two-stage validation for note sections (header first, then payload) -- ELFCLASS64 verification ensures uniform 64-bit structure sizes - -### Stack Walker Integration -- **walkFP/walkDwarf**: Return raw PCs → `convertNativeTrace()` → `populateRemoteFrame()` -- **walkVM/walkVMX**: Direct call to `resolveNativeFrameForWalkVM()` during stack walk - - No post-processing or reverse PC lookup needed - - Mark checking happens inline during frame resolution - - Terminates stack walk at JVM internal frames (marks != 0) - -### Design Evolution -- **Original approach**: Separate marked ranges index with O(log n) isMarkedAddress() -- **Current approach**: Simplified to binarySearch() + NativeFunc::read_mark() - - Same O(log n) performance but ~150 lines less code - - Eliminated complexity of maintaining separate index - - Marks packed into jmethodID for JFR serialization - -This implementation provides a solid foundation for remote symbolication while maintaining full backward compatibility and robust error handling. diff --git a/doc/reference/RemoteSymbolicationFrameTypes.md b/doc/reference/RemoteSymbolicationFrameTypes.md deleted file mode 100644 index 9511082ae..000000000 --- a/doc/reference/RemoteSymbolicationFrameTypes.md +++ /dev/null @@ -1,134 +0,0 @@ -# Frame Type vs Modifier: Design Decision for Remote Symbolication - -## Final Solution: Use Frame Type Instead of Modifier - -After evaluating multiple approaches, **we chose to use a new frame type (`FRAME_NATIVE_REMOTE = 7`) instead of a custom modifier flag**. This eliminates: -- ✅ All varint encoding overhead -- ✅ Any potential conflicts with Java modifiers -- ✅ Ambiguity in semantics - -Remote native frames now use: -- **Modifier**: `0x0100` (ACC_NATIVE, same as regular native frames) -- **Frame Type**: `FRAME_NATIVE_REMOTE` (7) - -## Java Access Modifiers (from JVM Spec) - -The Java Virtual Machine specification defines the following access modifiers for classes, methods, and fields: - -| Modifier | Value | Applies To | Description | -|----------|-------|------------|-------------| -| ACC_PUBLIC | 0x0001 | All | Public access | -| ACC_PRIVATE | 0x0002 | Methods/Fields | Private access | -| ACC_PROTECTED | 0x0004 | Methods/Fields | Protected access | -| ACC_STATIC | 0x0008 | Methods/Fields | Static member | -| ACC_FINAL | 0x0010 | All | Final/non-overridable | -| ACC_SYNCHRONIZED | 0x0020 | Methods | Synchronized method | -| ACC_SUPER | 0x0020 | Classes | Treat superclass invokes specially | -| ACC_BRIDGE | 0x0040 | Methods | Compiler-generated bridge method | -| ACC_VOLATILE | 0x0040 | Fields | Volatile field | -| ACC_VARARGS | 0x0080 | Methods | Variable arity method | -| ACC_TRANSIENT | 0x0080 | Fields | Not serialized | -| ACC_NATIVE | 0x0100 | Methods | Native implementation | -| **ACC_INTERFACE** | **0x0200** | **Classes** | **Interface declaration** | -| ACC_ABSTRACT | 0x0400 | Classes/Methods | Abstract class/method | -| ACC_STRICT | 0x0800 | Methods | Use strict floating-point | -| ACC_SYNTHETIC | 0x1000 | All | Compiler-generated | -| ACC_ANNOTATION | 0x2000 | Classes | Annotation type | -| ACC_ENUM | 0x4000 | Classes/Fields | Enum type/constant | -| ACC_MANDATED | 0x8000 | Parameters | Implicitly declared | - -## Profiler Custom Modifiers - -For the Java profiler's internal use, we define custom modifier flags that don't conflict with Java's standard modifiers: - -| Modifier | Value | Usage | Notes | -|----------|-------|-------|-------| -| ACC_NATIVE | 0x0100 | Native frames | Reuses Java's ACC_NATIVE for consistency | -| ACC_SYNTHETIC | 0x1000 | Compiler-generated | Reuses Java's ACC_SYNTHETIC | -| ACC_BRIDGE | 0x0040 | Bridge methods | Reuses Java's ACC_BRIDGE | -| **ACC_REMOTE_SYMBOLICATION** | **0x10000** | **Remote native frames** | **Custom profiler flag (bit 16, outside Java range)** | - -## Modifier Conflict Analysis - -### Evolution of the Design - -**Version 1 (Initial)**: Used `0x200` -- ❌ **CONFLICT**: Java's `ACC_INTERFACE` (0x0200) -- Issues: Could confuse JFR parsers, clash with standard modifiers - -**Version 2**: Changed to `0x2000` -- ⚠️ **CONFLICT**: Java's `ACC_ANNOTATION` (0x2000) -- While theoretically safe for methods (annotations only apply to classes), still within Java's reserved range - -**Version 3 (Final)**: Changed to `0x10000` (bit 16) -- ✅ **NO CONFLICTS**: Completely outside Java's standard modifier range (0x0001-0x8000) -- ✅ Clean separation from JVM specification -- ✅ Future-proof against new Java modifiers - -### Why 0x10000 is the Correct Choice - -**Java Modifier Range:** -- Java uses bits 0-15 (0x0001 to 0x8000) -- Highest standard modifier: `ACC_MANDATED = 0x8000` (bit 15) - -**Custom Profiler Range:** -- Bits 16-30 available for custom flags (0x10000 to 0x40000000) -- `0x10000` (bit 16) is first bit outside Java range -- Clean power of 2, easy to test and debug - -**Benefits:** -1. **Zero theoretical conflicts** with any Java modifier (current or future) -2. **Clear separation** between JVM standard (bits 0-15) and profiler custom (bits 16+) -3. **32-bit safe**: Well within `jint` range (signed 32-bit) -4. **JFR compatible**: `_modifiers` field supports full 32-bit values -5. **Extensible**: Room for additional custom flags (0x20000, 0x40000, etc.) - -### Varint Encoding Analysis - -JFR uses LEB128 variable-length encoding for modifiers. The encoding size depends on the value: - -| Value Range | Example | Bytes | Notes | -|-------------|---------|-------|-------| -| 0x00-0x7F | 0 | 1 | Most compact | -| 0x80-0x3FFF | 0x0100 (ACC_NATIVE) | 2 | Standard native frames | -| 0x4000-0x1FFFFF | 0x1000 (ACC_SYNTHETIC) | 2 | High standard modifiers | -| 0x10000+ | 0x10000 | 3 | **+1 byte overhead!** | - -**Critical insight**: Using `0x10000` would add **1 extra byte per remote native frame**. Over millions of frames, this becomes significant! - -### Alternative Approaches Rejected - -1. **Use modifier 0x0200 (bit 9)**: - - ❌ Conflicts with ACC_INTERFACE - -2. **Use modifier 0x2000 (bit 13)**: - - ❌ Conflicts with ACC_ANNOTATION (theoretically) - - ⚠️ Would be safe in practice (annotations only for classes) - -3. **Use modifier 0x10000 (bit 16)**: - - ❌ 3-byte varint encoding vs 2-byte for regular frames - - ❌ **+1 byte overhead per frame** = significant space impact - -4. **Use a separate field**: - - ❌ Would require JFR metadata changes - - ❌ Breaks backward compatibility - -5. **Use frame type FRAME_NATIVE_REMOTE (CHOSEN)**: - - ✅ Zero encoding overhead (type already serialized) - - ✅ No modifier conflicts - - ✅ Clear semantics - -## Best Practices - -When adding custom modifiers in the future: - -1. **Check JVM Spec**: Always verify against latest JVM specification -2. **Consider Context**: Modifiers for methods vs classes vs fields -3. **Document Clearly**: Explain why the bit is safe to use -4. **Test Compatibility**: Verify JFR parsers handle custom modifiers correctly - -## References - -- Java Virtual Machine Specification (JVMS §4.1, §4.5, §4.6) -- JFR Format Specification -- Original Implementation: [elfBuildId.cpp, flightRecorder.cpp] \ No newline at end of file diff --git a/doc/reference/TestFlakinessAnalysis.md b/doc/reference/TestFlakinessAnalysis.md deleted file mode 100644 index 4a74e4a46..000000000 --- a/doc/reference/TestFlakinessAnalysis.md +++ /dev/null @@ -1,718 +0,0 @@ -# Test Flakiness Analysis - -## Overview - -This document analyzes potential sources of flakiness in two profiler tests: -- `CpuDumpSmokeTest` (@RetryTest(3) - indicates moderate flakiness) -- `ContextWallClockTest` (@RetryTest(5) - indicates severe flakiness) - -Both tests are parameterized across 4 stack walking modes: `vm`, `vmx`, `fp`, `dwarf`. - ---- - -## CpuDumpSmokeTest Flakiness Sources - -### 1. **CRITICAL: Null Pointer Risk in method3() - JfrDumpTest.java:72** - -**Issue:** -```java -for (String s : new File("/tmp").list()) { - value += s.substring(0, Math.min(s.length(), 16)).hashCode(); -``` - -**Problems:** -- `File.list()` can return `null` if: - - Directory doesn't exist - - I/O error occurs - - Permission denied -- No null check before iteration → `NullPointerException` -- Empty string filenames would cause `substring()` issues - -**Impact:** Immediate test failure when /tmp is inaccessible or returns null. - -**Likelihood:** Medium on CI systems, low on developer machines. - ---- - -### 2. **File System Race Conditions - JfrDumpTest.java:72** - -**Issue:** -```java -for (String s : new File("/tmp").list()) { -``` - -**Problems:** -- /tmp directory is shared by all processes -- Files can be created/deleted by other processes during iteration -- File list can vary between test runs -- Different file counts affect loop execution time and sampling - -**Impact:** Variable test workload → variable CPU samples captured. - -**Likelihood:** High on shared CI systems. - ---- - -### 3. **Time-Based Workload Variability - JfrDumpTest.java:69-81** - -**Issue:** -```java -long ts = System.nanoTime(); -for (int i = 0; i < 1000; ++i) { - // ... work ... - if ((System.nanoTime() - ts) > 20000000L) { // 20ms timeout - break; - } -} -``` - -**Problems:** -- Loop breaks after 20ms regardless of iteration count -- Actual iterations vary based on: - - System load - - CPU frequency scaling - - Cache effects - - File system performance -- Variable workload → variable stack trace patterns - -**Impact:** Some runs might not execute enough iterations to be sampled. - -**Likelihood:** High under varying system load. - ---- - -### 4. **CPU Sampling Non-Determinism** - -**Issue:** CPU profiling with 1ms interval is probabilistic. - -**Problems:** -- Samples are taken at arbitrary execution points -- No guarantee specific methods will be sampled -- Short execution windows (50 iterations) might miss samples -- Stack walking can fail intermittently based on frame structure - -**Impact:** `verifyStackTraces()` might not find expected patterns. - -**Likelihood:** Medium - mitigated by 10 dump iterations and 500 final iterations. - ---- - -### 5. **Stack Walking Mode Differences** - -**Issue:** Different stack walking modes have different failure characteristics. - -**Problems:** -- **fp (frame pointer):** Fails if code compiled without `-fno-omit-frame-pointer` -- **dwarf:** Requires debug symbols, can be slow -- **vm/vmx:** Depend on JVM internal structures, can fail on unsupported frames -- Each mode has different success rates for native frames - -**Impact:** Some modes might fail to capture stack traces intermittently. - -**Likelihood:** Medium - mode-dependent. - ---- - -## ContextWallClockTest Flakiness Sources - -### 1. **CRITICAL: Known Weight Distribution Issues - BaseContextWallClockTest.java:163-178** - -**Issue:** Explicitly documented in comments: - -```java -// After async-profiler 4.2.1 integration and wall clock collapsing fixes, weight -// distribution changed across all unwinding modes (vm, vmx, fp, dwarf). All modes now -// show ~55% weight for method1Impl instead of expected ~33%. Root causes include: -// 1. DWARF: collects 10-20 native frames (vs 2-5 for FP), native frame PCs vary causing -// trace ID fragmentation -// 2. FP/VMX: async-profiler integration changed frame collection or attribution behavior -// 3. All modes: trace IDs hash all frames including native PCs with slight address variations -``` - -**Problems:** -- Expected 33% weight per method, actual ~55% for method1Impl -- Different modes show different distributions -- Root cause: trace ID fragmentation due to native frame PC variations -- "Proper fix requires architectural changes" per comment - -**Impact:** Weight assertions fail when distribution deviates beyond relaxed 30% margin. - -**Likelihood:** **HIGH** - this is a known issue with architectural root causes. - -**Current Mitigation:** Error margin increased to 30% for affected modes. - ---- - -### 2. **CRITICAL: Complex Multi-Threaded Coordination - BaseContextWallClockTest.java:209-225** - -**Issue:** Complex synchronization pattern with multiple race condition points: - -```java -public void method1Impl(int id, Tracing.Context context) throws ExecutionException, InterruptedException { - sleep(10); // Point A - Object monitor = new Object(); - Future wait = executor.submit(() -> method3(id, monitor)); // Point B - method2(id, monitor); // Point C - synchronized (monitor) { - monitor.wait(10); // Point D - only 10ms! - } - wait.get(); // Point E -} -``` - -**Race Conditions:** -1. **Point A:** Sleep can be interrupted or delayed by system scheduling -2. **Point B:** Executor thread might not start immediately -3. **Point C:** method2 might acquire monitor before method3 submitted -4. **Point D:** Only 10ms wait - if notify() delayed, wait times out -5. **Point E:** If method3 not complete, blocks indefinitely - -**Problems:** -- Monitor contention timing is non-deterministic -- 10ms wait timeout is too short for loaded systems -- Thread scheduling order not guaranteed -- Wall clock samples can hit any of these points → different thread states - -**Impact:** Thread state assertions (WAITING, PARKED, CONTENDED) might not all be observed. - -**Likelihood:** **HIGH** on loaded systems or slow platforms. - ---- - -### 3. **Race Condition in Sample Attribution - BaseContextWallClockTest.java:105-108** - -**Issue:** Explicitly documented in comments: - -```java -// a lot of care needs to be taken here with samples that fall between a context activation and -// a method call. E.g. not finding method2Impl in the stack trace doesn't mean the sample wasn't -// taken in the part of method2 between activation and invoking method2Impl, which complicates -// assertions when we only find method1Impl -``` - -**Problems:** -- Wall clock samples can arrive between context activation and method invocation -- Stack traces might show method1 frames when actually executing method2 setup -- Attribution logic has complex fallback cases -- This is inherently racy - profiler samples at arbitrary async points - -**Impact:** Weight attribution can be incorrect, causing assertion failures. - -**Likelihood:** **HIGH** - fundamental race condition in async sampling. - ---- - -### 4. **Custom Sleep Implementation - BaseContextWallClockTest.java:263-272** - -**Issue:** -```java -private void sleep(long millis) { - long target = System.nanoTime() + millis * 1_000_000L; - do { - try { - Thread.sleep((target - System.nanoTime()) / 1_000_000L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } while (System.nanoTime() < target); -} -``` - -**Problems:** -- Loop continues after interrupt, only sets interrupt flag -- No guarantee of exact sleep time -- System scheduling can cause significant delays -- On loaded systems, might loop many times -- Interrupt flag restoration doesn't prevent continued execution - -**Impact:** Timing skew in test execution → different sampling patterns. - -**Likelihood:** Medium - depends on system load. - ---- - -### 5. **Platform-Specific Behavior - BaseContextWallClockTest.java:163-165** - -**Issue:** -```java -// TODO: vmstructs unwinding on Liberica and aarch64 creates a higher number of broken frames -// it is under investigation but until it gets resolved we will just relax the error margin -double allowedError = Platform.isAarch64() && "BellSoft".equals(System.getProperty("java.vendor")) ? 0.4d : 0.2d; -``` - -**Problems:** -- Liberica on aarch64 requires 40% error margin (vs 20% baseline) -- vmstructs unwinding creates broken frames -- Still under investigation per comment -- Platform-specific issues not fully resolved - -**Impact:** Tests might fail on specific platform/JVM combinations. - -**Likelihood:** Medium on affected platforms. - ---- - -### 6. **Executor Shutdown Timing - BaseContextWallClockTest.java:50-51** - -**Issue:** -```java -executor.shutdownNow(); -executor.awaitTermination(30, TimeUnit.SECONDS); -``` - -**Problems:** -- `shutdownNow()` interrupts running tasks -- Tasks might be interrupted mid-execution -- Profiler state might be inconsistent if sampling during shutdown -- 30s timeout might not be enough on slow systems - -**Impact:** Final samples might capture inconsistent state. - -**Likelihood:** Low - but could happen on very slow systems. - ---- - -### 7. **Configuration-Dependent Assertions - BaseContextWallClockTest.java:185** - -**Issue:** -```java -if (config.equals("release") || config.equals("debug")) { - // ... weight assertions ... -} -``` - -**Problems:** -- Sanitizer configs (asan, tsan) skip weight assertions -- Test still runs and collects data for these configs -- Different code paths based on config -- Might hide issues in sanitizer builds - -**Impact:** Inconsistent test coverage across configurations. - -**Likelihood:** N/A - intentional behavior, but worth noting. - ---- - -## Common Flakiness Sources - -### 1. **Signal-Based Profiling Fundamentals** - -Both tests use signal-based profiling: -- **CPU:** SIGPROF signal -- **Wall:** SIGALRM signal - -**Problems:** -- Signal delivery not guaranteed -- Signals can be delayed on heavily loaded systems -- Signal handlers can be interrupted by other signals -- Some system calls block signals -- Kernel can merge or drop signals under load - -**Impact:** Missing or delayed samples → incomplete data. - ---- - -### 2. **Short Sampling Intervals (1ms)** - -Both tests use 1ms sampling intervals. - -**Problems:** -- Very aggressive sampling rate -- High profiler overhead -- More susceptible to timing variations -- Scheduler quantum effects more pronounced -- More signal delivery conflicts - -**Impact:** Increased likelihood of dropped or delayed samples. - ---- - -### 3. **Stack Walking Mode Variability** - -Tests run with 4 modes: vm, vmx, fp, dwarf - -**Problems:** -- **vm:** Relies on JVM internal structures, can fail on complex frames -- **vmx:** Extended VM structs, can fail on unsupported frame types -- **fp:** Requires frame pointers, fails if omitted during compilation -- **dwarf:** Requires debug symbols, slow to parse, can fail on stripped binaries - -**Impact:** Mode-specific failures not always reproducible. - ---- - -### 4. **JFR Event Buffering and Flush Timing** - -Both tests rely on JFR events being captured. - -**Problems:** -- Events are buffered in thread-local buffers -- Buffers flushed asynchronously or when full -- `dump()` and `stop()` might not capture all in-flight events -- Event loss can occur under high load - -**Impact:** Expected events might not appear in recordings. - ---- - -### 5. **Platform and JVM Differences** - -Tests run across multiple platforms and JVMs. - -**Problems:** -- Different schedulers: Linux CFS, macOS Mach -- Different architectures: x64, arm64 -- Different JVM implementations: HotSpot, J9, Zing -- Different signal handling behaviors -- Platform-specific stack walking issues - -**Impact:** Behavior varies across test matrix. - ---- - -## Recommendations - -### High Priority (CpuDumpSmokeTest) - -1. **Fix null pointer risk in method3():** - ```java - String[] files = new File("/tmp").list(); - if (files != null) { - for (String s : files) { - // ... existing code ... - } - } - ``` - -2. **Consider using fixed workload instead of time-based:** - - Replace 20ms timeout with fixed iteration count - - Reduces variability in sampling - -3. **Increase execution iterations:** - - Run method1/2/3 more times (e.g., 200 instead of 50) - - Increases probability of capturing expected samples - -### High Priority (ContextWallClockTest) - -1. **Address known weight distribution issue:** - - Document this is a known limitation - - Consider disabling strict weight assertions until architectural fix - - Or further relax error margins - -2. **Increase monitor wait timeout:** - ```java - monitor.wait(10); // Too short! - ``` - - Change to at least 100ms or higher - - Reduces likelihood of spurious timeouts - -3. **Add retry logic for thread state assertions:** - - Thread states might not all appear in single run - - Aggregate across multiple iterations before asserting - -4. **Fix sleep() interrupt handling:** - ```java - private void sleep(long millis) { - long target = System.nanoTime() + millis * 1_000_000L; - while (System.nanoTime() < target) { - try { - long remaining = (target - System.nanoTime()) / 1_000_000L; - if (remaining <= 0) break; - Thread.sleep(remaining); - break; // Success - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; // Exit on interrupt - } - } - } - ``` - -### Medium Priority (Both Tests) - -1. **Increase sampling intervals to 5-10ms:** - - Reduces profiler overhead - - Less susceptible to timing issues - - Still sufficient for testing - -2. **Add diagnostic logging:** - - Log when samples are missed - - Log actual vs expected weight distributions - - Log thread states observed - - Helps diagnose intermittent failures - -3. **Consider conditional assertions:** - - Some assertions might need to be advisory on CI - - Log warnings instead of failing - - Track failure rates over time - -4. **Add test warmup phase:** - - Run profiler for brief period before actual test - - Ensures profiler fully initialized - - Reduces cold-start effects - -### Low Priority - -1. **Consolidate retry logic:** - - Consider increasing retry counts further - - Add exponential backoff between retries - - Log retry reasons - -2. **Platform-specific test parameters:** - - Adjust timeouts based on platform - - Adjust error margins based on known issues - - Use @EnabledOnOs annotations for problematic platforms - ---- - -## Conclusion - -Both tests suffer from inherent flakiness due to: -1. **Probabilistic sampling** - can't guarantee specific patterns will be captured -2. **Timing dependencies** - multiple race conditions and time-based logic -3. **Known issues** - documented weight distribution problems and platform quirks -4. **Signal delivery** - non-deterministic nature of signal-based profiling - -The **ContextWallClockTest** is particularly flaky (@RetryTest(5)) due to: -- Complex multi-threaded coordination with tight timeouts -- Known weight distribution architectural issues -- Race conditions in sample attribution -- Platform-specific stack walking problems - -**Most Critical Issues:** -1. CpuDumpSmokeTest: Null pointer risk (easy fix, high impact) -2. ContextWallClockTest: 10ms wait timeout (easy fix, high impact) -3. ContextWallClockTest: Known weight distribution issue (hard fix, documented workaround exists) - -**Recommended Immediate Actions:** -1. Fix null pointer risk in method3() -2. Increase monitor wait timeout from 10ms to 100ms+ -3. Fix sleep() interrupt handling -4. Add diagnostic logging to both tests -5. Consider relaxing or removing strict weight assertions until architectural fix available - ---- - -## Implementation Tasks (Alternative 1: Tactical Quick Wins) - -**Strategy:** Fix critical bugs and immediate issues without architectural changes -**Expected Outcome:** 60-70% flakiness reduction - -### Task 1: Fix Null Pointer Risk in method3() ☐ - -**File:** `ddprof-test/src/test/java/com/datadoghq/profiler/jfr/JfrDumpTest.java:68-82` -**Priority:** CRITICAL -**Impact:** HIGH - Eliminates immediate crash potential - -**Current code:** -```java -for (String s : new File("/tmp").list()) { // Can return null! - value += s.substring(0, Math.min(s.length(), 16)).hashCode(); -} -``` - -**Fix:** -```java -String[] files = new File("/tmp").list(); -if (files != null) { - for (String s : files) { - if (s != null && !s.isEmpty()) { // Defensive check - value += s.substring(0, Math.min(s.length(), 16)).hashCode(); - } - } -} -``` - ---- - -### Task 2: Increase Monitor Wait Timeout ☐ - -**File:** `ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/BaseContextWallClockTest.java:221` -**Priority:** CRITICAL -**Impact:** HIGH - Reduces race condition likelihood by ~80% - -**Current code:** -```java -monitor.wait(10); // Too short! -``` - -**Fix:** -```java -monitor.wait(150); // 150ms = 10ms (method2) + 10ms (method3) + 130ms buffer -``` - -**Rationale:** method2Impl and method3Impl each have 10ms sleep, plus 130ms buffer for thread scheduling, lock contention, and context switching. - ---- - -### Task 3: Fix Sleep Interrupt Handling ☐ - -**File:** `ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/BaseContextWallClockTest.java:263-272` -**Priority:** HIGH -**Impact:** MEDIUM - Proper interrupt semantics, eliminates continuous re-throw cycles - -**Current code:** -```java -private void sleep(long millis) { - long target = System.nanoTime() + millis * 1_000_000L; - do { - try { - Thread.sleep((target - System.nanoTime()) / 1_000_000L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); // Sets flag but continues! - } - } while (System.nanoTime() < target); -} -``` - -**Fix:** -```java -private void sleep(long millis) { - long target = System.nanoTime() + millis * 1_000_000L; - while (System.nanoTime() < target) { - try { - long remaining = (target - System.nanoTime()) / 1_000_000L; - if (remaining <= 0) break; - Thread.sleep(remaining); - break; // Sleep completed successfully - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - break; // Exit immediately on interrupt - } - } -} -``` - ---- - -### Task 4: Replace Time-Based Loop with Fixed Iterations ☐ - -**File:** `ddprof-test/src/test/java/com/datadoghq/profiler/jfr/JfrDumpTest.java:68-82` -**Priority:** MEDIUM -**Impact:** MEDIUM - Deterministic sampling probability - -**Current code:** -```java -for (int i = 0; i < 1000; ++i) { - // ... work ... - if ((System.nanoTime() - ts) > 20000000L) break; // Variable iterations -} -``` - -**Fix:** -```java -for (int i = 0; i < 200; ++i) { // Fixed 200 iterations - // ... work ... - // No time-based break -} -``` - -**Rationale:** Provides deterministic workload regardless of system load. 200 iterations = reasonable CPU time for sampling (typically 15-25ms on modern hardware). - ---- - -### Task 5: Add Executor Shutdown Verification ☐ - -**File:** `ddprof-test/src/test/java/com/datadoghq/profiler/wallclock/BaseContextWallClockTest.java:49-53` -**Priority:** LOW -**Impact:** LOW - Explicit failure instead of silent continuation - -**Current code:** -```java -executor.shutdownNow(); -executor.awaitTermination(30, TimeUnit.SECONDS); -``` - -**Fix:** -```java -executor.shutdownNow(); -boolean terminated = executor.awaitTermination(30, TimeUnit.SECONDS); -if (!terminated) { - throw new IllegalStateException("Executor failed to terminate within 30 seconds"); -} -``` - ---- - -### Validation Tasks ☐ - -After implementing all fixes: - -1. Run `./gradlew testDebug` multiple times (10+ runs) -2. Measure retry rate before and after -3. Document results in this file -4. Consider next steps based on flakiness reduction - ---- - -### Progress Tracking - -- [x] Task 1: Fix null pointer risk -- [x] Task 2: Increase monitor timeout -- [x] Task 3: Fix sleep interrupt handling -- [x] Task 4: Replace time-based loop -- [x] Task 5: Add executor shutdown verification -- [x] Validation: Run tests and measure improvement - ---- - -## Implementation Results (2026-02-03) - -**Status:** All Alternative 1 tasks completed successfully - -### Changes Made - -1. **JfrDumpTest.java (method3):** - - Added null check for `File.list()` to prevent `NullPointerException` - - Replaced time-based 20ms timeout loop with fixed 200 iterations - - Added defensive null/empty string checks - - Result: Deterministic workload, eliminates crash risk - -2. **BaseContextWallClockTest.java (method1Impl):** - - Increased `monitor.wait()` timeout from 10ms to 150ms - - Rationale: method2 (10ms) + method3 (10ms) + 130ms scheduling buffer - - Result: Reduces race condition likelihood by ~80% - -3. **BaseContextWallClockTest.java (sleep):** - - Fixed interrupt handling to exit immediately on interrupt - - Changed from `do-while` to `while` loop with explicit breaks - - Eliminates continuous `InterruptedException` re-throw cycles - - Result: Proper Java interrupt semantics - -4. **BaseContextWallClockTest.java (after):** - - Added verification that executor terminates within 30 seconds - - Throws `IllegalStateException` if termination times out - - Result: Explicit failure instead of silent continuation - -### Test Validation - -**CpuDumpSmokeTest:** ✅ PASSED (all 4 modes) -- cstack=vm: PASSED (27.40s) -- cstack=vmx: PASSED (26.79s) -- cstack=fp: PASSED (25.48s) -- cstack=dwarf: PASSED (25.87s) - -**ContextWallClockTest:** ✅ PASSED (all 4 modes) -- cstack=vm: PASSED -- cstack=vmx: PASSED -- cstack=fp: PASSED -- cstack=dwarf: PASSED - -**Build Time:** 35 seconds (clean testDebug run) -**Total C++ Unit Tests:** 127/127 passed - -### Expected Impact - -Based on the fixes implemented: -- **CpuDumpSmokeTest:** Expected reduction from @RetryTest(3) to @RetryTest(1) -- **ContextWallClockTest:** Expected reduction from @RetryTest(5) to @RetryTest(3) -- **Overall flakiness reduction:** 60-70% estimated - -### Recommendations for Further Improvement - -If flakiness persists after monitoring CI runs: -1. Consider Alternative 2 (Statistical/Probabilistic Testing) from the analysis -2. Address the documented weight distribution architectural issue (trace ID fragmentation) -3. Implement diagnostic logging to track failure patterns - ---- diff --git a/docker/Dockerfile.fuzz b/docker/Dockerfile.fuzz deleted file mode 100644 index e6b3bb63b..000000000 --- a/docker/Dockerfile.fuzz +++ /dev/null @@ -1,39 +0,0 @@ -ARG FUZZYDOG_VERSION=0.28.0 - -FROM registry.ddbuild.io/images/base/gbi-ubuntu_2404:release AS build - -USER root - -ARG FUZZYDOG_VERSION - -ENV DEBIAN_FRONTEND=noninteractive -RUN apt-get update -qq && apt-get install -y -qq \ - clang llvm curl git openjdk-21-jdk \ - && rm -rf /var/lib/apt/lists/* - -ENV JAVA_HOME=/usr/lib/jvm/java-21-openjdk-amd64 -ENV PATH=$JAVA_HOME/bin:$PATH - -WORKDIR /src -COPY . . - -RUN ./gradlew :ddprof-lib:fuzz:buildFuzz --no-daemon - -RUN mkdir -p /fuzzer/builds && \ - find ddprof-lib/fuzz/build/bin/fuzz -maxdepth 2 -type f -executable \ - -exec cp {} /fuzzer/builds/ \; && \ - ls /fuzzer/builds/ > /fuzz_binaries.txt - -RUN curl -fsSL "https://binaries.ddbuild.io/fuzzing/fuzzydog/${FUZZYDOG_VERSION}/fuzzydog-tar.tar.gz" \ - | tar -xz -C /usr/local/bin fuzzydog-linux-amd64 && \ - mv /usr/local/bin/fuzzydog-linux-amd64 /usr/local/bin/fuzzydog - -CMD fuzzydog fuzzer run "$FUZZ_APP" "$FUZZ_BUILD_ID" \ - --type libfuzzer \ - --team profiling \ - --build-path /fuzzer/builds/ \ - --skip-dl-build \ - --repository-url https://github.com/DataDog/java-profiler - -FROM scratch AS manifest -COPY --from=build /fuzz_binaries.txt /fuzz_binaries.txt diff --git a/docs/sphinx/specs/2026-05-29-generate-hidden-classes-in-the-lookup.md b/docs/sphinx/specs/2026-05-29-generate-hidden-classes-in-the-lookup.md deleted file mode 100644 index 096363eb8..000000000 --- a/docs/sphinx/specs/2026-05-29-generate-hidden-classes-in-the-lookup.md +++ /dev/null @@ -1,20 +0,0 @@ -# generate-hidden-classes-in-the-lookup-package - -## Finding - -`HiddenClassChurnAntagonist.generateClass()` set `internalName = "chaos/hidden/Gen" + uid`, -placing the generated class in package `chaos.hidden`. The lookup object obtained from -`MethodHandles.lookup()` inside `HiddenClassChurnAntagonist` belongs to package -`com.datadoghq.profiler.chaos`. On Java 15+, `Lookup.defineHiddenClass` requires the -bytecode to declare the same package as the lookup class; a mismatch throws -`IllegalArgumentException`, which was silently swallowed by `catch (Throwable t)`. As a -result the antagonist looped without ever defining a hidden class. - -## Fix - -Change `internalName` to `"com/datadoghq/profiler/chaos/Gen" + uid` so the generated -class's package matches the lookup class. - -## File - -`ddprof-stresstest/src/chaos/java/com/datadoghq/profiler/chaos/HiddenClassChurnAntagonist.java`, line 111 diff --git a/gradle.properties.template b/gradle.properties.template deleted file mode 100644 index b56022b3d..000000000 --- a/gradle.properties.template +++ /dev/null @@ -1,75 +0,0 @@ -# Java Profiler - Gradle Properties Template -# Copy this file to gradle.properties and customize as needed. -# -# Note: Property values can be overridden via command line with -P flag -# Example: ./gradlew build -Pskip-tests - -# ============================================================================= -# VERSION MANAGEMENT -# ============================================================================= - -# Override the project version (useful for custom builds) -# ddprof_version=1.38.0-custom - -# ============================================================================= -# BUILD CONTROL -# ============================================================================= - -# Skip test execution (Java tests) -# skip-tests=true - -# Skip native C++ compilation -# skip-native=true - -# Skip Google Test (C++ unit tests) -# skip-gtest=true - -# Skip fuzz testing -# skip-fuzz=true - -# ============================================================================= -# COMPILER CONFIGURATION -# ============================================================================= - -# Force a specific C++ compiler (auto-detects clang++ or g++ if not set) -# native.forceCompiler=/usr/bin/clang++ - -# ============================================================================= -# EXTERNAL DEPENDENCIES -# ============================================================================= - -# Path to pre-built native libraries (skips native compilation if set) -# with-libs=/path/to/external/libs - -# ============================================================================= -# TESTING -# ============================================================================= - -# Keep JFR recordings after test runs (also respects KEEP_JFRS env var) -# keepJFRs=true - -# Arguments to pass to UnwindingValidator -# validatorArgs=--verbose --output-format=json - -# ============================================================================= -# CI / PUBLISHING -# ============================================================================= - -# CI environment flag (auto-detected from CI env var) -# CI=true - -# Use local Nexus for testing (requires running docker container) -# forceLocal=true - -# ============================================================================= -# MEMORY / PERFORMANCE -# ============================================================================= - -# Gradle JVM arguments (default values shown) -# org.gradle.jvmargs=-Xmx2g -XX:+HeapDumpOnOutOfMemoryError - -# Enable Gradle build cache -# org.gradle.caching=true - -# Enable parallel project execution -# org.gradle.parallel=true diff --git a/gradle/enforcement/.clang-format b/gradle/enforcement/.clang-format deleted file mode 100644 index d6b89741c..000000000 --- a/gradle/enforcement/.clang-format +++ /dev/null @@ -1,8 +0,0 @@ ---- -BasedOnStyle: LLVM -AlignOperands: false -AllowShortBlocksOnASingleLine: true -AlignConsecutiveBitFields: true -IndentPPDirectives: AfterHash -ColumnLimit: 100 ---- \ No newline at end of file diff --git a/gradle/enforcement/codenarc.groovy b/gradle/enforcement/codenarc.groovy deleted file mode 100644 index 47f3fa0e9..000000000 --- a/gradle/enforcement/codenarc.groovy +++ /dev/null @@ -1,411 +0,0 @@ -ruleset { - // rulesets/basic.xml - /* - AssertWithinFinallyBlock - AssignmentInConditional - BigDecimalInstantiation - BitwiseOperatorInConditional - BooleanGetBoolean - BrokenNullCheck - BrokenOddnessCheck - ClassForName - ComparisonOfTwoConstants - ComparisonWithSelf - ConstantAssertExpression - ConstantIfExpression - ConstantTernaryExpression - DeadCode - DoubleNegative - DuplicateCaseStatement - DuplicateMapKey - DuplicateSetValue - EmptyCatchBlock - EmptyClass - EmptyElseBlock - EmptyFinallyBlock - EmptyForStatement - EmptyIfStatement - EmptyInstanceInitializer - EmptyMethod - EmptyStaticInitializer - EmptySwitchStatement - EmptySynchronizedStatement - EmptyTryBlock - EmptyWhileStatement - EqualsAndHashCode - EqualsOverloaded - ExplicitGarbageCollection - ForLoopShouldBeWhileLoop - HardCodedWindowsFileSeparator - HardCodedWindowsRootDirectory - IntegerGetInteger - RandomDoubleCoercedToZero - RemoveAllOnSelf - ReturnFromFinallyBlock - ThrowExceptionFromFinallyBlock - */ - - // rulesets/braces.xml - ElseBlockBraces - ForStatementBraces - IfStatementBraces - WhileStatementBraces - - // rulesets/concurrency.xml - /* - BusyWait - DoubleCheckedLocking - InconsistentPropertyLocking - InconsistentPropertySynchronization - NestedSynchronization - StaticCalendarField - StaticConnection - StaticDateFormatField - StaticMatcherField - StaticSimpleDateFormatField - SynchronizedMethod - SynchronizedOnBoxedPrimitive - SynchronizedOnGetClass - SynchronizedOnReentrantLock - SynchronizedOnString - SynchronizedOnThis - SynchronizedReadObjectMethod - SystemRunFinalizersOnExit - ThisReferenceEscapesConstructor - ThreadGroup - ThreadLocalNotStaticFinal - ThreadYield - UseOfNotifyMethod - VolatileArrayField - VolatileLongOrDoubleField - WaitOutsideOfWhileLoop - */ - - // rulesets/convention.xml - /* - ConfusingTernary - CouldBeElvis - HashtableIsObsolete - IfStatementCouldBeTernary - InvertedIfElse - LongLiteralWithLowerCaseL - ParameterReassignment - TernaryCouldBeElvis - VectorIsObsolete - */ - - // rulesets/design.xml - /* - AbstractClassWithPublicConstructor - AbstractClassWithoutAbstractMethod - BooleanMethodReturnsNull - BuilderMethodWithSideEffects - CloneableWithoutClone - CloseWithoutCloseable - CompareToWithoutComparable - ConstantsOnlyInterface - EmptyMethodInAbstractClass - FinalClassWithProtectedMember - ImplementationAsType - LocaleSetDefault - PrivateFieldCouldBeFinal - PublicInstanceField - ReturnsNullInsteadOfEmptyArray - ReturnsNullInsteadOfEmptyCollection - SimpleDateFormatMissingLocale - StatelessSingleton - */ - - // rulesets/dry.xml - /* - DuplicateListLiteral - DuplicateMapLiteral - DuplicateNumberLiteral - DuplicateStringLiteral - */ - - // rulesets/enhanced.xml - /* - CloneWithoutCloneable - JUnitAssertEqualsConstantActualValue - UnsafeImplementationAsMap - */ - - // rulesets/exceptions.xml - /* - CatchArrayIndexOutOfBoundsException - CatchError - CatchException - CatchIllegalMonitorStateException - CatchIndexOutOfBoundsException - CatchNullPointerException - CatchRuntimeException - CatchThrowable - ConfusingClassNamedException - ExceptionExtendsError - ExceptionNotThrown - MissingNewInThrowStatement - ReturnNullFromCatchBlock - SwallowThreadDeath - ThrowError - ThrowException - ThrowNullPointerException - ThrowRuntimeException - ThrowThrowable - */ - - // rulesets/formatting.xml - /* - BracesForClass - BracesForForLoop - BracesForIfElse - BracesForMethod - BracesForTryCatchFinally - ClassJavadoc - ClosureStatementOnOpeningLineOfMultipleLineClosure - LineLength - SpaceAfterCatch - SpaceAfterClosingBrace - SpaceAfterComma - SpaceAfterFor - SpaceAfterIf - SpaceAfterOpeningBrace - SpaceAfterSemicolon - SpaceAfterSwitch - SpaceAfterWhile - SpaceAroundClosureArrow - SpaceAroundMapEntryColon - SpaceAroundOperator - SpaceBeforeClosingBrace - SpaceBeforeOpeningBrace - */ - - // rulesets/generic.xml - /* - IllegalClassMember - IllegalClassReference - IllegalPackageReference - IllegalRegex - IllegalString - RequiredRegex - RequiredString - StatelessClass - */ - - // rulesets/grails.xml - /* - GrailsDomainHasEquals - GrailsDomainHasToString - GrailsDomainReservedSqlKeywordName - GrailsDomainWithServiceReference - GrailsDuplicateConstraint - GrailsDuplicateMapping - GrailsPublicControllerMethod - GrailsServletContextReference - GrailsSessionReference // DEPRECATED - GrailsStatelessService - */ - - // rulesets/groovyism.xml - /* - AssignCollectionSort - AssignCollectionUnique - ClosureAsLastMethodParameter - CollectAllIsDeprecated - ConfusingMultipleReturns - ExplicitArrayListInstantiation - ExplicitCallToAndMethod - ExplicitCallToCompareToMethod - ExplicitCallToDivMethod - ExplicitCallToEqualsMethod - ExplicitCallToGetAtMethod - ExplicitCallToLeftShiftMethod - ExplicitCallToMinusMethod - ExplicitCallToModMethod - ExplicitCallToMultiplyMethod - ExplicitCallToOrMethod - ExplicitCallToPlusMethod - ExplicitCallToPowerMethod - ExplicitCallToRightShiftMethod - ExplicitCallToXorMethod - ExplicitHashMapInstantiation - ExplicitHashSetInstantiation - ExplicitLinkedHashMapInstantiation - ExplicitLinkedListInstantiation - ExplicitStackInstantiation - ExplicitTreeSetInstantiation - GStringAsMapKey - GStringExpressionWithinString - GetterMethodCouldBeProperty - GroovyLangImmutable - UseCollectMany - UseCollectNested - */ - - // rulesets/imports.xml - DuplicateImport - ImportFromSamePackage -// ImportFromSunPackages -// MisorderedStaticImports - UnnecessaryGroovyImport - UnusedImport - - // rulesets/jdbc.xml - /* - DirectConnectionManagement - JdbcConnectionReference - JdbcResultSetReference - JdbcStatementReference - */ - - // rulesets/junit.xml - /* - ChainedTest - CoupledTestCase - JUnitAssertAlwaysFails - JUnitAssertAlwaysSucceeds - JUnitFailWithoutMessage - JUnitLostTest - JUnitPublicField - JUnitPublicNonTestMethod - JUnitSetUpCallsSuper - JUnitStyleAssertions - JUnitTearDownCallsSuper - JUnitTestMethodWithoutAssert - JUnitUnnecessarySetUp - JUnitUnnecessaryTearDown - JUnitUnnecessaryThrowsException - SpockIgnoreRestUsed - UnnecessaryFail - UseAssertEqualsInsteadOfAssertTrue - UseAssertFalseInsteadOfNegation - UseAssertNullInsteadOfAssertEquals - UseAssertSameInsteadOfAssertTrue - UseAssertTrueInsteadOfAssertEquals - UseAssertTrueInsteadOfNegation - */ - - // rulesets/logging.xml - /* - LoggerForDifferentClass - LoggerWithWrongModifiers - LoggingSwallowsStacktrace - MultipleLoggers - PrintStackTrace - Println - SystemErrPrint - SystemOutPrint - */ - - // rulesets/naming.xml - AbstractClassName - ClassName { - regex = '^[A-Z][\\$a-zA-Z0-9]*$' - } - ClassNameSameAsFilename -// ConfusingMethodName -// FactoryMethodName - FieldName { - regex = '^_?[a-z][a-zA-Z0-9]*$' - finalRegex = '^_?[a-z][a-zA-Z0-9]*$' - staticFinalRegex = '^logger$|^[A-Z][A-Z_0-9]*$|^serialVersionUID$' - } - InterfaceName - MethodName { - regex = '^[a-z][\\$_a-zA-Z0-9]*$|^.*\\s.*$' - } - ObjectOverrideMisspelledMethodName - PackageName - ParameterName - PropertyName - VariableName { - finalRegex = '^[a-z][a-zA-Z0-9]*$' - } - - // rulesets/security.xml - /* - FileCreateTempFile - InsecureRandom - JavaIoPackageAccess - NonFinalPublicField - NonFinalSubclassOfSensitiveInterface - ObjectFinalize - PublicFinalizeMethod - SystemExit - UnsafeArrayDeclaration - */ - - // rulesets/serialization.xml - /* - EnumCustomSerializationIgnored - SerialPersistentFields - SerialVersionUID - SerializableClassMustDefineSerialVersionUID - */ - - // rulesets/size.xml - /* - AbcComplexity // DEPRECATED: Use the AbcMetric rule instead. Requires the GMetrics jar - AbcMetric // Requires the GMetrics jar - ClassSize - CrapMetric // Requires the GMetrics jar and a Cobertura coverage file - CyclomaticComplexity // Requires the GMetrics jar - MethodCount - MethodSize - NestedBlockDepth - */ - - // rulesets/unnecessary.xml - AddEmptyString - ConsecutiveLiteralAppends - ConsecutiveStringConcatenation - UnnecessaryBigDecimalInstantiation - UnnecessaryBigIntegerInstantiation - UnnecessaryBooleanExpression - UnnecessaryBooleanInstantiation -// UnnecessaryCallForLastElement - UnnecessaryCallToSubstring - UnnecessaryCatchBlock -// UnnecessaryCollectCall - UnnecessaryCollectionCall - UnnecessaryConstructor - UnnecessaryDefInFieldDeclaration - UnnecessaryDefInMethodDeclaration - UnnecessaryDefInVariableDeclaration - UnnecessaryDotClass - UnnecessaryDoubleInstantiation - UnnecessaryElseStatement - UnnecessaryFinalOnPrivateMethod - UnnecessaryFloatInstantiation -// UnnecessaryGString -// UnnecessaryGetter - UnnecessaryIfStatement - UnnecessaryInstanceOfCheck - UnnecessaryInstantiationToGetClass - UnnecessaryIntegerInstantiation - UnnecessaryLongInstantiation - UnnecessaryModOne - UnnecessaryNullCheck - UnnecessaryNullCheckBeforeInstanceOf -// UnnecessaryObjectReferences - UnnecessaryOverridingMethod -// UnnecessaryPackageReference - UnnecessaryParenthesesForMethodCallWithClosure - UnnecessaryPublicModifier -// UnnecessaryReturnKeyword -// UnnecessarySelfAssignment - UnnecessarySemicolon - UnnecessaryStringInstantiation -// UnnecessarySubstring - UnnecessaryTernaryExpression - UnnecessaryTransientModifier - - // rulesets/unused.xml - UnusedArray -// UnusedMethodParameter - UnusedObject - UnusedPrivateField - UnusedPrivateMethod - UnusedPrivateMethodParameter - UnusedVariable -} diff --git a/gradle/enforcement/codenarcTest.groovy b/gradle/enforcement/codenarcTest.groovy deleted file mode 100644 index 585b1ba17..000000000 --- a/gradle/enforcement/codenarcTest.groovy +++ /dev/null @@ -1,411 +0,0 @@ -ruleset { - // rulesets/basic.xml - /* - AssertWithinFinallyBlock - AssignmentInConditional - BigDecimalInstantiation - BitwiseOperatorInConditional - BooleanGetBoolean - BrokenNullCheck - BrokenOddnessCheck - ClassForName - ComparisonOfTwoConstants - ComparisonWithSelf - ConstantAssertExpression - ConstantIfExpression - ConstantTernaryExpression - DeadCode - DoubleNegative - DuplicateCaseStatement - DuplicateMapKey - DuplicateSetValue - EmptyCatchBlock - EmptyClass - EmptyElseBlock - EmptyFinallyBlock - EmptyForStatement - EmptyIfStatement - EmptyInstanceInitializer - EmptyMethod - EmptyStaticInitializer - EmptySwitchStatement - EmptySynchronizedStatement - EmptyTryBlock - EmptyWhileStatement - EqualsAndHashCode - EqualsOverloaded - ExplicitGarbageCollection - ForLoopShouldBeWhileLoop - HardCodedWindowsFileSeparator - HardCodedWindowsRootDirectory - IntegerGetInteger - RandomDoubleCoercedToZero - RemoveAllOnSelf - ReturnFromFinallyBlock - ThrowExceptionFromFinallyBlock - */ - - // rulesets/braces.xml - ElseBlockBraces - ForStatementBraces - IfStatementBraces - WhileStatementBraces - - // rulesets/concurrency.xml - /* - BusyWait - DoubleCheckedLocking - InconsistentPropertyLocking - InconsistentPropertySynchronization - NestedSynchronization - StaticCalendarField - StaticConnection - StaticDateFormatField - StaticMatcherField - StaticSimpleDateFormatField - SynchronizedMethod - SynchronizedOnBoxedPrimitive - SynchronizedOnGetClass - SynchronizedOnReentrantLock - SynchronizedOnString - SynchronizedOnThis - SynchronizedReadObjectMethod - SystemRunFinalizersOnExit - ThisReferenceEscapesConstructor - ThreadGroup - ThreadLocalNotStaticFinal - ThreadYield - UseOfNotifyMethod - VolatileArrayField - VolatileLongOrDoubleField - WaitOutsideOfWhileLoop - */ - - // rulesets/convention.xml - /* - ConfusingTernary - CouldBeElvis - HashtableIsObsolete - IfStatementCouldBeTernary - InvertedIfElse - LongLiteralWithLowerCaseL - ParameterReassignment - TernaryCouldBeElvis - VectorIsObsolete - */ - - // rulesets/design.xml - /* - AbstractClassWithPublicConstructor - AbstractClassWithoutAbstractMethod - BooleanMethodReturnsNull - BuilderMethodWithSideEffects - CloneableWithoutClone - CloseWithoutCloseable - CompareToWithoutComparable - ConstantsOnlyInterface - EmptyMethodInAbstractClass - FinalClassWithProtectedMember - ImplementationAsType - LocaleSetDefault - PrivateFieldCouldBeFinal - PublicInstanceField - ReturnsNullInsteadOfEmptyArray - ReturnsNullInsteadOfEmptyCollection - SimpleDateFormatMissingLocale - StatelessSingleton - */ - - // rulesets/dry.xml - /* - DuplicateListLiteral - DuplicateMapLiteral - DuplicateNumberLiteral - DuplicateStringLiteral - */ - - // rulesets/enhanced.xml - /* - CloneWithoutCloneable - JUnitAssertEqualsConstantActualValue - UnsafeImplementationAsMap - */ - - // rulesets/exceptions.xml - /* - CatchArrayIndexOutOfBoundsException - CatchError - CatchException - CatchIllegalMonitorStateException - CatchIndexOutOfBoundsException - CatchNullPointerException - CatchRuntimeException - CatchThrowable - ConfusingClassNamedException - ExceptionExtendsError - ExceptionNotThrown - MissingNewInThrowStatement - ReturnNullFromCatchBlock - SwallowThreadDeath - ThrowError - ThrowException - ThrowNullPointerException - ThrowRuntimeException - ThrowThrowable - */ - - // rulesets/formatting.xml - /* - BracesForClass - BracesForForLoop - BracesForIfElse - BracesForMethod - BracesForTryCatchFinally - ClassJavadoc - ClosureStatementOnOpeningLineOfMultipleLineClosure - LineLength - SpaceAfterCatch - SpaceAfterClosingBrace - SpaceAfterComma - SpaceAfterFor - SpaceAfterIf - SpaceAfterOpeningBrace - SpaceAfterSemicolon - SpaceAfterSwitch - SpaceAfterWhile - SpaceAroundClosureArrow - SpaceAroundMapEntryColon - SpaceAroundOperator - SpaceBeforeClosingBrace - SpaceBeforeOpeningBrace - */ - - // rulesets/generic.xml - /* - IllegalClassMember - IllegalClassReference - IllegalPackageReference - IllegalRegex - IllegalString - RequiredRegex - RequiredString - StatelessClass - */ - - // rulesets/grails.xml - /* - GrailsDomainHasEquals - GrailsDomainHasToString - GrailsDomainReservedSqlKeywordName - GrailsDomainWithServiceReference - GrailsDuplicateConstraint - GrailsDuplicateMapping - GrailsPublicControllerMethod - GrailsServletContextReference - GrailsSessionReference // DEPRECATED - GrailsStatelessService - */ - - // rulesets/groovyism.xml - /* - AssignCollectionSort - AssignCollectionUnique - ClosureAsLastMethodParameter - CollectAllIsDeprecated - ConfusingMultipleReturns - ExplicitArrayListInstantiation - ExplicitCallToAndMethod - ExplicitCallToCompareToMethod - ExplicitCallToDivMethod - ExplicitCallToEqualsMethod - ExplicitCallToGetAtMethod - ExplicitCallToLeftShiftMethod - ExplicitCallToMinusMethod - ExplicitCallToModMethod - ExplicitCallToMultiplyMethod - ExplicitCallToOrMethod - ExplicitCallToPlusMethod - ExplicitCallToPowerMethod - ExplicitCallToRightShiftMethod - ExplicitCallToXorMethod - ExplicitHashMapInstantiation - ExplicitHashSetInstantiation - ExplicitLinkedHashMapInstantiation - ExplicitLinkedListInstantiation - ExplicitStackInstantiation - ExplicitTreeSetInstantiation - GStringAsMapKey - GStringExpressionWithinString - GetterMethodCouldBeProperty - GroovyLangImmutable - UseCollectMany - UseCollectNested - */ - - // rulesets/imports.xml - DuplicateImport - ImportFromSamePackage -// ImportFromSunPackages -// MisorderedStaticImports - UnnecessaryGroovyImport - UnusedImport - - // rulesets/jdbc.xml - /* - DirectConnectionManagement - JdbcConnectionReference - JdbcResultSetReference - JdbcStatementReference - */ - - // rulesets/junit.xml - /* - ChainedTest - CoupledTestCase - JUnitAssertAlwaysFails - JUnitAssertAlwaysSucceeds - JUnitFailWithoutMessage - JUnitLostTest - JUnitPublicField - JUnitPublicNonTestMethod - JUnitSetUpCallsSuper - JUnitStyleAssertions - JUnitTearDownCallsSuper - JUnitTestMethodWithoutAssert - JUnitUnnecessarySetUp - JUnitUnnecessaryTearDown - JUnitUnnecessaryThrowsException - SpockIgnoreRestUsed - UnnecessaryFail - UseAssertEqualsInsteadOfAssertTrue - UseAssertFalseInsteadOfNegation - UseAssertNullInsteadOfAssertEquals - UseAssertSameInsteadOfAssertTrue - UseAssertTrueInsteadOfAssertEquals - UseAssertTrueInsteadOfNegation - */ - - // rulesets/logging.xml - /* - LoggerForDifferentClass - LoggerWithWrongModifiers - LoggingSwallowsStacktrace - MultipleLoggers - PrintStackTrace - Println - SystemErrPrint - SystemOutPrint - */ - - // rulesets/naming.xml - AbstractClassName - ClassName { - regex = '^[A-Z][\\$a-zA-Z0-9]*(? -# race:__tsan_atomic64_compare_exchange_strong -# race:strncpy -# race:free -# race:libjvm.so$ClassAllocator::initialize -# race:libjvm.so$G1BlockOffsetTablePart::alloc_block_work -# race:libjvm.so$ClassAllocator::initialize -# race:libjvm.so$G1BlockOffsetTablePart::alloc_block_work -# race:libjvm.so$G1BlockOffsetTablePart::alloc_block_work -# race:libjvm.so$JavaThread::~JavaThread diff --git a/gradle/sanitizers/ubsan.supp b/gradle/sanitizers/ubsan.supp deleted file mode 100644 index e69de29bb..000000000 diff --git a/gradle/scm.gradle b/gradle/scm.gradle deleted file mode 100644 index b5bf8c849..000000000 --- a/gradle/scm.gradle +++ /dev/null @@ -1,20 +0,0 @@ -scmVersion { - tag { - prefix = 'v_' - versionSeparator = '' - initialVersion({config, position -> '0.31.0'}) - } - - versionIncrementer 'incrementMinor' - - checks { - uncommittedChanges = true - aheadOfRemote = true - snapshotDependencies = false - } - - nextVersion { - suffix.set("SNAPSHOT") - separator.set("-") - } -} diff --git a/gradle/semantic-version.gradle b/gradle/semantic-version.gradle deleted file mode 100644 index beb07ad49..000000000 --- a/gradle/semantic-version.gradle +++ /dev/null @@ -1 +0,0 @@ -version = "0.58.0" \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar deleted file mode 100644 index b1b8ef56b..000000000 Binary files a/gradle/wrapper/gradle-wrapper.jar and /dev/null differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties deleted file mode 100644 index 95a673c6f..000000000 --- a/gradle/wrapper/gradle-wrapper.properties +++ /dev/null @@ -1,9 +0,0 @@ -distributionBase=GRADLE_USER_HOME -distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-9.6.1-bin.zip -networkTimeout=30000 -retries=5 -retryBackOffMs=2000 -validateDistributionUrl=true -zipStoreBase=GRADLE_USER_HOME -zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew deleted file mode 100755 index 249efbb03..000000000 --- a/gradlew +++ /dev/null @@ -1,248 +0,0 @@ -#!/bin/sh - -# -# Copyright © 2015 the original authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 -# - -############################################################################## -# -# gradlew start up script for POSIX generated by Gradle. -# -# Important for running: -# -# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is -# noncompliant, but you have some other compliant shell such as ksh or -# bash, then to run this script, type that shell name before the whole -# command line, like: -# -# ksh gradlew -# -# Busybox and similar reduced shells will NOT work, because this script -# requires all of these POSIX shell features: -# * functions; -# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», -# «${var#prefix}», «${var%suffix}», and «$( cmd )»; -# * compound commands having a testable exit status, especially «case»; -# * various built-in commands including «command», «set», and «ulimit». -# -# Important for patching: -# -# (2) This script targets any POSIX shell, so it avoids extensions provided -# by Bash, Ksh, etc; in particular arrays are avoided. -# -# The "traditional" practice of packing multiple parameters into a -# space-separated string is a well documented source of bugs and security -# problems, so this is (mostly) avoided, by progressively accumulating -# options in "$@", and eventually passing that to Java. -# -# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, -# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; -# see the in-line comments for details. -# -# There are tweaks for specific operating systems such as AIX, CygWin, -# Darwin, MinGW, and NonStop. -# -# (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt -# within the Gradle project. -# -# You can find Gradle at https://github.com/gradle/gradle/. -# -############################################################################## - -# Attempt to set APP_HOME - -# Resolve links: $0 may be a link -app_path=$0 - -# Need this for daisy-chained symlinks. -while - APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path - [ -h "$app_path" ] -do - ls=$( ls -ld "$app_path" ) - link=${ls#*' -> '} - case $link in #( - /*) app_path=$link ;; #( - *) app_path=$APP_HOME$link ;; - esac -done - -# This is normally unused -# shellcheck disable=SC2034 -APP_BASE_NAME=${0##*/} -# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036) -APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s\n' "$PWD" ) || exit - -# Use the maximum available, or set MAX_FD != -1 to use that value. -MAX_FD=maximum - -warn () { - echo "$*" -} >&2 - -die () { - echo - echo "$*" - echo - exit 1 -} >&2 - -# OS specific support (must be 'true' or 'false'). -cygwin=false -msys=false -darwin=false -nonstop=false -case "$( uname )" in #( - CYGWIN* ) cygwin=true ;; #( - Darwin* ) darwin=true ;; #( - MSYS* | MINGW* ) msys=true ;; #( - NONSTOP* ) nonstop=true ;; -esac - - - -# Determine the Java command to use to start the JVM. -if [ -n "$JAVA_HOME" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD=$JAVA_HOME/jre/sh/java - else - JAVACMD=$JAVA_HOME/bin/java - fi - if [ ! -x "$JAVACMD" ] ; then - die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -else - JAVACMD=java - if ! command -v java >/dev/null 2>&1 - then - die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. - -Please set the JAVA_HOME variable in your environment to match the -location of your Java installation." - fi -fi - -# Increase the maximum file descriptors if we can. -if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then - case $MAX_FD in #( - max*) - # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC2039,SC3045 - MAX_FD=$( ulimit -H -n ) || - warn "Could not query maximum file descriptor limit" - esac - case $MAX_FD in #( - '' | soft) :;; #( - *) - # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. - # shellcheck disable=SC2039,SC3045 - ulimit -n "$MAX_FD" || - warn "Could not set maximum file descriptor limit to $MAX_FD" - esac -fi - -# Collect all arguments for the java command, stacking in reverse order: -# * args from the command line -# * the main class name -# * -classpath -# * -D...appname settings -# * --module-path (only if needed) -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. - -# For Cygwin or MSYS, switch paths to Windows format before running java -if "$cygwin" || "$msys" ; then - APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - - JAVACMD=$( cygpath --unix "$JAVACMD" ) - - # Now convert the arguments - kludge to limit ourselves to /bin/sh - for arg do - if - case $arg in #( - -*) false ;; # don't mess with options #( - /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath - [ -e "$t" ] ;; #( - *) false ;; - esac - then - arg=$( cygpath --path --ignore --mixed "$arg" ) - fi - # Roll the args list around exactly as many times as the number of - # args, so each arg winds up back in the position where it started, but - # possibly modified. - # - # NB: a `for` loop captures its iteration list before it begins, so - # changing the positional parameters here affects neither the number of - # iterations, nor the values presented in `arg`. - shift # remove old arg - set -- "$@" "$arg" # push replacement arg - done -fi - - -# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' - -# Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, -# and any embedded shellness will be escaped. -# * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be -# treated as '${Hostname}' itself on the command line. - -set -- \ - "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ - "$@" - -# Stop when "xargs" is not available. -if ! command -v xargs >/dev/null 2>&1 -then - die "xargs is not available" -fi - -# Use "xargs" to parse quoted args. -# -# With -n1 it outputs one arg per line, with the quotes and backslashes removed. -# -# In Bash we could simply go: -# -# readarray ARGS < <( xargs -n1 <<<"$var" ) && -# set -- "${ARGS[@]}" "$@" -# -# but POSIX shell has neither arrays nor command substitution, so instead we -# post-process each arg (as a line of input to sed) to backslash-escape any -# character that might be a shell metacharacter, then use eval to reverse -# that process (while maintaining the separation between arguments), and wrap -# the whole thing up as a single "set" statement. -# -# This will of course break if any of these variables contains a newline or -# an unmatched quote. -# - -eval "set -- $( - printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | - xargs -n1 | - sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | - tr '\n' ' ' - )" '"$@"' - -exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat deleted file mode 100644 index 8508ef684..000000000 --- a/gradlew.bat +++ /dev/null @@ -1,82 +0,0 @@ -@rem -@rem Copyright 2015 the original author or authors. -@rem -@rem Licensed under the Apache License, Version 2.0 (the "License"); -@rem you may not use this file except in compliance with the License. -@rem You may obtain a copy of the License at -@rem -@rem https://www.apache.org/licenses/LICENSE-2.0 -@rem -@rem Unless required by applicable law or agreed to in writing, software -@rem distributed under the License is distributed on an "AS IS" BASIS, -@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -@rem See the License for the specific language governing permissions and -@rem limitations under the License. -@rem -@rem SPDX-License-Identifier: Apache-2.0 -@rem - -@if "%DEBUG%"=="" @echo off -@rem ########################################################################## -@rem -@rem gradlew startup script for Windows -@rem -@rem ########################################################################## - -@rem Set local scope for the variables, and ensure extensions are enabled -setlocal EnableExtensions - -set DIRNAME=%~dp0 -if "%DIRNAME%"=="" set DIRNAME=. -@rem This is normally unused -set APP_BASE_NAME=%~n0 -set APP_HOME=%DIRNAME% - -@rem Resolve any "." and ".." in APP_HOME to make it shorter. -for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi - -@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. -set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" - -@rem Find java.exe -if defined JAVA_HOME goto findJavaFromJavaHome - -set JAVA_EXE=java.exe -%JAVA_EXE% -version >NUL 2>&1 -if %ERRORLEVEL% equ 0 goto execute - -echo. 1>&2 -echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2 -echo. 1>&2 -echo Please set the JAVA_HOME variable in your environment to match the 1>&2 -echo location of your Java installation. 1>&2 - -"%COMSPEC%" /c exit 1 - -:findJavaFromJavaHome -set JAVA_HOME=%JAVA_HOME:"=% -set JAVA_EXE=%JAVA_HOME%/bin/java.exe - -if exist "%JAVA_EXE%" goto execute - -echo. 1>&2 -echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2 -echo. 1>&2 -echo Please set the JAVA_HOME variable in your environment to match the 1>&2 -echo location of your Java installation. 1>&2 - -"%COMSPEC%" /c exit 1 - -:execute -@rem Setup the command line - - - -@rem Execute gradlew -@rem endlocal doesn't take effect until after the line is parsed and variables are expanded -@rem which allows us to clear the local environment before executing the java command -endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel - -:exitWithErrorLevel -@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts -"%COMSPEC%" /c exit %ERRORLEVEL% diff --git a/index.md b/index.md new file mode 100644 index 000000000..f4d3def44 --- /dev/null +++ b/index.md @@ -0,0 +1,48 @@ +--- +layout: default +title: Java Profiler Build - Test Dashboard +--- + +# Java Profiler Build - Test Dashboard + +> **Last Updated:** 2026-06-30 16:29 UTC + +## Quick Status + +| Test Type | Latest | Status | Branch | PR | +|-----------|--------|--------|--------|-----| +| [Integration](integration/) | [#121891935](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121891935) | ❌ | main | - | +| [Benchmarks](benchmarks/) | - | - | - | - | +| [Reliability](reliability/) | - | - | - | - | + +--- + +## Test Types + +### Integration Tests +dd-trace-java compatibility tests verifying profiler works correctly with the Datadog tracer. +Tests run on every main branch build across multiple JDK versions and platforms. + +### Benchmarks +Performance regression testing using Renaissance benchmark suite. +Compares profiler overhead against baseline (no profiling). + +### Reliability Tests +Long-running stability tests checking for memory leaks and crashes. +Tests multiple allocator configurations (gmalloc, tcmalloc, jemalloc). + +--- + +## Recent Runs (All Types) + +| Date | Type | Pipeline | Branch | PR | Status | +|------|------|----------|--------|-----|--------| +| 2026-06-30 | Integration | [#121891935](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121891935) | main | - | ❌ | +| 2026-06-30 | Integration | [#121891837](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121891837) | main | - | ❌ | +| 2026-06-30 | Integration | [#121890528](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121890528) | main | - | ❌ | +| 2026-06-30 | Integration | [#121876849](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121876849) | main | - | ❌ | +| 2026-06-30 | Integration | [#121872420](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121872420) | main | - | ❌ | + +--- + +[Repository](https://github.com/DataDog/java-profiler) | [java-profiler](https://github.com/DataDog/java-profiler) | [View history](https://github.com/DataDog/java-profiler/commits/gh-pages) diff --git a/integration/index.md b/integration/index.md new file mode 100644 index 000000000..679f5ae46 --- /dev/null +++ b/integration/index.md @@ -0,0 +1,197 @@ +--- +layout: default +title: DD-Trace Integration Test History +--- + +# DD-Trace Integration Test History + +[← Back to Dashboard](../) + +Tests dd-trace-java compatibility with ddprof across multiple JDK versions and platforms. + +## Last 10 Runs + +

+ +2026-06-30 16:29 | ❌ | main | Pipeline [#121891935](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121891935) + + +**Version:** unknown +**Commit:** 984428f0 + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ +
+ +2026-06-30 15:51 | ❌ | main | Pipeline [#121891837](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121891837) + + +**Version:** unknown +**Commit:** e4975db2 + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ +
+ +2026-06-30 15:45 | ❌ | main | Pipeline [#121890528](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121890528) + + +**Version:** unknown +**Commit:** 76b61aba + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ +
+ +2026-06-30 15:04 | ❌ | main | Pipeline [#121876849](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121876849) + + +**Version:** unknown +**Commit:** c7046666 + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ +
+ +2026-06-30 15:01 | ❌ | main | Pipeline [#121872420](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121872420) + + +**Version:** unknown +**Commit:** c6718586 + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ +
+ +2026-06-30 14:25 | ❌ | main | Pipeline [#121862090](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121862090) + + +**Version:** unknown +**Commit:** 13222ae6 + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ +
+ +2026-06-30 13:38 | ❌ | main | Pipeline [#121848831](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121848831) + + +**Version:** unknown +**Commit:** b31f2705 + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ +
+ +2026-06-30 13:38 | ❌ | main | Pipeline [#121847651](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121847651) + + +**Version:** unknown +**Commit:** dc9612ce + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ +
+ +2026-06-30 09:50 | ❌ | main | Pipeline [#121802370](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121802370) + + +**Version:** unknown +**Commit:** 6f4cbb0b + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ +
+ +2026-06-30 09:45 | ❌ | main | Pipeline [#121800513](https://gitlab.ddbuild.io/DataDog/java-profiler/-/pipelines/121800513) + + +**Version:** unknown +**Commit:** 1c61f181 + +| Metric | Value | +|--------|-------| +| Jobs | 40 | +| Passed | 0 | +| Failed | 40 | + +**Failed Configs:** glibc-arm64-hotspot-jdk11, glibc-arm64-hotspot-jdk17, glibc-arm64-hotspot-jdk21, glibc-arm64-hotspot-jdk25, glibc-arm64-hotspot-jdk8, glibc-arm64-openj9-jdk11, glibc-arm64-openj9-jdk17, glibc-arm64-openj9-jdk21, glibc-arm64-openj9-jdk25, glibc-arm64-openj9-jdk8, glibc-x64-hotspot-jdk11, glibc-x64-hotspot-jdk17, glibc-x64-hotspot-jdk21, glibc-x64-hotspot-jdk25, glibc-x64-hotspot-jdk8, glibc-x64-openj9-jdk11, glibc-x64-openj9-jdk17, glibc-x64-openj9-jdk21, glibc-x64-openj9-jdk25, glibc-x64-openj9-jdk8, musl-arm64-hotspot-jdk11, musl-arm64-hotspot-jdk17, musl-arm64-hotspot-jdk21, musl-arm64-hotspot-jdk25, musl-arm64-hotspot-jdk8, musl-arm64-openj9-jdk11, musl-arm64-openj9-jdk17, musl-arm64-openj9-jdk21, musl-arm64-openj9-jdk25, musl-arm64-openj9-jdk8, musl-x64-hotspot-jdk11, musl-x64-hotspot-jdk17, musl-x64-hotspot-jdk21, musl-x64-hotspot-jdk25, musl-x64-hotspot-jdk8, musl-x64-openj9-jdk11, musl-x64-openj9-jdk17, musl-x64-openj9-jdk21, musl-x64-openj9-jdk25, musl-x64-openj9-jdk8 + +
+ + +--- + +[← Back to Dashboard](../) | [View git history](https://github.com/DataDog/java-profiler/commits/gh-pages) diff --git a/legacy_tests/include.sh b/legacy_tests/include.sh deleted file mode 100644 index 363b47118..000000000 --- a/legacy_tests/include.sh +++ /dev/null @@ -1,18 +0,0 @@ -#! /bin/bash - -function collapseJfr() { - local event=$1 - local input=$2 - local output=$3 - - local json_out="${input}.json" - $JAVA_HOME/bin/jfr print --events $1 --stack-depth 10 --json $input > $json_out - jq -r '[.recording.events[]?.values.stackTrace | select(.truncated == false) | .frames | map("\(.method.type.name).\(.method.name)") | reverse | join(";")]' $json_out | grep -v '.no_Java_frame' | sed 's/"//g' | sed 's/,/ /g' > $output -} - - function assert_string() { - if ! grep -q "$1" $2; then - echo "Required text "$1" was not found in $2" - exit 1 - fi - } diff --git a/legacy_tests/load-libraries-test.sh b/legacy_tests/load-libraries-test.sh deleted file mode 100755 index 4d61b2b1a..000000000 --- a/legacy_tests/load-libraries-test.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -set -e # exit on any failure -set -x # print all executed lines - -if [ -z "${JAVA_HOME}" ]; then - echo "JAVA_HOME is not set" - exit 1 -fi - -HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) - -source ${HERE}/include.sh - -cd ${HERE}/loadlibs -# build some dynamic libraries to load -g++ -c -fPIC -o increment.o increment.cpp -gcc -shared -o libincrement.so increment.o - -g++ -ldl -c -fPIC -I$JAVA_HOME/include/linux/ -I$JAVA_HOME/include/ com_datadoghq_loader_DynamicLibraryLoader.cpp -o com_datadoghq_loader_DynamicLibraryLoader.o -g++ -shared -fPIC -o libloader.so com_datadoghq_loader_DynamicLibraryLoader.o -lc - -JFR=/tmp/load-libraries-test.jfr -rm -f $JFR - -# need to package the profiler JAR with the native artifacts -# will skip tests and native build because they will be initiated elsewhere -if [ -f ${HERE}/../build/libjavaProfielr.so ]; then - SKIP_NATIVE_ARG="-Dskip-native" -fi - -CLASSPATH=$(find ${HERE}/../target -name 'ddprof-*.jar') - -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:. -$JAVA_HOME/bin/javac -cp $CLASSPATH com/datadoghq/loader/DynamicLibraryLoader.java -$JAVA_HOME/bin/java -cp .:$CLASSPATH \ - -Djava.library.path=. com.datadoghq.loader.DynamicLibraryLoader \ - $JFR ./libincrement.so:increment - -# $JAVA_HOME/bin/jfr print --json $JFR -# $JAVA_HOME/bin/jfr summary $JFR - - diff --git a/legacy_tests/loadlibs/com/datadoghq/loader/DynamicLibraryLoader.java b/legacy_tests/loadlibs/com/datadoghq/loader/DynamicLibraryLoader.java deleted file mode 100644 index 4812f1667..000000000 --- a/legacy_tests/loadlibs/com/datadoghq/loader/DynamicLibraryLoader.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.datadoghq.loader; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.ThreadLocalRandom; - -import com.datadoghq.profiler.JavaProfiler; - -public class DynamicLibraryLoader { - - static { - System.loadLibrary("loader"); - } - - public static void main(String... args) throws Exception { - DynamicLibraryLoader loader = new DynamicLibraryLoader(); - String jfrDump = args[0]; - for (int i = 1; i < args.length; i++) { - String[] split = args[i].split(":"); - if (!loader.loadLibrary(split[0], split[1])) { - System.err.println("Could not load " + split[0] + " and invoke " + split[1]); - System.exit(1); - } - } - startProfilerAndDoWork(jfrDump); - } - - private static void startProfilerAndDoWork(String jfrFile) throws Exception { - JavaProfiler ap = JavaProfiler.getInstance(); - Path jfrDump = Paths.get(jfrFile); - ap.execute("start,loglevel=TRACE,cpu=1m,wall=1ms,filter=0,jfr,file=" + jfrDump.toAbsolutePath()); - ap.addThread(); - work(); - ap.stop(); - } - - private static void work() throws Exception { - Thread.sleep(10); - long blackhole = System.nanoTime(); - for (int i = 0; i < 10_000_000; i++) { - blackhole ^= ThreadLocalRandom.current().nextLong(); - } - Thread.sleep(10); - System.err.println(blackhole); - } - - - private native boolean loadLibrary(String libraryFile, String functionName); - -} diff --git a/legacy_tests/loadlibs/com_datadoghq_loader_DynamicLibraryLoader.cpp b/legacy_tests/loadlibs/com_datadoghq_loader_DynamicLibraryLoader.cpp deleted file mode 100644 index d6a1c66d8..000000000 --- a/legacy_tests/loadlibs/com_datadoghq_loader_DynamicLibraryLoader.cpp +++ /dev/null @@ -1,21 +0,0 @@ -#include -#include "com_datadoghq_loader_DynamicLibraryLoader.h" -#include -#include - -JNIEXPORT jboolean JNICALL Java_com_datadoghq_loader_DynamicLibraryLoader_loadLibrary(JNIEnv* env, jobject unused, jstring library, jstring name) { - int (*function)(int i); - const char* libraryName = env->GetStringUTFChars(library, 0); - void* handle = dlopen(libraryName, RTLD_LAZY); - if (NULL == handle) { - std::cout << "could not load " << libraryName << std::endl; - return false; - } else { - const char *functionName = env->GetStringUTFChars(name, 0); - function = (int(*)(int)) dlsym(handle, functionName); - int next = (*function)(1); - std::cout << "loaded " << libraryName << " and computed: " << next << std::endl; - return true; - } -} - diff --git a/legacy_tests/loadlibs/com_datadoghq_loader_DynamicLibraryLoader.h b/legacy_tests/loadlibs/com_datadoghq_loader_DynamicLibraryLoader.h deleted file mode 100644 index 6cbdc68dd..000000000 --- a/legacy_tests/loadlibs/com_datadoghq_loader_DynamicLibraryLoader.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef JAVA_PROFILER_COM_DATADOGHQ_LOADER_DYNAMICLIBRARYLOADER_H -#define JAVA_PROFILER_COM_DATADOGHQ_LOADER_DYNAMICLIBRARYLOADER_H - -#include - -extern "C" { - JNIEXPORT jboolean JNICALL Java_com_datadoghq_loader_DynamicLibraryLoader_loadLibrary(JNIEnv *, jobject, jstring, jstring); -} - -#endif //JAVA_PROFILER_COM_DATADOGHQ_LOADER_DYNAMICLIBRARYLOADER_H diff --git a/legacy_tests/loadlibs/increment.cpp b/legacy_tests/loadlibs/increment.cpp deleted file mode 100644 index b006c6e11..000000000 --- a/legacy_tests/loadlibs/increment.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "increment.h" - -int increment(int x) { - return x + 1; -} diff --git a/legacy_tests/loadlibs/increment.h b/legacy_tests/loadlibs/increment.h deleted file mode 100644 index f8be23a59..000000000 --- a/legacy_tests/loadlibs/increment.h +++ /dev/null @@ -1,8 +0,0 @@ -#ifndef JAVA_PROFILER_INCREMENT_H -#define JAVA_PROFILER_INCREMENT_H - -extern "C" { - int increment(int x); -} - -#endif //JAVA_PROFILER_INCREMENT_H diff --git a/legacy_tests/run_renaissance.sh b/legacy_tests/run_renaissance.sh deleted file mode 100755 index fe0d63b05..000000000 --- a/legacy_tests/run_renaissance.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -set -e # exit on any failure -set -x # print all executed lines - -if [ -z "${JAVA_HOME}" ]; then - echo "JAVA_HOME is not set" - exit 1 -fi - -WGET=$(which wget) -if [ -z "$WGET" ]; then - echo "Missing wget" - exit 1 -fi - -function help() { - echo "Usage: run_renaissance.sh -pa " -} - -if [ $# -eq 0 ]; then - help - exit 1 -fi - -while [ $# -gt 1 ]; do - KEY=$1 - case "$KEY" in - "-pa") - shift - PROFILER_ARGS=$1 - ;; - *) - if [ -z "$PROFILER_ARGS" ]; then - help - exit 1 - fi - BENCHMARK_ARGS=("$@") - break - ;; - esac - shift -done - -mkdir -p .resources -(cd .resources && if [ ! -e renaissance.jar ]; then wget https://github.com/renaissance-benchmarks/renaissance/releases/download/v0.14.0/renaissance-mit-0.14.0.jar -nv -O renaissance.jar; fi) - -HERE=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -BASEDIR="${HERE}/.." - -AGENT_PATH=${BASEDIR}/build/libjavaProfiler.so -if [ ! -f "$AGENT_PATH" ]; then - # we are running in CI - the library will be in a different place - AGENT_PATH=${BASEDIR}/target/classes/META-INF/native/linux-x64/libjavaProfiler.so -fi -echo "${JAVA_HOME}/bin/java -agentpath:${AGENT_PATH}=start,${PROFILER_ARGS} -jar .resources/renaissance.jar ${BENCHMARK_ARGS[@]}" -${JAVA_HOME}/bin/java -agentpath:${AGENT_PATH}=start,${PROFILER_ARGS} -jar .resources/renaissance.jar "${BENCHMARK_ARGS[@]}" \ No newline at end of file diff --git a/legacy_tests/test-all.sh b/legacy_tests/test-all.sh deleted file mode 100755 index 016fcb26f..000000000 --- a/legacy_tests/test-all.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - -set -e - -for TEST in $(find . -name '*-test.sh'); do - echo "=== $TEST" - bash $TEST -done - -echo "All tests passed" \ No newline at end of file diff --git a/malloc-shim/build.gradle.kts b/malloc-shim/build.gradle.kts deleted file mode 100644 index 8317701ef..000000000 --- a/malloc-shim/build.gradle.kts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Memory allocation interceptor for malloc debugging (Linux only). - */ - -import com.datadoghq.native.model.Platform -import com.datadoghq.native.util.PlatformUtils - -plugins { - base - id("com.datadoghq.simple-native-lib") -} - -group = "com.datadoghq" -version = "0.1" - -simpleNativeLib { - // malloc-shim is Linux-only - enabled.set(PlatformUtils.currentPlatform == Platform.LINUX) - - libraryName.set("debug") - sourceDir.set(file("src/main/cpp")) - includeDirs.set(listOf(file("src/main/public").absolutePath)) - - compilerArgs.set( - listOf( - "-O3", - "-fno-omit-frame-pointer", - "-fvisibility=hidden", - "-std=c++17", - "-DPROFILER_VERSION=\"${project.version}\"", - "-fPIC", - ), - ) - - linkerArgs.set(listOf("-ldl")) -} diff --git a/malloc-shim/settings.gradle.kts b/malloc-shim/settings.gradle.kts deleted file mode 100644 index 0b00a8c6e..000000000 --- a/malloc-shim/settings.gradle.kts +++ /dev/null @@ -1 +0,0 @@ -rootProject.name = "malloc-shim" diff --git a/malloc-shim/src/main/cpp/malloc_intercept.cpp b/malloc-shim/src/main/cpp/malloc_intercept.cpp deleted file mode 100644 index df117acda..000000000 --- a/malloc-shim/src/main/cpp/malloc_intercept.cpp +++ /dev/null @@ -1,32 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include - -#include "debug.h" - -#ifdef __linux__ -static int sighandler_tid = -1; - -extern void *__libc_malloc(size_t size); - -void *malloc(size_t size) { - void *result = NULL; - int tid = sighandler_tid != -1 ? syscall(__NR_gettid) : -2; - if (tid == sighandler_tid) { - fprintf(stderr, "!!!! MALLOC in signal handler !!!\n"); - raise(SIGSEGV); - } else { - result = __libc_malloc(size); - } - - return result; -} - -void set_sighandler_tid(int tid) { - sighandler_tid = tid; -} -#endif // __linux__ diff --git a/malloc-shim/src/main/public/debug.h b/malloc-shim/src/main/public/debug.h deleted file mode 100644 index d1617f68b..000000000 --- a/malloc-shim/src/main/public/debug.h +++ /dev/null @@ -1,6 +0,0 @@ -#ifndef _DEBUG_H -#define _DEBUG_H - -void set_sighandler_tid(int tid); - -#endif // _DEBUG_H \ No newline at end of file diff --git a/pom.xml b/pom.xml deleted file mode 100644 index e69de29bb..000000000 diff --git a/reliability/index.md b/reliability/index.md new file mode 100644 index 000000000..99d371b45 --- /dev/null +++ b/reliability/index.md @@ -0,0 +1,18 @@ +--- +layout: default +title: Reliability Test History +--- + +# Reliability Test History + +[← Back to Dashboard](../) + +Long-running stability tests for memory leaks and crashes. + +## Last 10 Runs + +*No test runs recorded yet.* + +--- + +[← Back to Dashboard](../) | [View git history](https://github.com/DataDog/java-profiler/commits/gh-pages) diff --git a/reports/glibc-arm64-hotspot-jdk11.md b/reports/glibc-arm64-hotspot-jdk11.md new file mode 100644 index 000000000..9df87051c --- /dev/null +++ b/reports/glibc-arm64-hotspot-jdk11.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-hotspot-jdk11 +--- + +## glibc-arm64-hotspot-jdk11 - ✅ PASS + +**Date:** 2026-06-30 12:29:23 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | hotspot | +| Java | jdk11 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 52 | +| CPU Cores (end) | 52 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 52-52 cores) + +``` +1782836652 52 +1782836657 52 +1782836662 52 +1782836667 52 +1782836672 52 +1782836677 52 +1782836682 52 +1782836687 52 +1782836692 52 +1782836697 52 +1782836702 52 +1782836707 52 +1782836712 52 +1782836717 52 +1782836722 52 +1782836727 52 +1782836732 52 +1782836737 52 +1782836742 52 +1782836747 52 +``` +
+ +--- + diff --git a/reports/glibc-arm64-hotspot-jdk17.md b/reports/glibc-arm64-hotspot-jdk17.md new file mode 100644 index 000000000..1dee0650c --- /dev/null +++ b/reports/glibc-arm64-hotspot-jdk17.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-hotspot-jdk17 +--- + +## glibc-arm64-hotspot-jdk17 - ✅ PASS + +**Date:** 2026-06-30 12:29:23 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | hotspot | +| Java | jdk17 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 40 | +| CPU Cores (end) | 40 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 40-40 cores) + +``` +1782836652 40 +1782836657 40 +1782836662 40 +1782836667 40 +1782836672 40 +1782836677 40 +1782836682 40 +1782836687 40 +1782836692 40 +1782836697 40 +1782836702 40 +1782836707 40 +1782836712 40 +1782836717 40 +1782836722 40 +1782836727 40 +1782836732 40 +1782836737 40 +1782836742 40 +1782836747 40 +``` +
+ +--- + diff --git a/reports/glibc-arm64-hotspot-jdk21.md b/reports/glibc-arm64-hotspot-jdk21.md new file mode 100644 index 000000000..e297b72f7 --- /dev/null +++ b/reports/glibc-arm64-hotspot-jdk21.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-hotspot-jdk21 +--- + +## glibc-arm64-hotspot-jdk21 - ✅ PASS + +**Date:** 2026-06-30 12:29:23 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | hotspot | +| Java | jdk21 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 64 | +| CPU Cores (end) | 64 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 64-64 cores) + +``` +1782836628 64 +1782836633 64 +1782836638 64 +1782836643 64 +1782836648 64 +1782836653 64 +1782836658 64 +1782836663 64 +1782836668 64 +1782836673 64 +1782836678 64 +1782836683 64 +1782836688 64 +1782836693 64 +1782836698 64 +1782836703 64 +1782836708 64 +1782836713 64 +1782836718 64 +1782836723 64 +``` +
+ +--- + diff --git a/reports/glibc-arm64-hotspot-jdk25.md b/reports/glibc-arm64-hotspot-jdk25.md new file mode 100644 index 000000000..636479432 --- /dev/null +++ b/reports/glibc-arm64-hotspot-jdk25.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-hotspot-jdk25 +--- + +## glibc-arm64-hotspot-jdk25 - ✅ PASS + +**Date:** 2026-06-30 12:29:23 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | hotspot | +| Java | jdk25 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 52 | +| CPU Cores (end) | 64 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 52-64 cores) + +``` +1782836621 52 +1782836626 52 +1782836631 52 +1782836636 52 +1782836641 52 +1782836646 52 +1782836651 52 +1782836656 52 +1782836661 52 +1782836666 52 +1782836671 52 +1782836676 52 +1782836681 52 +1782836686 52 +1782836691 52 +1782836696 52 +1782836701 52 +1782836706 52 +1782836711 52 +1782836716 52 +``` +
+ +--- + diff --git a/reports/glibc-arm64-hotspot-jdk8.md b/reports/glibc-arm64-hotspot-jdk8.md new file mode 100644 index 000000000..b4c138303 --- /dev/null +++ b/reports/glibc-arm64-hotspot-jdk8.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-hotspot-jdk8 +--- + +## glibc-arm64-hotspot-jdk8 - ✅ PASS + +**Date:** 2026-06-30 12:29:23 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | hotspot | +| Java | jdk8 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 40 | +| CPU Cores (end) | 52 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 40-52 cores) + +``` +1782836625 40 +1782836630 40 +1782836635 40 +1782836641 40 +1782836646 40 +1782836651 40 +1782836656 40 +1782836661 40 +1782836666 40 +1782836671 40 +1782836676 40 +1782836681 40 +1782836686 40 +1782836691 40 +1782836696 40 +1782836701 52 +1782836706 52 +1782836711 52 +1782836716 52 +1782836721 52 +``` +
+ +--- + diff --git a/reports/glibc-arm64-openj9-jdk11.md b/reports/glibc-arm64-openj9-jdk11.md new file mode 100644 index 000000000..ecd57d28a --- /dev/null +++ b/reports/glibc-arm64-openj9-jdk11.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-openj9-jdk11 +--- + +## glibc-arm64-openj9-jdk11 - ✅ PASS + +**Date:** 2026-06-30 12:29:23 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | openj9 | +| Java | jdk11 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 64 | +| CPU Cores (end) | 64 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 64-64 cores) + +``` +1782836645 64 +1782836650 64 +1782836655 64 +1782836660 64 +1782836665 64 +1782836670 64 +1782836675 64 +1782836680 64 +1782836685 64 +1782836690 64 +1782836695 64 +1782836700 64 +1782836705 64 +1782836710 64 +1782836715 64 +1782836720 64 +1782836725 64 +1782836730 64 +1782836735 64 +1782836740 64 +``` +
+ +--- + diff --git a/reports/glibc-arm64-openj9-jdk17.md b/reports/glibc-arm64-openj9-jdk17.md new file mode 100644 index 000000000..b91142e59 --- /dev/null +++ b/reports/glibc-arm64-openj9-jdk17.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-openj9-jdk17 +--- + +## glibc-arm64-openj9-jdk17 - ✅ PASS + +**Date:** 2026-06-30 12:29:23 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | openj9 | +| Java | jdk17 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 64 | +| CPU Cores (end) | 64 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 64-64 cores) + +``` +1782836715 64 +1782836720 64 +1782836725 64 +1782836730 64 +1782836735 64 +1782836740 64 +1782836745 64 +1782836750 64 +1782836755 64 +1782836760 64 +1782836765 64 +1782836770 64 +1782836775 64 +1782836780 64 +1782836785 64 +1782836790 64 +1782836795 64 +1782836800 64 +1782836805 64 +1782836810 64 +``` +
+ +--- + diff --git a/reports/glibc-arm64-openj9-jdk21.md b/reports/glibc-arm64-openj9-jdk21.md new file mode 100644 index 000000000..a0d9ed75a --- /dev/null +++ b/reports/glibc-arm64-openj9-jdk21.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-openj9-jdk21 +--- + +## glibc-arm64-openj9-jdk21 - ✅ PASS + +**Date:** 2026-06-30 12:29:23 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | openj9 | +| Java | jdk21 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 55 | +| CPU Cores (end) | 37 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (3 unique values: 37-55 cores) + +``` +1782836744 55 +1782836749 55 +1782836754 55 +1782836759 46 +1782836764 46 +1782836769 46 +1782836774 46 +1782836779 46 +1782836784 46 +1782836789 46 +1782836794 46 +1782836799 46 +1782836804 46 +1782836809 46 +1782836814 46 +1782836819 46 +1782836824 46 +1782836829 46 +1782836834 46 +1782836839 46 +``` +
+ +--- + diff --git a/reports/glibc-arm64-openj9-jdk25.md b/reports/glibc-arm64-openj9-jdk25.md new file mode 100644 index 000000000..7aa6e47a4 --- /dev/null +++ b/reports/glibc-arm64-openj9-jdk25.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-openj9-jdk25 +--- + +## glibc-arm64-openj9-jdk25 - ✅ PASS + +**Date:** 2026-06-30 12:29:23 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | openj9 | +| Java | jdk25 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 64 | +| CPU Cores (end) | 59 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 59-64 cores) + +``` +1782836624 64 +1782836629 64 +1782836634 64 +1782836639 64 +1782836644 64 +1782836649 64 +1782836654 64 +1782836659 64 +1782836664 64 +1782836669 64 +1782836674 64 +1782836679 64 +1782836684 64 +1782836689 64 +1782836694 64 +1782836699 64 +1782836704 64 +1782836709 64 +1782836714 64 +1782836719 64 +``` +
+ +--- + diff --git a/reports/glibc-arm64-openj9-jdk8.md b/reports/glibc-arm64-openj9-jdk8.md new file mode 100644 index 000000000..0edfe62af --- /dev/null +++ b/reports/glibc-arm64-openj9-jdk8.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-arm64-openj9-jdk8 +--- + +## glibc-arm64-openj9-jdk8 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-arm64 | +| JVM | openj9 | +| Java | jdk8 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 40 | +| CPU Cores (end) | 40 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 40-40 cores) + +``` +1782836635 40 +1782836640 40 +1782836645 40 +1782836650 40 +1782836655 40 +1782836660 40 +1782836665 40 +1782836670 40 +1782836675 40 +1782836680 40 +1782836685 40 +1782836690 40 +1782836695 40 +1782836700 40 +1782836705 40 +1782836710 40 +1782836715 40 +1782836720 40 +1782836725 40 +1782836730 40 +``` +
+ +--- + diff --git a/reports/glibc-x64-hotspot-jdk11.md b/reports/glibc-x64-hotspot-jdk11.md new file mode 100644 index 000000000..d1dd3eff8 --- /dev/null +++ b/reports/glibc-x64-hotspot-jdk11.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-hotspot-jdk11 +--- + +## glibc-x64-hotspot-jdk11 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | hotspot | +| Java | jdk11 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 88 | +| CPU Cores (end) | 92 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 88-92 cores) + +``` +1782836638 88 +1782836643 88 +1782836648 88 +1782836653 88 +1782836658 92 +1782836663 92 +1782836668 92 +1782836673 92 +1782836678 92 +1782836683 92 +1782836688 92 +1782836693 92 +1782836698 92 +1782836703 92 +1782836708 92 +1782836713 92 +1782836718 92 +1782836724 92 +1782836729 92 +1782836734 92 +``` +
+ +--- + diff --git a/reports/glibc-x64-hotspot-jdk17.md b/reports/glibc-x64-hotspot-jdk17.md new file mode 100644 index 000000000..d782a68a1 --- /dev/null +++ b/reports/glibc-x64-hotspot-jdk17.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-hotspot-jdk17 +--- + +## glibc-x64-hotspot-jdk17 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | hotspot | +| Java | jdk17 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 68 | +| CPU Cores (end) | 69 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (3 unique values: 65-69 cores) + +``` +1782836633 68 +1782836638 68 +1782836643 68 +1782836648 68 +1782836653 68 +1782836658 65 +1782836663 65 +1782836668 65 +1782836673 65 +1782836678 65 +1782836683 65 +1782836688 65 +1782836693 65 +1782836698 65 +1782836703 69 +1782836708 69 +1782836713 69 +1782836718 69 +1782836723 69 +1782836728 69 +``` +
+ +--- + diff --git a/reports/glibc-x64-hotspot-jdk21.md b/reports/glibc-x64-hotspot-jdk21.md new file mode 100644 index 000000000..b907eb6bc --- /dev/null +++ b/reports/glibc-x64-hotspot-jdk21.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-hotspot-jdk21 +--- + +## glibc-x64-hotspot-jdk21 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | hotspot | +| Java | jdk21 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 56 | +| CPU Cores (end) | 64 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (6 unique values: 56-64 cores) + +``` +1782836647 56 +1782836652 56 +1782836657 56 +1782836662 56 +1782836667 62 +1782836672 62 +1782836677 60 +1782836682 60 +1782836687 58 +1782836692 58 +1782836697 58 +1782836702 58 +1782836707 58 +1782836712 58 +1782836717 58 +1782836722 61 +1782836727 61 +1782836732 64 +1782836737 64 +1782836742 64 +``` +
+ +--- + diff --git a/reports/glibc-x64-hotspot-jdk25.md b/reports/glibc-x64-hotspot-jdk25.md new file mode 100644 index 000000000..0fdda3f68 --- /dev/null +++ b/reports/glibc-x64-hotspot-jdk25.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-hotspot-jdk25 +--- + +## glibc-x64-hotspot-jdk25 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | hotspot | +| Java | jdk25 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 71 | +| CPU Cores (end) | 76 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 71-76 cores) + +``` +1782836607 71 +1782836612 71 +1782836617 76 +1782836622 76 +1782836627 76 +1782836632 76 +1782836637 76 +1782836642 76 +1782836647 76 +1782836652 76 +1782836657 76 +1782836662 76 +1782836667 76 +1782836672 76 +1782836677 76 +1782836682 76 +1782836687 76 +1782836692 76 +1782836697 76 +1782836702 76 +``` +
+ +--- + diff --git a/reports/glibc-x64-hotspot-jdk8.md b/reports/glibc-x64-hotspot-jdk8.md new file mode 100644 index 000000000..d9ab6aa2a --- /dev/null +++ b/reports/glibc-x64-hotspot-jdk8.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-hotspot-jdk8 +--- + +## glibc-x64-hotspot-jdk8 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | hotspot | +| Java | jdk8 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 24 | +| CPU Cores (end) | 28 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 24-28 cores) + +``` +1782836613 24 +1782836618 24 +1782836623 24 +1782836628 24 +1782836633 24 +1782836638 24 +1782836643 24 +1782836648 24 +1782836653 24 +1782836658 24 +1782836663 24 +1782836668 28 +1782836673 28 +1782836678 28 +1782836684 28 +1782836689 28 +1782836694 28 +1782836699 28 +1782836704 28 +1782836709 28 +``` +
+ +--- + diff --git a/reports/glibc-x64-openj9-jdk11.md b/reports/glibc-x64-openj9-jdk11.md new file mode 100644 index 000000000..a6121c2c6 --- /dev/null +++ b/reports/glibc-x64-openj9-jdk11.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-openj9-jdk11 +--- + +## glibc-x64-openj9-jdk11 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | openj9 | +| Java | jdk11 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 56 | +| CPU Cores (end) | 56 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 56-56 cores) + +``` +1782836794 56 +1782836799 56 +1782836804 56 +1782836809 56 +1782836814 56 +1782836819 56 +1782836824 56 +1782836829 56 +1782836834 56 +1782836839 56 +1782836844 56 +1782836849 56 +1782836854 56 +1782836859 56 +1782836864 56 +1782836869 56 +1782836874 56 +1782836879 56 +1782836884 56 +1782836889 56 +``` +
+ +--- + diff --git a/reports/glibc-x64-openj9-jdk17.md b/reports/glibc-x64-openj9-jdk17.md new file mode 100644 index 000000000..115bf667a --- /dev/null +++ b/reports/glibc-x64-openj9-jdk17.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-openj9-jdk17 +--- + +## glibc-x64-openj9-jdk17 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | openj9 | +| Java | jdk17 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 68 | +| CPU Cores (end) | 69 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (3 unique values: 65-69 cores) + +``` +1782836635 68 +1782836640 68 +1782836645 68 +1782836650 68 +1782836655 68 +1782836660 65 +1782836665 65 +1782836670 65 +1782836675 65 +1782836680 65 +1782836685 65 +1782836690 65 +1782836695 65 +1782836700 69 +1782836705 69 +1782836710 69 +1782836716 69 +1782836721 69 +1782836726 69 +1782836731 69 +``` +
+ +--- + diff --git a/reports/glibc-x64-openj9-jdk21.md b/reports/glibc-x64-openj9-jdk21.md new file mode 100644 index 000000000..7491159f9 --- /dev/null +++ b/reports/glibc-x64-openj9-jdk21.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-openj9-jdk21 +--- + +## glibc-x64-openj9-jdk21 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | openj9 | +| Java | jdk21 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 52 | +| CPU Cores (end) | 62 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (4 unique values: 50-58 cores) + +``` +1782836610 52 +1782836616 52 +1782836621 52 +1782836626 52 +1782836631 52 +1782836636 52 +1782836641 52 +1782836646 52 +1782836651 52 +1782836656 52 +1782836661 52 +1782836666 52 +1782836671 54 +1782836676 54 +1782836681 50 +1782836686 50 +1782836691 50 +1782836696 50 +1782836701 58 +1782836706 58 +``` +
+ +--- + diff --git a/reports/glibc-x64-openj9-jdk25.md b/reports/glibc-x64-openj9-jdk25.md new file mode 100644 index 000000000..272eab9a4 --- /dev/null +++ b/reports/glibc-x64-openj9-jdk25.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-openj9-jdk25 +--- + +## glibc-x64-openj9-jdk25 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | openj9 | +| Java | jdk25 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 89 | +| CPU Cores (end) | 82 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (5 unique values: 82-91 cores) + +``` +1782836628 89 +1782836633 89 +1782836638 89 +1782836643 91 +1782836648 91 +1782836653 83 +1782836658 83 +1782836663 83 +1782836668 83 +1782836673 85 +1782836678 85 +1782836683 85 +1782836688 85 +1782836693 85 +1782836698 85 +1782836703 85 +1782836708 85 +1782836713 85 +1782836718 85 +1782836723 85 +``` +
+ +--- + diff --git a/reports/glibc-x64-openj9-jdk8.md b/reports/glibc-x64-openj9-jdk8.md new file mode 100644 index 000000000..c90026648 --- /dev/null +++ b/reports/glibc-x64-openj9-jdk8.md @@ -0,0 +1,75 @@ +--- +layout: default +title: glibc-x64-openj9-jdk8 +--- + +## glibc-x64-openj9-jdk8 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | glibc-x64 | +| JVM | openj9 | +| Java | jdk8 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 66 | +| CPU Cores (end) | 88 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (4 unique values: 58-88 cores) + +``` +1782836605 66 +1782836610 58 +1782836615 58 +1782836620 76 +1782836625 76 +1782836630 76 +1782836635 76 +1782836640 76 +1782836645 76 +1782836650 76 +1782836655 76 +1782836660 76 +1782836665 76 +1782836670 76 +1782836675 76 +1782836680 88 +1782836685 88 +1782836690 88 +1782836695 88 +1782836700 88 +``` +
+ +--- + diff --git a/reports/musl-arm64-hotspot-jdk11.md b/reports/musl-arm64-hotspot-jdk11.md new file mode 100644 index 000000000..78d503bda --- /dev/null +++ b/reports/musl-arm64-hotspot-jdk11.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-hotspot-jdk11 +--- + +## musl-arm64-hotspot-jdk11 - ✅ PASS + +**Date:** 2026-06-30 12:29:24 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | hotspot | +| Java | jdk11 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 53 | +| CPU Cores (end) | 64 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (6 unique values: 53-64 cores) + +``` +1782836618 53 +1782836623 53 +1782836628 53 +1782836633 56 +1782836638 56 +1782836643 54 +1782836648 54 +1782836653 54 +1782836658 57 +1782836663 57 +1782836668 62 +1782836673 62 +1782836678 62 +1782836683 62 +1782836688 62 +1782836693 62 +1782836698 62 +1782836703 62 +1782836709 62 +1782836714 62 +``` +
+ +--- + diff --git a/reports/musl-arm64-hotspot-jdk17.md b/reports/musl-arm64-hotspot-jdk17.md new file mode 100644 index 000000000..5767a8de7 --- /dev/null +++ b/reports/musl-arm64-hotspot-jdk17.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-hotspot-jdk17 +--- + +## musl-arm64-hotspot-jdk17 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | hotspot | +| Java | jdk17 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 57 | +| CPU Cores (end) | 57 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 57-57 cores) + +``` +1782836631 57 +1782836636 57 +1782836641 57 +1782836646 57 +1782836651 57 +1782836656 57 +1782836661 57 +1782836666 57 +1782836671 57 +1782836676 57 +1782836681 57 +1782836686 57 +1782836691 57 +1782836696 57 +1782836701 57 +1782836706 57 +1782836711 57 +1782836716 57 +1782836721 57 +1782836726 57 +``` +
+ +--- + diff --git a/reports/musl-arm64-hotspot-jdk21.md b/reports/musl-arm64-hotspot-jdk21.md new file mode 100644 index 000000000..9a2e7d66d --- /dev/null +++ b/reports/musl-arm64-hotspot-jdk21.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-hotspot-jdk21 +--- + +## musl-arm64-hotspot-jdk21 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | hotspot | +| Java | jdk21 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 60 | +| CPU Cores (end) | 60 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 60-60 cores) + +``` +1782836606 60 +1782836611 60 +1782836616 60 +1782836621 60 +1782836626 60 +1782836631 60 +1782836636 60 +1782836641 60 +1782836646 60 +1782836651 60 +1782836656 60 +1782836661 60 +1782836666 60 +1782836671 60 +1782836676 60 +1782836681 60 +1782836686 60 +1782836691 60 +1782836696 60 +1782836701 60 +``` +
+ +--- + diff --git a/reports/musl-arm64-hotspot-jdk25.md b/reports/musl-arm64-hotspot-jdk25.md new file mode 100644 index 000000000..fc2ec4e32 --- /dev/null +++ b/reports/musl-arm64-hotspot-jdk25.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-hotspot-jdk25 +--- + +## musl-arm64-hotspot-jdk25 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | hotspot | +| Java | jdk25 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 61 | +| CPU Cores (end) | 43 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 43-61 cores) + +``` +1782836667 61 +1782836672 61 +1782836677 61 +1782836682 61 +1782836687 61 +1782836692 61 +1782836697 61 +1782836702 61 +1782836707 61 +1782836712 61 +1782836717 61 +1782836722 61 +1782836727 61 +1782836732 61 +1782836737 61 +1782836742 61 +1782836747 61 +1782836752 61 +1782836757 61 +1782836762 43 +``` +
+ +--- + diff --git a/reports/musl-arm64-hotspot-jdk8.md b/reports/musl-arm64-hotspot-jdk8.md new file mode 100644 index 000000000..f537d0627 --- /dev/null +++ b/reports/musl-arm64-hotspot-jdk8.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-hotspot-jdk8 +--- + +## musl-arm64-hotspot-jdk8 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | hotspot | +| Java | jdk8 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 55 | +| CPU Cores (end) | 37 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (3 unique values: 37-55 cores) + +``` +1782836728 55 +1782836733 55 +1782836738 55 +1782836743 55 +1782836748 55 +1782836753 55 +1782836758 55 +1782836763 46 +1782836768 46 +1782836773 46 +1782836778 46 +1782836783 46 +1782836788 46 +1782836793 46 +1782836798 46 +1782836803 46 +1782836808 46 +1782836813 46 +1782836818 46 +1782836823 46 +``` +
+ +--- + diff --git a/reports/musl-arm64-openj9-jdk11.md b/reports/musl-arm64-openj9-jdk11.md new file mode 100644 index 000000000..911abb958 --- /dev/null +++ b/reports/musl-arm64-openj9-jdk11.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-openj9-jdk11 +--- + +## musl-arm64-openj9-jdk11 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | openj9 | +| Java | jdk11 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 60 | +| CPU Cores (end) | 60 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 60-60 cores) + +``` +1782836607 60 +1782836612 60 +1782836617 60 +1782836622 60 +1782836627 60 +1782836632 60 +1782836637 60 +1782836642 60 +1782836647 60 +1782836652 60 +1782836657 60 +1782836662 60 +1782836667 60 +1782836672 60 +1782836677 60 +1782836682 60 +1782836687 60 +1782836692 60 +1782836697 60 +1782836702 60 +``` +
+ +--- + diff --git a/reports/musl-arm64-openj9-jdk17.md b/reports/musl-arm64-openj9-jdk17.md new file mode 100644 index 000000000..51e719d08 --- /dev/null +++ b/reports/musl-arm64-openj9-jdk17.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-openj9-jdk17 +--- + +## musl-arm64-openj9-jdk17 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | openj9 | +| Java | jdk17 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 40 | +| CPU Cores (end) | 40 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 40-40 cores) + +``` +1782836622 40 +1782836627 40 +1782836632 40 +1782836637 40 +1782836642 40 +1782836647 40 +1782836652 40 +1782836657 40 +1782836662 40 +1782836667 40 +1782836672 40 +1782836677 40 +1782836682 40 +1782836687 40 +1782836692 40 +1782836697 40 +1782836702 40 +1782836707 40 +1782836712 40 +1782836717 40 +``` +
+ +--- + diff --git a/reports/musl-arm64-openj9-jdk21.md b/reports/musl-arm64-openj9-jdk21.md new file mode 100644 index 000000000..162fc4b10 --- /dev/null +++ b/reports/musl-arm64-openj9-jdk21.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-openj9-jdk21 +--- + +## musl-arm64-openj9-jdk21 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | openj9 | +| Java | jdk21 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 64 | +| CPU Cores (end) | 17 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 17-64 cores) + +``` +1782836624 64 +1782836629 64 +1782836634 64 +1782836640 64 +1782836645 64 +1782836650 64 +1782836655 64 +1782836660 64 +1782836665 64 +1782836670 64 +1782836675 64 +1782836680 64 +1782836685 64 +1782836690 17 +1782836695 17 +1782836700 17 +1782836705 17 +1782836710 17 +1782836715 17 +1782836720 17 +``` +
+ +--- + diff --git a/reports/musl-arm64-openj9-jdk25.md b/reports/musl-arm64-openj9-jdk25.md new file mode 100644 index 000000000..fa0723dde --- /dev/null +++ b/reports/musl-arm64-openj9-jdk25.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-openj9-jdk25 +--- + +## musl-arm64-openj9-jdk25 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | openj9 | +| Java | jdk25 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 53 | +| CPU Cores (end) | 64 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (6 unique values: 53-64 cores) + +``` +1782836613 53 +1782836618 53 +1782836623 53 +1782836628 56 +1782836633 56 +1782836638 56 +1782836643 56 +1782836648 54 +1782836653 54 +1782836658 57 +1782836663 57 +1782836668 62 +1782836673 62 +1782836678 62 +1782836683 62 +1782836688 62 +1782836693 62 +1782836698 62 +1782836703 62 +1782836708 62 +``` +
+ +--- + diff --git a/reports/musl-arm64-openj9-jdk8.md b/reports/musl-arm64-openj9-jdk8.md new file mode 100644 index 000000000..6cc8f0a54 --- /dev/null +++ b/reports/musl-arm64-openj9-jdk8.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-arm64-openj9-jdk8 +--- + +## musl-arm64-openj9-jdk8 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-arm64 | +| JVM | openj9 | +| Java | jdk8 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 44 | +| CPU Cores (end) | 39 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 39-44 cores) + +``` +1782836619 44 +1782836624 44 +1782836629 44 +1782836634 44 +1782836639 44 +1782836644 39 +1782836649 39 +1782836654 39 +1782836659 39 +1782836664 39 +1782836669 39 +1782836674 39 +1782836679 39 +1782836684 39 +1782836689 39 +1782836694 39 +1782836699 39 +1782836704 39 +1782836709 39 +1782836714 39 +``` +
+ +--- + diff --git a/reports/musl-x64-hotspot-jdk11.md b/reports/musl-x64-hotspot-jdk11.md new file mode 100644 index 000000000..18d75b869 --- /dev/null +++ b/reports/musl-x64-hotspot-jdk11.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-hotspot-jdk11 +--- + +## musl-x64-hotspot-jdk11 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | hotspot | +| Java | jdk11 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 52 | +| CPU Cores (end) | 58 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (4 unique values: 50-58 cores) + +``` +1782836599 52 +1782836604 52 +1782836609 52 +1782836614 52 +1782836619 52 +1782836624 52 +1782836629 52 +1782836634 52 +1782836639 52 +1782836644 52 +1782836649 52 +1782836654 52 +1782836659 52 +1782836664 52 +1782836669 54 +1782836674 54 +1782836679 50 +1782836684 50 +1782836689 50 +1782836694 50 +``` +
+ +--- + diff --git a/reports/musl-x64-hotspot-jdk17.md b/reports/musl-x64-hotspot-jdk17.md new file mode 100644 index 000000000..a743dcd19 --- /dev/null +++ b/reports/musl-x64-hotspot-jdk17.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-hotspot-jdk17 +--- + +## musl-x64-hotspot-jdk17 - ✅ PASS + +**Date:** 2026-06-30 12:29:25 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | hotspot | +| Java | jdk17 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 46 | +| CPU Cores (end) | 51 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (3 unique values: 46-56 cores) + +``` +1782836610 46 +1782836615 46 +1782836620 46 +1782836625 46 +1782836630 46 +1782836636 56 +1782836641 56 +1782836646 56 +1782836651 56 +1782836656 56 +1782836661 56 +1782836666 56 +1782836671 56 +1782836676 56 +1782836681 56 +1782836686 56 +1782836691 56 +1782836696 56 +1782836701 56 +1782836706 56 +``` +
+ +--- + diff --git a/reports/musl-x64-hotspot-jdk21.md b/reports/musl-x64-hotspot-jdk21.md new file mode 100644 index 000000000..cbe211081 --- /dev/null +++ b/reports/musl-x64-hotspot-jdk21.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-hotspot-jdk21 +--- + +## musl-x64-hotspot-jdk21 - ✅ PASS + +**Date:** 2026-06-30 12:29:26 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | hotspot | +| Java | jdk21 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 44 | +| CPU Cores (end) | 44 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 44-44 cores) + +``` +1782836613 44 +1782836618 44 +1782836623 44 +1782836628 44 +1782836633 44 +1782836638 44 +1782836643 44 +1782836648 44 +1782836653 44 +1782836658 44 +1782836663 44 +1782836668 44 +1782836673 44 +1782836678 44 +1782836683 44 +1782836688 44 +1782836693 44 +1782836698 44 +1782836703 44 +1782836708 44 +``` +
+ +--- + diff --git a/reports/musl-x64-hotspot-jdk25.md b/reports/musl-x64-hotspot-jdk25.md new file mode 100644 index 000000000..2f2e2d2fc --- /dev/null +++ b/reports/musl-x64-hotspot-jdk25.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-hotspot-jdk25 +--- + +## musl-x64-hotspot-jdk25 - ✅ PASS + +**Date:** 2026-06-30 12:29:26 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | hotspot | +| Java | jdk25 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 78 | +| CPU Cores (end) | 94 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (7 unique values: 77-94 cores) + +``` +1782836697 78 +1782836703 78 +1782836708 78 +1782836713 78 +1782836718 78 +1782836723 78 +1782836728 78 +1782836733 78 +1782836738 84 +1782836743 84 +1782836748 84 +1782836753 82 +1782836758 82 +1782836763 82 +1782836768 77 +1782836773 77 +1782836778 81 +1782836783 81 +1782836788 81 +1782836793 89 +``` +
+ +--- + diff --git a/reports/musl-x64-hotspot-jdk8.md b/reports/musl-x64-hotspot-jdk8.md new file mode 100644 index 000000000..cd3e6d088 --- /dev/null +++ b/reports/musl-x64-hotspot-jdk8.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-hotspot-jdk8 +--- + +## musl-x64-hotspot-jdk8 - ✅ PASS + +**Date:** 2026-06-30 12:29:26 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | hotspot | +| Java | jdk8 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 49 | +| CPU Cores (end) | 69 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 49-69 cores) + +``` +1782836606 49 +1782836611 49 +1782836616 49 +1782836621 49 +1782836626 49 +1782836631 49 +1782836636 49 +1782836641 69 +1782836646 69 +1782836651 69 +1782836656 69 +1782836661 69 +1782836666 69 +1782836671 69 +1782836676 69 +1782836681 69 +1782836686 69 +1782836691 69 +1782836696 69 +1782836701 69 +``` +
+ +--- + diff --git a/reports/musl-x64-openj9-jdk11.md b/reports/musl-x64-openj9-jdk11.md new file mode 100644 index 000000000..c8e060321 --- /dev/null +++ b/reports/musl-x64-openj9-jdk11.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-openj9-jdk11 +--- + +## musl-x64-openj9-jdk11 - ✅ PASS + +**Date:** 2026-06-30 12:29:26 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | openj9 | +| Java | jdk11 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 56 | +| CPU Cores (end) | 58 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (2 unique values: 56-58 cores) + +``` +1782836603 56 +1782836608 56 +1782836613 56 +1782836618 56 +1782836623 56 +1782836628 56 +1782836633 56 +1782836638 56 +1782836643 56 +1782836648 56 +1782836653 56 +1782836658 56 +1782836663 56 +1782836668 56 +1782836673 56 +1782836678 56 +1782836683 56 +1782836688 56 +1782836693 56 +1782836698 56 +``` +
+ +--- + diff --git a/reports/musl-x64-openj9-jdk17.md b/reports/musl-x64-openj9-jdk17.md new file mode 100644 index 000000000..e4954a177 --- /dev/null +++ b/reports/musl-x64-openj9-jdk17.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-openj9-jdk17 +--- + +## musl-x64-openj9-jdk17 - ✅ PASS + +**Date:** 2026-06-30 12:29:26 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | openj9 | +| Java | jdk17 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 52 | +| CPU Cores (end) | 58 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (4 unique values: 50-58 cores) + +``` +1782836599 52 +1782836604 52 +1782836609 52 +1782836614 52 +1782836619 52 +1782836624 52 +1782836629 52 +1782836634 52 +1782836639 52 +1782836644 52 +1782836649 52 +1782836654 52 +1782836659 52 +1782836664 52 +1782836669 54 +1782836674 54 +1782836679 50 +1782836684 50 +1782836689 50 +1782836694 50 +``` +
+ +--- + diff --git a/reports/musl-x64-openj9-jdk21.md b/reports/musl-x64-openj9-jdk21.md new file mode 100644 index 000000000..96cec1236 --- /dev/null +++ b/reports/musl-x64-openj9-jdk21.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-openj9-jdk21 +--- + +## musl-x64-openj9-jdk21 - ✅ PASS + +**Date:** 2026-06-30 12:29:26 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | openj9 | +| Java | jdk21 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 65 | +| CPU Cores (end) | 73 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (3 unique values: 65-73 cores) + +``` +1782836612 65 +1782836617 67 +1782836622 67 +1782836627 67 +1782836632 67 +1782836637 67 +1782836642 67 +1782836647 67 +1782836652 67 +1782836657 67 +1782836662 67 +1782836667 67 +1782836672 67 +1782836677 67 +1782836682 67 +1782836687 67 +1782836692 67 +1782836697 67 +1782836702 65 +1782836707 65 +``` +
+ +--- + diff --git a/reports/musl-x64-openj9-jdk25.md b/reports/musl-x64-openj9-jdk25.md new file mode 100644 index 000000000..dc40321b2 --- /dev/null +++ b/reports/musl-x64-openj9-jdk25.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-openj9-jdk25 +--- + +## musl-x64-openj9-jdk25 - ✅ PASS + +**Date:** 2026-06-30 12:29:26 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | openj9 | +| Java | jdk25 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 69 | +| CPU Cores (end) | 73 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (4 unique values: 69-74 cores) + +``` +1782836763 69 +1782836768 69 +1782836773 69 +1782836778 69 +1782836783 69 +1782836788 69 +1782836793 69 +1782836798 69 +1782836803 74 +1782836808 74 +1782836813 74 +1782836818 74 +1782836823 71 +1782836828 71 +1782836833 71 +1782836838 71 +1782836843 71 +1782836848 71 +1782836853 71 +1782836858 71 +``` +
+ +--- + diff --git a/reports/musl-x64-openj9-jdk8.md b/reports/musl-x64-openj9-jdk8.md new file mode 100644 index 000000000..d63e0ca1e --- /dev/null +++ b/reports/musl-x64-openj9-jdk8.md @@ -0,0 +1,75 @@ +--- +layout: default +title: musl-x64-openj9-jdk8 +--- + +## musl-x64-openj9-jdk8 - ✅ PASS + +**Date:** 2026-06-30 12:29:26 EDT + +### Configuration +| Setting | Value | +|---------|-------| +| Platform | musl-x64 | +| JVM | openj9 | +| Java | jdk8 | +| Container | false | + +### System Diagnostics +| Metric | Value | +|--------|-------| +| CPU Cores (start) | 22 | +| CPU Cores (end) | 22 | +| Throttling | 0% | + +### Test Results + +#### Scenario 1: Profiler-Only ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +#### Scenario 2: Tracer+Profiler ⚠️ +| Metric | Value | +|--------|-------| +| Status | N/A | +| CPU Samples | N/A | +| Sample Rate | N/A/sec | +| Health Score | N/A% | +| Threads | N/A | +| Allocations | N/A | + +
+CPU Timeline (1 unique values: 22-22 cores) + +``` +1782836611 22 +1782836616 22 +1782836621 22 +1782836626 22 +1782836631 22 +1782836636 22 +1782836641 22 +1782836646 22 +1782836651 22 +1782836656 22 +1782836661 22 +1782836666 22 +1782836671 22 +1782836676 22 +1782836681 22 +1782836686 22 +1782836691 22 +1782836696 22 +1782836701 22 +1782836706 22 +``` +
+ +--- + diff --git a/repository.datadog.yml b/repository.datadog.yml deleted file mode 100644 index b4ce7a233..000000000 --- a/repository.datadog.yml +++ /dev/null @@ -1,3 +0,0 @@ -schema-version: v1 -kind: adms -auto-version-updates-enabled: true diff --git a/settings.gradle.kts b/settings.gradle.kts deleted file mode 100644 index 15ec93800..000000000 --- a/settings.gradle.kts +++ /dev/null @@ -1,35 +0,0 @@ -pluginManagement { - includeBuild("build-logic") - val mavenRepositoryProxy = providers.gradleProperty("mavenRepositoryProxy").orNull - repositories { - if (mavenRepositoryProxy != null) { - maven { url = uri(mavenRepositoryProxy) } - } - gradlePluginPortal() - mavenCentral() - } -} - -// Centralized dependency resolution - subprojects should not define their own repositories -dependencyResolutionManagement { - repositoriesMode.set(RepositoriesMode.PREFER_SETTINGS) - val mavenRepositoryProxy = providers.gradleProperty("mavenRepositoryProxy").orNull - repositories { - if (mavenRepositoryProxy != null) { - maven { url = uri(mavenRepositoryProxy) } - } - mavenCentral() - gradlePluginPortal() - } -} - -rootProject.name = "java-profiler" - -include(":ddprof-lib") -include(":ddprof-lib:fuzz") -include(":ddprof-lib:benchmarks") -include(":ddprof-test-tracer") -include(":ddprof-test") -include(":ddprof-test-native") -include(":malloc-shim") -include(":ddprof-stresstest") diff --git a/test-validation/README.md b/test-validation/README.md deleted file mode 100644 index 0a73d0415..000000000 --- a/test-validation/README.md +++ /dev/null @@ -1,363 +0,0 @@ -# JFR Validation - -This directory contains JFR (Java Flight Recorder) validation scripts for profiler integration tests. - -## Overview - -The validation system uses [jfr-shell](https://github.com/btraceio/jfr-shell) to perform deep inspection of JFR recordings, verifying that the profiler is collecting expected events with correct data. - -## Files - -### validate-jfr.jfrs - -Main JFR validation script that performs comprehensive event analysis. - -**Usage**: -```bash -jbang jfr-shell@btraceio script validate-jfr.jfrs \ - \ - \ - [threshold_multiplier] -``` - -**Arguments**: -- `recording.jfr` - Path to JFR recording file -- `scenario` - Test scenario (`profiler-only` or `tracer+profiler`) -- `threshold_multiplier` - Optional multiplier for thresholds (default: 1.0) - -**Example**: -```bash -# Validate profiler-only recording with default thresholds -jbang jfr-shell@btraceio script validate-jfr.jfrs \ - profiler-only.jfr \ - profiler-only \ - 1.0 - -# Validate with adjusted thresholds for OpenJ9 -jbang jfr-shell@btraceio script validate-jfr.jfrs \ - recording.jfr \ - profiler-only \ - 0.5 -``` - -**Exit codes**: -- `0` - All validations passed -- `1` - Validation failed (with specific error message) -- `2` - JFR file cannot be opened - -### thresholds.env - -Configuration file defining minimum event count thresholds. - -**Base thresholds** (30-second recording, HotSpot, x64-glibc): -- `BASE_EXECUTION_SAMPLES=100` - Minimum ExecutionSample events -- `BASE_ALLOCATIONS=500` - Minimum ObjectAllocationSample events -- `BASE_THREAD_COUNT=4` - Minimum unique threads sampled - -**Platform multipliers**: -- `THRESHOLD_X64_GLIBC=1.0` - x64 with glibc (baseline) -- `THRESHOLD_X64_MUSL=1.0` - x64 with musl (Alpine) -- `THRESHOLD_ARM64_GLIBC=0.8` - ARM64 with glibc -- `THRESHOLD_ARM64_MUSL=0.8` - ARM64 with musl - -**JVM type multipliers**: -- `THRESHOLD_HOTSPOT=1.0` - HotSpot JVM (baseline) -- `THRESHOLD_OPENJ9=0.5` - OpenJ9 JVM (produces fewer samples) - -**Threshold calculation**: -``` -final_threshold = base_threshold × platform_mult × jvm_mult -``` - -## Validation Categories - -### 1. ExecutionSample Events (CPU Profiling) - -**What it validates**: -- Minimum event count (≥100 for 30s recording) -- Events are present and parseable - -**Why it matters**: -- Verifies CPU profiling is working -- Ensures sampling rate is sufficient - -**Failure scenarios**: -- Profiler not started -- Native library failed to load -- Sampling configuration incorrect - -### 2. Stack Traces - -**What it validates**: -- Stack traces present in ExecutionSample events -- Minimum 95% of samples have valid stack traces - -**Why it matters**: -- Stack traces are essential for identifying hot code paths -- Missing stack traces indicate profiler integration issues - -**Failure scenarios**: -- Async profiler integration broken -- JVM internal issues -- Stack unwinding failures - -### 3. Thread Diversity - -**What it validates**: -- Multiple unique threads are sampled (≥4 threads) -- Thread names are captured correctly - -**Why it matters**: -- Verifies profiler samples across all active threads -- Ensures multi-threaded applications are profiled correctly - -**Failure scenarios**: -- Thread sampling configuration incorrect -- Single-threaded profiling only -- Thread metadata not captured - -### 4. ObjectAllocationSample Events - -**What it validates**: -- Minimum event count (≥500 for 30s recording) -- Allocation types are captured - -**Why it matters**: -- Verifies allocation profiling is working -- Ensures memory profiling data is collected - -**Failure scenarios**: -- Allocation profiling disabled -- TLAB events not configured -- Sampling threshold too high - -### 5. ThreadAllocationStatistics Events - -**What it validates**: -- Per-thread allocation statistics present -- Events contain valid data - -**Why it matters**: -- Provides per-thread allocation totals -- Useful for identifying allocation-heavy threads - -**Note**: This event type is optional and may not be available on all JDK versions. - -### 6. Scenario-Specific Validation - -**profiler-only**: -- Validates profiler operates correctly without tracing -- Baseline profiler functionality test - -**tracer+profiler**: -- Validates profiler operates correctly with tracing enabled -- Future: will validate span context propagation in JFR events - -## Threshold Tuning - -Thresholds are conservative starting points. They should be tuned based on actual CI run data. - -### Tuning Process - -1. **Collect baseline data**: - ```bash - # Run tests and collect event counts - for recording in *.jfr; do - jbang jfr-shell@btraceio show $recording "events/jdk.ExecutionSample | count()" - done - ``` - -2. **Calculate P5 (5th percentile)**: - - Sort event counts from all successful runs - - Find 5th percentile value (95% of runs exceed this) - -3. **Set threshold**: - ``` - threshold = P5 × 0.8 # 80% of P5 provides safety margin - ``` - -4. **Update thresholds.env**: - ```bash - # For platform with P5 = 125 ExecutionSample events - THRESHOLD_ARM64_GLIBC=0.8 # 100 × 0.8 = 80 events minimum - ``` - -5. **Monitor and iterate**: - - Track false positive/negative rates - - Adjust multipliers as needed - - Document changes in thresholds.env - -### Platform-Specific Considerations - -**ARM64 platforms** (0.8× multiplier): -- Lower sampling rates due to hardware characteristics -- JIT compiler differences -- Conservative threshold provides stability - -**OpenJ9 JVM** (0.5× multiplier): -- Different JIT compilation strategy -- Typically produces 50% fewer samples than HotSpot -- This is expected behavior, not a bug - -**Alpine Linux** (musl): -- Same thresholds as glibc for same architecture -- Musl libc has minimal impact on profiling - -## Troubleshooting - -### Validation Fails: Insufficient ExecutionSample Events - -**Problem**: `ERROR: Insufficient ExecutionSample events` - -**Diagnostic steps**: -1. Check JFR file size: `ls -lh recording.jfr` - - Should be >1MB for 30s recording - - If <100KB, profiler likely not started - -2. Inspect JFR manually: - ```bash - jbang jfr-shell@btraceio show recording.jfr "events/jdk.ExecutionSample | count()" - ``` - -3. Check agent logs for errors: - ```bash - grep -i "error\|exception" agent.log - ``` - -**Possible causes**: -- Profiler not enabled: Check `-Ddd.profiling.enabled=true` -- Native library not loaded: Check for libjavaProfiler.so errors -- Short test duration: Ensure test runs for at least 30 seconds -- Platform-specific: May need to adjust threshold multiplier - -### Validation Fails: Missing Stack Traces - -**Problem**: `ERROR: Missing stack traces in ExecutionSample events` - -**Diagnostic steps**: -1. Check if any samples have stack traces: - ```bash - jbang jfr-shell@btraceio show recording.jfr \ - "events/jdk.ExecutionSample[exists(stackTrace)] | count()" - ``` - -2. Compare to total samples: - ```bash - jbang jfr-shell@btraceio show recording.jfr \ - "events/jdk.ExecutionSample | count()" - ``` - -**Possible causes**: -- Async profiler integration issue -- JVM internal stack unwinding failure -- Profiler configuration error - -### Validation Fails: Insufficient Thread Diversity - -**Problem**: `ERROR: Insufficient thread diversity` - -**Diagnostic steps**: -1. List sampled threads: - ```bash - jbang jfr-shell@btraceio show recording.jfr \ - "events/jdk.ExecutionSample | groupBy(sampledThread/javaName)" - ``` - -2. Check test app thread creation: - - Ensure test app actually creates multiple threads - - Verify threads are doing work (not blocked) - -**Possible causes**: -- Test app not creating threads -- Threads blocked/sleeping -- Single-threaded profiling configuration - -### jbang Not Found - -**Problem**: `jbang: command not found` - -**Solution**: Install jbang: -```bash -curl -Ls https://sh.jbang.dev | bash -s - app setup -export PATH="$HOME/.jbang/bin:$PATH" -``` - -Or use the prerequisite installation script: -```bash -./.gitlab/dd-trace-integration/install-prerequisites.sh -``` - -## Integration with CI - -The validation scripts are automatically run in the GitLab CI pipeline: - -```yaml -script: - # Run test - - java -javaagent:dd-java-agent.jar \ - -Ddd.profiling.jfr-template-override-file=recording.jfr \ - ProfilerTestApp --duration 30 - - # Validate JFR recording - - jbang jfr-shell@btraceio script \ - test-validation/validate-jfr.jfrs \ - recording.jfr \ - profiler-only \ - ${THRESHOLD_MULTIPLIER} -``` - -See `.gitlab/dd-trace-integration/run-integration-test.sh` for the complete implementation. - -## Advanced Usage - -### Custom Validation Scripts - -Create custom validation scripts for specific scenarios: - -```bash -#!/usr/bin/env jbang jfr-shell@btraceio script - -# custom-validation.jfrs -open $1 - -# Custom validation logic -set my_events = events/my.custom.Event | count() -if ${my_events.count} < 10 - echo "ERROR: Not enough custom events" - exit 1 -endif - -echo "SUCCESS" -close -``` - -### JSON Output for Programmatic Processing - -```bash -jbang jfr-shell@btraceio show recording.jfr \ - "events/jdk.ExecutionSample | groupBy(sampledThread/javaName) | top(10, by=count)" \ - --format json > thread-samples.json -``` - -### Correlation Analysis - -```bash -#!/usr/bin/env jbang jfr-shell@btraceio script - -# Correlate allocations with GC events -open $1 - -show events/jdk.ObjectAllocationSample | \ - decorateByTime(events/jdk.GarbageCollection, -100ms, +100ms) | \ - groupBy(objectClass/name) | \ - top(20, by=sum(weight)) - -close -``` - -## Related Documentation - -- [Test Applications README](../test-apps/README.md) - Test workload documentation -- [jfr-shell Documentation](https://github.com/btraceio/jfr-shell) - JFR-shell user guide -- [JFR Event Reference](https://sap.github.io/SapMachine/jfrevents/) - JDK JFR events -- [Integration Tests CI](./.gitlab/dd-trace-integration/.gitlab-ci.yml) - CI configuration diff --git a/test-validation/conformance.yaml b/test-validation/conformance.yaml deleted file mode 100644 index 1c0df2989..000000000 --- a/test-validation/conformance.yaml +++ /dev/null @@ -1,162 +0,0 @@ -# JFR Event Conformance Specification -# -# This file defines expected JFR events based on profiler configuration. -# Used by validate-jfr.jfrs to ensure the correct events are emitted. - -# Event type definitions -event_types: - cpu_profiling: - jdk: "jdk.ExecutionSample" - datadog: "datadog.ExecutionSample" - fields: - jdk: - thread: "sampledThread" - stack: "stackTrace" - datadog: - thread: "eventThread" - stack: "stackTrace" - - allocation_profiling: - jdk: "jdk.ObjectAllocationSample" - datadog: "datadog.ObjectSample" - fields: - jdk: - class: "objectClass/name" - weight: "weight" - datadog: - class: "objectClass/name" - weight: "weight" - - thread_stats: - jdk: "jdk.ThreadAllocationStatistics" - # No datadog equivalent - - endpoint_events: - datadog: "datadog.EndpointEvent" - # Only present when tracer is enabled - -# Configuration profiles -# Each profile defines which events should be present and which implementation to use -profiles: - - # Profile: ddprof enabled, tracer disabled - ddprof_only: - description: "Datadog profiler enabled, tracer disabled" - matches: - dd.profiling.ddprof.enabled: true - dd.trace.enabled: false - expected_events: - cpu_profiling: - implementation: datadog - required: true - min_count_formula: "2 * test_duration * threshold_multiplier" - min_threads: 3 - allocation_profiling: - implementation: datadog - required: false # May be disabled - min_count_formula: "10 * test_duration * threshold_multiplier" - thread_stats: - implementation: jdk - required: false # JDK-version dependent - unexpected_events: - - "jdk.ExecutionSample" - - "jdk.ObjectAllocationSample" - - "datadog.EndpointEvent" - - # Profile: ddprof enabled, tracer enabled - ddprof_with_tracer: - description: "Datadog profiler and tracer both enabled" - matches: - dd.profiling.ddprof.enabled: true - dd.trace.enabled: true - expected_events: - cpu_profiling: - implementation: datadog - required: true - min_count_formula: "2 * test_duration * threshold_multiplier" - min_threads: 3 - allocation_profiling: - implementation: datadog - required: false - min_count_formula: "10 * test_duration * threshold_multiplier" - endpoint_events: - implementation: datadog - required: true - min_count: 1 - thread_stats: - implementation: jdk - required: false - unexpected_events: - - "jdk.ExecutionSample" - - "jdk.ObjectAllocationSample" - - # Profile: ddprof disabled (fallback to JDK events) - jdk_fallback: - description: "Datadog profiler disabled, using JDK profiling" - matches: - dd.profiling.ddprof.enabled: false - expected_events: - cpu_profiling: - implementation: jdk - required: true - min_count_formula: "2 * test_duration * threshold_multiplier" - min_threads: 3 - allocation_profiling: - implementation: jdk - required: false - min_count_formula: "10 * test_duration * threshold_multiplier" - thread_stats: - implementation: jdk - required: false - unexpected_events: - - "datadog.ExecutionSample" - - "datadog.ObjectSample" - - "datadog.EndpointEvent" - - # Profile: Auto-detect (current behavior - accept either) - auto_detect: - description: "Auto-detect profiler implementation (for compatibility)" - matches: - auto: true - expected_events: - cpu_profiling: - implementation: any # Accept jdk or datadog - required: true - min_count_formula: "2 * test_duration * threshold_multiplier" - min_threads: 3 - allocation_profiling: - implementation: any - required: false - min_count_formula: "10 * test_duration * threshold_multiplier" - thread_stats: - implementation: jdk - required: false - unexpected_events: [] # Don't fail on unexpected events - -# Threshold multipliers (REFERENCE ONLY) -# ======================================== -# SINGLE SOURCE OF TRUTH: validate-jfr-conformance.sh -# This section is documentation only. The bash script calculates actual -# thresholds and passes them to validate-jfr.jfrs. Do not duplicate logic. -# ======================================== -# -# Multipliers are combined as: platform_arch * libc * jvm -multipliers: - platform_arch: - x64: 1.0 - arm64: 0.5 # ARM64 emulation has lower sampling rates - libc: - glibc: 1.0 - musl: 0.15 # Docker-on-runner overhead + Alpine scheduler differences - jvm: - hotspot: 1.0 - openj9: 0.3 # Significantly fewer samples due to different JIT behavior - combined_formula: "platform_arch * libc * jvm" - # Worst case (arm64-musl-openj9): 0.5 * 0.15 * 0.3 = 0.0225 (2.25% of baseline) - -# Test duration scaling -# Events scale linearly with test duration -duration_scaling: - base_duration: 60 # seconds - cpu_samples_per_second: 2 - allocation_samples_per_second: 10 diff --git a/test-validation/thresholds.env b/test-validation/thresholds.env deleted file mode 100644 index 7568156c5..000000000 --- a/test-validation/thresholds.env +++ /dev/null @@ -1,117 +0,0 @@ -# JFR Validation Thresholds Configuration -# -# ======================================== -# SINGLE SOURCE OF TRUTH: validate-jfr-conformance.sh -# ======================================== -# This file is DOCUMENTATION ONLY. The actual threshold calculation is in: -# test-validation/validate-jfr-conformance.sh -# -# The bash script: -# 1. Defines multipliers for platform, libc, and JVM -# 2. Calculates actual min thresholds -# 3. Passes them directly to validate-jfr.jfrs -# -# This eliminates duplicate logic and ensures consistency. -# ======================================== -# -# Formula: final_threshold = base_rate × duration × (arch_mult × libc_mult × jvm_mult) - -# ======================================== -# Base Thresholds (30-second recording) -# ======================================== -# These are baseline expectations for HotSpot on x64-glibc - -# CPU profiling: ExecutionSample events -# Target: ~3-5 samples/second = 90-150 for 30s -BASE_EXECUTION_SAMPLES=100 - -# Memory profiling: ObjectAllocationSample events -# Allocation sampling is unreliable - just sanity check >=1 event -BASE_ALLOCATIONS=1 - -# Thread diversity: minimum unique threads sampled -# Target: At least 4 worker threads + timer thread -BASE_THREAD_COUNT=4 - -# Stack trace coverage: percentage of samples with stack traces -# Target: 95%+ should have valid stack traces -MIN_STACK_TRACE_PCT=95.0 - -# ======================================== -# Platform Multipliers -# ======================================== -# Different platforms may have different sampling characteristics - -# x64 glibc (Debian/Ubuntu) - baseline -THRESHOLD_X64_GLIBC=1.0 - -# x64 musl (Alpine) - Docker-on-runner overhead -THRESHOLD_X64_MUSL=0.15 - -# ARM64 glibc - emulation has lower sampling rates -THRESHOLD_ARM64_GLIBC=0.5 - -# ARM64 musl (Alpine on ARM) - combined emulation + Docker overhead -THRESHOLD_ARM64_MUSL=0.075 - -# ======================================== -# JVM Type Multipliers -# ======================================== -# Different JVM implementations have different profiling characteristics - -# HotSpot - baseline -THRESHOLD_HOTSPOT=1.0 - -# OpenJ9 - significantly fewer samples due to different JIT and GC behavior -THRESHOLD_OPENJ9=0.3 - -# ======================================== -# JDK Version Multipliers -# ======================================== -# Different JDK versions may have different event rates - -THRESHOLD_JDK8=1.0 -THRESHOLD_JDK11=1.0 -THRESHOLD_JDK17=1.0 -THRESHOLD_JDK21=1.0 -THRESHOLD_JDK25=1.0 - -# ======================================== -# Usage Examples -# ======================================== -# Formula: base × arch × libc × jvm × jdk -# -# Example 1: x64-glibc, HotSpot, JDK 11 -# final_execution_samples = 100 × 1.0 × 1.0 × 1.0 × 1.0 = 100 -# -# Example 2: x64-musl, HotSpot, JDK 17 -# final_execution_samples = 100 × 1.0 × 0.15 × 1.0 × 1.0 = 15 -# -# Example 3: ARM64-glibc, OpenJ9, JDK 17 -# final_execution_samples = 100 × 0.5 × 1.0 × 0.3 × 1.0 = 15 -# -# Example 4: ARM64-musl, OpenJ9, JDK 17 (worst case) -# final_execution_samples = 100 × 0.5 × 0.15 × 0.3 × 1.0 = 2.25 - -# ======================================== -# Threshold Adjustment Notes -# ======================================== -# -# These thresholds are conservative starting points. They should be tuned based on: -# -# 1. Actual observed event counts across CI runs -# 2. False positive/negative rates -# 3. Platform-specific characteristics discovered during testing -# -# To tune thresholds: -# 1. Collect event counts from passing tests across all platforms -# 2. Calculate P5 (5th percentile) for each platform/JVM combination -# 3. Set threshold to 80% of P5 value (provides safety margin) -# 4. Monitor flakiness rates and adjust as needed -# -# Threshold adjustment history: -# - 2026-01-16: Initial conservative thresholds based on expected workload -# - 2026-01-30: Significantly reduced thresholds for problematic platforms: -# - arm64: 0.8 → 0.5 (emulation overhead) -# - musl: 1.0 → 0.15 (Docker-on-runner + Alpine scheduler) -# - openj9: 0.5 → 0.3 (JIT/GC behavior differences) diff --git a/test-validation/validate-jfr-conformance.sh b/test-validation/validate-jfr-conformance.sh deleted file mode 100755 index a9f11538c..000000000 --- a/test-validation/validate-jfr-conformance.sh +++ /dev/null @@ -1,463 +0,0 @@ -#!/bin/bash - -# validate-jfr-conformance.sh - Conformance-based JFR validation -# -# This script selects the appropriate conformance profile based on profiler -# configuration and validates JFR recordings against expected events. -# -# Usage: -# validate-jfr-conformance.sh [options] -# -# Options: -# --ddprof-enabled= Whether dd.profiling.ddprof.enabled (default: auto-detect) -# --tracer-enabled= Whether dd.trace.enabled (default: false) -# --test-duration= Test duration in seconds (default: 60) -# --arch= Architecture (default: x64) -# --libc= libc variant (default: glibc) -# --jvm-type= JVM type (default: hotspot) -# --output= Output log file (default: stdout) -# -# Exit codes: -# 0 - Validation passed -# 1 - Validation failed -# 2 - Configuration error - -set -euo pipefail - -# Ensure consistent decimal handling regardless of system locale -export LC_NUMERIC=C - -# Defaults -JFR_FILE="" -DDPROF_ENABLED="auto" -TRACER_ENABLED="false" -TEST_DURATION="60" -ARCH="x64" -LIBC="glibc" -JVM_TYPE="hotspot" -OUTPUT_FILE="" - -# Colors -GREEN='\033[0;32m' -YELLOW='\033[0;33m' -RED='\033[0;31m' -NC='\033[0m' - -function log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -function log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -function log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} - -# Parse arguments -while [[ $# -gt 0 ]]; do - case $1 in - --ddprof-enabled=*) - DDPROF_ENABLED="${1#*=}" - shift - ;; - --tracer-enabled=*) - TRACER_ENABLED="${1#*=}" - shift - ;; - --test-duration=*) - TEST_DURATION="${1#*=}" - shift - ;; - --arch=*) - ARCH="${1#*=}" - shift - ;; - --libc=*) - LIBC="${1#*=}" - shift - ;; - --jvm-type=*) - JVM_TYPE="${1#*=}" - shift - ;; - --output=*) - OUTPUT_FILE="${1#*=}" - shift - ;; - *) - if [ -z "${JFR_FILE}" ]; then - JFR_FILE="$1" - else - log_error "Unknown argument: $1" - exit 2 - fi - shift - ;; - esac -done - -# Validate required arguments -if [ -z "${JFR_FILE}" ]; then - log_error "Missing required argument: " - echo "" - echo "Usage: $0 [options]" - exit 2 -fi - -if [ ! -f "${JFR_FILE}" ]; then - log_error "JFR file not found: ${JFR_FILE}" - exit 2 -fi - -# Determine conformance profile -PROFILE="auto_detect" -PROFILE_DESC="Auto-detect profiler implementation" - -if [ "${DDPROF_ENABLED}" = "true" ]; then - if [ "${TRACER_ENABLED}" = "true" ]; then - PROFILE="ddprof_with_tracer" - PROFILE_DESC="Datadog profiler with tracer" - else - PROFILE="ddprof_only" - PROFILE_DESC="Datadog profiler only" - fi -elif [ "${DDPROF_ENABLED}" = "false" ]; then - PROFILE="jdk_fallback" - PROFILE_DESC="JDK profiling (ddprof disabled)" -fi - -log_info "==========================================" -log_info " JFR Conformance Validation" -log_info "==========================================" -log_info "" -log_info "Configuration:" -log_info " JFR file: ${JFR_FILE}" -log_info " Profile: ${PROFILE} (${PROFILE_DESC})" -log_info " ddprof enabled: ${DDPROF_ENABLED}" -log_info " Tracer enabled: ${TRACER_ENABLED}" -log_info " Test duration: ${TEST_DURATION}s" -log_info " Platform: ${ARCH}-${LIBC}" -log_info " JVM: ${JVM_TYPE}" -log_info "" - -# Calculate threshold multiplier -PLATFORM_MULT="1.0" -case "${ARCH}" in - arm64) - # ARM64 emulation has lower sampling rates - PLATFORM_MULT="0.5" - ;; -esac - -JVM_MULT="1.0" -case "${JVM_TYPE}" in - openj9) - # OpenJ9 produces significantly fewer samples due to different JIT behavior - JVM_MULT="0.3" - ;; -esac - -LIBC_MULT="1.0" -case "${LIBC}" in - musl) - # musl runners use Docker-on-runner with significant overhead - # Additional reduction due to Alpine's different scheduler behavior - LIBC_MULT="0.15" - ;; -esac - -THRESHOLD_MULTIPLIER=$(awk "BEGIN {print ${PLATFORM_MULT} * ${JVM_MULT} * ${LIBC_MULT}}") -log_info "Threshold multiplier: ${THRESHOLD_MULTIPLIER} (arch=${PLATFORM_MULT}, jvm=${JVM_MULT}, libc=${LIBC_MULT})" - -# ======================================== -# Calculate actual minimum thresholds -# ======================================== -# Base rates per second (HotSpot, x64, glibc baseline): -# ExecutionSample: ~2/sec -# ObjectAllocationSample: ~10/sec -# Thread count: minimum 3 unique threads -BASE_EXEC_RATE=2 -BASE_ALLOC_RATE=10 -BASE_THREAD_COUNT=3 - -# Calculate minimum thresholds based on test duration and multiplier -MIN_EXECUTION_SAMPLES=$(awk "BEGIN {val = ${BASE_EXEC_RATE} * ${TEST_DURATION} * ${THRESHOLD_MULTIPLIER}; print (val < 2) ? 2 : int(val)}") -# Allocation sampling is unreliable - just sanity check that at least 1 event exists -MIN_ALLOCATION_SAMPLES=1 - -# Thread count doesn't scale with duration, but reduced for challenging platforms -if awk "BEGIN {exit (${THRESHOLD_MULTIPLIER} < 0.3) ? 0 : 1}"; then - MIN_THREAD_COUNT=2 -else - MIN_THREAD_COUNT=${BASE_THREAD_COUNT} -fi - -log_info "" -log_info "Calculated thresholds (single source of truth):" -log_info " ExecutionSample: >=${MIN_EXECUTION_SAMPLES} (${BASE_EXEC_RATE}/sec × ${TEST_DURATION}s × ${THRESHOLD_MULTIPLIER})" -log_info " AllocationSample: >=${MIN_ALLOCATION_SAMPLES} (sanity check only)" -log_info " Thread count: >=${MIN_THREAD_COUNT}" - -# Load diagnostic data if available -DIAGNOSTICS_DIR="$(dirname "${JFR_FILE}")/diagnostics" -HAVE_DIAGNOSTICS="false" - -if [ -d "${DIAGNOSTICS_DIR}" ] && [ -f "${DIAGNOSTICS_DIR}/summary.txt" ]; then - HAVE_DIAGNOSTICS="true" - log_info "System diagnostics available" - - # Extract key metrics - THROTTLE_PCT=$(grep '"percentage"' "${DIAGNOSTICS_DIR}/system-metrics-end.json" 2>/dev/null | awk -F': ' '{print $2}' | tr -d ',' || echo "0") - CPU_START=$(grep '"cpu_count"' "${DIAGNOSTICS_DIR}/system-metrics-start.json" 2>/dev/null | awk -F': ' '{print $2}' | tr -d ',' || echo "0") - CPU_END=$(grep '"cpu_count"' "${DIAGNOSTICS_DIR}/system-metrics-end.json" 2>/dev/null | awk -F': ' '{print $2}' | tr -d ',' || echo "0") - - log_info " CPU: ${CPU_START} → ${CPU_END} cores" - log_info " Throttling: ${THROTTLE_PCT}%" -fi - -# Set expected event types based on profile -case "${PROFILE}" in - ddprof_only) - EXPECTED_CPU_EVENT="datadog" - EXPECTED_ALLOC_EVENT="datadog" - REQUIRE_CPU="true" - REQUIRE_ALLOC="false" - CHECK_ENDPOINT="false" - UNEXPECTED_JDK_EXEC="true" - UNEXPECTED_JDK_ALLOC="true" - UNEXPECTED_DD_EXEC="false" - UNEXPECTED_DD_ALLOC="false" - UNEXPECTED_ENDPOINT="true" - ;; - - ddprof_with_tracer) - EXPECTED_CPU_EVENT="datadog" - EXPECTED_ALLOC_EVENT="datadog" - REQUIRE_CPU="true" - REQUIRE_ALLOC="false" - CHECK_ENDPOINT="true" - UNEXPECTED_JDK_EXEC="true" - UNEXPECTED_JDK_ALLOC="true" - UNEXPECTED_DD_EXEC="false" - UNEXPECTED_DD_ALLOC="false" - UNEXPECTED_ENDPOINT="false" - ;; - - jdk_fallback) - EXPECTED_CPU_EVENT="jdk" - EXPECTED_ALLOC_EVENT="jdk" - REQUIRE_CPU="true" - REQUIRE_ALLOC="false" - CHECK_ENDPOINT="false" - UNEXPECTED_JDK_EXEC="false" - UNEXPECTED_JDK_ALLOC="false" - UNEXPECTED_DD_EXEC="true" - UNEXPECTED_DD_ALLOC="true" - UNEXPECTED_ENDPOINT="true" - ;; - - auto_detect) - EXPECTED_CPU_EVENT="any" - EXPECTED_ALLOC_EVENT="any" - REQUIRE_CPU="true" - REQUIRE_ALLOC="false" - CHECK_ENDPOINT="false" - UNEXPECTED_JDK_EXEC="false" - UNEXPECTED_JDK_ALLOC="false" - UNEXPECTED_DD_EXEC="false" - UNEXPECTED_DD_ALLOC="false" - UNEXPECTED_ENDPOINT="false" - ;; - - *) - log_error "Unknown profile: ${PROFILE}" - exit 2 - ;; -esac - -log_info "Expected events:" -log_info " CPU profiling: ${EXPECTED_CPU_EVENT} (required: ${REQUIRE_CPU})" -log_info " Allocation profiling: ${EXPECTED_ALLOC_EVENT} (required: ${REQUIRE_ALLOC})" -log_info " Endpoint events: ${CHECK_ENDPOINT}" -log_info "Unexpected events:" -log_info " jdk.ExecutionSample: ${UNEXPECTED_JDK_EXEC}" -log_info " jdk.ObjectAllocationSample: ${UNEXPECTED_JDK_ALLOC}" -log_info " datadog.ExecutionSample: ${UNEXPECTED_DD_EXEC}" -log_info " datadog.ObjectSample: ${UNEXPECTED_DD_ALLOC}" -log_info " datadog.EndpointEvent: ${UNEXPECTED_ENDPOINT}" -log_info "" - -# Prepare validation command -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -VALIDATION_SCRIPT="${SCRIPT_DIR}/validate-jfr.jfrs" - -if [ ! -f "${VALIDATION_SCRIPT}" ]; then - log_error "Validation script not found: ${VALIDATION_SCRIPT}" - exit 2 -fi - -# Run validation -log_info "Running JFR validation..." -log_info "" - -# Check if JFR validation should be skipped (JDK 25 unavailable) -if [ -f /tmp/skip-jfr-validation ]; then - SKIP_REASON=$(cat /tmp/skip-jfr-validation 2>/dev/null || echo "prerequisite unavailable") - log_warn "Skipping JFR validation: ${SKIP_REASON}" - if [ -n "${OUTPUT_FILE}" ]; then - echo "VALIDATION_SKIPPED: ${SKIP_REASON}" > "${OUTPUT_FILE}" - fi - exit 0 -fi - -# jfr-shell (jafar) requires Java 25 (class file version 69.0) -# Always use --java 25 to let jbang download the correct JDK -unset JAVA_VERSION # Prevent jbang from picking up test JDK version - -log_info "Using Java 25 for jbang (required by jfr-shell)" - -# Pass calculated thresholds directly (single source of truth - no duplicate logic in jfrs) -VALIDATION_CMD="jbang --java 25 jfr-shell@btraceio script \"${VALIDATION_SCRIPT}\" \"${JFR_FILE}\" \"${PROFILE}\" \"${MIN_EXECUTION_SAMPLES}\" \"${MIN_ALLOCATION_SAMPLES}\" \"${MIN_THREAD_COUNT}\" \"${EXPECTED_CPU_EVENT}\" \"${EXPECTED_ALLOC_EVENT}\" \"${CHECK_ENDPOINT}\" \"${UNEXPECTED_JDK_EXEC}\" \"${UNEXPECTED_JDK_ALLOC}\" \"${UNEXPECTED_DD_EXEC}\" \"${UNEXPECTED_DD_ALLOC}\" \"${UNEXPECTED_ENDPOINT}\"" - -if [ -n "${OUTPUT_FILE}" ]; then - # Set up trap to write failure marker if script is killed/crashes - trap 'echo "VALIDATION_FAILED: Script terminated unexpectedly (exit code: $?)" >> "${OUTPUT_FILE}" 2>/dev/null || true' EXIT ERR - - # Write START marker so we know validation began - { - echo "VALIDATION_STARTED: $(date -u +"%Y-%m-%dT%H:%M:%SZ")" - echo "Configuration: ddprof=${DDPROF_ENABLED}, tracer=${TRACER_ENABLED}, duration=${TEST_DURATION}s" - echo "Platform: arch=${ARCH}, libc=${LIBC}, jvm-type=${JVM_TYPE}" - echo "JFR file: ${JFR_FILE}" - echo "Profile: ${PROFILE}" - echo "---" - } > "${OUTPUT_FILE}" - - # Run validation and append output - eval "${VALIDATION_CMD}" >> "${OUTPUT_FILE}" 2>&1 - JBANG_EXIT=$? - - # Always write completion marker based on exit code - if [ ${JBANG_EXIT} -eq 0 ]; then - # Check if validation script itself reported failure - if grep -q "VALIDATION_FAILED" "${OUTPUT_FILE}"; then - echo "VALIDATION_FAILED: Validation checks did not pass" >> "${OUTPUT_FILE}" - else - echo "SUCCESS: All validations passed" >> "${OUTPUT_FILE}" - fi - else - echo "VALIDATION_FAILED: jbang/jfr-shell exit code ${JBANG_EXIT}" >> "${OUTPUT_FILE}" - fi - - # Clear trap now that we've written the marker - trap - EXIT ERR - - # Check if validation actually failed by examining output - if grep -q "VALIDATION_FAILED" "${OUTPUT_FILE}"; then - log_error "" - log_error "==========================================" - log_error " VALIDATION FAILED" - log_error " Profile: ${PROFILE}" - log_error "==========================================" - log_error "" - - # Show diagnostics if available - if [ "${HAVE_DIAGNOSTICS}" = "true" ]; then - log_error "System Diagnostics:" - log_error " CPU range: ${CPU_START} → ${CPU_END} cores" - log_error " Throttling: ${THROTTLE_PCT}%" - log_error " Full diagnostics: ${DIAGNOSTICS_DIR}/" - log_error "" - fi - - # Show the full output to see what failed - cat "${OUTPUT_FILE}" - exit 1 - fi - - # Show summary from output - tail -20 "${OUTPUT_FILE}" -else - VALIDATION_OUTPUT=$(eval "${VALIDATION_CMD}" 2>&1) - JBANG_EXIT=$? - - # Print output - echo "${VALIDATION_OUTPUT}" - - # Check if validation failed - if echo "${VALIDATION_OUTPUT}" | grep -q "VALIDATION_FAILED"; then - log_error "" - log_error "==========================================" - log_error " VALIDATION FAILED" - log_error " Profile: ${PROFILE}" - log_error "==========================================" - - # Show diagnostics if available - if [ "${HAVE_DIAGNOSTICS}" = "true" ]; then - log_error "" - log_error "System Diagnostics:" - log_error " CPU range: ${CPU_START} → ${CPU_END} cores" - log_error " Throttling: ${THROTTLE_PCT}%" - log_error " Full diagnostics: ${DIAGNOSTICS_DIR}/" - fi - - exit 1 - fi -fi - -# Check if jbang itself failed -if [ ${JBANG_EXIT} -ne 0 ]; then - log_error "" - log_error "==========================================" - log_error " JBANG EXECUTION FAILED" - log_error " Profile: ${PROFILE}" - log_error " Exit code: ${JBANG_EXIT}" - log_error "==========================================" - exit 1 -fi - -# Success - show enhanced summary -log_info "" -log_info "==========================================" -log_info " VALIDATION PASSED" -log_info " Profile: ${PROFILE}" -log_info "==========================================" - -if [ "${HAVE_DIAGNOSTICS}" = "true" ]; then - log_info "" - log_info "System Performance:" - log_info " CPU: ${CPU_START} → ${CPU_END} cores" - log_info " Throttling: ${THROTTLE_PCT}%" - - # Calculate actual sample rate from validation output - # Read from OUTPUT_FILE if set, otherwise use VALIDATION_OUTPUT variable - if [ -n "${OUTPUT_FILE}" ] && [ -f "${OUTPUT_FILE}" ]; then - ACTUAL_SAMPLES=$(grep "ExecutionSample:" "${OUTPUT_FILE}" | grep -oE '[0-9]+ events' | awk '{print $1}') - else - ACTUAL_SAMPLES=$(echo "${VALIDATION_OUTPUT}" | grep "ExecutionSample:" | grep -oE '[0-9]+ events' | awk '{print $1}') - fi - if [ -n "${ACTUAL_SAMPLES}" ]; then - ACTUAL_RATE=$(awk "BEGIN {printf \"%.2f\", ${ACTUAL_SAMPLES} / ${TEST_DURATION}}") - EXPECTED_BASE=1.6 - EXPECTED_ADJUSTED=$(awk "BEGIN {printf \"%.2f\", ${EXPECTED_BASE} * ${THRESHOLD_MULTIPLIER}}") - HEALTH_SCORE=$(awk "BEGIN {printf \"%.0f\", (${ACTUAL_RATE} / ${EXPECTED_ADJUSTED}) * 100}") - - # Classification - if [ "${HEALTH_SCORE}" -ge 90 ]; then - CLASSIFICATION="EXCELLENT" - elif [ "${HEALTH_SCORE}" -ge 75 ]; then - CLASSIFICATION="GOOD" - elif [ "${HEALTH_SCORE}" -ge 60 ]; then - CLASSIFICATION="MARGINAL" - else - CLASSIFICATION="POOR" - fi - - log_info " Sample rate: ${ACTUAL_RATE}/sec (expected: ${EXPECTED_ADJUSTED}/sec)" - log_info " Health score: ${HEALTH_SCORE}% (${CLASSIFICATION})" - fi -fi - -exit 0 diff --git a/test-validation/validate-jfr.jfrs b/test-validation/validate-jfr.jfrs deleted file mode 100755 index 087bcdc4b..000000000 --- a/test-validation/validate-jfr.jfrs +++ /dev/null @@ -1,504 +0,0 @@ -#!/usr/bin/env jbang jfr-shell@btraceio script - -# validate-jfr.jfrs - Deep JFR validation for profiler integration tests -# -# Arguments: -# $1 - JFR recording file path -# $2 - Test scenario/profile name -# $3 - Minimum ExecutionSample count (calculated by caller) -# $4 - Minimum allocation sample count (calculated by caller) -# $5 - Minimum unique thread count (calculated by caller) -# $6 - Expected CPU event type: jdk|datadog|any (optional, default any) -# $7 - Expected alloc event type: jdk|datadog|any (optional, default any) -# $8 - Check endpoint events: true|false (optional, default false) -# $9 - Unexpected jdk.ExecutionSample: true|false (optional, default false) -# $10 - Unexpected jdk.ObjectAllocationSample: true|false (optional, default false) -# $11 - Unexpected datadog.ExecutionSample: true|false (optional, default false) -# $12 - Unexpected datadog.ObjectSample: true|false (optional, default false) -# $13 - Unexpected datadog.EndpointEvent: true|false (optional, default false) -# -# Thresholds are calculated by the calling bash script (validate-jfr-conformance.sh) -# based on platform, JVM type, and libc variant. This eliminates duplicate -# threshold logic and provides a single source of truth. -# -# The script prints SUCCESS or VALIDATION_FAILED at the end. -# The calling script should check for "VALIDATION_FAILED" in output. - -# Parse arguments -set scenario = "$2" -set min_execution_samples = "$3" -set min_allocation_samples = "$4" -set min_thread_count = "$5" -set expected_cpu = "$6" -set expected_alloc = "$7" -set check_endpoint = "$8" -set unexpected_jdk_exec = "$9" -set unexpected_jdk_alloc = "$10" -set unexpected_dd_exec = "$11" -set unexpected_dd_alloc = "$12" -set unexpected_endpoint = "$13" - -# Default min values if not provided (legacy compatibility) -if "${min_execution_samples}" == "" - set min_execution_samples = 10 -endif - -if "${min_allocation_samples}" == "" - set min_allocation_samples = 10 -endif - -if "${min_thread_count}" == "" - set min_thread_count = 2 -endif - -# Default to 'any' for backwards compatibility -if "${expected_cpu}" == "" - set expected_cpu = "any" -endif - -if "${expected_alloc}" == "" - set expected_alloc = "any" -endif - -if "${check_endpoint}" == "" - set check_endpoint = "false" -endif - -# Default unexpected event flags to false -if "${unexpected_jdk_exec}" == "" - set unexpected_jdk_exec = "false" -endif - -if "${unexpected_jdk_alloc}" == "" - set unexpected_jdk_alloc = "false" -endif - -if "${unexpected_dd_exec}" == "" - set unexpected_dd_exec = "false" -endif - -if "${unexpected_dd_alloc}" == "" - set unexpected_dd_alloc = "false" -endif - -if "${unexpected_endpoint}" == "" - set unexpected_endpoint = "false" -endif - -echo "=== JFR Validation ===" -echo "Recording: $1" -echo "Scenario: ${scenario}" -echo "Expected CPU events: ${expected_cpu}" -echo "Expected allocation events: ${expected_alloc}" -echo "Check endpoint events: ${check_endpoint}" -echo "" - -# Open JFR recording (open command doesn't support variable expansion) -open $1 - -echo "✓ JFR recording opened successfully" -echo "" - -echo "Validation requirements:" -echo " ExecutionSample: >=1 (presence check)" -echo " Stack traces: 100% coverage (all present events)" -echo " ObjectAllocationSample: >=${min_allocation_samples} (soft threshold)" -echo " Unique threads: >=${min_thread_count}" -echo "" - -# Initialize validation state -set validation_failed = "false" - -# ======================================== -# 1. Validate ExecutionSample events (CPU profiling) -# ======================================== -echo "[1/8] Validating ExecutionSample events..." - -# Check both jdk.ExecutionSample and datadog.ExecutionSample -set jdk_exec_count = events/jdk.ExecutionSample | count() -set dd_exec_count = events/datadog.ExecutionSample | count() - -echo " Found: ${jdk_exec_count.count} jdk.ExecutionSample events" -echo " Found: ${dd_exec_count.count} datadog.ExecutionSample events" - -# Determine which implementation to validate based on expected_cpu -# Validation strategy: require at least 1 event of the expected type -if "${expected_cpu}" == "jdk" - # Conformance mode: expect ONLY jdk events - set exec_event_type = "jdk" - set exec_count = ${jdk_exec_count.count} - echo " Expected: jdk.ExecutionSample (conformance mode)" - - if ${dd_exec_count.count} > 0 - echo " ERROR: Found unexpected datadog.ExecutionSample events (${dd_exec_count.count})" - set validation_failed = "true" - endif - - if ${jdk_exec_count.count} < 1 - echo " ERROR: No jdk.ExecutionSample events found" - set validation_failed = "true" - else - echo " ✓ ExecutionSample events present (${jdk_exec_count.count})" - endif - -elif "${expected_cpu}" == "datadog" - # Conformance mode: expect ONLY datadog events - set exec_event_type = "datadog" - set exec_count = ${dd_exec_count.count} - echo " Expected: datadog.ExecutionSample (conformance mode)" - - if ${jdk_exec_count.count} > 0 - echo " ERROR: Found unexpected jdk.ExecutionSample events (${jdk_exec_count.count})" - set validation_failed = "true" - endif - - if ${dd_exec_count.count} < 1 - echo " ERROR: No datadog.ExecutionSample events found" - set validation_failed = "true" - else - echo " ✓ ExecutionSample events present (${dd_exec_count.count})" - endif - -else - # Auto-detect mode: accept either (backwards compatibility) - if ${jdk_exec_count.count} >= ${dd_exec_count.count} - set exec_event_type = "jdk" - set exec_count = ${jdk_exec_count.count} - echo " Total: ${jdk_exec_count.count} ExecutionSample events (using jdk.ExecutionSample)" - - if ${jdk_exec_count.count} < 1 - echo " ERROR: No ExecutionSample events found" - set validation_failed = "true" - else - echo " ✓ ExecutionSample events present (${jdk_exec_count.count})" - endif - else - set exec_event_type = "datadog" - set exec_count = ${dd_exec_count.count} - echo " Total: ${dd_exec_count.count} ExecutionSample events (using datadog.ExecutionSample)" - - if ${dd_exec_count.count} < 1 - echo " ERROR: No ExecutionSample events found" - set validation_failed = "true" - else - echo " ✓ ExecutionSample events present (${dd_exec_count.count})" - endif - endif -endif - -# ======================================== -# 2. Validate stack traces present -# ======================================== -echo "[2/8] Validating stack traces..." - -# Use the event type we detected in step 1 -# Validation: all present ExecutionSample events should have stack traces -if "${exec_event_type}" == "jdk" - set samples_with_stack = events/jdk.ExecutionSample[exists(stackTrace)] | count() - echo " Samples with stack traces: ${samples_with_stack.count} of ${jdk_exec_count.count}" - - if ${samples_with_stack.count} < ${jdk_exec_count.count} - echo " ERROR: Some ExecutionSample events are missing stack traces" - echo " Expected: ${jdk_exec_count.count}, Got: ${samples_with_stack.count}" - set validation_failed = "true" - else - echo " ✓ All ExecutionSample events have stack traces (${samples_with_stack.count})" - endif -else - set samples_with_stack = events/datadog.ExecutionSample[exists(stackTrace)] | count() - echo " Samples with stack traces: ${samples_with_stack.count} of ${dd_exec_count.count}" - - if ${samples_with_stack.count} < ${dd_exec_count.count} - echo " ERROR: Some ExecutionSample events are missing stack traces" - echo " Expected: ${dd_exec_count.count}, Got: ${samples_with_stack.count}" - set validation_failed = "true" - else - echo " ✓ All ExecutionSample events have stack traces (${samples_with_stack.count})" - endif -endif - -# ======================================== -# 3. Validate thread diversity -# ======================================== -echo "[3/8] Validating thread diversity..." - -# Use the event type we detected in step 1 -# Note: datadog.ExecutionSample uses eventThread, jdk.ExecutionSample uses sampledThread -if "${exec_event_type}" == "jdk" - set unique_threads = events/jdk.ExecutionSample | groupBy(sampledThread/javaName) | count() -else - set unique_threads = events/datadog.ExecutionSample | groupBy(eventThread/javaName) | count() -endif - -echo " Unique threads sampled: ${unique_threads.count}" - -if ${unique_threads.count} < ${min_thread_count} - echo " ERROR: Insufficient thread diversity" - echo " Expected: >=${min_thread_count}, Got: ${unique_threads.count}" - set validation_failed = "true" -else - echo " ✓ Thread diversity OK (${unique_threads.count} threads)" -endif - -# Show top threads by sample count -echo " Top threads by samples:" -if "${exec_event_type}" == "jdk" - show events/jdk.ExecutionSample | groupBy(sampledThread/javaName) | top(5, by=count) -else - show events/datadog.ExecutionSample | groupBy(eventThread/javaName) | top(5, by=count) -endif - -# ======================================== -# 4. Validate ObjectAllocationSample events -# ======================================== -echo "" -echo "[4/8] Validating ObjectAllocationSample events..." - -# Check both jdk.ObjectAllocationSample and datadog.ObjectSample -set jdk_alloc_count = events/jdk.ObjectAllocationSample | count() -set dd_alloc_count = events/datadog.ObjectSample | count() - -echo " Found: ${jdk_alloc_count.count} jdk.ObjectAllocationSample events" -echo " Found: ${dd_alloc_count.count} datadog.ObjectSample events" - -# Determine which implementation to validate based on expected_alloc -if "${expected_alloc}" == "jdk" - # Conformance mode: expect ONLY jdk events - echo " Expected: jdk.ObjectAllocationSample (conformance mode)" - - if ${dd_alloc_count.count} > 0 - echo " ERROR: Found unexpected datadog.ObjectSample events (${dd_alloc_count.count})" - set validation_failed = "true" - endif - - if ${jdk_alloc_count.count} < ${min_allocation_samples} - echo " WARNING: Insufficient jdk.ObjectAllocationSample events" - echo " Expected: >=${min_allocation_samples}, Got: ${jdk_alloc_count.count}" - echo " Note: Allocation profiling may not be enabled" - else - echo " ✓ Allocation sample count OK (${jdk_alloc_count.count})" - endif - - # Show top allocation sites if we have any - if ${jdk_alloc_count.count} > 0 - echo " Top allocation types:" - show events/jdk.ObjectAllocationSample | groupBy(objectClass/name) | top(5, by=sum(weight)) - endif - -elif "${expected_alloc}" == "datadog" - # Conformance mode: expect ONLY datadog events - echo " Expected: datadog.ObjectSample (conformance mode)" - - if ${jdk_alloc_count.count} > 0 - echo " ERROR: Found unexpected jdk.ObjectAllocationSample events (${jdk_alloc_count.count})" - set validation_failed = "true" - endif - - if ${dd_alloc_count.count} < ${min_allocation_samples} - echo " WARNING: Insufficient datadog.ObjectSample events" - echo " Expected: >=${min_allocation_samples}, Got: ${dd_alloc_count.count}" - echo " Note: Allocation profiling may not be enabled" - else - echo " ✓ Allocation sample count OK (${dd_alloc_count.count})" - endif - - # Show top allocation sites if we have any - if ${dd_alloc_count.count} > 0 - echo " Top allocation types:" - show events/datadog.ObjectSample | groupBy(objectClass/name) | top(5, by=sum(weight)) - endif - -else - # Auto-detect mode: accept either (backwards compatibility) - if ${jdk_alloc_count.count} >= ${dd_alloc_count.count} - echo " Total: ${jdk_alloc_count.count} allocation events (using jdk.ObjectAllocationSample)" - - if ${jdk_alloc_count.count} < ${min_allocation_samples} - echo " WARNING: Insufficient allocation sample events" - echo " Expected: >=${min_allocation_samples}, Got: ${jdk_alloc_count.count}" - echo " Note: Allocation profiling may not be enabled" - else - echo " ✓ Allocation sample count OK (${jdk_alloc_count.count})" - endif - - # Show top allocation sites if we have any - if ${jdk_alloc_count.count} > 0 - echo " Top allocation types:" - show events/jdk.ObjectAllocationSample | groupBy(objectClass/name) | top(5, by=sum(weight)) - endif - else - echo " Total: ${dd_alloc_count.count} allocation events (using datadog.ObjectSample)" - - if ${dd_alloc_count.count} < ${min_allocation_samples} - echo " WARNING: Insufficient allocation sample events" - echo " Expected: >=${min_allocation_samples}, Got: ${dd_alloc_count.count}" - echo " Note: Allocation profiling may not be enabled" - else - echo " ✓ Allocation sample count OK (${dd_alloc_count.count})" - endif - - # Show top allocation sites if we have any - if ${dd_alloc_count.count} > 0 - echo " Top allocation types:" - show events/datadog.ObjectSample | groupBy(objectClass/name) | top(5, by=sum(weight)) - endif - endif -endif - -# ======================================== -# 5. Validate ThreadAllocationStatistics -# ======================================== -echo "" -echo "[5/8] Validating ThreadAllocationStatistics..." - -set thread_alloc = events/jdk.ThreadAllocationStatistics | count() - -echo " Found: ${thread_alloc.count} ThreadAllocationStatistics events" - -if ${thread_alloc.count} > 0 - echo " ✓ ThreadAllocationStatistics present (${thread_alloc.count})" -else - echo " WARNING: No ThreadAllocationStatistics events found" - echo " This may be expected depending on JDK version and configuration" -endif - -# ======================================== -# 6. Validate endpoint events (if tracer enabled) -# ======================================== -echo "" -echo "[6/8] Validating endpoint events..." - -if "${check_endpoint}" == "true" - set endpoint_count = events/datadog.EndpointEvent | count() - echo " Found: ${endpoint_count.count} datadog.EndpointEvent events" - - if ${endpoint_count.count} > 0 - echo " ✓ Endpoint events present (tracer integration working)" - else - echo " WARNING: No endpoint events found" - echo " This may indicate tracer is not capturing any requests" - endif -else - echo " Skipped (tracer not expected in this configuration)" -endif - -# ======================================== -# 7. Check for unexpected events -# ======================================== -echo "" -echo "[7/8] Checking for unexpected events..." - -set has_unexpected = "false" - -# Check for unexpected jdk.ExecutionSample -if "${unexpected_jdk_exec}" == "true" - set check_jdk_exec = events/jdk.ExecutionSample | count() - if ${check_jdk_exec.count} > 0 - echo " ERROR: Found unexpected jdk.ExecutionSample events (${check_jdk_exec.count})" - set validation_failed = "true" - set has_unexpected = "true" - endif -endif - -# Check for unexpected jdk.ObjectAllocationSample -if "${unexpected_jdk_alloc}" == "true" - set check_jdk_alloc = events/jdk.ObjectAllocationSample | count() - if ${check_jdk_alloc.count} > 0 - echo " ERROR: Found unexpected jdk.ObjectAllocationSample events (${check_jdk_alloc.count})" - set validation_failed = "true" - set has_unexpected = "true" - endif -endif - -# Check for unexpected datadog.ExecutionSample -if "${unexpected_dd_exec}" == "true" - set check_dd_exec = events/datadog.ExecutionSample | count() - if ${check_dd_exec.count} > 0 - echo " ERROR: Found unexpected datadog.ExecutionSample events (${check_dd_exec.count})" - set validation_failed = "true" - set has_unexpected = "true" - endif -endif - -# Check for unexpected datadog.ObjectSample -if "${unexpected_dd_alloc}" == "true" - set check_dd_alloc = events/datadog.ObjectSample | count() - if ${check_dd_alloc.count} > 0 - echo " ERROR: Found unexpected datadog.ObjectSample events (${check_dd_alloc.count})" - set validation_failed = "true" - set has_unexpected = "true" - endif -endif - -# Check for unexpected datadog.EndpointEvent -if "${unexpected_endpoint}" == "true" - set check_endpoint_evt = events/datadog.EndpointEvent | count() - if ${check_endpoint_evt.count} > 0 - echo " ERROR: Found unexpected datadog.EndpointEvent events (${check_endpoint_evt.count})" - set validation_failed = "true" - set has_unexpected = "true" - endif -endif - -if "${has_unexpected}" == "false" - echo " ✓ No unexpected events found" -endif - -# ======================================== -# 8. Scenario-specific validation -# ======================================== -echo "" -echo "[8/8] Scenario-specific validation (${scenario})..." - -# Note: jfr-shell's "or" operator is broken, use nested ifs instead -if "${scenario}" == "tracer+profiler" - echo " Validating tracer+profiler scenario..." - echo " ✓ Tracer+profiler scenario checks passed" -elif "${scenario}" == "ddprof_with_tracer" - echo " Validating tracer+profiler scenario..." - echo " ✓ Tracer+profiler scenario checks passed" -elif "${scenario}" == "profiler-only" - echo " Validating profiler-only scenario..." - echo " ✓ Profiler-only scenario checks passed" -elif "${scenario}" == "ddprof_only" - echo " Validating profiler-only scenario..." - echo " ✓ Profiler-only scenario checks passed" -elif "${scenario}" == "jdk_fallback" - echo " Validating JDK fallback scenario..." - echo " ✓ JDK fallback scenario checks passed" -else - echo " Profile: ${scenario}" - echo " ✓ Conformance validation passed" -endif - -# ======================================== -# Summary -# ======================================== -echo "" -echo "=== Validation Summary ===" - -if "${exec_event_type}" == "jdk" - echo "ExecutionSample: ${jdk_exec_count.count} events (jdk.ExecutionSample)" -else - echo "ExecutionSample: ${dd_exec_count.count} events (datadog.ExecutionSample)" -endif - -echo "Stack traces: ${samples_with_stack.count} samples" -echo "Thread diversity: ${unique_threads.count} threads" - -if ${jdk_alloc_count.count} >= ${dd_alloc_count.count} - echo "Allocation samples: ${jdk_alloc_count.count} events (jdk.ObjectAllocationSample)" -else - echo "Allocation samples: ${dd_alloc_count.count} events (datadog.ObjectSample)" -endif - -echo "ThreadAllocationStatistics: ${thread_alloc.count} events" -echo "" - -if "${validation_failed}" == "true" - echo "VALIDATION_FAILED: One or more checks did not pass" -else - echo "SUCCESS: All validations passed" -endif - -close diff --git a/test/native/libs/reladyn.c b/test/native/libs/reladyn.c deleted file mode 100644 index 2133004c5..000000000 --- a/test/native/libs/reladyn.c +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -// Force pthread_setspecific into .rela.dyn with R_X86_64_GLOB_DAT. -int (*indirect_pthread_setspecific)(pthread_key_t, const void*); - -// Force pthread_exit into .rela.dyn with R_X86_64_64. -void (*static_pthread_exit)(void*) = pthread_exit; - -void* thread_function(void* arg) { - printf("Thread running\n"); - return NULL; -} - -// Not indended to be executed. -int reladyn() { - pthread_t thread; - pthread_key_t key; - - pthread_key_create(&key, NULL); - - // Direct call, forces into .rela.plt. - pthread_create(&thread, NULL, thread_function, NULL); - - // Assign to a function pointer at runtime, forces into .rela.dyn as R_X86_64_GLOB_DAT. - indirect_pthread_setspecific = pthread_setspecific; - indirect_pthread_setspecific(key, "Thread-specific value"); - - // Use pthread_exit via the static pointer, forces into .rela.dyn as R_X86_64_64. - static_pthread_exit(NULL); - - return 0; -} diff --git a/test/native/symbolsLinuxTest.cpp b/test/native/symbolsLinuxTest.cpp deleted file mode 100644 index 537cf808b..000000000 --- a/test/native/symbolsLinuxTest.cpp +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#ifdef __linux__ - -#include "codeCache.h" -#include "profiler.h" -#include "testRunner.hpp" -#include - -#define ASSERT_RESOLVE(id) \ - { \ - void* result = dlopen("libreladyn.so", RTLD_NOW); /* see reladyn.c */ \ - ASSERT(result); \ - Profiler::instance()->updateSymbols(false); \ - CodeCache* libreladyn = Profiler::instance()->findLibraryByName("libreladyn"); \ - ASSERT(libreladyn); \ - void* sym = libreladyn->findImport(id); \ - ASSERT(sym); \ - } - -TEST_CASE(ResolveFromRela_plt) { - ASSERT_RESOLVE(im_pthread_create); -} - -TEST_CASE(ResolveFromRela_dyn_R_GLOB_DAT) { - ASSERT_RESOLVE(im_pthread_setspecific); -} - -TEST_CASE(ResolveFromRela_dyn_R_ABS64) { - ASSERT_RESOLVE(im_pthread_exit); -} - -#endif // __linux__ diff --git a/test/test/nativemem/malloc_plt_dyn.c b/test/test/nativemem/malloc_plt_dyn.c deleted file mode 100644 index d61d20e80..000000000 --- a/test/test/nativemem/malloc_plt_dyn.c +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright The async-profiler authors - * SPDX-License-Identifier: Apache-2.0 - */ - -#include -#include - -const int MALLOC_SIZE = 1999993; -const int MALLOC_DYN_SIZE = 2000003; - -// A global pointer referencing malloc as data -> .rela.dyn -static void* (*malloc_dyn)(size_t) = malloc; - -int main(void) { - // Direct call -> .rela.plt - void* p = malloc(MALLOC_SIZE); - - void* q = malloc_dyn(MALLOC_DYN_SIZE); - - free(p); - free(q); - - return 0; -} diff --git a/upstream-tracker-state.txt b/upstream-tracker-state.txt new file mode 100644 index 000000000..b5862f1ae --- /dev/null +++ b/upstream-tracker-state.txt @@ -0,0 +1 @@ +656d96b57f1bcd7e54dee04d61985d340d6b93c3 diff --git a/utils/README.md b/utils/README.md deleted file mode 100644 index 605adbdcc..000000000 --- a/utils/README.md +++ /dev/null @@ -1,159 +0,0 @@ -# Utility Scripts - -This directory contains utility scripts for managing the java-profiler project. - ---- - -## Release - -### `release.sh` - -Triggers the Validated Release workflow using GitHub CLI to create a new release. - -**Prerequisites:** -- [GitHub CLI](https://cli.github.com/) installed and authenticated -- Git repository is up to date -- You are on the correct branch for the release type - -**Usage:** -```bash -./utils/release.sh [options] -``` - -**Arguments:** -- `release_type`: Type of release (`major`, `minor`, or `patch`) - -**Options:** -- `--no-dry-run`: Actually perform the release (default is dry-run) -- `--skip-tests`: Skip pre-release tests (emergency releases only) -- `--branch `: Specify branch to release from (default: current branch) -- `--commit `: Specify commit SHA to release (default: interactive selection) -- `--help`: Show help message - -**Branch rules:** -- **Major/Minor releases**: must be run from `main` -- **Patch releases**: must be run from a `release/X.Y._` branch - -**Release flow:** -1. Validates inputs and branch rules -2. Interactive commit selection (or use `--commit`) -3. Triggers GitHub Actions "Validated Release" workflow -4. Workflow runs pre-release tests, creates annotated git tag -5. Tag push triggers GitLab build pipeline -6. GitLab builds multi-platform artifacts and publishes to Maven Central -7. GitHub workflows create release with assets - ---- - -## Backport - -### `backport-pr.sh` - -Cherry-picks a merged PR onto a release branch, pushes the backport branch, and opens a PR. - -**Prerequisites:** -- [GitHub CLI](https://cli.github.com/) installed and authenticated -- [jq](https://jqlang.github.io/jq/) installed -- Clean working tree - -**Usage:** -```bash -./utils/backport-pr.sh [--dry-run] [] -``` - -**Arguments:** -- ``: Target release branch suffix, e.g. `1.9._` (maps to `release/1.9._`). If omitted, an interactive picker is shown. -- ``: PR number (`420`) or full GitHub URL. -- `--dry-run`: Preview without making changes. - -**Examples:** -```bash -./utils/backport-pr.sh 1.9._ 420 -./utils/backport-pr.sh 420 # interactive branch selection -./utils/backport-pr.sh --dry-run 1.9._ 420 -``` - ---- - -## Testing - -### `run-containers-tests.sh` - -Runs tests in containers across various OS/libc/JDK combinations, mirroring the CI matrix locally. Defaults to Podman; use `--container=docker` to use Docker. - -**Usage:** -```bash -./utils/run-containers-tests.sh [options] - --libc=glibc|musl (default: glibc) - --jdk=8|11|17|21|25|8-j9|... (default: 21) - --arch=x64|aarch64 (default: auto-detect) - --config=debug|release|asan|tsan (default: debug) - --container=podman|docker (default: podman) - --tests="TestPattern" (optional) - --gtest (enable C++ gtests) - --gtest-task=Task (run one C++ gtest task) - --shell (drop to shell instead of running tests) - --mount (mount local repo instead of cloning) - --rebuild (force rebuild of container images) -``` - -Examples: -```bash -# Run a single C++ gtest binary in ASan mode -./utils/run-containers-tests.sh --config=asan --gtest-task=elfparser_ut - -# Use Docker instead of the default Podman runtime -./utils/run-containers-tests.sh --container=docker --libc=glibc --jdk=21 -``` - -### `patch-dd-java-agent.sh` - -Patches a `dd-java-agent.jar` with a locally-built ddprof library for quick local testing without a full dd-trace-java rebuild. - -**Usage:** -```bash -DD_AGENT_JAR=path/to/dd-java-agent.jar DDPROF_JAR=path/to/ddprof.jar \ - ./utils/patch-dd-java-agent.sh -``` - ---- - -## Upstream Tracking - -See [README_UPSTREAM_TRACKER.md](README_UPSTREAM_TRACKER.md) for full documentation. - -### `check_upstream_changes.sh` - -Wrapper to compare local files against a given upstream async-profiler commit and produce a change report. - -### `track_upstream_changes.sh` - -Core change detection and report generation logic. - -### `generate_tracked_files.sh` - -Identifies which local files should be tracked against upstream (based on async-profiler copyright headers). - -### `check_contribution_candidates.sh` - -Identifies divergences from upstream async-profiler that could be contributed back. - -### `find_contribution_candidates.sh` - -Core diff analysis and report generation for contribution candidate detection. - ---- - -## CI / Ops - -### `update-sonatype-credentials.sh` - -Updates the Sonatype (Maven Central) OSSRH credentials stored in AWS SSM, used by the CI publish pipeline. - -**Prerequisites:** -- AWS CLI authenticated with `ssm:PutParameter` permission - -**Usage:** -```bash -./utils/update-sonatype-credentials.sh -``` diff --git a/utils/README_UPSTREAM_TRACKER.md b/utils/README_UPSTREAM_TRACKER.md deleted file mode 100644 index 88eeca531..000000000 --- a/utils/README_UPSTREAM_TRACKER.md +++ /dev/null @@ -1,112 +0,0 @@ -# Upstream Async-Profiler Change Tracker - -This directory contains scripts to track changes in the upstream async-profiler repository. - -## Scripts - -All scripts support `-h` or `--help` for detailed usage information. - -### check_upstream_changes.sh -Convenient wrapper to check upstream changes locally. - -**Usage:** -```bash -# Show help -./utils/check_upstream_changes.sh --help - -# Compare against v4.2.1 (current baseline) -./utils/check_upstream_changes.sh - -# Compare against specific commit -./utils/check_upstream_changes.sh abc1234 - -# Compare against 10 commits ago -./utils/check_upstream_changes.sh HEAD~10 -``` - -**Output:** -- Markdown report: `build/upstream-reports/upstream_changes_YYYYMMDD_HHMMSS.md` -- JSON report: `build/upstream-reports/upstream_changes_YYYYMMDD_HHMMSS.json` -- Tracked files list: `build/upstream-reports/tracked_files.txt` - -### generate_tracked_files.sh -Identifies which local files should be tracked against upstream. - -**Usage:** -```bash -# Show help -./utils/generate_tracked_files.sh --help - -# Standard usage -./utils/generate_tracked_files.sh -``` - -**Tracking criteria:** -- File contains async-profiler copyright header -- File exists in upstream repository - -### track_upstream_changes.sh -Core logic for change detection and report generation. - -**Usage:** -```bash -# Show help -./utils/track_upstream_changes.sh --help - -# Standard usage -./utils/track_upstream_changes.sh \ - \ - \ - \ - \ - \ - -``` - -## Automated CI Workflow - -A GitHub Actions workflow runs daily at 3 AM UTC (see `.github/workflows/upstream-tracker.yml`): - -- Compares against async-profiler master branch HEAD -- Creates GitHub issues with labels: `upstream-tracking`, `needs-review` -- Tracks state using GitHub repository variable: `UPSTREAM_LAST_COMMIT` - -No additional configuration required - the workflow works out of the box. - -**Manual trigger:** -```bash -# Normal run (no report if no changes) -gh workflow run upstream-tracker.yml - -# Force report generation (for testing) -gh workflow run upstream-tracker.yml -f force_report=true -``` - -## Report Format - -### Markdown Report -- Summary of changes -- List of changed files sorted by commit count -- Recent commits (up to 20) -- Referenced pull requests -- Direct links to GitHub compare views - -### JSON Report -- Machine-readable format for automation -- Contains file list, commit counts, date ranges - -## Example Output - -Based on the latest run (v4.2.1 to a071e8a): -- **27 files changed** -- **71 commits** in range -- Most impacted: profiler.cpp (14 commits), profiler.h (4 commits), stackWalker.cpp (3 commits) - -## Compatibility - -All scripts are POSIX-compliant and work on: -- macOS (bash 3.2+) -- Linux (bash 3.2+) -- GitHub Actions (Ubuntu) - -No bash 4+ features required (no associative arrays or mapfile). diff --git a/utils/backport-pr.sh b/utils/backport-pr.sh deleted file mode 100755 index 80e0419b7..000000000 --- a/utils/backport-pr.sh +++ /dev/null @@ -1,352 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# --- Colors & helpers -------------------------------------------------------- -if [ -t 1 ]; then - RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[0;33m' - CYAN='\033[0;36m'; BOLD='\033[1m'; RESET='\033[0m' -else - RED=''; GREEN=''; YELLOW=''; CYAN=''; BOLD=''; RESET='' -fi -info() { echo -e "${GREEN}✓${RESET} $*"; } -warn() { echo -e "${YELLOW}⚠${RESET} $*"; } -error() { echo -e "${RED}✗${RESET} $*" >&2; } -step() { echo -e "${CYAN}→${RESET} ${BOLD}$*${RESET}"; } - -# --- Cleanup trap ------------------------------------------------------------ -CURRENT_BRANCH="" -BACKPORT_BRANCH="" -CHERRY_PICK_IN_PROGRESS=0 -DRY_RUN=0 - -cleanup() { - local exit_code=$? - if [ $exit_code -ne 0 ]; then - if [ $CHERRY_PICK_IN_PROGRESS -eq 1 ]; then - echo "" - error "Cherry-pick failed — likely a conflict." - echo "" - echo -e " You have two options:" - echo "" - echo -e " ${BOLD}Option 1: Resolve manually${RESET}" - echo -e " 1. Fix the conflicts in the listed files" - echo -e " 2. ${CYAN}git add ${RESET}" - echo -e " 3. ${CYAN}git cherry-pick --continue${RESET}" - echo -e " 4. ${CYAN}git push -u origin $BACKPORT_BRANCH${RESET}" - echo -e " 5. Create the PR manually or re-run this script" - echo "" - echo -e " ${BOLD}Option 2: Abort and go back${RESET}" - echo -e " 1. ${CYAN}git cherry-pick --abort${RESET}" - echo -e " 2. ${CYAN}git checkout $CURRENT_BRANCH${RESET}" - echo -e " 3. ${CYAN}git branch -D $BACKPORT_BRANCH${RESET}" - echo "" - return - fi - # Generic failure — try to restore original branch - if [ -n "$CURRENT_BRANCH" ]; then - warn "Restoring original branch ($CURRENT_BRANCH)" - git checkout "$CURRENT_BRANCH" 2>/dev/null || true - fi - fi -} -trap cleanup EXIT - -# --- Argument parsing -------------------------------------------------------- -usage() { - echo -e "${BOLD}Usage:${RESET} $0 [--dry-run] [] " - echo "" - echo " e.g. 1.9._ (if omitted, you'll pick from a list)" - echo " PR number (420) or full URL (https://github.com/.../pull/420)" - echo " --dry-run Show what would happen without making changes" - exit 1 -} - -RELEASE_NAME="" -PR_INPUT="" - -for arg in "$@"; do - case "$arg" in - --dry-run) DRY_RUN=1 ;; - --help|-h) usage ;; - *) - if [ -z "$PR_INPUT" ] && [[ "$arg" =~ (^[0-9]+$|/pull/[0-9]+) ]]; then - PR_INPUT="$arg" - elif [ -z "$RELEASE_NAME" ]; then - RELEASE_NAME="$arg" - elif [ -z "$PR_INPUT" ]; then - PR_INPUT="$arg" - else - usage - fi - ;; - esac -done - -if [ -z "$PR_INPUT" ]; then - usage -fi - -# Extract PR number from URL or plain number -if [[ "$PR_INPUT" =~ /pull/([0-9]+) ]]; then - PR_NUMBER="${BASH_REMATCH[1]}" -elif [[ "$PR_INPUT" =~ ^[0-9]+$ ]]; then - PR_NUMBER="$PR_INPUT" -else - error "Cannot parse PR number from: $PR_INPUT" - exit 1 -fi - -# --- Check requirements ------------------------------------------------------ -step "Checking prerequisites" - -CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) - -for cmd in gh jq; do - $cmd --version 1>/dev/null 2>&1 || { error "$cmd is not installed"; exit 1; } -done -info "gh and jq are available" - -if [ -n "$(git status --porcelain)" ]; then - error "Working tree is not clean. Please stash or commit your changes first." - exit 1 -fi -info "Working tree is clean" - -git fetch --quiet -info "Fetched latest from origin" - -# --- Release branch selection ------------------------------------------------ -if [ -z "$RELEASE_NAME" ]; then - step "Select a release branch" - BRANCHES=() - while IFS= read -r ref; do - branch="${ref#refs/remotes/origin/release/}" - BRANCHES+=("$branch") - done < <(git for-each-ref --sort=-version:refname --format='%(refname)' 'refs/remotes/origin/release/*') - - if [ ${#BRANCHES[@]} -eq 0 ]; then - error "No release branches found" - exit 1 - fi - - # Show the 10 most recent - SHOW_COUNT=10 - if [ ${#BRANCHES[@]} -lt $SHOW_COUNT ]; then - SHOW_COUNT=${#BRANCHES[@]} - fi - echo "" - for i in $(seq 1 "$SHOW_COUNT"); do - echo -e " ${BOLD}$i)${RESET} release/${BRANCHES[$((i-1))]}" - done - echo "" - echo -n "Pick a branch [1-$SHOW_COUNT]: " - read -r PICK - if ! [[ "$PICK" =~ ^[0-9]+$ ]] || [ "$PICK" -lt 1 ] || [ "$PICK" -gt "$SHOW_COUNT" ]; then - error "Invalid selection" - exit 1 - fi - RELEASE_NAME="${BRANCHES[$((PICK-1))]}" -fi - -if [[ ! "$RELEASE_NAME" =~ ^[0-9]+\.[0-9]+\._ ]]; then - error "Release name should be in the format X.Y._ (e.g. 1.9._)" - exit 1 -fi - -RELEASE_BRANCH="release/$RELEASE_NAME" -git show-ref --verify --quiet "refs/remotes/origin/$RELEASE_BRANCH" 2>/dev/null || { - error "Branch $RELEASE_BRANCH does not exist on origin" - exit 1 -} -info "Target branch: $RELEASE_BRANCH" - -# --- Fetch PR details (single API call) -------------------------------------- -step "Fetching PR #$PR_NUMBER details" - -PR_DATA=$(gh pr view "$PR_NUMBER" --json commits,mergeCommit,title,labels,state) -PR_STATE=$(echo "$PR_DATA" | jq -r '.state') -if [ "$PR_STATE" == "null" ] || [ -z "$PR_STATE" ]; then - error "PR #$PR_NUMBER does not exist" - exit 1 -fi -if [ "$PR_STATE" != "MERGED" ]; then - warn "PR #$PR_NUMBER is $PR_STATE (not merged). Proceed anyway? (y/n)" - read -r ANSWER - [ "$ANSWER" == "y" ] || { echo "Aborting."; exit 1; } -fi - -PR_TITLE=$(echo "$PR_DATA" | jq -r '.title') -PR_LABELS=$(echo "$PR_DATA" | jq -r '[.labels[].name] | join(",")') -PR_COMMITS=$(echo "$PR_DATA" | jq -r '.commits[].oid') -PR_MERGE_COMMIT=$(echo "$PR_DATA" | jq -r '.mergeCommit.oid // empty') - -info "PR: $PR_TITLE" - -# --- Determine commits to cherry-pick ---------------------------------------- -USE_MERGE_COMMIT=0 - -for PR_COMMIT in $PR_COMMITS; do - if ! git cat-file -e "$PR_COMMIT" 2>/dev/null; then - warn "Commit $PR_COMMIT is no longer present (garbage collected after squash)." - USE_MERGE_COMMIT=1 - break - fi -done - -if [ $USE_MERGE_COMMIT -eq 0 ]; then - for PR_COMMIT in $PR_COMMITS; do - PARENT_COUNT=$(git rev-list --parents -n 1 "$PR_COMMIT" 2>/dev/null | wc -w) - if [ "$PARENT_COUNT" -gt 2 ]; then - warn "PR contains a merge commit ($PR_COMMIT)." - USE_MERGE_COMMIT=1 - break - fi - done -fi - -if [ $USE_MERGE_COMMIT -eq 1 ]; then - if [ -z "$PR_MERGE_COMMIT" ]; then - error "Need merge commit but PR has not been merged yet." - exit 1 - fi - echo -n "Cherry-pick the merge commit instead of individual commits? (y/n) " - read -r ANSWER - if [ "$ANSWER" == "y" ]; then - PR_COMMITS="$PR_MERGE_COMMIT" - else - echo "Aborting. Please backport manually." - exit 1 - fi -fi - -COMMIT_COUNT=$(echo "$PR_COMMITS" | wc -w | tr -d ' ') -info "Will cherry-pick $COMMIT_COUNT commit(s)" - -# --- Handle existing backport branch ----------------------------------------- -BACKPORT_BRANCH="$USER/backport-pr-$PR_NUMBER" -SKIP_CHERRY_PICK=0 - -EXISTING_REMOTE=0 -EXISTING_LOCAL=0 -EXISTING_PR=0 -git show-ref --verify --quiet "refs/remotes/origin/$BACKPORT_BRANCH" 2>/dev/null && EXISTING_REMOTE=1 -git show-ref --verify --quiet "refs/heads/$BACKPORT_BRANCH" 2>/dev/null && EXISTING_LOCAL=1 -if [ $EXISTING_REMOTE -eq 1 ]; then - gh pr view "$BACKPORT_BRANCH" --json url 1>/dev/null 2>&1 && EXISTING_PR=1 -fi - -if [ $EXISTING_REMOTE -eq 1 ] && [ $EXISTING_PR -eq 1 ]; then - EXISTING_PR_URL=$(gh pr view "$BACKPORT_BRANCH" --json url --jq '.url') - error "A backport PR already exists: $EXISTING_PR_URL" - exit 1 -fi - -if [ $EXISTING_REMOTE -eq 1 ] && [ $EXISTING_PR -eq 0 ]; then - warn "Remote branch $BACKPORT_BRANCH exists but has no open PR." - echo -n " Create the PR from the existing branch? (y/n) " - read -r ANSWER - if [ "$ANSWER" == "y" ]; then - SKIP_CHERRY_PICK=1 - info "Will reuse existing branch" - else - echo -n " Delete it and start fresh instead? (y/n) " - read -r ANSWER - if [ "$ANSWER" == "y" ]; then - if [ $DRY_RUN -eq 0 ]; then - git push origin --delete "$BACKPORT_BRANCH" 2>/dev/null || true - fi - info "Deleted remote branch $BACKPORT_BRANCH" - else - echo "Aborting." - exit 1 - fi - fi -fi - -if [ $SKIP_CHERRY_PICK -eq 0 ] && [ $EXISTING_LOCAL -eq 1 ]; then - warn "Local branch $BACKPORT_BRANCH already exists." - echo -n "Delete it and start fresh? (y/n) " - read -r ANSWER - if [ "$ANSWER" == "y" ]; then - if [ $DRY_RUN -eq 0 ]; then - git branch -D "$BACKPORT_BRANCH" - fi - info "Deleted local branch $BACKPORT_BRANCH" - else - echo "Aborting." - exit 1 - fi -fi - -# --- Dry-run summary --------------------------------------------------------- -if [ $DRY_RUN -eq 1 ]; then - echo "" - step "Dry-run summary (no changes will be made)" - echo "" - echo -e " PR: ${BOLD}#$PR_NUMBER${RESET} — $PR_TITLE" - echo -e " Target: ${BOLD}$RELEASE_BRANCH${RESET}" - echo -e " Branch: ${BOLD}$BACKPORT_BRANCH${RESET}" - echo -e " Commits: $COMMIT_COUNT" - for c in $PR_COMMITS; do - echo -e " $c" - done - echo -e " Labels: ${PR_LABELS:-}" - echo -e " PR title: 🍒 $PR_NUMBER - $PR_TITLE" - if [ $SKIP_CHERRY_PICK -eq 1 ]; then - echo -e " Mode: ${YELLOW}Resume${RESET} (reuse existing remote branch)" - fi - echo "" - info "Dry run complete. Re-run without --dry-run to execute." - exit 0 -fi - -# --- Backport ---------------------------------------------------------------- -if [ $SKIP_CHERRY_PICK -eq 1 ]; then - step "Skipping cherry-pick (reusing existing branch)" -else - step "Creating backport" - - git checkout "$RELEASE_BRANCH" - git pull --quiet - git checkout -b "$BACKPORT_BRANCH" - - CHERRY_PICK_IN_PROGRESS=1 - for PR_COMMIT in $PR_COMMITS; do - git cherry-pick -x "$PR_COMMIT" - done - CHERRY_PICK_IN_PROGRESS=0 - - git push -u origin "$BACKPORT_BRANCH" - info "Pushed $BACKPORT_BRANCH" -fi - -# --- Create PR --------------------------------------------------------------- -step "Creating pull request" - -LABEL_ARGS=() -if [ -n "$PR_LABELS" ]; then - LABEL_ARGS=(--label "$PR_LABELS") -fi - -BACKPORT_PR_URL=$(gh pr create --base "$RELEASE_BRANCH" \ - --head "$BACKPORT_BRANCH" \ - --title "🍒 $PR_NUMBER - $PR_TITLE" \ - --body "Backport of #$PR_NUMBER to \`$RELEASE_BRANCH\`" \ - "${LABEL_ARGS[@]+"${LABEL_ARGS[@]}"}") -if [ -z "$BACKPORT_PR_URL" ]; then - error "gh pr create did not return a URL" - exit 1 -fi -info "Created: $BACKPORT_PR_URL" - -# Comment on the original PR for traceability -gh pr comment "$PR_NUMBER" --body "Backported to \`$RELEASE_BRANCH\` via $BACKPORT_PR_URL" 2>/dev/null || true - -# --- Restore ------------------------------------------------------------------ -step "Restoring original branch" -git checkout "$CURRENT_BRANCH" -info "Back on $CURRENT_BRANCH" - -echo "" -echo -e "${GREEN}${BOLD}Done!${RESET}" -echo -e " ${BOLD}Backport PR:${RESET} $BACKPORT_PR_URL" diff --git a/utils/check_contribution_candidates.sh b/utils/check_contribution_candidates.sh deleted file mode 100755 index 06b09e8c2..000000000 --- a/utils/check_contribution_candidates.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash -# Check divergences from upstream async-profiler for potential contributions - -set -e - -show_help() { - cat < "$TRACKED_FILES" - -FILE_COUNT=$(wc -l < "$TRACKED_FILES" | tr -d ' ') -echo " Tracking $FILE_COUNT files" -echo "" - -# Analyze contribution candidates -echo "[3/4] Analyzing divergences from upstream..." -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -MARKDOWN_REPORT="$REPORT_DIR/contribution_candidates_${TIMESTAMP}.md" -JSON_REPORT="$REPORT_DIR/contribution_candidates_${TIMESTAMP}.json" - -"$SCRIPT_DIR/find_contribution_candidates.sh" \ - "$UPSTREAM_CLONE_DIR" \ - "$BASELINE" \ - "$LOCAL_CPP_DIR" \ - "$TRACKED_FILES" \ - "$MARKDOWN_REPORT" \ - "$JSON_REPORT" - -# Check results -if [ ! -f "$JSON_REPORT" ]; then - echo " No contribution candidates detected." - echo "" - echo "All divergences are Datadog-specific." -else - CANDIDATES=$(jq -r '.files_with_candidates' "$JSON_REPORT") - TOTAL_CHANGED=$(jq -r '.files_with_changes' "$JSON_REPORT") - - echo "" - echo " Candidates found!" - echo " Files with divergences: $TOTAL_CHANGED" - echo " Files with contributable hunks: $CANDIDATES" - echo "" - echo "Reports generated:" - echo " Markdown: $MARKDOWN_REPORT" - echo " JSON: $JSON_REPORT" - echo "" - echo "View the markdown report:" - echo " cat $MARKDOWN_REPORT" -fi - -# Cleanup -echo "" -echo "[4/4] Cleaning up..." -rm -rf "$UPSTREAM_CLONE_DIR" -echo " Removed temporary clone" - -echo "" -echo "Done!" -echo "" -echo "Tracked files list saved to:" -echo " $TRACKED_FILES" diff --git a/utils/check_upstream_changes.sh b/utils/check_upstream_changes.sh deleted file mode 100755 index aa3221310..000000000 --- a/utils/check_upstream_changes.sh +++ /dev/null @@ -1,154 +0,0 @@ -#!/bin/bash -# Convenient wrapper to check upstream async-profiler changes locally - -set -e - -show_help() { - cat < "$TRACKED_FILES" - -FILE_COUNT=$(wc -l < "$TRACKED_FILES" | tr -d ' ') -echo " Tracking $FILE_COUNT files" -echo "" - -# Check for changes -echo "[3/5] Analyzing changes..." -MARKDOWN_REPORT="$REPORT_DIR/upstream_changes_$(date +%Y%m%d_%H%M%S).md" -JSON_REPORT="$REPORT_DIR/upstream_changes_$(date +%Y%m%d_%H%M%S).json" - -"$SCRIPT_DIR/track_upstream_changes.sh" \ - "$UPSTREAM_CLONE_DIR" \ - "$LAST_COMMIT" \ - "$CURRENT_HEAD" \ - "$TRACKED_FILES" \ - "$MARKDOWN_REPORT" \ - "$JSON_REPORT" - -# Check if reports were generated -if [ ! -f "$JSON_REPORT" ]; then - echo " No changes detected!" - echo "" - echo "✓ All tracked files are up to date with upstream." -else - FILES_CHANGED=$(jq -r '.files_changed' "$JSON_REPORT") - COMMIT_COUNT=$(jq -r '.commit_count' "$JSON_REPORT") - - echo " Changes detected!" - echo " Files changed: $FILES_CHANGED" - echo " Commits: $COMMIT_COUNT" - echo "" - echo "✓ Reports generated:" - echo " Markdown: $MARKDOWN_REPORT" - echo " JSON: $JSON_REPORT" - echo "" - echo "View the markdown report:" - echo " cat $MARKDOWN_REPORT" -fi - -# Cleanup -echo "" -echo "[4/5] Cleaning up..." -rm -rf "$UPSTREAM_CLONE_DIR" -echo " Removed temporary clone" - -echo "" -echo "[5/5] Done!" -echo "" -echo "Tracked files list saved to:" -echo " $TRACKED_FILES" diff --git a/utils/find_contribution_candidates.sh b/utils/find_contribution_candidates.sh deleted file mode 100755 index 662346641..000000000 --- a/utils/find_contribution_candidates.sh +++ /dev/null @@ -1,271 +0,0 @@ -#!/bin/bash -# Analyze divergences from upstream async-profiler and identify contribution candidates - -set -e - -show_help() { - cat <&2 - show_help >&2 - exit 1 -fi - -# DD-specific markers used to classify hunks -DD_MARKERS='DD_\|ddprof\|Datadog\|datadog\|DDPROF\|context\.h\|counters\.h\|tagger\|QueueItem' - -TEMP_DATA_DIR="$(mktemp -d)" -trap "rm -rf $TEMP_DATA_DIR" EXIT - -candidate_count=0 -total_with_changes=0 - -while IFS= read -r file; do - if [ -z "$file" ]; then - continue - fi - - basename_file=$(basename "$file") - - # Extract upstream version at baseline - upstream_content=$(cd "$UPSTREAM_REPO" && git show "${BASELINE_COMMIT}:${file}" 2>/dev/null) || continue - local_file="$LOCAL_CPP_DIR/$basename_file" - - if [ ! -f "$local_file" ]; then - continue - fi - - # Write upstream content to temp file for diffing - echo "$upstream_content" > "$TEMP_DATA_DIR/upstream_${basename_file}" - - # Diff upstream (baseline) vs local; non-zero exit means differences exist - diff_output=$(diff -u "$TEMP_DATA_DIR/upstream_${basename_file}" "$local_file" 2>/dev/null) || true - - if [ -z "$diff_output" ]; then - continue - fi - - total_with_changes=$((total_with_changes + 1)) - - # Split diff into hunks and classify each using awk (fast single-pass) - echo "$diff_output" > "$TEMP_DATA_DIR/full_diff_${basename_file}" - - # awk splits on @@ lines, classifies each hunk by DD markers, writes - # contrib hunks to contrib_file, and prints "contrib_count dd_count" at end - read -r contrib_hunks dd_hunks <<< "$(awk -v contrib_file="$TEMP_DATA_DIR/contrib_hunks_${basename_file}" \ - -v dd_pat='DD_|ddprof|Datadog|datadog|DDPROF|context\\.h|counters\\.h|tagger|QueueItem' \ - ' - BEGIN { in_hunk = 0; contrib = 0; dd = 0; hunk = "" ; is_dd = 0 } - /^@@/ { - if (in_hunk) { - if (is_dd) { dd++ } - else { contrib++; printf "%s\n---HUNK_BOUNDARY---\n", hunk >> contrib_file } - } - hunk = $0; is_dd = 0; in_hunk = 1; next - } - /^---( |$)/ && !in_hunk { next } - /^\+\+\+( |$)/ && !in_hunk { next } - in_hunk { - hunk = hunk "\n" $0 - if ($0 ~ dd_pat) is_dd = 1 - } - END { - if (in_hunk) { - if (is_dd) { dd++ } - else { contrib++; printf "%s\n---HUNK_BOUNDARY---\n", hunk >> contrib_file } - } - print contrib, dd - } - ' "$TEMP_DATA_DIR/full_diff_${basename_file}")" - - total_hunks=$((contrib_hunks + dd_hunks)) - - if [ $contrib_hunks -gt 0 ]; then - file_id=$(printf "%03d" $candidate_count) - echo "$basename_file" > "$TEMP_DATA_DIR/${file_id}.path" - echo "$contrib_hunks" > "$TEMP_DATA_DIR/${file_id}.contrib" - echo "$dd_hunks" > "$TEMP_DATA_DIR/${file_id}.dd" - echo "$total_hunks" > "$TEMP_DATA_DIR/${file_id}.total" - cp "$TEMP_DATA_DIR/contrib_hunks_${basename_file}" "$TEMP_DATA_DIR/${file_id}.hunks" 2>/dev/null || true - - candidate_count=$((candidate_count + 1)) - fi -done < "$TRACKED_FILES" - -if [ $candidate_count -eq 0 ]; then - echo "No contribution candidates detected" - exit 0 -fi - -# Build sorted index (by contrib hunk count, descending) -sorted_ids="" -for i in $(seq 0 $((candidate_count - 1))); do - file_id=$(printf "%03d" $i) - contrib=$(cat "$TEMP_DATA_DIR/${file_id}.contrib") - sorted_ids="${sorted_ids}${file_id}:${contrib} -" -done -SORTED_FILE_IDS=$(echo "$sorted_ids" | grep -v '^$' | sort -t':' -k2 -rn | cut -d':' -f1) - -# --- Markdown Report --- -cat > "$OUTPUT_MD" <> "$OUTPUT_MD" <> "$OUTPUT_MD" - hunk_shown=0 - while IFS= read -r line; do - if [ "$line" = "---HUNK_BOUNDARY---" ]; then - hunk_shown=$((hunk_shown + 1)) - if [ $hunk_shown -ge 3 ]; then - break - fi - echo "" >> "$OUTPUT_MD" - continue - fi - echo "$line" >> "$OUTPUT_MD" - done < "$TEMP_DATA_DIR/${file_id}.hunks" - echo '```' >> "$OUTPUT_MD" - if [ $contrib -gt 3 ]; then - echo "_... and $((contrib - 3)) more contributable hunks_" >> "$OUTPUT_MD" - fi - echo "" >> "$OUTPUT_MD" - fi -done - -cat >> "$OUTPUT_MD" < "$OUTPUT_JSON" <> "$OUTPUT_JSON" - fi - - cat >> "$OUTPUT_JSON" <> "$OUTPUT_JSON" < tracked.txt - - # Count tracked files - $(basename "$0") ddprof-lib/src/main/cpp /tmp/async-profiler/src | wc -l - -EXIT STATUS: - 0 Success (tracked files listed) - 1 Error (missing directories, etc.) - -EOF -} - -# Parse arguments -if [ "$1" = "-h" ] || [ "$1" = "--help" ]; then - show_help - exit 0 -fi - -LOCAL_DIR="$1" -UPSTREAM_DIR="$2" - -if [ ! -d "$LOCAL_DIR" ]; then - echo "Error: Local directory not found: $LOCAL_DIR" >&2 - exit 1 -fi - -if [ ! -d "$UPSTREAM_DIR" ]; then - echo "Error: Upstream directory not found: $UPSTREAM_DIR" >&2 - exit 1 -fi - -# Find files with async-profiler copyright using grep -grep -rl "Copyright.*async-profiler authors" "$LOCAL_DIR" --include="*.cpp" --include="*.h" 2>/dev/null | while read -r local_file; do - basename_file=$(basename "$local_file") - - # Check if corresponding file exists in upstream - upstream_file="$UPSTREAM_DIR/$basename_file" - if [ ! -f "$upstream_file" ]; then - continue - fi - - # Output relative path from upstream repo root - echo "src/$basename_file" -done | sort -u diff --git a/utils/patch-dd-java-agent.sh b/utils/patch-dd-java-agent.sh deleted file mode 100755 index f8244440e..000000000 --- a/utils/patch-dd-java-agent.sh +++ /dev/null @@ -1,385 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -# Patch dd-java-agent.jar with ddprof contents - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" - -# Input JARs -DD_AGENT_JAR="${DD_AGENT_JAR:-${PROJECT_ROOT}/dd-java-agent-original.jar}" -DDPROF_JAR="${DDPROF_JAR:-${PROJECT_ROOT}/ddprof.jar}" - -# Output JAR -OUTPUT_JAR="${OUTPUT_JAR:-${PROJECT_ROOT}/dd-java-agent-patched.jar}" - -# Working directory for extraction -# Use mktemp for guaranteed unique directory, fall back to PID-based if mktemp unavailable -WORK_DIR="${WORK_DIR:-$(mktemp -d 2>/dev/null || echo "/tmp/jar-patch-$$")}" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -function log_info() { - echo -e "${GREEN}[INFO]${NC} $*" -} - -function log_warn() { - echo -e "${YELLOW}[WARN]${NC} $*" -} - -function log_error() { - echo -e "${RED}[ERROR]${NC} $*" -} - -function log_debug() { - if [ "${DEBUG:-false}" = "true" ]; then - echo -e "${BLUE}[DEBUG]${NC} $*" - fi -} - -function cleanup() { - if [ -d "${WORK_DIR}" ]; then - log_debug "Cleaning up work directory: ${WORK_DIR}" - rm -rf "${WORK_DIR}" - fi -} - -trap cleanup EXIT - -# Validate required tools are available -for tool in unzip zip dirname basename; do - if ! command -v "$tool" &> /dev/null; then - log_error "Required tool not found: $tool" - log_error "Please install $tool to continue" - exit 1 - fi -done - -function usage() { - cat << EOF -Usage: $0 [OPTIONS] - -Patch dd-java-agent.jar with ddprof contents. - -Mapping rules: - - Native libraries: ddprof.jar:META-INF/native-libs/** → dd-java-agent.jar:shared/META-INF/native-libs/** - - Class files: ddprof.jar:**/*.class → dd-java-agent.jar:shared/**/*.classdata (same package structure) - -OPTIONS: - --dd-agent-jar Path to dd-java-agent-original.jar (default: ${DD_AGENT_JAR}) - --ddprof-jar Path to ddprof.jar (default: ${DDPROF_JAR}) - --output-jar Path to output patched jar (default: ${OUTPUT_JAR}) - --work-dir Working directory for extraction (default: /tmp/jar-patch-\$\$) - --debug Enable debug output - --help Show this help message - -ENVIRONMENT VARIABLES: - DD_AGENT_JAR Path to dd-java-agent-original.jar - DDPROF_JAR Path to ddprof.jar - OUTPUT_JAR Path to output patched jar - WORK_DIR Working directory for extraction - DEBUG Enable debug output (true/false) - -EXAMPLES: - # Patch with default paths - $0 - - # Patch with custom paths - $0 --dd-agent-jar /path/to/agent.jar --ddprof-jar /path/to/ddprof.jar - - # Enable debug output - DEBUG=true $0 -EOF -} - -# Parse command line arguments -while [ $# -gt 0 ]; do - case "$1" in - --dd-agent-jar) - DD_AGENT_JAR="$2" - shift 2 - ;; - --ddprof-jar) - DDPROF_JAR="$2" - shift 2 - ;; - --output-jar) - OUTPUT_JAR="$2" - shift 2 - ;; - --work-dir) - WORK_DIR="$2" - shift 2 - ;; - --debug) - DEBUG=true - shift - ;; - --help) - usage - exit 0 - ;; - *) - log_error "Unknown option: $1" - usage - exit 1 - ;; - esac -done - -# Validate input JARs exist -if [ ! -f "${DD_AGENT_JAR}" ]; then - log_error "dd-java-agent JAR not found: ${DD_AGENT_JAR}" - exit 1 -fi - -if [ ! -f "${DDPROF_JAR}" ]; then - log_error "ddprof JAR not found: ${DDPROF_JAR}" - exit 1 -fi - -log_info "Starting JAR patching process" -log_info " dd-java-agent: ${DD_AGENT_JAR}" -log_info " ddprof: ${DDPROF_JAR}" -log_info " output: ${OUTPUT_JAR}" -log_info " work dir: ${WORK_DIR}" - -# Create working directories -if ! mkdir -p "${WORK_DIR}/agent" "${WORK_DIR}/ddprof"; then - log_error "Failed to create working directories in: ${WORK_DIR}" - log_error "Check disk space and permissions" - exit 1 -fi - -# Extract dd-java-agent -log_info "Extracting dd-java-agent..." -if ! unzip -q "${DD_AGENT_JAR}" -d "${WORK_DIR}/agent/"; then - log_error "Failed to extract dd-java-agent: ${DD_AGENT_JAR}" - log_error "Check if file is corrupted or disk is full" - exit 1 -fi -log_info "✓ Extracted dd-java-agent" - -# Extract ddprof -log_info "Extracting ddprof..." -if ! unzip -q "${DDPROF_JAR}" -d "${WORK_DIR}/ddprof/"; then - log_error "Failed to extract ddprof: ${DDPROF_JAR}" - log_error "Check if file is corrupted or disk is full" - exit 1 -fi -log_info "✓ Extracted ddprof" - -# Create shared directory structure in agent -mkdir -p "${WORK_DIR}/agent/shared/META-INF" - -# Copy native libraries -log_info "Copying native libraries..." -if [ -d "${WORK_DIR}/ddprof/META-INF/native-libs" ]; then - cp -r "${WORK_DIR}/ddprof/META-INF/native-libs" "${WORK_DIR}/agent/shared/META-INF/" - - # Count native libraries - NATIVE_COUNT=$(find "${WORK_DIR}/agent/shared/META-INF/native-libs" -type f -name "*.so" | wc -l | tr -d ' ') - log_info "✓ Copied ${NATIVE_COUNT} native libraries" - - # List platforms - if [ "${DEBUG:-false}" = "true" ]; then - log_debug "Native library platforms:" - find "${WORK_DIR}/agent/shared/META-INF/native-libs" -type d -mindepth 1 -maxdepth 1 -exec basename {} \; | while read -r platform; do - log_debug " - ${platform}" - done - fi -else - log_warn "No native libraries found in ddprof (META-INF/native-libs missing)" -fi - -# Copy and rename class files -log_info "Copying and renaming class files..." -CLASS_COUNT=0 -SKIPPED_COUNT=0 - -# Count total class files first for progress tracking -TOTAL_CLASSES=$(find "${WORK_DIR}/ddprof" -name "*.class" -type f | wc -l | tr -d ' ') - -# Validate TOTAL_CLASSES is a number -if ! [[ "${TOTAL_CLASSES}" =~ ^[0-9]+$ ]]; then - log_error "Failed to count class files, got: '${TOTAL_CLASSES}'" - exit 1 -fi - -log_info "Found ${TOTAL_CLASSES} class files to process" - -# Verify ddprof extraction directory exists and has content -if [ ! -d "${WORK_DIR}/ddprof" ]; then - log_error "ddprof extraction directory not found: ${WORK_DIR}/ddprof" - exit 1 -fi - -# Exit early if no files to process -if [ "${TOTAL_CLASSES}" -eq 0 ]; then - log_warn "No class files found to process" - # Skip loop but don't fail - might be expected for some builds -fi - -log_info "Starting class file copy loop..." - -# Use find to locate all .class files, excluding META-INF -while IFS= read -r -d '' classfile; do - log_debug "Processing: ${classfile}" - - # Get relative path from ddprof root - relpath="${classfile#"${WORK_DIR}"/ddprof/}" - - # Skip META-INF directory - if [[ "$relpath" == META-INF/* ]]; then - log_debug "Skipping META-INF class: ${relpath}" - SKIPPED_COUNT=$((SKIPPED_COUNT + 1)) - continue - fi - - # Convert path: foo/bar/Baz.class → shared/foo/bar/Baz.classdata - targetpath="${WORK_DIR}/agent/shared/${relpath%.class}.classdata" - targetdir=$(dirname "$targetpath") - - # Create target directory if needed - if ! mkdir -p "$targetdir"; then - log_error "Failed to create directory: $targetdir" - log_error "Check disk space and permissions" - df -h "${WORK_DIR}" >&2 - exit 1 - fi - - # Copy file - if ! cp "$classfile" "$targetpath"; then - log_error "Failed to copy class file: $classfile" - log_error "Target: $targetpath" - log_error "Check disk space and permissions" - df -h "${WORK_DIR}" >&2 - exit 1 - fi - CLASS_COUNT=$((CLASS_COUNT + 1)) - - # Show progress every 100 files - if [ $((CLASS_COUNT % 100)) -eq 0 ]; then - log_debug "Progress: ${CLASS_COUNT}/${TOTAL_CLASSES} files copied" - fi - - log_debug "Copied: ${relpath} → shared/${relpath%.class}.classdata" -done < <(find "${WORK_DIR}/ddprof" -name "*.class" -type f -print0) - -# Verify loop completed successfully -if [ "${CLASS_COUNT}" -eq 0 ] && [ "${TOTAL_CLASSES}" -gt 0 ]; then - log_error "Class file loop failed - no files were copied despite ${TOTAL_CLASSES} files found" - exit 1 -fi - -log_info "✓ Copied and renamed ${CLASS_COUNT} class files (${SKIPPED_COUNT} skipped from META-INF)" - -# List some sample class files for verification -if [ "${DEBUG:-false}" = "true" ]; then - log_debug "Sample classdata files:" - # Use process substitution to avoid SIGPIPE from head in pipeline - count=0 - while IFS= read -r f && [ "$count" -lt 5 ]; do - relpath="${f#"${WORK_DIR}"/agent/}" - log_debug " - ${relpath}" - count=$((count + 1)) - done < <(find "${WORK_DIR}/agent/shared" -name "*.classdata" -type f) -fi - -# Repackage JAR -log_info "Repackaging JAR..." - -# Save original directory and change to agent directory -ORIG_DIR=$(pwd) -if ! cd "${WORK_DIR}/agent"; then - log_error "Failed to change directory to ${WORK_DIR}/agent" - log_error "Check if directory exists and is accessible" - exit 1 -fi - -if ! zip -r -q "${OUTPUT_JAR}" .; then - log_error "Failed to repackage JAR" - log_error "Check disk space and write permissions for: ${OUTPUT_JAR}" - cd "$ORIG_DIR" || true - exit 1 -fi - -# Restore original directory -cd "$ORIG_DIR" || log_warn "Failed to return to original directory" - -log_info "✓ Created patched JAR: ${OUTPUT_JAR}" - -# Validate patched JAR -log_info "Validating patched JAR..." - -if ! unzip -t "${OUTPUT_JAR}" > /dev/null 2>&1; then - log_error "Patched JAR is corrupted" - exit 1 -fi -log_info "✓ JAR integrity check passed" - -# Verify shared directory structure -log_info "Verifying structure..." -SHARED_NATIVE_COUNT=$(unzip -l "${OUTPUT_JAR}" | grep -c "shared/META-INF/native-libs/.*\.so$" || echo "0") -TOTAL_CLASSDATA_COUNT=$(unzip -l "${OUTPUT_JAR}" | grep -c "shared/.*\.classdata$" || echo "0") - -if [ "${SHARED_NATIVE_COUNT}" -eq 0 ] && [ "${NATIVE_COUNT}" -gt 0 ]; then - log_error "Native libraries missing in patched JAR" - exit 1 -fi - -# Diagnostic: Check for unexpected classes in ddprof-owned package namespaces -# Only check com/datadoghq/profiler - this is the actual ddprof package -log_debug "Checking for unexpected classes in com/datadoghq/profiler package..." -DDPROF_CLASSES=$(unzip -l "${DDPROF_JAR}" '*.class' 2>/dev/null | \ - awk '{print $NF}' | \ - grep "^com/datadoghq/profiler/" | \ - grep "\.class$" | \ - sed 's/\.class$//' | \ - sort || true) - -# Get classes from patched JAR in the ddprof package -PATCHED_CLASSES=$(unzip -l "${OUTPUT_JAR}" 2>/dev/null | \ - grep "shared/com/datadoghq/profiler/.*\.classdata$" | \ - awk '{print $NF}' | \ - sed 's|^shared/||' | \ - sed 's/\.classdata$//' | \ - sort || true) - -# Find classes in patched JAR that weren't in original ddprof.jar -UNEXPECTED_CLASSES=$(comm -13 <(echo "${DDPROF_CLASSES}") <(echo "${PATCHED_CLASSES}") || true) - -if [ -n "${UNEXPECTED_CLASSES}" ]; then - UNEXPECTED_COUNT=$(echo "${UNEXPECTED_CLASSES}" | grep -c . || echo "0") - log_warn "Found ${UNEXPECTED_COUNT} unexpected classes in com/datadoghq/profiler:" - log_warn "These classes exist in dd-java-agent but not in ddprof.jar:" - echo "${UNEXPECTED_CLASSES}" | while IFS= read -r cls; do - log_debug " - ${cls}" - done | head -10 - if [ "${UNEXPECTED_COUNT}" -gt 10 ]; then - log_debug " ... and $((UNEXPECTED_COUNT - 10)) more" - fi - log_warn "This may indicate:" - log_warn " - Package namespace overlap between dd-java-agent and ddprof" - log_warn " - Classes added to ddprof.jar (valid scenario)" -fi - -log_info "✓ Structure verification passed" -log_info " - ${SHARED_NATIVE_COUNT} native libraries in shared/META-INF/native-libs/" -log_info " - ${CLASS_COUNT} ddprof classes added to patched JAR" -log_info " - ${TOTAL_CLASSDATA_COUNT} total classdata files in patched JAR" - -# Print summary -JAR_SIZE=$(du -h "${OUTPUT_JAR}" | cut -f1) -log_info "" -log_info "Patching complete!" -log_info " Output: ${OUTPUT_JAR} (${JAR_SIZE})" -log_info "" -log_info "To inspect the patched JAR structure:" -log_info " unzip -l ${OUTPUT_JAR} | grep shared/" diff --git a/utils/release.sh b/utils/release.sh deleted file mode 100755 index 1c2be09fc..000000000 --- a/utils/release.sh +++ /dev/null @@ -1,618 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2025, Datadog, Inc - -# Script to trigger the Validated Release workflow using GitHub CLI -# -# Usage: -# ./utils/release.sh [options] -# -# Arguments: -# release_type: major, minor, or patch -# -# Options: -# --no-dry-run Actually perform the release (default is dry-run) -# --skip-tests Skip pre-release tests (emergency releases only) -# --branch Specify branch to release from (default: current branch) -# -# Examples: -# ./utils/release.sh minor # Dry-run of minor release -# ./utils/release.sh minor --no-dry-run # Actual minor release -# ./utils/release.sh patch --skip-tests # Emergency patch without tests -# ./utils/release.sh major --branch main # Specify branch explicitly - -set -e - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Default values -DRY_RUN="true" -SKIP_TESTS="false" -BRANCH="" -RELEASE_TYPE="" -COMMIT_SHA="" - -# Function to print colored output -print_error() { - echo -e "${RED}ERROR: $1${NC}" >&2 -} - -print_success() { - echo -e "${GREEN}$1${NC}" -} - -print_warning() { - echo -e "${YELLOW}WARNING: $1${NC}" -} - -print_info() { - echo -e "${BLUE}$1${NC}" -} - -# Read a single keypress (arrow keys, enter, q) from /dev/tty -read_key() { - local key - IFS= read -rsn1 key /dev/null) - - if [ ${#branches[@]} -eq 0 ]; then - print_error "No release branches found matching release/X.Y._" >&2 - exit 1 - fi - - if [ ! -t 0 ]; then - print_error "Interactive mode requires a terminal" >&2 - print_error "Use --branch to specify a release branch" >&2 - exit 1 - fi - - local selected=0 - local total=${#branches[@]} - - display_branch_menu() { - clear >&2 - echo "" >&2 - echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════════${NC}" >&2 - echo -e "${BLUE} Select Release Branch for Patch${NC}" >&2 - echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════════${NC}" >&2 - echo "" >&2 - echo "Use ↑/↓ arrow keys to navigate, Enter to select, 'q' to quit" >&2 - echo "" >&2 - - for i in "${!branches[@]}"; do - if [ $i -eq $selected ]; then - echo -e "${GREEN}→ ${branches[$i]}${NC}" >&2 - else - echo -e " ${branches[$i]}" >&2 - fi - done - - echo "" >&2 - echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════════${NC}" >&2 - } - - while true; do - display_branch_menu - key=$(read_key) - case $key in - up) - [ $selected -gt 0 ] && ((selected--)) - ;; - down) - [ $selected -lt $((total - 1)) ] && ((selected++)) - ;; - enter) - echo "${branches[$selected]}" - return 0 - ;; - quit) - echo "" >&2 - print_info "Selection cancelled" >&2 - exit 0 - ;; - esac - done -} - -# Function to show interactive commit selector -select_commit() { - local branch=$1 - - # Get last 10 commits with format: SHA | DATE | AUTHOR | MESSAGE - mapfile -t commits < <(git log "$branch" -n 10 --pretty=format:"%H|%ar|%an|%s" 2>&1) - - if [ ${#commits[@]} -eq 0 ]; then - print_error "No commits found on branch $branch" >&2 - exit 1 - fi - - # Check if we're running in a terminal - if [ ! -t 0 ]; then - print_error "Interactive mode requires a terminal" >&2 - print_error "Use --commit to specify a commit" >&2 - exit 1 - fi - - local selected=0 - local total=${#commits[@]} - - # Function to display menu - display_menu() { - clear >&2 - echo "" >&2 - echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════════${NC}" >&2 - echo -e "${BLUE} Select Commit for Release${NC}" >&2 - echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════════${NC}" >&2 - echo "" >&2 - echo "Use ↑/↓ arrow keys to navigate, Enter to select, 'q' to quit" >&2 - echo "" >&2 - - for i in "${!commits[@]}"; do - IFS='|' read -r sha date author message <<< "${commits[$i]}" - local short_sha="${sha:0:8}" - - if [ $i -eq $selected ]; then - echo -e "${GREEN}→ ${short_sha}${NC} ${YELLOW}${date}${NC} ${BLUE}${author:0:20}${NC} ${message:0:60}" >&2 - else - echo -e " ${short_sha} ${date} ${author:0:20} ${message:0:60}" >&2 - fi - done - - echo "" >&2 - echo -e "${BLUE}═══════════════════════════════════════════════════════════════════════════${NC}" >&2 - } - - # Main selection loop - while true; do - display_menu - - key=$(read_key) - - case $key in - up) - if [ $selected -gt 0 ]; then - ((selected--)) - fi - ;; - down) - if [ $selected -lt $((total - 1)) ]; then - ((selected++)) - fi - ;; - enter) - IFS='|' read -r sha _ _ _ <<< "${commits[$selected]}" - echo "$sha" - return 0 - ;; - quit) - echo "" >&2 - print_info "Selection cancelled" >&2 - exit 0 - ;; - esac - done -} - -# Function to show usage -show_usage() { - cat << EOF -Usage: $0 [options] - -Arguments: - release_type Type of release: major, minor, or patch - -Options: - --no-dry-run Actually perform the release (default is dry-run) - --skip-tests Skip pre-release tests (emergency releases only) - --branch Specify branch to release from (default: current branch) - --commit Specify commit SHA to release (default: interactive selection) - --help Show this help message - -Examples: - $0 minor # Dry-run, interactive commit selection - $0 minor --no-dry-run # Actual minor release - $0 patch # Interactive release-branch picker, then commit selection - $0 patch --branch release/1.2._ # Patch on a specific release branch - $0 patch --commit abc123 # Release specific commit (branch picked interactively if needed) - $0 patch --skip-tests # Emergency patch without tests (dry-run) - $0 patch --no-dry-run --skip-tests # Emergency patch without tests (real) - $0 major --branch main # Specify branch explicitly - -Release Flow: - 1. Validates inputs and branch rules - 2. Runs pre-release tests (testDebug + testAsan) unless skipped - 3. Creates annotated git tag - 4. Triggers GitLab build pipeline - 5. GitLab publishes to Maven Central - 6. GitHub creates release with assets - -Branch Rules: - - major/minor: Must be run from 'main' branch - - patch: Must be run from 'release/X.Y._' branch -EOF -} - -# Parse arguments first to handle --help -if [ $# -eq 0 ]; then - print_error "No release type specified" - show_usage - exit 1 -fi - -# Check for --help early -for arg in "$@"; do - if [ "$arg" == "--help" ]; then - show_usage - exit 0 - fi -done - -RELEASE_TYPE=$1 -shift - -while [ $# -gt 0 ]; do - case "$1" in - --no-dry-run) - DRY_RUN="false" - shift - ;; - --skip-tests) - SKIP_TESTS="true" - shift - ;; - --branch) - if [ -z "$2" ]; then - print_error "--branch requires a branch name" - exit 1 - fi - BRANCH="$2" - shift 2 - ;; - --commit) - if [ -z "$2" ]; then - print_error "--commit requires a commit SHA" - exit 1 - fi - COMMIT_SHA="$2" - shift 2 - ;; - *) - print_error "Unknown option: $1" - show_usage - exit 1 - ;; - esac -done - -# Validate release type -if [[ ! "$RELEASE_TYPE" =~ ^(major|minor|patch)$ ]]; then - print_error "Invalid release type: $RELEASE_TYPE" - echo "Must be one of: major, minor, patch" - exit 1 -fi - -# Get current branch if not specified -if [ -z "$BRANCH" ]; then - BRANCH=$(git branch --show-current) - if [ -z "$BRANCH" ]; then - print_error "Could not determine current branch" - echo "Please specify branch with --branch option" - exit 1 - fi -fi - -# Validate branch rules BEFORE commit selection -if [ "$RELEASE_TYPE" == "patch" ]; then - if [[ ! "$BRANCH" =~ ^release/[0-9]+\.[0-9]+\._$ ]]; then - print_info "Patch releases require a release branch. Fetching available branches..." - git fetch --prune origin 'refs/heads/release/*:refs/remotes/origin/release/*' 2>/dev/null || true - echo "" - BRANCH=$(select_release_branch) - clear - print_info "Branch selected: $BRANCH" - echo "" - fi -else - if [ "$BRANCH" != "main" ]; then - print_error "Major/minor releases can ONLY be performed from 'main' branch" - echo "Current branch: $BRANCH" - echo "" - echo "To create a $RELEASE_TYPE release:" - echo " 1. Switch to main: git checkout main" - echo " 2. Run: $0 $RELEASE_TYPE" - exit 1 - fi -fi - -# Get commit SHA - either from option or interactive selection -# This happens BEFORE gh authentication check so users can browse commits -if [ -z "$COMMIT_SHA" ]; then - print_info "No commit specified. Showing recent commits on branch: $BRANCH" - echo "" - COMMIT_SHA=$(select_commit "$BRANCH") - clear - print_info "Commit selected. Validating..." - echo "" -fi - -# Validate commit exists -print_info "Validating commit SHA..." -if ! git rev-parse --verify "$COMMIT_SHA" >/dev/null 2>&1; then - print_error "Invalid commit SHA: $COMMIT_SHA" - exit 1 -fi - -# Get full commit SHA -print_info "Resolving full commit SHA..." -COMMIT_SHA=$(git rev-parse "$COMMIT_SHA" 2>&1) || { - print_error "Failed to resolve commit SHA: $COMMIT_SHA" - exit 1 -} -SHORT_SHA="${COMMIT_SHA:0:8}" -print_info "Commit: $SHORT_SHA" - -# Verify the commit is on the selected branch -print_info "Verifying commit is on branch $BRANCH..." -if ! git merge-base --is-ancestor "$COMMIT_SHA" "$BRANCH" 2>&1; then - print_error "Commit $SHORT_SHA is not on branch '$BRANCH'" - echo "" - echo "The selected commit must be part of the branch history." - echo "Please select a commit that exists on $BRANCH" - exit 1 -fi - -# Verify the commit exists on remote -print_info "Verifying commit exists on remote..." -REMOTE_HEAD=$(git rev-parse "origin/$BRANCH" 2>&1) || { - print_error "Failed to get remote branch HEAD" - exit 1 -} - -if [ "$COMMIT_SHA" != "$REMOTE_HEAD" ]; then - print_warning "Selected commit $SHORT_SHA is not at the HEAD of remote branch origin/$BRANCH" - echo "" - echo "The GitHub Actions workflow will run against the remote HEAD:" - echo " Remote HEAD: ${REMOTE_HEAD:0:8}" - echo " Selected: $SHORT_SHA" - echo "" - print_warning "You need to either:" - echo " 1. Push your local branch: git push origin $BRANCH" - echo " 2. Select the remote HEAD commit: ${REMOTE_HEAD:0:8}" - exit 1 -fi - -# Get commit info for display -print_info "Retrieving commit information..." -COMMIT_MESSAGE=$(git log -1 --pretty=format:"%s" "$COMMIT_SHA" 2>&1) || { - print_error "Failed to get commit message" - exit 1 -} -COMMIT_AUTHOR=$(git log -1 --pretty=format:"%an" "$COMMIT_SHA" 2>&1) || { - print_error "Failed to get commit author" - exit 1 -} -COMMIT_DATE=$(git log -1 --pretty=format:"%ar" "$COMMIT_SHA" 2>&1) || { - print_error "Failed to get commit date" - exit 1 -} - -# NOW check GitHub CLI authentication (after commit selection) -# Check if gh CLI is installed -print_info "Checking GitHub CLI installation..." -if ! command -v gh &> /dev/null; then - print_error "GitHub CLI (gh) is not installed" - echo "Install it from: https://cli.github.com/" - exit 1 -fi - -# Check if user is authenticated -print_info "Checking GitHub authentication..." -# Note: gh auth status may return non-zero even when authenticated, so check the output -AUTH_STATUS=$(gh auth status 2>&1 || true) -if ! echo "$AUTH_STATUS" | grep -q "Logged in"; then - print_error "Not authenticated with GitHub CLI" - echo "Run: gh auth login" - echo "" - echo "Current auth status:" - echo "$AUTH_STATUS" - exit 1 -fi -print_info "GitHub authentication verified" - -# Branch validation already done earlier (before commit selection) - -# Show summary -echo "" -print_info "═══════════════════════════════════════════════════════" -print_info " Release Configuration" -print_info "═══════════════════════════════════════════════════════" -echo " Release Type: $RELEASE_TYPE" -echo " Branch: $BRANCH" -echo " Commit: $SHORT_SHA" -echo " Message: $COMMIT_MESSAGE" -echo " Author: $COMMIT_AUTHOR ($COMMIT_DATE)" -echo " Dry Run: $DRY_RUN" -echo " Skip Tests: $SKIP_TESTS" -print_info "═══════════════════════════════════════════════════════" -echo "" - -if [ "$DRY_RUN" == "false" ]; then - print_warning "This will perform an ACTUAL release!" - if [ "$SKIP_TESTS" == "true" ]; then - print_warning "Tests will be SKIPPED!" - fi - echo "" - read -p "Are you sure you want to continue? (yes/no): " -r "$WORKFLOW_OUTPUT" 2> "$WORKFLOW_ERROR"; then - - WORKFLOW_SUCCESS=true - echo "" - print_success "✓ Workflow triggered successfully!" - REPO_URL=$(gh repo view --json url -q .url) - - # Wait for the run to appear and capture its ID - print_info "Waiting for workflow run to appear..." - RUN_ID="" - for i in $(seq 1 15); do - sleep 2 - RUN_ID=$(gh run list --workflow=release-validated.yml --limit 1 --json databaseId,status -q '.[0].databaseId // empty') - if [ -n "$RUN_ID" ]; then - break - fi - done - - if [ -n "$RUN_ID" ]; then - echo "" - print_info "Watching workflow run ${RUN_ID}..." - echo " ${REPO_URL}/actions/runs/${RUN_ID}" - echo "" - if gh run watch "$RUN_ID" --exit-status; then - WORKFLOW_CONCLUSION="success" - print_success "✓ Workflow completed successfully!" - else - WORKFLOW_CONCLUSION="failure" - print_error "✗ Workflow failed!" - echo " View logs: gh run view $RUN_ID --log-failed" - fi - else - WORKFLOW_CONCLUSION="unknown" - print_warning "Could not detect the workflow run. Monitor manually:" - echo " gh run list --workflow=release-validated.yml --limit 1" - fi - - echo "" - if [ "$DRY_RUN" == "false" ]; then - print_info "Next Steps:" - echo " 1. Verify Maven: https://repo1.maven.org/maven2/com/datadoghq/ddprof/" - echo " 2. Check release: ${REPO_URL}/releases" - else - print_info "This was a dry-run. Review the output and run again with --no-dry-run" - fi -else - WORKFLOW_SUCCESS=false - echo "" - print_error "Failed to trigger workflow" -fi - -# Print comprehensive summary -echo "" -echo "" -print_info "═══════════════════════════════════════════════════════════════════════════" -print_info " RELEASE EXECUTION SUMMARY" -print_info "═══════════════════════════════════════════════════════════════════════════" -echo "" - -# Action performed -echo "Actions Performed:" -echo " ✓ Validated release type: $RELEASE_TYPE" -echo " ✓ Validated branch rules: $BRANCH" -echo " ✓ Selected commit: $SHORT_SHA" - -if [ "$WORKFLOW_SUCCESS" = true ]; then - echo " ✓ Triggered GitHub Actions workflow" -else - echo " ✗ FAILED to trigger GitHub Actions workflow" -fi - -echo "" - -# Configuration summary -echo "Configuration:" -echo " Release Type: $RELEASE_TYPE" -echo " Branch: $BRANCH" -echo " Commit: $SHORT_SHA ($COMMIT_DATE)" -echo " Message: $COMMIT_MESSAGE" -echo " Author: $COMMIT_AUTHOR" -echo " Dry Run: $DRY_RUN" -echo " Skip Tests: $SKIP_TESTS" - -echo "" - -# Status and next steps -if [ "$WORKFLOW_SUCCESS" = true ]; then - if [ "${WORKFLOW_CONCLUSION:-}" = "success" ]; then - print_success "Status: WORKFLOW SUCCEEDED" - elif [ "${WORKFLOW_CONCLUSION:-}" = "failure" ]; then - print_error "Status: WORKFLOW FAILED" - echo " View logs: gh run view $RUN_ID --log-failed" - elif [ "$DRY_RUN" == "true" ]; then - print_success "Status: DRY-RUN COMPLETED" - echo "" - echo " → No actual changes were made." - echo " → To perform the release, run: $0 $RELEASE_TYPE --no-dry-run --commit $SHORT_SHA" - else - print_warning "Status: WORKFLOW STATUS UNKNOWN" - echo " Check manually: gh run list --workflow=release-validated.yml --limit 1" - fi -else - print_error "Status: FAILED TO TRIGGER WORKFLOW" - echo "" - echo "Error Details:" - if [ -s "$WORKFLOW_ERROR" ]; then - cat "$WORKFLOW_ERROR" | sed 's/^/ /' - else - echo " Unknown error. Check GitHub CLI authentication and repository access." - fi - echo "" - echo "Troubleshooting:" - echo " • Verify authentication: gh auth status" - echo " • Check permissions: gh repo view --json viewerPermission" - echo " • Verify commit exists: git show $SHORT_SHA" -fi - -echo "" -print_info "═══════════════════════════════════════════════════════════════════════════" - -# Cleanup temp files -rm -f "$WORKFLOW_OUTPUT" "$WORKFLOW_ERROR" - -# Exit with appropriate code -if [ "$WORKFLOW_SUCCESS" = true ]; then - exit 0 -else - exit 1 -fi diff --git a/utils/run-containers-tests.sh b/utils/run-containers-tests.sh deleted file mode 100755 index 4d43c6ad8..000000000 --- a/utils/run-containers-tests.sh +++ /dev/null @@ -1,602 +0,0 @@ -#!/bin/bash -# Copyright 2026, Datadog, Inc -# -# Run tests in containers with various OS/libc/JDK combinations (similar to CI) -# Defaults to Podman; use --container=docker to use Docker. -# Uses two-level container image caching: -# 1. Base image with OS + build tools (java-profiler-base:-) -# 2. JDK-specific image on top (java-profiler-test:-jdk-) -# -# Usage: ./utils/run-containers-tests.sh [options] -# --libc=glibc|musl (default: glibc) -# --jdk=8|11|17|21|25|8-j9|11-j9|17-j9|21-j9|17-graal|21-graal|25-graal (default: 21) -# --arch=x64|aarch64 (default: auto-detect) -# --config=debug|release|asan|tsan (default: debug) -# --container=podman|docker (default: podman) -# --tests="TestPattern" (optional, specific test to run) -# --gtest (enable C++ gtests, disabled by default) -# --gtest-task=Task (run one C++ gtest task; accepts elfparser_ut or :ddprof-lib:gtestAsan_elfparser_ut) -# --shell (drop to shell instead of running tests; enables SYS_PTRACE for gdb) -# --mount (mount local repo instead of cloning - faster but may have stale artifacts) -# --rebuild (force rebuild of container images) -# --rebuild-base (force rebuild of base image only) -# --help (show this help) - -set -e - -# Defaults -CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-podman}" -LIBC="glibc" -JDK_VERSION="21" -ARCH="" -CONFIG="debug" -TESTS="" -GTEST_TASK="" -SHELL_MODE=false -MOUNT_MODE=false -GTEST_ENABLED=false -REBUILD=false -REBUILD_BASE=false -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -PROJECT_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" -BASE_IMAGE_PREFIX="java-profiler-base" -IMAGE_PREFIX="java-profiler-test" - -selinux_enforcing() { - if [[ -r /sys/fs/selinux/enforce ]]; then - [[ "$(cat /sys/fs/selinux/enforce)" == "1" ]] - elif command -v getenforce >/dev/null 2>&1; then - [[ "$(getenforce)" == "Enforcing" ]] - else - false - fi -} - -# Auto-detect architecture -detect_arch() { - local machine - machine=$(uname -m) - case "$machine" in - x86_64|amd64) - echo "x64" - ;; - aarch64|arm64) - echo "aarch64" - ;; - *) - echo "x64" # default fallback - ;; - esac -} - -# JDK Download URLs (Bellsoft Liberica for musl) -get_musl_jdk_url() { - local version=$1 - local arch=$2 - - case "$version-$arch" in - 8-x64) echo "https://download.bell-sw.com/java/8u462+11/bellsoft-jdk8u462+11-linux-x64-musl-lite.tar.gz" ;; - 8-aarch64) echo "https://download.bell-sw.com/java/8u462+11/bellsoft-jdk8u462+11-linux-aarch64-musl-lite.tar.gz" ;; - 11-x64) echo "https://download.bell-sw.com/java/11.0.28+12/bellsoft-jdk11.0.28+12-linux-x64-musl-lite.tar.gz" ;; - 11-aarch64) echo "https://download.bell-sw.com/java/11.0.28+12/bellsoft-jdk11.0.28+12-linux-aarch64-musl-lite.tar.gz" ;; - 17-x64) echo "https://download.bell-sw.com/java/17.0.16+12/bellsoft-jdk17.0.16+12-linux-x64-musl-lite.tar.gz" ;; - 17-aarch64) echo "https://download.bell-sw.com/java/17.0.16+12/bellsoft-jdk17.0.16+12-linux-aarch64-musl-lite.tar.gz" ;; - 21-x64) echo "https://download.bell-sw.com/java/21.0.8+12/bellsoft-jdk21.0.8+12-linux-x64-musl-lite.tar.gz" ;; - 21-aarch64) echo "https://download.bell-sw.com/java/21.0.8+12/bellsoft-jdk21.0.8+12-linux-aarch64-musl-lite.tar.gz" ;; - 25-x64) echo "https://download.bell-sw.com/java/25.0.2+12/bellsoft-jdk25.0.2+12-linux-x64-musl-lite.tar.gz" ;; - 25-aarch64) echo "https://download.bell-sw.com/java/25.0.2+12/bellsoft-jdk25.0.2+12-linux-aarch64-musl-lite.tar.gz" ;; - *) echo "" ;; - esac -} - -# JDK Download URLs (Eclipse Temurin for glibc) -get_glibc_jdk_url() { - local version=$1 - local arch=$2 - - case "$version-$arch" in - 8-x64) echo "https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u462-b08/OpenJDK8U-jdk_x64_linux_hotspot_8u462b08.tar.gz" ;; - 8-aarch64) echo "https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u462-b08/OpenJDK8U-jdk_aarch64_linux_hotspot_8u462b08.tar.gz" ;; - 11-x64) echo "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.28%2B6/OpenJDK11U-jdk_x64_linux_hotspot_11.0.28_6.tar.gz" ;; - 11-aarch64) echo "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.28%2B6/OpenJDK11U-jdk_aarch64_linux_hotspot_11.0.28_6.tar.gz" ;; - 17-x64) echo "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.16%2B8/OpenJDK17U-jdk_x64_linux_hotspot_17.0.16_8.tar.gz" ;; - 17-aarch64) echo "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.16%2B8/OpenJDK17U-jdk_aarch64_linux_hotspot_17.0.16_8.tar.gz" ;; - 21-x64) echo "https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.8%2B9/OpenJDK21U-jdk_x64_linux_hotspot_21.0.8_9.tar.gz" ;; - 21-aarch64) echo "https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.8%2B9/OpenJDK21U-jdk_aarch64_linux_hotspot_21.0.8_9.tar.gz" ;; - 25-x64) echo "https://github.com/adoptium/temurin25-binaries/releases/download/jdk-25.0.2%2B10/OpenJDK25U-jdk_x64_linux_hotspot_25.0.2_10.tar.gz" ;; - 25-aarch64) echo "https://github.com/adoptium/temurin25-binaries/releases/download/jdk-25.0.2%2B10/OpenJDK25U-jdk_aarch64_linux_hotspot_25.0.2_10.tar.gz" ;; - *) echo "" ;; - esac -} - -# JDK Download URLs (Oracle GraalVM) -# Versions match the SDKMAN-installed builds used in CI (java_setup.sh + -# cache_java.yml: JAVA__GRAAL_VERSION). Using Oracle's stable archive -# URLs (not "latest") so the docker images match the CI configuration. -get_graal_jdk_url() { - local version=$1 - local arch=$2 - # Oracle GraalVM archive arch labels: x64 / aarch64 (same as our $arch) - case "$version-$arch" in - 17-x64) echo "https://download.oracle.com/graalvm/17/archive/graalvm-jdk-17.0.12_linux-x64_bin.tar.gz" ;; - 17-aarch64) echo "https://download.oracle.com/graalvm/17/archive/graalvm-jdk-17.0.12_linux-aarch64_bin.tar.gz" ;; - 21-x64) echo "https://download.oracle.com/graalvm/21/archive/graalvm-jdk-21.0.4_linux-x64_bin.tar.gz" ;; - 21-aarch64) echo "https://download.oracle.com/graalvm/21/archive/graalvm-jdk-21.0.4_linux-aarch64_bin.tar.gz" ;; - 25-x64) echo "https://download.oracle.com/graalvm/25/archive/graalvm-jdk-25_linux-x64_bin.tar.gz" ;; - 25-aarch64) echo "https://download.oracle.com/graalvm/25/archive/graalvm-jdk-25_linux-aarch64_bin.tar.gz" ;; - *) echo "" ;; - esac -} - -# JDK Download URLs (IBM Semeru OpenJ9) -get_j9_jdk_url() { - local version=$1 - local arch=$2 - - case "$version-$arch" in - 8-x64) echo "https://github.com/ibmruntimes/semeru8-binaries/releases/download/jdk8u482-b08_openj9-0.57.0/ibm-semeru-open-jdk_x64_linux_8u482b08_openj9-0.57.0.tar.gz" ;; - 8-aarch64) echo "https://github.com/ibmruntimes/semeru8-binaries/releases/download/jdk8u482-b08_openj9-0.57.0/ibm-semeru-open-jdk_aarch64_linux_8u482b08_openj9-0.57.0.tar.gz" ;; - 11-x64) echo "https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.30%2B7_openj9-0.57.0/ibm-semeru-open-jdk_x64_linux_11.0.30_7_openj9-0.57.0.tar.gz" ;; - 11-aarch64) echo "https://github.com/ibmruntimes/semeru11-binaries/releases/download/jdk-11.0.30%2B7_openj9-0.57.0/ibm-semeru-open-jdk_aarch64_linux_11.0.30_7_openj9-0.57.0.tar.gz" ;; - 17-x64) echo "https://github.com/ibmruntimes/semeru17-binaries/releases/download/jdk-17.0.18%2B8_openj9-0.57.0/ibm-semeru-open-jdk_x64_linux_17.0.18_8_openj9-0.57.0.tar.gz" ;; - 17-aarch64) echo "https://github.com/ibmruntimes/semeru17-binaries/releases/download/jdk-17.0.18%2B8_openj9-0.57.0/ibm-semeru-open-jdk_aarch64_linux_17.0.18_8_openj9-0.57.0.tar.gz" ;; - 21-x64) echo "https://github.com/ibmruntimes/semeru21-binaries/releases/download/jdk-21.0.10%2B7_openj9-0.57.0/ibm-semeru-open-jdk_x64_linux_21.0.9_10_openj9-0.56.0.tar.gz" ;; - 21-aarch64) echo "https://github.com/ibmruntimes/semeru21-binaries/releases/download/jdk-21.0.10%2B7_openj9-0.57.0/ibm-semeru-open-jdk_aarch64_linux_21.0.9_10_openj9-0.56.0.tar.gz" ;; - *) echo "" ;; - esac -} - -usage() { - head -n 23 "$0" | tail -n 20 - exit 0 -} - -# Parse arguments -while [[ $# -gt 0 ]]; do - case $1 in - --libc=*) - LIBC="${1#*=}" - shift - ;; - --jdk=*) - JDK_VERSION="${1#*=}" - shift - ;; - --arch=*) - ARCH="${1#*=}" - shift - ;; - --config=*) - CONFIG="${1#*=}" - shift - ;; - --container=*) - CONTAINER_RUNTIME="${1#*=}" - shift - ;; - --tests=*) - TESTS="${1#*=}" - shift - ;; - --shell) - SHELL_MODE=true - shift - ;; - --mount) - MOUNT_MODE=true - shift - ;; - --gtest) - GTEST_ENABLED=true - shift - ;; - --gtest-task=*) - GTEST_TASK="${1#*=}" - GTEST_ENABLED=true - shift - ;; - --rebuild) - REBUILD=true - shift - ;; - --rebuild-base) - REBUILD_BASE=true - shift - ;; - --help|-h) - usage - ;; - *) - echo "Unknown option: $1" - usage - ;; - esac -done - -# Auto-detect architecture if not specified -if [[ -z "$ARCH" ]]; then - ARCH=$(detect_arch) -fi - -# Validate arguments -if [[ "$LIBC" != "musl" && "$LIBC" != "glibc" ]]; then - echo "Error: --libc must be 'musl' or 'glibc'" - exit 1 -fi - -if [[ "$ARCH" != "x64" && "$ARCH" != "aarch64" ]]; then - echo "Error: --arch must be 'x64' or 'aarch64'" - exit 1 -fi - -if [[ "$CONFIG" != "debug" && "$CONFIG" != "release" && "$CONFIG" != "asan" && "$CONFIG" != "tsan" ]]; then - echo "Error: --config must be 'debug', 'release', 'asan', or 'tsan'" - exit 1 -fi - -if [[ "$CONTAINER_RUNTIME" != "podman" && "$CONTAINER_RUNTIME" != "docker" ]]; then - echo "Error: --container must be 'podman' or 'docker'" - exit 1 -fi - -if [[ -n "$GTEST_TASK" && -n "$TESTS" ]]; then - echo "Error: --tests cannot be combined with --gtest-task" - exit 1 -fi - -if ! command -v "$CONTAINER_RUNTIME" >/dev/null 2>&1; then - echo "Error: container runtime '$CONTAINER_RUNTIME' not found" - echo "Use --container to select the runtime, e.g. $0 --container=docker ..." - exit 1 -fi - -# Parse JDK version and variant (e.g., "21-j9" -> version="21", variant="j9") -JDK_BASE_VERSION="${JDK_VERSION%%-*}" -JDK_VARIANT="${JDK_VERSION#*-}" -if [[ "$JDK_VARIANT" == "$JDK_VERSION" ]]; then - JDK_VARIANT="" # No variant specified -fi - -# Get JDK URL based on variant and libc -if [[ "$JDK_VARIANT" == "j9" ]]; then - if [[ "$LIBC" == "musl" ]]; then - echo "Error: J9/OpenJ9 is not available for musl libc" - exit 1 - fi - JDK_URL=$(get_j9_jdk_url "$JDK_BASE_VERSION" "$ARCH") -elif [[ "$JDK_VARIANT" == "graal" ]]; then - if [[ "$LIBC" == "musl" ]]; then - echo "Error: GraalVM is not available for musl libc" - exit 1 - fi - JDK_URL=$(get_graal_jdk_url "$JDK_BASE_VERSION" "$ARCH") -elif [[ "$LIBC" == "musl" ]]; then - JDK_URL=$(get_musl_jdk_url "$JDK_BASE_VERSION" "$ARCH") -else - JDK_URL=$(get_glibc_jdk_url "$JDK_BASE_VERSION" "$ARCH") -fi - -if [[ -z "$JDK_URL" ]]; then - echo "Error: --jdk must be one of: 8, 11, 17, 21, 25, 8-j9, 11-j9, 17-j9, 21-j9, 17-graal, 21-graal, 25-graal" - exit 1 -fi - -# Image names for caching -BASE_IMAGE_NAME="${BASE_IMAGE_PREFIX}:${LIBC}-${ARCH}" -IMAGE_NAME="${IMAGE_PREFIX}:${LIBC}-jdk${JDK_VERSION}-${ARCH}" - -# Container platform for cross-architecture support -CONTAINER_PLATFORM="" -if [[ "$ARCH" == "aarch64" ]]; then - CONTAINER_PLATFORM="--platform linux/arm64" -elif [[ "$ARCH" == "x64" ]]; then - CONTAINER_PLATFORM="--platform linux/amd64" -fi - -echo "=== Container Test Runner ===" -echo "LIBC: $LIBC" -echo "Build JDK: 21 (Gradle 9 requirement)" -echo "Test JDK: $JDK_VERSION" -echo "Arch: $ARCH" -echo "Config: $CONFIG" -echo "Runtime: $CONTAINER_RUNTIME" -if [[ -n "$GTEST_TASK" ]]; then - echo "GTest task: $GTEST_TASK" -fi -echo "Tests: ${TESTS:-}" -echo "GTest: $(if $GTEST_ENABLED; then echo 'enabled'; else echo 'disabled'; fi)" -echo "Mode: $(if $SHELL_MODE; then echo 'shell'; else echo 'test'; fi)" -echo "Source: $(if $MOUNT_MODE; then echo 'mount (local)'; else echo 'clone (clean)'; fi)" -echo "Base Image: $BASE_IMAGE_NAME" -echo "Image: $IMAGE_NAME" -echo "==========================" - -# Create temporary Dockerfile directory -DOCKERFILE_DIR=$(mktemp -d) -trap "rm -rf $DOCKERFILE_DIR" EXIT - -# Copy gradle wrapper for caching -cp "$PROJECT_ROOT/gradlew" "$DOCKERFILE_DIR/" -cp -r "$PROJECT_ROOT/gradle" "$DOCKERFILE_DIR/" - -# ========== Build Base Image (if needed) ========== -BASE_IMAGE_EXISTS=false -if [[ "$REBUILD" == "false" && "$REBUILD_BASE" == "false" ]]; then - if "$CONTAINER_RUNTIME" image inspect "$BASE_IMAGE_NAME" >/dev/null 2>&1; then - BASE_IMAGE_EXISTS=true - echo ">>> Using cached base image: $BASE_IMAGE_NAME" - fi -fi - -if [[ "$BASE_IMAGE_EXISTS" == "false" ]]; then - echo ">>> Building base image: $BASE_IMAGE_NAME" - - if [[ "$LIBC" == "musl" ]]; then - cat > "$DOCKERFILE_DIR/Dockerfile.base" <<'EOF' -FROM alpine:3.21 - -# Install build dependencies -# - linux-headers provides linux/limits.h -# - compiler-rt provides sanitizer runtimes (ASan, TSan) for clang -# - llvm provides libFuzzer -# - openssh-client for git clone over SSH -RUN apk update && \ - apk add --no-cache \ - curl wget bash make g++ clang git jq cmake coreutils \ - gtest-dev gmock tar binutils musl-dbg linux-headers \ - compiler-rt llvm openssh-client gdb - -# Set up Gradle cache directory -ENV GRADLE_USER_HOME=/gradle-cache -RUN mkdir -p /gradle-cache - -WORKDIR /workspace -EOF - else - # libclang-rt-dev is only available on x64, not arm64 - if [[ "$ARCH" == "x64" ]]; then - CLANG_RT_PKG="libclang-dev" - else - CLANG_RT_PKG="" - fi - # On aarch64, install clang-17 + libclang-rt-17-dev from the LLVM apt repository. - # GCC 11's libtsan only knows 39-bit VMA; the linuxkit kernel (Docker Desktop) - # uses 48-bit VMA, so GCC 11's TSan crashes with "unexpected memory mapping". - # Clang-17's compiler-rt TSan runtime handles both 39-bit and 48-bit VMA. - # libclang-rt-17-dev provides libclang_rt.tsan-aarch64.{a,so} that clang-17 - # needs when linking executables with -fsanitize=thread. - if [[ "$ARCH" == "aarch64" ]]; then - CLANG17_BLOCK=' -# Install clang-17 + compiler-rt for TSan 48-bit VMA support (aarch64 only) -RUN wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ - | gpg --dearmor -o /etc/apt/trusted.gpg.d/llvm.gpg \ - && echo "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-17 main" \ - > /etc/apt/sources.list.d/llvm.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends clang-17 libclang-rt-17-dev \ - && rm -rf /var/lib/apt/lists/*' - else - CLANG17_BLOCK='' - fi - cat > "$DOCKERFILE_DIR/Dockerfile.base" <>> Base image built: $BASE_IMAGE_NAME" -fi - -# ========== Get Build JDK URL (always JDK 21 for Gradle 9) ========== -# Gradle 9 requires JDK 17+ to run; we use JDK 21 (LTS) as the build JDK -BUILD_JDK_VERSION="21" -if [[ "$LIBC" == "musl" ]]; then - BUILD_JDK_URL=$(get_musl_jdk_url "$BUILD_JDK_VERSION" "$ARCH") -else - BUILD_JDK_URL=$(get_glibc_jdk_url "$BUILD_JDK_VERSION" "$ARCH") -fi - -# ========== Build JDK Image (if needed) ========== -IMAGE_EXISTS=false -if [[ "$REBUILD" == "false" ]]; then - if "$CONTAINER_RUNTIME" image inspect "$IMAGE_NAME" >/dev/null 2>&1; then - IMAGE_EXISTS=true - echo ">>> Using cached image: $IMAGE_NAME" - fi -fi - -if [[ "$IMAGE_EXISTS" == "false" ]]; then - echo ">>> Building JDK image: $IMAGE_NAME" - echo ">>> Build JDK (for Gradle): $BUILD_JDK_VERSION" - echo ">>> Test JDK: $JDK_VERSION" - - # Determine if we need two JDKs or just one - if [[ "$JDK_BASE_VERSION" == "$BUILD_JDK_VERSION" && -z "$JDK_VARIANT" ]]; then - # Test JDK is same as build JDK - use single installation - cat > "$DOCKERFILE_DIR/Dockerfile" < "$DOCKERFILE_DIR/Dockerfile" <>> JDK image built: $IMAGE_NAME" -fi - -# ========== Run Tests ========== - -# Build gradle test command -# Capitalize first letter for gradle task names (testDebug, testAsan, etc.) -# Note: -Ptests property works uniformly across all platforms (glibc, musl, macOS) -CONFIG_CAPITALIZED="$(tr '[:lower:]' '[:upper:]' <<< ${CONFIG:0:1})${CONFIG:1}" -if [[ -n "$GTEST_TASK" ]]; then - if [[ "$GTEST_TASK" == :* ]]; then - GRADLE_CMD="./gradlew -PCI -PkeepJFRs $GTEST_TASK" - else - GRADLE_CMD="./gradlew -PCI -PkeepJFRs :ddprof-lib:gtest${CONFIG_CAPITALIZED}_${GTEST_TASK}" - fi -else - GRADLE_CMD="./gradlew -PCI -PkeepJFRs :ddprof-test:test${CONFIG_CAPITALIZED}" - if [[ -n "$TESTS" ]]; then - # No need for quotes around $TESTS - Gradle property values don't require quoting - GRADLE_CMD="$GRADLE_CMD -Ptests=$TESTS" - fi - if ! $GTEST_ENABLED; then - GRADLE_CMD="$GRADLE_CMD -Pskip-gtest" - fi -fi -# On aarch64 glibc, TSan needs clang-17's embedded runtime (supports 48-bit VMA). -# GCC 11's libtsan is linked by default but only knows 39-bit VMA, causing a crash. -if [[ "$CONFIG" == "tsan" ]] && [[ "$ARCH" == "aarch64" ]] && [[ "$LIBC" == "glibc" ]]; then - GRADLE_CMD="$GRADLE_CMD -Pnative.forceCompiler=clang++-17" -fi -GRADLE_CMD="$GRADLE_CMD --no-daemon --parallel --build-cache --no-watch-fs" - -# On aarch64 glibc TSan: reduce ASLR entropy so TSan's shadow doesn't conflict with -# initial library load addresses. Requires --privileged for sysctl in the container. -# Do NOT set kernel.randomize_va_space=0: ld-linux-aarch64.so loads at 0x2000000000 -# (TSan's shadow start) with full ASLR off. -SYSCTL_PREP="" -NEEDS_PRIVILEGED=false -if [[ "$CONFIG" == "tsan" ]] && [[ "$ARCH" == "aarch64" ]] && [[ "$LIBC" == "glibc" ]]; then - SYSCTL_PREP="sysctl -w vm.mmap_rnd_bits=28 2>/dev/null || true && " - NEEDS_PRIVILEGED=true -fi - -# Build container run command base -CONTAINER_CMD="$CONTAINER_RUNTIME run --rm" -CONTAINER_VOLUME_RW_OPTIONS="" -CONTAINER_VOLUME_RO_OPTIONS=":ro" -if [[ "${CONTAINER_RUNTIME##*/}" == "podman" ]] && selinux_enforcing; then - CONTAINER_VOLUME_RW_OPTIONS=":z" - CONTAINER_VOLUME_RO_OPTIONS=":ro,z" -fi -if $SHELL_MODE; then - CONTAINER_CMD="$CONTAINER_CMD -it --init --ulimit core=-1 --cap-add=SYS_PTRACE" -fi -if $NEEDS_PRIVILEGED; then - CONTAINER_CMD="$CONTAINER_CMD --privileged" -fi -CONTAINER_CMD="$CONTAINER_CMD $CONTAINER_PLATFORM" -CONTAINER_CMD="$CONTAINER_CMD -e LIBC=$LIBC" -CONTAINER_CMD="$CONTAINER_CMD -e SANITIZER=$CONFIG" -CONTAINER_CMD="$CONTAINER_CMD -e TEST_CONFIGURATION=$LIBC/${JDK_VERSION}-$CONFIG-$ARCH" -CONTAINER_CMD="$CONTAINER_CMD -e GRADLE_USER_HOME=/gradle-cache" - -if $MOUNT_MODE; then - # Mount mode: use local repo directly (faster, but may have stale artifacts) - CONTAINER_CMD="$CONTAINER_CMD -v \"$PROJECT_ROOT\":/workspace${CONTAINER_VOLUME_RW_OPTIONS}" - CONTAINER_CMD="$CONTAINER_CMD $IMAGE_NAME" - - if $SHELL_MODE; then - SHELL_CMD="/bin/bash" - else - SHELL_CMD="${SYSCTL_PREP}${GRADLE_CMD}" - fi - - echo "" - echo ">>> Running in container (mount mode)..." - echo ">>> Command: $SHELL_CMD" - eval "$CONTAINER_CMD /bin/bash -c '$SHELL_CMD'" -else - # Clone mode: shallow clone from mounted local repo for clean builds (default) - # Mount the local repo as source, then clone from it to /workspace - CONTAINER_CMD="$CONTAINER_CMD -v \"$PROJECT_ROOT\":/source${CONTAINER_VOLUME_RO_OPTIONS}" - CONTAINER_CMD="$CONTAINER_CMD $IMAGE_NAME" - - # Build clone and test command - clone from local mounted source - if $SHELL_MODE; then - CLONE_CMD="git clone --depth 1 file:///source /workspace && cd /workspace && /bin/bash" - else - CLONE_CMD="git clone --depth 1 file:///source /workspace && cd /workspace && ${SYSCTL_PREP}${GRADLE_CMD}" - fi - - echo "" - echo ">>> Running in container (clone mode)..." - echo ">>> Cloning from local source to /workspace" - eval "$CONTAINER_CMD /bin/bash -c '$CLONE_CMD'" -fi diff --git a/utils/track_upstream_changes.sh b/utils/track_upstream_changes.sh deleted file mode 100755 index d43b3c93d..000000000 --- a/utils/track_upstream_changes.sh +++ /dev/null @@ -1,285 +0,0 @@ -#!/bin/bash -# Track upstream async-profiler changes and generate reports - -set -e - -show_help() { - cat </dev/null) - -if [ -z "$ALL_COMMITS" ]; then - echo "No new commits found" - exit 0 -fi - -# Count total commits -TOTAL_COMMITS=$(echo "$ALL_COMMITS" | wc -l | tr -d ' ') - -# Extract PR numbers from commit messages -extract_pr_numbers() { - local commits="$1" - echo "$commits" | grep -o '(#[0-9]\+)' | sed 's/(#\([0-9]*\))/\1/' | sort -u || true -} - -# Get list of commits for display (limited to 20) -COMMITS_DISPLAY=$(echo "$ALL_COMMITS" | head -20) -if [ $TOTAL_COMMITS -gt 20 ]; then - COMMITS_DISPLAY="${COMMITS_DISPLAY} -... and $((TOTAL_COMMITS - 20)) more commits" -fi - -# Extract all PR numbers -PR_NUMBERS=$(extract_pr_numbers "$ALL_COMMITS") - -# Generate short commit range for display -SHORT_LAST=$(echo "$LAST_COMMIT" | cut -c1-7) -SHORT_CURRENT=$(echo "$CURRENT_COMMIT" | cut -c1-7) -COMMIT_RANGE="${SHORT_LAST}...${SHORT_CURRENT}" - -# Track which files were modified - use temp directory for per-file data -TEMP_DATA_DIR="$(mktemp -d)" -trap "rm -rf $TEMP_DATA_DIR" EXIT - -changed_files_list=() -changed_files_count=0 - -# For each tracked file, check if it was modified -while IFS= read -r file; do - if [ -z "$file" ]; then - continue - fi - - # Check if file was modified in commit range - if git diff --quiet "$LAST_COMMIT" "$CURRENT_COMMIT" -- "$file" 2>/dev/null; then - continue # No changes to this file - fi - - # Get commit list for this file (limited to 20) - commits_for_file=$(git log --oneline "$LAST_COMMIT..$CURRENT_COMMIT" -- "$file" | head -20) - commit_count=$(echo "$commits_for_file" | wc -l | tr -d ' ') - - # Store commits in a file, and track file:count - file_id=$(printf "%03d" $changed_files_count) - echo "$commits_for_file" > "$TEMP_DATA_DIR/${file_id}.commits" - echo "$file" > "$TEMP_DATA_DIR/${file_id}.path" - echo "$commit_count" > "$TEMP_DATA_DIR/${file_id}.count" - - changed_files_list+=("$file_id:$commit_count") - changed_files_count=$((changed_files_count + 1)) -done < "$TRACKED_FILES" - -if [ $changed_files_count -eq 0 ]; then - echo "No changes to tracked files" - exit 0 -fi - -# Sort file IDs by commit count (descending) -SORTED_FILE_IDS=$(printf '%s\n' "${changed_files_list[@]}" | sort -t':' -k2 -rn | cut -d':' -f1) - -# Generate Markdown Report -cat > "$OUTPUT_MD" <> "$OUTPUT_MD" <> "$OUTPUT_MD" - done - - commit_count_for_file=$(wc -l < "$TEMP_DATA_DIR/${file_id}.commits" | tr -d ' ') - if [ "$commit_count_for_file" -gt 5 ]; then - echo "- ... and $((commit_count_for_file - 5)) more commits" >> "$OUTPUT_MD" - fi - - echo "" >> "$OUTPUT_MD" - echo "**View diff**: [${basename_file}](https://github.com/async-profiler/async-profiler/commits/${CURRENT_COMMIT}/${file})" >> "$OUTPUT_MD" - echo "" >> "$OUTPUT_MD" -done - -# Add recent commits section -cat >> "$OUTPUT_MD" <> "$OUTPUT_MD" -done - -# Add PR references if any -if [ -n "$PR_NUMBERS" ]; then - cat >> "$OUTPUT_MD" <> "$OUTPUT_MD" - done -fi - -# Add action items -cat >> "$OUTPUT_MD" < "$OUTPUT_JSON" <> "$OUTPUT_JSON" - fi - - cat >> "$OUTPUT_JSON" <> "$OUTPUT_JSON" < - -AWS_REGION=us-east-1 -SSM_PREFIX=ci.java-profiler -AWS_VAULT_PROFILE=sso-build-stable-developer - -usage() { - echo "Usage: $0 " - echo "" - echo "Updates Sonatype OSSRH credentials in AWS SSM:" - echo " ${SSM_PREFIX}.sonatype_token_user" - echo " ${SSM_PREFIX}.sonatype_token" - exit 1 -} - -if [ $# -ne 2 ]; then - usage -fi - -USERNAME="$1" -TOKEN="$2" - -aws-vault login sso-build-stable-developer - -# Verify AWS authentication -if ! aws-vault exec "${AWS_VAULT_PROFILE}" -- aws sts get-caller-identity --query "Arn" --output text 2>/dev/null; then - echo "ERROR: Not authenticated with AWS. Run 'aws-vault login ${AWS_VAULT_PROFILE}' and retry." - exit 1 -fi - -echo "Updating ${SSM_PREFIX}.sonatype_token_user ..." -aws-vault exec "${AWS_VAULT_PROFILE}" -- aws ssm put-parameter \ - --region "${AWS_REGION}" \ - --name "${SSM_PREFIX}.sonatype_token_user" \ - --value "${USERNAME}" \ - --type SecureString \ - --overwrite - -echo "Updating ${SSM_PREFIX}.sonatype_token ..." -aws-vault exec "${AWS_VAULT_PROFILE}" -- aws ssm put-parameter \ - --region "${AWS_REGION}" \ - --name "${SSM_PREFIX}.sonatype_token" \ - --value "${TOKEN}" \ - --type SecureString \ - --overwrite - -echo "Done."