From 457171b35d3f89a530faf8428a3f179de59ac1e4 Mon Sep 17 00:00:00 2001 From: Daniel Leech Date: Sun, 31 May 2026 09:23:53 +0100 Subject: [PATCH] Report Derivations - Derive columns from existing columns. - One or multiple passes. - New "aggregate-preview" report which includes the new "diff" column making use of this feature. - Add documentation - Deprecates the `BASELINE_EXPRESSIONS` option - nl2br in HTML report for description --- docs/report-generators/expression.rst | 30 ++++ .../Command/report-generators-column-diff | 11 +- .../Command/report-generators-derivations | 22 +++ .../Command/report-generators-expressions | 6 +- lib/Extension/config/report/generators.php | 66 ++++++++ lib/Report/Generator/ExpressionGenerator.php | 142 +++++++++++++++--- templates/html/Report.phtml | 2 +- .../generator/aggregate-preview-console | 13 ++ .../generator/aggregate-preview-csv | 2 + .../generator/aggregate-preview-delimited | 2 + .../generator/aggregate-preview-html | 1 + .../generator/aggregate-preview-json | 1 + .../Generator/expression/derivations-multiple | 36 +++++ .../Generator/expression/derivations-single | 32 ++++ 14 files changed, 333 insertions(+), 33 deletions(-) create mode 100644 examples/Command/report-generators-derivations create mode 100644 tests/Integration/generator/aggregate-preview-console create mode 100644 tests/Integration/generator/aggregate-preview-csv create mode 100644 tests/Integration/generator/aggregate-preview-delimited create mode 100644 tests/Integration/generator/aggregate-preview-html create mode 100644 tests/Integration/generator/aggregate-preview-json create mode 100644 tests/Unit/Report/Generator/expression/derivations-multiple create mode 100644 tests/Unit/Report/Generator/expression/derivations-single diff --git a/docs/report-generators/expression.rst b/docs/report-generators/expression.rst index 47c8aafc9..2407e6bca 100644 --- a/docs/report-generators/expression.rst +++ b/docs/report-generators/expression.rst @@ -129,6 +129,36 @@ Which yields: :language: bash :section: 2 + +.. _generator_expression_derivations: + +Derivations +----------- + +:ref:`Expressions` effectively enable you to +populate a *table* based on the benchmarking :ref:`data `. +**Derivations** allow you to derive new columns from that table in one or more +passes. + +Derivations have access to two variables and each expression is mapped on a row: + +- ``table``: a data frame containing the entire table. ``table['mean']`` for example will yield a **list** containing all of the values in the `mean` column. +- ``row``: the current row to which the derivation will be mapped. ``row['mean']`` for example will yield the value of `mean` in the current row. + +In the following example we derive a new column ``super mean``: + +.. approved:: ../../examples/Command/report-generators-derivations + :language: javascript + :section: 0 + +Which yields: + +.. approved:: ../../examples/Command/report-generators-derivations + :language: bash + :section: 2 + +.. _generator_expression_data: + Data ---- diff --git a/examples/Command/report-generators-column-diff b/examples/Command/report-generators-column-diff index 4468bcb92..082136eff 100644 --- a/examples/Command/report-generators-column-diff +++ b/examples/Command/report-generators-column-diff @@ -11,10 +11,13 @@ "my-report": { "generator": "expression", "aggregate": ["iteration_index"], - "cols": { - "subject": null, - "mode": null, - "diff": "format('%.2fx', mean(result_time_avg) / min(suite['result_time_avg']))" + "cols": [ + "subject", + "mode", + "diff" + ], + "derivations": { + "diff": "if(min(table['mode']) > 0, format('%.2fx', row['mode'] / min(table['mode'])), 'N/A')" } } } diff --git a/examples/Command/report-generators-derivations b/examples/Command/report-generators-derivations new file mode 100644 index 000000000..275c0fba4 --- /dev/null +++ b/examples/Command/report-generators-derivations @@ -0,0 +1,22 @@ +{ + "report.generators": { + "my-report": { + "generator": "expression", + "cols": [ "benchmark", "subject", "mode", "super mode" ], + "derivations": { + "super mode": "row['mode'] ~ ' of ' ~ sum(table['mode'])" + } + } + } +} +--- +phpbench run --report=my-report --executor=debug MultipleSubjects.php --progress=none +--- ++----------------------+---------------+----------+------------+ +| benchmark | subject | mode | super mode | ++----------------------+---------------+----------+------------+ +| MultipleSubjectBench | benchSubject1 | 10.000μs | 10 of 30 | +| MultipleSubjectBench | benchSubject2 | 10.000μs | 10 of 30 | +| MultipleSubjectBench | benchSubject3 | 10.000μs | 10 of 30 | ++----------------------+---------------+----------+------------+ + diff --git a/examples/Command/report-generators-expressions b/examples/Command/report-generators-expressions index c7bc001eb..84b13b602 100644 --- a/examples/Command/report-generators-expressions +++ b/examples/Command/report-generators-expressions @@ -3,9 +3,9 @@ "my-report": { "generator": "expression", "expressions": { - "mode": "\"This is the mode: \" ~ mode(result_time_avg)" + "new_mode": "\"This is the mode: \" ~ mode(result_time_avg)" }, - "cols": [ "benchmark", "subject", "mode" ] + "cols": [ "benchmark", "subject", "new_mode" ] } } } @@ -13,7 +13,7 @@ phpbench run --report=my-report --executor=debug NothingBench.php --progress=none --- +--------------+--------------+----------------------+ -| benchmark | subject | mode | +| benchmark | subject | new_mode | +--------------+--------------+----------------------+ | NothingBench | benchNothing | This is the mode: 10 | +--------------+--------------+----------------------+ diff --git a/lib/Extension/config/report/generators.php b/lib/Extension/config/report/generators.php index 78aee8ba8..ee25da723 100644 --- a/lib/Extension/config/report/generators.php +++ b/lib/Extension/config/report/generators.php @@ -1,5 +1,23 @@ [ 'generator' => 'bare', @@ -21,6 +39,54 @@ 'rstdev', ] ], + 'aggregate-preview' => [ + 'title' => 'Aggregate by Subject', + 'description' => "Aggregate results by subject and group by benchmark class.\nThis is a preview report and may change in minor version updates.", + 'extends' => 'aggregate', + 'break' => [ + 'benchmark', + ], + 'expressions' => [ + 'name' => sprintf( + 'if(first(variant_name) != "", %s, %s)', + 'first(subject_name) ~ " (" ~ first(variant_name) ~ ")"', + 'first(subject_name)' + ), + '_mode' => 'mode(result_time_avg)', + ], + 'cols' => [ + 'name', + 'revs', + 'its', + 'mem_peak', + 'mode', + 'rstdev', + 'diff', + ], + 'baseline_expressions' => [], + 'derivations' => [ + 'diff' => sprintf( + 'if(min(table["_mode"]) > 0, %s, %s)', + 'format("%.2fx", row["_mode"] / min(table["_mode"]))', + '"N/A"' + ), + 'mode' => sprintf( + 'if(row["baseline_exists"], %s, %s)', + $formatTime('row["_mode"]') . ' ~ " " ~ percent_diff(row["baseline_mode"], row["_mode"])', + $formatTime('row["_mode"]'), + ), + 'mem_peak' => sprintf( + 'if(row["baseline_exists"], %s, %s)', + '(row["mem_peak"] as memory) ~ " " ~ percent_diff(row["baseline_mem_peak"], row["mem_peak"])', + 'row["mem_peak"] as memory', + ), + 'rstdev' => sprintf( + 'if(row["baseline_exists"], %s, %s)', + '(row["rstdev"] as "rstdev") ~ " " ~ percent_diff(row["baseline_rstdev"], row["rstdev"])', + 'row["rstdev"] as "rstdev"', + ), + ] + ], 'bar_chart_time' => [ 'generator' => 'component', 'components' => [ diff --git a/lib/Report/Generator/ExpressionGenerator.php b/lib/Report/Generator/ExpressionGenerator.php index 18dace24a..b0c6f8caf 100644 --- a/lib/Report/Generator/ExpressionGenerator.php +++ b/lib/Report/Generator/ExpressionGenerator.php @@ -8,6 +8,7 @@ use PhpBench\Data\DataFrames; use PhpBench\Data\Row; use PhpBench\Expression\Ast\Node; +use PhpBench\Expression\Ast\ScalarValue; use PhpBench\Expression\Ast\StringNode; use PhpBench\Expression\Exception\EvaluationError; use PhpBench\Expression\ExpressionEvaluator; @@ -36,6 +37,7 @@ class ExpressionGenerator implements GeneratorInterface final public const PARAM_AGGREGATE = 'aggregate'; final public const PARAM_BREAK = 'break'; final public const PARAM_INCLUDE_BASELINE = 'include_baseline'; + final public const PARAM_DERIVATIONS = 'derivations'; public function __construct(private readonly ExpressionEvaluator $evaluator, private readonly SuiteCollectionTransformer $transformer, private readonly LoggerInterface $logger) { @@ -69,6 +71,7 @@ public function configure(OptionsResolver $options): void self::PARAM_AGGREGATE => ['suite_tag', 'benchmark_class', 'subject_name', 'variant_index'], self::PARAM_BREAK => [], self::PARAM_INCLUDE_BASELINE => false, + self::PARAM_DERIVATIONS => [], ]); $options->setAllowedTypes(self::PARAM_TITLE, ['null', 'string']); @@ -97,6 +100,15 @@ public function configure(OptionsResolver $options): void ], $expressions); }); $options->setNormalizer(self::PARAM_BASELINE_EXPRESSIONS, function (Options $options, array $expressions) use ($formatTime) { + if (count($expressions) > 0) { + trigger_error(sprintf( + 'The `%s` option is deprecated and will be removed in PHPBench 2.0. '. + 'The `%s` option can be used as an equivalent.', + self::PARAM_BASELINE_EXPRESSIONS, + self::PARAM_DERIVATIONS + ), E_USER_DEPRECATED); + } + return array_merge([ 'best' => $formatTime('min(result_time_avg)'), 'worst' => $formatTime('max(result_time_avg)'), @@ -124,6 +136,7 @@ public function configure(OptionsResolver $options): void self::PARAM_AGGREGATE => 'Group rows by these columns', self::PARAM_BREAK => 'Group tables by these columns', self::PARAM_INCLUDE_BASELINE => 'If the baseline should be included as additional rows, or if it should be inlined', + self::PARAM_DERIVATIONS => 'Derive new columns from existing columns', ]); } @@ -133,6 +146,7 @@ public function configure(OptionsResolver $options): void public function generate(SuiteCollection $collection, Config $config): Reports { $expressionMap = $this->resolveExpressionMap($config); + $baselineExpressionMap = $this->resolveBaselineExpressionMap($config, array_keys($expressionMap)); /** @var bool $includeBaseline */ @@ -160,6 +174,8 @@ public function generate(SuiteCollection $collection, Config $config): Reports // split the evaluated tables by "break" for display $tables = $this->partition($table, $breakCols); + $tables = $this->derivations($tables, $config[self::PARAM_DERIVATIONS], $baselineExpressionMap, $config[self::PARAM_COLS]); + $tables = $this->columns($tables, $config[self::PARAM_COLS], $breakCols); // convert the array into Table instances for rendering return $this->generateReports($tables, $config); @@ -276,6 +292,29 @@ private function partition(array $table, array $breakCols): array return $partitioned; } + /** + * @param string[] $visibleCols + * + * @return array + */ + private function resolveBaselineExpressionMap(Config $config, array $visibleCols): array + { + $map = []; + + /** @var array $baselineExpressions */ + $baselineExpressions = $config[self::PARAM_BASELINE_EXPRESSIONS]; + + foreach ($baselineExpressions as $name => $baselineExpression) { + if (!in_array($name, $visibleCols)) { + continue; + } + $map[$name] = $baselineExpression; + } + + return $map; + } + + /** * @return array */ @@ -283,25 +322,16 @@ private function resolveExpressionMap(Config $config): array { /** @var array $expressions */ $expressions = $config[self::PARAM_EXPRESSIONS]; - $map = []; + /** @var array $cols */ $cols = $config[self::PARAM_COLS]; + $map = $expressions; foreach ($cols as $key => $expr) { if (is_int($key) || null === $expr) { - $expr ??= $key; - - if (!isset($expressions[$expr])) { - throw new RuntimeException(sprintf( - 'No expression with name "%s" is available, available expressions: "%s"', - $expr, - implode('", "', array_keys($expressions)) - )); - } - $map[(string)$expr] = $expressions[$expr]; - continue; } + $map[$key] = $expr; } @@ -309,24 +339,86 @@ private function resolveExpressionMap(Config $config): array } /** - * @param string[] $visibleCols + * @param array>> $tables + * @param array|list> $derivations + * @param array $baselineExprMap + * @param array $cols * - * @return array + * @return array>> */ - private function resolveBaselineExpressionMap(Config $config, array $visibleCols): array + private function derivations(array $tables, array $derivations, array $baselineExprMap, array $cols): array { - $map = []; - - /** @var array $baselineExpressions */ - $baselineExpressions = $config[self::PARAM_BASELINE_EXPRESSIONS]; + if (!array_is_list($derivations)) { + $derivations = [ $derivations ]; + } - foreach ($baselineExpressions as $name => $baselineExpression) { - if (!in_array($name, $visibleCols)) { - continue; + return array_map(function (array $table) use ($derivations, $baselineExprMap, $cols) { + + foreach ($derivations as $derivation) { + $bareTable = array_map(function (array $row) { + return array_map(function (Node $value) { + if ($value instanceof ScalarValue) { + return $value->value(); + } + + return null; + }, $row); + }, $table); + + foreach ($derivation as $col => $expression) { + + // the "baseline" expressions are deprecated but we can't + // drop support for them now. IN the case the derivation + // conflcits with one we ignore the derivation. + if (array_key_exists($col, $baselineExprMap)) { + continue; + } + + // if the user specified a custom expression for a column + // then that takes precedence over derivations + if (array_key_exists($col, $cols) && $cols[$col] !== null) { + continue; + } + + foreach ($table as $i => &$row) { + $frame = DataFrame::fromRecords($bareTable); + $row[$col] = $this->evaluator->evaluate($expression, [ + 'row' => $bareTable[$i], + 'table' => $frame, + ]); + } + } } - $map[$name] = $baselineExpression; - } - return $map; + return $table; + }, $tables); + } + + /** + * @param array>> $tables + * @param list|array $cols + * @param list $breakCols + * + * @return array>> + */ + private function columns(array $tables, array $cols, array $breakCols): array + { + $cols = array_is_list($cols) ? $cols : array_keys($cols); + + return array_map(function (array $table) use ($cols, $breakCols) { + return array_map(function (array $row) use ($cols, $breakCols) { + $new = []; + + foreach ($cols as $col) { + if (in_array($col, $breakCols, true)) { + continue; + } + + $new[$col] = $row[$col] ?? null; + } + + return $new; + }, $table); + }, $tables); } } diff --git a/templates/html/Report.phtml b/templates/html/Report.phtml index 9c5ed285f..b5b833399 100644 --- a/templates/html/Report.phtml +++ b/templates/html/Report.phtml @@ -4,7 +4,7 @@

title() ?>

description()): ?> -

description() ?>

+

description()) ?>

tabbed()): ?> objects() as $object): ?> diff --git a/tests/Integration/generator/aggregate-preview-console b/tests/Integration/generator/aggregate-preview-console new file mode 100644 index 000000000..622d523f1 --- /dev/null +++ b/tests/Integration/generator/aggregate-preview-console @@ -0,0 +1,13 @@ +Aggregate by Subject +==================== + +Aggregate results by subject and group by benchmark class. +This is a preview report and may change in minor version updates. + +TestBench ++--------------+------+-----+----------+-------------+----------+-------+ +| name | revs | its | mem_peak | mode | rstdev | diff | ++--------------+------+-----+----------+-------------+----------+-------+ +| benchOne (0) | 5 | 2 | 200b | 5.0000000μs | ±100.00% | 1.00x | ++--------------+------+-----+----------+-------------+----------+-------+ + diff --git a/tests/Integration/generator/aggregate-preview-csv b/tests/Integration/generator/aggregate-preview-csv new file mode 100644 index 000000000..88c79f0af --- /dev/null +++ b/tests/Integration/generator/aggregate-preview-csv @@ -0,0 +1,2 @@ +name,revs,its,mem_peak,mode,rstdev,diff +"benchOne (0)",5,2,200,5,100,1.00x diff --git a/tests/Integration/generator/aggregate-preview-delimited b/tests/Integration/generator/aggregate-preview-delimited new file mode 100644 index 000000000..6658aeead --- /dev/null +++ b/tests/Integration/generator/aggregate-preview-delimited @@ -0,0 +1,2 @@ +name revs its mem_peak mode rstdev diff +"benchOne (0)" 5 2 200 5 100 1.00x diff --git a/tests/Integration/generator/aggregate-preview-html b/tests/Integration/generator/aggregate-preview-html new file mode 100644 index 000000000..99e30d775 --- /dev/null +++ b/tests/Integration/generator/aggregate-preview-html @@ -0,0 +1 @@ +Written report to: /.phpbench/html/index.html diff --git a/tests/Integration/generator/aggregate-preview-json b/tests/Integration/generator/aggregate-preview-json new file mode 100644 index 000000000..b5c1d7644 --- /dev/null +++ b/tests/Integration/generator/aggregate-preview-json @@ -0,0 +1 @@ +[{"name":"benchOne (0)","revs":5,"its":2,"mem_peak":200,"mode":5,"rstdev":100,"diff":"1.00x"}] diff --git a/tests/Unit/Report/Generator/expression/derivations-multiple b/tests/Unit/Report/Generator/expression/derivations-multiple new file mode 100644 index 000000000..e3ce7a4ee --- /dev/null +++ b/tests/Unit/Report/Generator/expression/derivations-multiple @@ -0,0 +1,36 @@ +{ + "subjects": [ + "one", + "two" + ], + "basetime": 0, + "iterations": [ + 10 + ] +} +--- +{ + "cols": [ + "subject", + "benchmark", + "mode", + "sum", + "sumsum" + ], + "derivations": [ + { + "sum": "sum(table['mode'])" + }, + { + "sumsum": "row['sum'] + row['sum']" + } + ] +} +--- ++---------+-----------+----------+-----+--------+ +| subject | benchmark | mode | sum | sumsum | ++---------+-----------+----------+-----+--------+ +| one | TestBench | 10.000μs | 20 | 40 | +| two | TestBench | 10.000μs | 20 | 40 | ++---------+-----------+----------+-----+--------+ + diff --git a/tests/Unit/Report/Generator/expression/derivations-single b/tests/Unit/Report/Generator/expression/derivations-single new file mode 100644 index 000000000..8d73e1154 --- /dev/null +++ b/tests/Unit/Report/Generator/expression/derivations-single @@ -0,0 +1,32 @@ +{ + "subjects": [ + "one", + "two" + ], + "basetime": 0, + "iterations": [ + 10 + ] +} +--- +{ + "cols": { + "subject": "\"hello\"", + "benchmark": "\"hello\"", + "mode": "mode(result_time_avg)", + "subbench": null + }, + "derivations": [ + { + "subbench": "row['subject'] ~ ' ' ~ row['mode']" + } + ] +} +--- ++---------+-----------+------+----------+ +| subject | benchmark | mode | subbench | ++---------+-----------+------+----------+ +| hello | hello | 10 | hello 10 | +| hello | hello | 10 | hello 10 | ++---------+-----------+------+----------+ +