From b4dfeef6dfef344dcc60a89f45493aec450b2fb5 Mon Sep 17 00:00:00 2001 From: Maja Kabus Date: Wed, 23 Sep 2020 12:59:15 +0200 Subject: [PATCH 1/4] Added helper fill for expressions --- .../include/Framework/HistogramRegistry.h | 27 +++++++++++ .../Core/test/test_HistogramRegistry.cxx | 45 +++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/Framework/Core/include/Framework/HistogramRegistry.h b/Framework/Core/include/Framework/HistogramRegistry.h index fdaf9f982a712..4bbdc6c3c7ca2 100644 --- a/Framework/Core/include/Framework/HistogramRegistry.h +++ b/Framework/Core/include/Framework/HistogramRegistry.h @@ -344,6 +344,33 @@ class HistogramRegistry std::vector mHistogramCreationCallbacks; }; +template +void fill(TH1* hist, const T& table, const o2::framework::expressions::Filter& filter) +{ + auto filtered = o2::soa::Filtered{{table.asArrowTable()}, o2::framework::expressions::createSelection(table.asArrowTable(), filter)}; + for (auto& t : filtered) { + hist->Fill(*(static_cast(t).getIterator()), *(static_cast(t).getIterator()), *(static_cast(t).getIterator())); + } +} + +template +void fill(TH1* hist, const T& table, const o2::framework::expressions::Filter& filter) +{ + auto filtered = o2::soa::Filtered{{table.asArrowTable()}, o2::framework::expressions::createSelection(table.asArrowTable(), filter)}; + for (auto& t : filtered) { + hist->Fill(*(static_cast(t).getIterator()), *(static_cast(t).getIterator())); + } +} + +template +void fill(TH1* hist, const T& table, const o2::framework::expressions::Filter& filter) +{ + auto filtered = o2::soa::Filtered{{table.asArrowTable()}, o2::framework::expressions::createSelection(table.asArrowTable(), filter)}; + for (auto& t : filtered) { + hist->Fill(*(static_cast(t).getIterator())); + } +} + } // namespace framework } // namespace o2 diff --git a/Framework/Core/test/test_HistogramRegistry.cxx b/Framework/Core/test/test_HistogramRegistry.cxx index 53d4bb0b59900..79280d4113fdb 100644 --- a/Framework/Core/test/test_HistogramRegistry.cxx +++ b/Framework/Core/test/test_HistogramRegistry.cxx @@ -17,6 +17,12 @@ using namespace o2; using namespace o2::framework; +namespace test +{ +DECLARE_SOA_COLUMN_FULL(X, x, float, "x"); +DECLARE_SOA_COLUMN_FULL(Y, y, float, "y"); +} // namespace test + HistogramRegistry foo() { return {"r", true, {{"histo", "histo", {HistogramType::kTH1F, {{100, 0, 1}}}}}}; @@ -48,3 +54,42 @@ BOOST_AUTO_TEST_CASE(HistogramRegistryLookup) auto histo2 = r.get("histo").get(); BOOST_REQUIRE_EQUAL(histo2->GetNbinsX(), 100); } + +BOOST_AUTO_TEST_CASE(HistogramRegistryExpressionFill) +{ + TableBuilder builderA; + auto rowWriterA = builderA.persist({"x", "y"}); + rowWriterA(0, 0.0f, -2.0f); + rowWriterA(0, 1.0f, -4.0f); + rowWriterA(0, 2.0f, -1.0f); + rowWriterA(0, 3.0f, -5.0f); + rowWriterA(0, 4.0f, 0.0f); + rowWriterA(0, 5.0f, -9.0f); + rowWriterA(0, 6.0f, -7.0f); + rowWriterA(0, 7.0f, -4.0f); + auto tableA = builderA.finalize(); + BOOST_REQUIRE_EQUAL(tableA->num_rows(), 8); + using TestA = o2::soa::Table, test::X, test::Y>; + TestA tests{tableA}; + BOOST_REQUIRE_EQUAL(8, tests.size()); + + /// Construct a registry object with direct declaration + HistogramRegistry registry{"registry", true, { + {"x", "test x", {HistogramType::kTH1F, {{100, 0.0f, 10.0f}}}}, // + {"xy", "test xy", {HistogramType::kTH2F, {{100, -10.0f, 10.01f}, {100, -10.0f, 10.01f}}}} // + }}; + + /// Get a pointer to the histogram + auto histx = registry.get("x").get(); + + /// Fill histogram with expression and table + fill(histx, tests, test::x > 3.0f); + BOOST_CHECK_EQUAL(histx->GetEntries(), 4); + + /// Get a pointer to the histogram + auto histxy = registry.get("xy").get(); + + /// Fill histogram with expression and table + fill(histxy, tests, test::x > 3.0f && test::y > -5.0f); + BOOST_CHECK_EQUAL(histxy->GetEntries(), 2); +} From 364b31ddd0026320f3bde1e332e96b6208ae8e1c Mon Sep 17 00:00:00 2001 From: Maja Kabus Date: Wed, 23 Sep 2020 13:52:56 +0200 Subject: [PATCH 2/4] Added custom fill() to hist registry --- .../include/Framework/HistogramRegistry.h | 79 ++++++++++++------- .../Core/test/test_HistogramRegistry.cxx | 14 +--- 2 files changed, 55 insertions(+), 38 deletions(-) diff --git a/Framework/Core/include/Framework/HistogramRegistry.h b/Framework/Core/include/Framework/HistogramRegistry.h index 4bbdc6c3c7ca2..bd7871e22f9e4 100644 --- a/Framework/Core/include/Framework/HistogramRegistry.h +++ b/Framework/Core/include/Framework/HistogramRegistry.h @@ -161,6 +161,34 @@ std::unique_ptr createTH3FromSpec(HistogramSpec const& spec) return std::make_unique(spec.name.data(), spec.readableName.data(), spec.config.axes[0].nBins, spec.config.axes[0].bins.data(), spec.config.axes[1].nBins, spec.config.axes[1].bins.data(), spec.config.axes[2].nBins, spec.config.axes[2].bins.data()); } +/// Helper functions to fill histograms with expressions +template +void fill(TH1* hist, const T& table, const o2::framework::expressions::Filter& filter) +{ + auto filtered = o2::soa::Filtered{{table.asArrowTable()}, o2::framework::expressions::createSelection(table.asArrowTable(), filter)}; + for (auto& t : filtered) { + hist->Fill(*(static_cast(t).getIterator()), *(static_cast(t).getIterator()), *(static_cast(t).getIterator())); + } +} + +template +void fill(TH1* hist, const T& table, const o2::framework::expressions::Filter& filter) +{ + auto filtered = o2::soa::Filtered{{table.asArrowTable()}, o2::framework::expressions::createSelection(table.asArrowTable(), filter)}; + for (auto& t : filtered) { + hist->Fill(*(static_cast(t).getIterator()), *(static_cast(t).getIterator())); + } +} + +template +void fill(TH1* hist, const T& table, const o2::framework::expressions::Filter& filter) +{ + auto filtered = o2::soa::Filtered{{table.asArrowTable()}, o2::framework::expressions::createSelection(table.asArrowTable(), filter)}; + for (auto& t : filtered) { + hist->Fill(*(static_cast(t).getIterator())); + } +} + using HistogramCreationCallback = std::function(HistogramSpec const& spec)>; // Wrapper to avoid multiple function definitinions error @@ -278,7 +306,7 @@ class HistogramRegistry return get(name); } - // @return the associated OutputSpec + /// @return the associated OutputSpec OutputSpec const spec() { ConcreteDataMatcher matcher{"HIST", "\0", 0}; @@ -308,6 +336,28 @@ class HistogramRegistry return folder; } + /// fill the histogram with an expression + template + void fill(char const* const name, const T& table, const o2::framework::expressions::Filter& filter) + { + TH1* hist = get(name).get(); + framework::fill(hist, table, filter); + } + + template + void fill(char const* const name, const T& table, const o2::framework::expressions::Filter& filter) + { + TH1* hist = get(name).get(); + framework::fill(hist, table, filter); + } + + template + void fill(char const* const name, const T& table, const o2::framework::expressions::Filter& filter) + { + TH1* hist = get(name).get(); + framework::fill(hist, table, filter); + } + /// lookup distance counter for benchmarking mutable uint32_t lookup = 0; @@ -344,33 +394,6 @@ class HistogramRegistry std::vector mHistogramCreationCallbacks; }; -template -void fill(TH1* hist, const T& table, const o2::framework::expressions::Filter& filter) -{ - auto filtered = o2::soa::Filtered{{table.asArrowTable()}, o2::framework::expressions::createSelection(table.asArrowTable(), filter)}; - for (auto& t : filtered) { - hist->Fill(*(static_cast(t).getIterator()), *(static_cast(t).getIterator()), *(static_cast(t).getIterator())); - } -} - -template -void fill(TH1* hist, const T& table, const o2::framework::expressions::Filter& filter) -{ - auto filtered = o2::soa::Filtered{{table.asArrowTable()}, o2::framework::expressions::createSelection(table.asArrowTable(), filter)}; - for (auto& t : filtered) { - hist->Fill(*(static_cast(t).getIterator()), *(static_cast(t).getIterator())); - } -} - -template -void fill(TH1* hist, const T& table, const o2::framework::expressions::Filter& filter) -{ - auto filtered = o2::soa::Filtered{{table.asArrowTable()}, o2::framework::expressions::createSelection(table.asArrowTable(), filter)}; - for (auto& t : filtered) { - hist->Fill(*(static_cast(t).getIterator())); - } -} - } // namespace framework } // namespace o2 diff --git a/Framework/Core/test/test_HistogramRegistry.cxx b/Framework/Core/test/test_HistogramRegistry.cxx index 79280d4113fdb..fc394c20efccc 100644 --- a/Framework/Core/test/test_HistogramRegistry.cxx +++ b/Framework/Core/test/test_HistogramRegistry.cxx @@ -79,17 +79,11 @@ BOOST_AUTO_TEST_CASE(HistogramRegistryExpressionFill) {"xy", "test xy", {HistogramType::kTH2F, {{100, -10.0f, 10.01f}, {100, -10.0f, 10.01f}}}} // }}; - /// Get a pointer to the histogram - auto histx = registry.get("x").get(); - /// Fill histogram with expression and table - fill(histx, tests, test::x > 3.0f); - BOOST_CHECK_EQUAL(histx->GetEntries(), 4); - - /// Get a pointer to the histogram - auto histxy = registry.get("xy").get(); + registry.fill("x", tests, test::x > 3.0f); + BOOST_CHECK_EQUAL(registry.get("x")->GetEntries(), 4); /// Fill histogram with expression and table - fill(histxy, tests, test::x > 3.0f && test::y > -5.0f); - BOOST_CHECK_EQUAL(histxy->GetEntries(), 2); + registry.fill("xy", tests, test::x > 3.0f && test::y > -5.0f); + BOOST_CHECK_EQUAL(registry.get("xy")->GetEntries(), 2); } From 9e7ca724200934370038f19b526e3481992a5929 Mon Sep 17 00:00:00 2001 From: Maja Kabus Date: Wed, 23 Sep 2020 17:42:42 +0200 Subject: [PATCH 3/4] Move partition filter creation to standalone function --- .../Core/include/Framework/AnalysisHelpers.h | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/Framework/Core/include/Framework/AnalysisHelpers.h b/Framework/Core/include/Framework/AnalysisHelpers.h index 5907634f8873c..046e0f2ea600f 100644 --- a/Framework/Core/include/Framework/AnalysisHelpers.h +++ b/Framework/Core/include/Framework/AnalysisHelpers.h @@ -460,6 +460,25 @@ struct Service { } }; +template +o2::soa::Filtered* getTableFromFilter(const T& table, const expressions::Filter& filter) +{ + auto schema = table.asArrowTable()->schema(); + expressions::Operations ops = createOperations(filter); + gandiva::NodePtr tree = nullptr; + if (isSchemaCompatible(schema, ops)) { + tree = createExpressionTree(ops, schema); + } else { + throw std::runtime_error("Partition filter does not match declared table type"); + } + + if constexpr (soa::is_soa_filtered_t>::value) { + return new o2::soa::Filtered{{table}, tree}; + } else { + return new o2::soa::Filtered{{table.asArrowTable()}, tree}; + } +} + template struct Partition { Partition(expressions::Node&& filter_) : filter{std::move(filter_)} @@ -468,19 +487,7 @@ struct Partition { void setTable(const T& table) { - auto schema = table.asArrowTable()->schema(); - expressions::Operations ops = createOperations(filter); - if (isSchemaCompatible(schema, ops)) { - mTree = createExpressionTree(ops, schema); - } else { - throw std::runtime_error("Partition filter does not match declared table type"); - } - - if constexpr (soa::is_soa_filtered_t>::value) { - mFiltered.reset(new o2::soa::Filtered{{table}, mTree}); - } else { - mFiltered.reset(new o2::soa::Filtered{{table.asArrowTable()}, mTree}); - } + mFiltered.reset(getTableFromFilter(table, filter)); } template @@ -505,7 +512,6 @@ struct Partition { } expressions::Filter filter; - gandiva::NodePtr mTree = nullptr; std::unique_ptr> mFiltered = nullptr; using filtered_iterator = typename o2::soa::Filtered::iterator; From e4494de0baf0e15e483b0c1c12e8902a624e25e2 Mon Sep 17 00:00:00 2001 From: Maja Kabus Date: Thu, 24 Sep 2020 15:32:47 +0200 Subject: [PATCH 4/4] Added analysis task for new hist registry fill --- Analysis/Tutorials/src/histogramRegistry.cxx | 21 +++++++++++++++- .../Core/test/test_HistogramRegistry.cxx | 24 +++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/Analysis/Tutorials/src/histogramRegistry.cxx b/Analysis/Tutorials/src/histogramRegistry.cxx index f2ee48279b2d0..6a14f665de918 100644 --- a/Analysis/Tutorials/src/histogramRegistry.cxx +++ b/Analysis/Tutorials/src/histogramRegistry.cxx @@ -41,8 +41,27 @@ struct ATask { } }; +struct BTask { + /// Construct a registry object with direct declaration + HistogramRegistry registry{ + "registry", + true, + { + {"eta", "#eta", {HistogramType::kTH1F, {{102, -2.01, 2.01}}}}, // + {"ptToPt", "#ptToPt", {HistogramType::kTH2F, {{100, -0.01, 10.01}, {100, -0.01, 10.01}}}} // + } // + }; + + void process(aod::Tracks const& tracks) + { + registry.fill("eta", tracks, aod::track::eta > 0.0f); + registry.fill("ptToPt", tracks, aod::track::pt < 5.0f); + } +}; + WorkflowSpec defineDataProcessing(ConfigContext const&) { return WorkflowSpec{ - adaptAnalysisTask("eta-and-phi-histograms")}; + adaptAnalysisTask("eta-and-phi-histograms"), + adaptAnalysisTask("filtered-histograms")}; } diff --git a/Framework/Core/test/test_HistogramRegistry.cxx b/Framework/Core/test/test_HistogramRegistry.cxx index fc394c20efccc..0ecb7c367f16f 100644 --- a/Framework/Core/test/test_HistogramRegistry.cxx +++ b/Framework/Core/test/test_HistogramRegistry.cxx @@ -31,12 +31,14 @@ HistogramRegistry foo() BOOST_AUTO_TEST_CASE(HistogramRegistryLookup) { /// Construct a registry object with direct declaration - HistogramRegistry registry{"registry", true, { - {"eta", "#Eta", {HistogramType::kTH1F, {{100, -2.0, 2.0}}}}, // - {"phi", "#Phi", {HistogramType::kTH1D, {{102, 0, 2 * M_PI}}}}, // - {"pt", "p_{T}", {HistogramType::kTH1D, {{1002, -0.01, 50.1}}}}, // - {"ptToPt", "#ptToPt", {HistogramType::kTH2F, {{100, -0.01, 10.01}, {100, -0.01, 10.01}}}} // - }}; + HistogramRegistry registry{ + "registry", true, { + {"eta", "#Eta", {HistogramType::kTH1F, {{100, -2.0, 2.0}}}}, // + {"phi", "#Phi", {HistogramType::kTH1D, {{102, 0, 2 * M_PI}}}}, // + {"pt", "p_{T}", {HistogramType::kTH1D, {{1002, -0.01, 50.1}}}}, // + {"ptToPt", "#ptToPt", {HistogramType::kTH2F, {{100, -0.01, 10.01}, {100, -0.01, 10.01}}}} // + } // + }; /// Get histograms by name BOOST_REQUIRE_EQUAL(registry.get("eta")->GetNbinsX(), 100); @@ -74,10 +76,12 @@ BOOST_AUTO_TEST_CASE(HistogramRegistryExpressionFill) BOOST_REQUIRE_EQUAL(8, tests.size()); /// Construct a registry object with direct declaration - HistogramRegistry registry{"registry", true, { - {"x", "test x", {HistogramType::kTH1F, {{100, 0.0f, 10.0f}}}}, // - {"xy", "test xy", {HistogramType::kTH2F, {{100, -10.0f, 10.01f}, {100, -10.0f, 10.01f}}}} // - }}; + HistogramRegistry registry{ + "registry", true, { + {"x", "test x", {HistogramType::kTH1F, {{100, 0.0f, 10.0f}}}}, // + {"xy", "test xy", {HistogramType::kTH2F, {{100, -10.0f, 10.01f}, {100, -10.0f, 10.01f}}}} // + } // + }; /// Fill histogram with expression and table registry.fill("x", tests, test::x > 3.0f);