diff --git a/CMakeLists.txt b/CMakeLists.txt index a5baef7..48824a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,7 @@ find_package(Common REQUIRED) find_package(ReadoutCard REQUIRED) find_package(LLA REQUIRED) find_package(DIM REQUIRED) +find_package(DimRpcParallel REQUIRED) if(NOT APPLE) find_package(Python3 3.6 COMPONENTS Interpreter Development) @@ -127,6 +128,7 @@ target_link_libraries(ALF ${LINK_LIBS} PRIVATE dim::dim + DimRpcParallel ) set_target_properties(ALF @@ -176,6 +178,7 @@ foreach(i RANGE ${count}) PRIVATE ALF dim::dim + DimRpcParallel ) endforeach() diff --git a/apps/Alf.cxx b/apps/Alf.cxx index 8cc9469..6e7730b 100644 --- a/apps/Alf.cxx +++ b/apps/Alf.cxx @@ -58,6 +58,9 @@ class Alf : public AliceO2::Common::Program options.add_options()("no-fw-check", po::bool_switch(&mOptions.noFirmwareCheck)->default_value(false), "Disable firmware compatibility check"); + options.add_options()("sequential", + po::bool_switch(&mOptions.sequentialRpcs)->default_value(false), + "Switch to force DIM RPCs to be executed sequentially"); } virtual void run(const po::variables_map&) override @@ -126,7 +129,7 @@ class Alf : public AliceO2::Common::Program Logger::get() << link.alfId << " " << link.serialId << " " << link.linkId << LogDebugDevel << endm; } } - alfServer.makeRpcServers(links); + alfServer.makeRpcServers(links, mOptions.sequentialRpcs); } // main thread @@ -139,6 +142,7 @@ class Alf : public AliceO2::Common::Program struct OptionsStruct { std::string dimDnsNode = ""; bool noFirmwareCheck = false; + bool sequentialRpcs = false; } mOptions; }; diff --git a/apps/AlfClient.cxx b/apps/AlfClient.cxx index 9d67e23..6a6c78b 100644 --- a/apps/AlfClient.cxx +++ b/apps/AlfClient.cxx @@ -61,6 +61,9 @@ class AlfClient : public AliceO2::Common::Program options.add_options()("link", po::value(&mOptions.link), "Link number"); + options.add_options()("link2", + po::value(&mOptions.link2), + "Link2 number"); options.add_options()("alf-id", po::value(&mOptions.alfId)->default_value(""), "Hostname of node running the ALF server(required)"); @@ -85,6 +88,9 @@ class AlfClient : public AliceO2::Common::Program options.add_options()("swt-stress", po::bool_switch(&mOptions.swtStress)->default_value(false), "Flag enabling the swt-stress tests"); + options.add_options()("parallel-sc", + po::bool_switch(&mOptions.parallelSc)->default_value(false), + "Flag enabling the parallel SC tests"); options.add_options()("swt-stress-cycles", po::value(&mOptions.swtStressCycles)->default_value(2), "Number of cycles for which to run the SWT stress test"); @@ -123,8 +129,10 @@ class AlfClient : public AliceO2::Common::Program std::cout << "Starting the DIM Client using ALF ID=" << alfId << ", card=" << mOptions.serial << ":" << mOptions.endpoint << " and link=" << mOptions.link << std::endl; AlfLink link = AlfLink{ alfId, roc::SerialId(mOptions.serial, mOptions.endpoint), mOptions.link, mOptions.endpoint * 12 + mOptions.link, nullptr, roc::CardType::Cru }; + AlfLink link2 = AlfLink{ alfId, roc::SerialId(mOptions.serial, mOptions.endpoint), mOptions.link2, mOptions.endpoint * 12 + mOptions.link2, nullptr, roc::CardType::Cru }; ServiceNames names(link); + ServiceNames names2(link2); if (mOptions.crorc) { link.cardType = roc::CardType::Crorc; @@ -146,6 +154,7 @@ class AlfClient : public AliceO2::Common::Program LlaSessionStopRpc llaSessionStopRpc(names.llaSessionStop()); SwtSequenceRpc swtSequence(names.swtSequence()); + SwtSequenceRpc swtSequence2(names2.swtSequence()); ScaSequenceRpc scaSequence(names.scaSequence()); IcSequenceRpc icSequence(names.icSequence()); IcGbtI2cWriteRpc icGbtI2cWriteRpc(names.icGbtI2cWrite()); @@ -261,6 +270,27 @@ class AlfClient : public AliceO2::Common::Program } } + if (mOptions.parallelSc) { + std::thread t1( + [&] {auto swtOut = swtSequence.write({ std::make_pair("", "lock"), + std::make_pair("0x00000000000deadbeef", "write"), + std::make_pair("4", "read") }); + std::cout << "[SWT_SEQUENCE L" << mOptions.link << "] output: " << swtOut << std::endl; } + ); + + std::thread t2( + [&] {auto swtOut2 = swtSequence2.write({ std::make_pair("", "lock"), + std::make_pair("0x000000000000badcafe", "write"), + std::make_pair("4", "read") }); + std::cout << "[SWT_SEQUENCE L" << mOptions.link2 << "] output: " << swtOut2 << std::endl; } + ); + + t1.join(); + t2.join(); + + + } + std::cout << "See ya!" << std::endl; } @@ -270,6 +300,7 @@ class AlfClient : public AliceO2::Common::Program int serial = -1; int endpoint = 0; int link = -1; + int link2 = -1; std::string alfId = ""; bool crorc = false; bool ic = false; @@ -278,6 +309,7 @@ class AlfClient : public AliceO2::Common::Program bool swt = false; bool patternPlayer = false; bool swtStress = false; + bool parallelSc = false; int swtStressCycles = 2; int swtStressWords = 1000; } mOptions; diff --git a/include/Alf/ScBase.h b/include/Alf/ScBase.h index bc38a1f..bcc6dc4 100644 --- a/include/Alf/ScBase.h +++ b/include/Alf/ScBase.h @@ -65,7 +65,7 @@ class ScBase void barWrite(uint32_t index, uint32_t data); AlfLink mLink; - std::unique_ptr mLlaSession; + std::shared_ptr mLlaSession; private: /// Does the necessary initializations after an object creating diff --git a/src/AlfServer.cxx b/src/AlfServer.cxx index 1aa7e94..8716408 100644 --- a/src/AlfServer.cxx +++ b/src/AlfServer.cxx @@ -543,13 +543,15 @@ std::vector> AlfServer::parseStringToIcPairs( return pairs; } -void AlfServer::makeRpcServers(std::vector links) +void AlfServer::makeRpcServers(std::vector links, bool sequentialRpcs) { for (auto& link : links) { + // Set a unique parallel dim rpc bank for every link + int parallelDimRpcBank = sequentialRpcs ? 0 : (link.serialId.getSerial() * 100) + link.rawLinkId; // Function to create RPC server auto makeServer = [&](std::string name, auto callback) { - return std::make_unique(name, callback); + return std::make_unique(name, callback, parallelDimRpcBank); }; // Object for generating DNS names for the AlfLink diff --git a/src/AlfServer.h b/src/AlfServer.h index 6f3c860..1e079c7 100644 --- a/src/AlfServer.h +++ b/src/AlfServer.h @@ -48,7 +48,7 @@ class AlfServer { public: AlfServer(); - void makeRpcServers(std::vector links); + void makeRpcServers(std::vector links, bool sequentialRpcs = false); private: std::string scaBlobWrite(const std::string& parameter, AlfLink link); diff --git a/src/DimServices/DimServices.h b/src/DimServices/DimServices.h index d2720dd..f51fa24 100644 --- a/src/DimServices/DimServices.h +++ b/src/DimServices/DimServices.h @@ -26,6 +26,8 @@ #include #include +#include + #include "Alf/Exception.h" #include "Alf/Common.h" #include "ReadoutCard/Register.h" @@ -64,13 +66,13 @@ bool isSuccess(const std::string& str); bool isFailure(const std::string& str); std::string stripPrefix(const std::string& str); -class StringRpcServer : public DimRpc +class StringRpcServer : public DimRpcParallel { public: using Callback = std::function; - StringRpcServer(const std::string& serviceName, Callback callback) - : DimRpc(serviceName.c_str(), "C", "C"), mCallback(callback), mServiceName(serviceName) + StringRpcServer(const std::string& serviceName, Callback callback, int bank) + : DimRpcParallel(serviceName.c_str(), "C", "C", bank), mCallback(callback), mServiceName(serviceName) { } diff --git a/src/Ic.cxx b/src/Ic.cxx index d420b40..57c50af 100644 --- a/src/Ic.cxx +++ b/src/Ic.cxx @@ -45,22 +45,13 @@ namespace o2 namespace alf { -namespace ic_regs -{ -static constexpr roc::Register IC_BASE(0x00f00000); -static constexpr roc::Register IC_WR_DATA(IC_BASE.address + 0x20); -static constexpr roc::Register IC_WR_CFG(IC_BASE.address + 0x24); -static constexpr roc::Register IC_WR_CMD(IC_BASE.address + 0x28); -static constexpr roc::Register IC_RD_DATA(IC_BASE.address + 0x30); -} // namespace ic_regs - Ic::Ic(AlfLink link, std::shared_ptr llaSession) : ScBase(link, llaSession) { Logger::setFacility("ALF/IC"); // Set CFG to 0x3 by default - barWrite(ic_regs::IC_WR_CFG.index, 0x3); + barWrite(sc_regs::IC_WR_CFG.index, 0x3); } Ic::Ic(const roc::Parameters::CardIdType& cardId, int linkId) @@ -69,7 +60,7 @@ Ic::Ic(const roc::Parameters::CardIdType& cardId, int linkId) Logger::setFacility("ALF/IC"); // Set CFG to 0x3 by default - barWrite(ic_regs::IC_WR_CFG.index, 0x3); + barWrite(sc_regs::IC_WR_CFG.index, 0x3); } Ic::Ic(std::string cardId, int linkId) @@ -78,7 +69,7 @@ Ic::Ic(std::string cardId, int linkId) Logger::setFacility("ALF/IC"); // Set CFG to 0x3 by default - barWrite(ic_regs::IC_WR_CFG.index, 0x3); + barWrite(sc_regs::IC_WR_CFG.index, 0x3); } uint32_t Ic::read(uint32_t address) @@ -91,20 +82,22 @@ uint32_t Ic::read(uint32_t address) data = data + address; // Write to the FIFO - barWrite(ic_regs::IC_WR_DATA.index, data); - barWrite(ic_regs::IC_WR_CMD.index, 0x1); - barWrite(ic_regs::IC_WR_CMD.index, 0x0); + barWrite(sc_regs::IC_WR_DATA.index, data); + barWrite(sc_regs::IC_WR_CMD.index, 0x1); + barWrite(sc_regs::IC_WR_CMD.index, 0x0); // Execute the RD State Machine - barWrite(ic_regs::IC_WR_CMD.index, 0x8); - barWrite(ic_regs::IC_WR_CMD.index, 0x0); + barWrite(sc_regs::IC_WR_CMD.index, 0x8); + barWrite(sc_regs::IC_WR_CMD.index, 0x0); + + std::this_thread::sleep_for(std::chrono::milliseconds(10)); // Pulse the READ - barWrite(ic_regs::IC_WR_CMD.index, 0x2); - barWrite(ic_regs::IC_WR_CMD.index, 0x0); + barWrite(sc_regs::IC_WR_CMD.index, 0x2); + barWrite(sc_regs::IC_WR_CMD.index, 0x0); // Read the status of the FIFO - uint32_t ret = barRead(ic_regs::IC_RD_DATA.index); + uint32_t ret = barRead(sc_regs::IC_RD_DATA.index); //uint32_t gbtAddress = (ret >> 8) & 0xff; uint32_t retData = ret & 0xff; //uint32_t empty = (ret >> 16) & 0x1; @@ -124,18 +117,18 @@ uint32_t Ic::write(uint32_t address, uint32_t data) data += address; // Write to the FIFO - barWrite(ic_regs::IC_WR_DATA.index, data); - barWrite(ic_regs::IC_WR_CMD.index, 0x1); - barWrite(ic_regs::IC_WR_CMD.index, 0x0); + barWrite(sc_regs::IC_WR_DATA.index, data); + barWrite(sc_regs::IC_WR_CMD.index, 0x1); + barWrite(sc_regs::IC_WR_CMD.index, 0x0); // Execute the WR State Machine - barWrite(ic_regs::IC_WR_CMD.index, 0x4); - barWrite(ic_regs::IC_WR_CMD.index, 0x0); + barWrite(sc_regs::IC_WR_CMD.index, 0x4); + barWrite(sc_regs::IC_WR_CMD.index, 0x0); std::this_thread::sleep_for(std::chrono::milliseconds(10)); // Read the status of the FIFO - uint32_t ret = barRead(ic_regs::IC_RD_DATA.index); + uint32_t ret = barRead(sc_regs::IC_RD_DATA.index); //uint32_t gbtAddress = (ret >> 8) & 0xff; //uint32_t retData = ret & 0xff; uint32_t empty = (ret >> 16) & 0x1; @@ -149,7 +142,7 @@ uint32_t Ic::write(uint32_t address, uint32_t data) void Ic::writeGbtI2c(uint32_t data) { - barWrite(ic_regs::IC_WR_CFG.index, data); + barWrite(sc_regs::IC_WR_CFG.index, data); } std::vector> Ic::executeSequence(std::vector> ops, bool lock) @@ -184,7 +177,7 @@ std::vector> Ic::executeSequence(std::vector< // If an IC error occurs, we stop executing the sequence of commands and return the results as far as we got them, plus // the error message. IcData icData = boost::get(data); - std::string meaningfulMessage = (boost::format("ic_regs::IC_SEQUENCE address=0x%08x data=0x%08x serialId=%s link=%d, error='%s'") % icData.address % icData.data % mLink.serialId % mLink.linkId % e.what()).str(); + std::string meaningfulMessage = (boost::format("sc_regs::IC_SEQUENCE address=0x%08x data=0x%08x serialId=%s link=%d, error='%s'") % icData.address % icData.data % mLink.serialId % mLink.linkId % e.what()).str(); //Logger::get().err() << meaningfulMessage << endm; ret.push_back({ Operation::Error, meaningfulMessage }); diff --git a/src/Lla.cxx b/src/Lla.cxx index 52719f7..ed7416c 100644 --- a/src/Lla.cxx +++ b/src/Lla.cxx @@ -58,7 +58,9 @@ void LlaSession::start() void LlaSession::stop() { - mSession->stop(); + if (mSession) { + mSession->stop(); + } } } // namespace alf diff --git a/src/ScBase.cxx b/src/ScBase.cxx index 401ea85..e8f80fd 100644 --- a/src/ScBase.cxx +++ b/src/ScBase.cxx @@ -86,7 +86,6 @@ void ScBase::setChannel(int gbtChannel) mLink.linkId = gbtChannel; mLink.rawLinkId = mLink.serialId.getEndpoint() * kCruNumLinks + gbtChannel; - barWrite(sc_regs::SC_LINK.index, mLink.rawLinkId); } void ScBase::checkChannelSet() @@ -95,11 +94,7 @@ void ScBase::checkChannelSet() BOOST_THROW_EXCEPTION(ScException() << ErrorInfo::Message("No channel selected")); } - int channel = (barRead(sc_regs::SWT_MON.index) >> 8) & 0xff; - - if (channel != mLink.rawLinkId) { - setChannel(mLink.linkId); - } + setChannel(mLink.linkId); } void ScBase::scReset() @@ -110,12 +105,14 @@ void ScBase::scReset() void ScBase::barWrite(uint32_t index, uint32_t data) { - mBar2->writeRegister(index, data); + uint32_t linkIndex = (0x00f00000 + (mLink.rawLinkId << 8)) / 4 + index; + mBar2->writeRegister(linkIndex, data); } uint32_t ScBase::barRead(uint32_t index) { - return mBar2->readRegister(index); + uint32_t linkIndex = (0x00f00000 + (mLink.rawLinkId << 8)) / 4 + index; + return mBar2->readRegister(linkIndex); } } // namespace alf diff --git a/src/Swt.cxx b/src/Swt.cxx index f9d1c13..789e2d7 100644 --- a/src/Swt.cxx +++ b/src/Swt.cxx @@ -81,9 +81,6 @@ std::vector Swt::read(SwtWord::Size wordSize, TimeOut msTimeOut) for (int i = 0; i < (int)numWords; i++) { SwtWord tempWord; - barWrite(sc_regs::SWT_CMD.index, 0x2); - barWrite(sc_regs::SWT_CMD.index, 0x0); // void cmd to sync clocks - tempWord.setLow(barRead(sc_regs::SWT_RD_WORD_L.index)); if (wordSize == SwtWord::Size::Medium || wordSize == SwtWord::Size::High) {