From 82cad99d7014347333465d1df272b29214038af9 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Tue, 22 Oct 2024 15:15:14 +0200 Subject: [PATCH 1/7] [QC-1206] Make the database available to the UserCodeInterface + refactor *Config to extract common part - needed for the CTP Scalers - it required adding the database param to all the *Config.h I decided to extract the common part to a new interface `UserCodeConfig`. --- .../include/QualityControl/AggregatorConfig.h | 9 +---- Framework/include/QualityControl/Check.h | 5 --- .../include/QualityControl/CheckConfig.h | 9 +---- .../include/QualityControl/CheckInterface.h | 8 ---- .../QualityControl/PostProcessingConfig.h | 12 +----- .../include/QualityControl/TaskRunnerConfig.h | 10 +---- .../include/QualityControl/UserCodeConfig.h | 39 +++++++++++++++++++ .../QualityControl/UserCodeInterface.h | 5 ++- Framework/src/Aggregator.cxx | 9 +++-- Framework/src/Check.cxx | 8 +++- Framework/src/CheckRunner.cxx | 1 - Framework/src/PostProcessingConfig.cxx | 17 +++++--- Framework/src/PostProcessingFactory.cxx | 1 + Framework/src/PostProcessingRunner.cxx | 2 +- Framework/src/TaskRunner.cxx | 2 + Framework/src/TaskRunnerFactory.cxx | 15 +++---- Framework/src/TriggerHelpers.cxx | 6 +-- Framework/src/UserCodeInterface.cxx | 24 +++++++++++- Framework/test/testCheck.cxx | 10 +++-- Framework/test/testCheckWorkflow.cxx | 4 +- Framework/test/testTaskInterface.cxx | 9 ++++- Framework/test/testTriggerHelpers.cxx | 4 +- 22 files changed, 129 insertions(+), 80 deletions(-) create mode 100644 Framework/include/QualityControl/UserCodeConfig.h diff --git a/Framework/include/QualityControl/AggregatorConfig.h b/Framework/include/QualityControl/AggregatorConfig.h index 2ec3c56b66..83230948bc 100644 --- a/Framework/include/QualityControl/AggregatorConfig.h +++ b/Framework/include/QualityControl/AggregatorConfig.h @@ -24,25 +24,20 @@ #include #include "QualityControl/UpdatePolicyType.h" #include "QualityControl/AggregatorSource.h" -#include "QualityControl/CustomParameters.h" +#include "QualityControl/UserCodeConfig.h" namespace o2::quality_control::checker { /// \brief Container for the configuration of an Aggregator. -struct AggregatorConfig { +struct AggregatorConfig : public o2::quality_control::core::UserCodeConfig { std::string name; - std::string moduleName; - std::string className; - std::string detectorName = "MISC"; // intended to be the 3 letters code; - core::CustomParameters customParameters; UpdatePolicyType policyType = UpdatePolicyType::OnAny; std::vector objectNames{}; // fixme: if object names are empty, allObjects are true, consider reducing to one var // fixme: duplicates "sources" bool allObjects = false; framework::Inputs inputSpecs{}; framework::OutputSpec qoSpec{ "XXX", "INVALID" }; std::vector sources; - std::string conditionUrl{}; }; } // namespace o2::quality_control::checker diff --git a/Framework/include/QualityControl/Check.h b/Framework/include/QualityControl/Check.h index 2d9b969c29..147b45ccd5 100644 --- a/Framework/include/QualityControl/Check.h +++ b/Framework/include/QualityControl/Check.h @@ -94,11 +94,6 @@ class Check static CheckConfig extractConfig(const core::CommonSpec&, const CheckSpec&); static framework::OutputSpec createOutputSpec(const std::string& detector, const std::string& checkName); - void setDatabase(std::shared_ptr database) - { - mCheckInterface->setDatabase(database); - } - private: void beautify(std::map>& moMap, const core::Quality& quality); diff --git a/Framework/include/QualityControl/CheckConfig.h b/Framework/include/QualityControl/CheckConfig.h index 7524f462d4..1b70dfae77 100644 --- a/Framework/include/QualityControl/CheckConfig.h +++ b/Framework/include/QualityControl/CheckConfig.h @@ -22,25 +22,20 @@ #include #include "QualityControl/UpdatePolicyType.h" -#include "QualityControl/CustomParameters.h" +#include "QualityControl/UserCodeConfig.h" namespace o2::quality_control::checker { /// \brief Container for the configuration of a Check. -struct CheckConfig { +struct CheckConfig : public o2::quality_control::core::UserCodeConfig { std::string name; - std::string moduleName; - std::string className; - std::string detectorName = "MISC"; // intended to be the 3 letters code; - core::CustomParameters customParameters; UpdatePolicyType policyType = UpdatePolicyType::OnAny; std::vector objectNames{}; // fixme: if object names are empty, allObjects are true, consider reducing to one var bool allObjects = false; bool allowBeautify = false; framework::Inputs inputSpecs{}; framework::OutputSpec qoSpec{ "XXX", "INVALID" }; - std::string conditionUrl{}; }; } // namespace o2::quality_control::checker diff --git a/Framework/include/QualityControl/CheckInterface.h b/Framework/include/QualityControl/CheckInterface.h index b1bb206167..ce19d1ff97 100644 --- a/Framework/include/QualityControl/CheckInterface.h +++ b/Framework/include/QualityControl/CheckInterface.h @@ -83,11 +83,6 @@ class CheckInterface : public core::UserCodeInterface virtual void startOfActivity(const core::Activity& activity); // not fully abstract because we don't want to change all the existing subclasses virtual void endOfActivity(const core::Activity& activity); // not fully abstract because we don't want to change all the existing subclasses - void setDatabase(std::shared_ptr database) - { - mDatabase = database; - } - protected: /// \brief Called each time mCustomParameters is updated. virtual void configure() override; @@ -100,9 +95,6 @@ class CheckInterface : public core::UserCodeInterface /// \return std::shared_ptr retrieveReference(std::string path, Activity referenceActivity); - private: - std::shared_ptr mDatabase; - ClassDef(CheckInterface, 6) }; diff --git a/Framework/include/QualityControl/PostProcessingConfig.h b/Framework/include/QualityControl/PostProcessingConfig.h index b1532fa68b..88a13d2e62 100644 --- a/Framework/include/QualityControl/PostProcessingConfig.h +++ b/Framework/include/QualityControl/PostProcessingConfig.h @@ -17,12 +17,11 @@ #ifndef QUALITYCONTROL_POSTPROCESSINGCONFIG_H #define QUALITYCONTROL_POSTPROCESSINGCONFIG_H -#include "QualityControl/CustomParameters.h" - #include #include #include #include "QualityControl/Activity.h" +#include "QualityControl/UserCodeConfig.h" namespace o2::quality_control::postprocessing { @@ -30,27 +29,20 @@ namespace o2::quality_control::postprocessing // todo pretty print /// \brief Post-processing configuration structure -struct PostProcessingConfig { +struct PostProcessingConfig : public o2::quality_control::core::UserCodeConfig { PostProcessingConfig() = default; PostProcessingConfig(const std::string& id, const boost::property_tree::ptree& config); ~PostProcessingConfig() = default; std::string id; std::string taskName; - std::string moduleName; - std::string className; - std::string detectorName = "MISC"; std::vector initTriggers = {}; std::vector updateTriggers = {}; std::vector stopTriggers = {}; - std::string qcdbUrl; - std::string ccdbUrl; - std::string consulUrl; std::string kafkaBrokersUrl; std::string kafkaTopic; core::Activity activity; bool matchAnyRunNumber = false; bool critical; - core::CustomParameters customParameters; }; } // namespace o2::quality_control::postprocessing diff --git a/Framework/include/QualityControl/TaskRunnerConfig.h b/Framework/include/QualityControl/TaskRunnerConfig.h index 3411cefb3a..eefb5f88d2 100644 --- a/Framework/include/QualityControl/TaskRunnerConfig.h +++ b/Framework/include/QualityControl/TaskRunnerConfig.h @@ -24,7 +24,7 @@ #include #include "QualityControl/Activity.h" #include "QualityControl/LogDiscardParameters.h" -#include "QualityControl/CustomParameters.h" +#include "QualityControl/UserCodeConfig.h" namespace o2::base { @@ -40,23 +40,17 @@ namespace o2::quality_control::core { /// \brief Container for the configuration of a Task -struct TaskRunnerConfig { +struct TaskRunnerConfig : public UserCodeConfig { std::string deviceName; std::string taskName; - std::string moduleName; - std::string className; std::vector> cycleDurations = {}; int maxNumberCycles; bool critical; - std::string consulUrl{}; - std::string conditionUrl{}; std::string monitoringUrl{}; std::string bookkeepingUrl{}; framework::Inputs inputSpecs{}; framework::OutputSpec moSpec{ "XXX", "INVALID" }; framework::Options options{}; - CustomParameters customParameters; - std::string detectorName = "MISC"; // intended to be the 3 letters code int parallelTaskID = 0; // ID to differentiate parallel local Tasks from one another. 0 means this is the only one. std::string saveToFile{}; int resetAfterCycles = 0; diff --git a/Framework/include/QualityControl/UserCodeConfig.h b/Framework/include/QualityControl/UserCodeConfig.h new file mode 100644 index 0000000000..b829dd53e2 --- /dev/null +++ b/Framework/include/QualityControl/UserCodeConfig.h @@ -0,0 +1,39 @@ +// Copyright 2019-2020 CERN and copyright holders of ALICE O2. +// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders. +// All rights not expressly granted are reserved. +// +// This software is distributed under the terms of the GNU General Public +// License v3 (GPL Version 3), copied verbatim in the file "COPYING". +// +// In applying this license CERN does not waive the privileges and immunities +// granted to it by virtue of its status as an Intergovernmental Organization +// or submit itself to any jurisdiction. + +/// +/// \file UserCodeConfig.h +/// \author Barthelemy von Haller +/// + +#ifndef QUALITYCONTROL_USERCODECONFIG_H +#define QUALITYCONTROL_USERCODECONFIG_H + +#include "QualityControl/CustomParameters.h" +#include "QualityControl/stringUtils.h" + +namespace o2::quality_control::core +{ + +/// \brief Container for the configuration of a Task +struct UserCodeConfig { + std::string moduleName; + std::string className; + std::string detectorName = "MISC"; // intended to be the 3 letters code; + std::string consulUrl; + CustomParameters customParameters; + std::string conditionUrl{}; + std::unordered_map database; +}; + +} // namespace o2::quality_control::core + +#endif // QUALITYCONTROL_USERCODECONFIG_H diff --git a/Framework/include/QualityControl/UserCodeInterface.h b/Framework/include/QualityControl/UserCodeInterface.h index 42dacb5c04..8cd26dd310 100644 --- a/Framework/include/QualityControl/UserCodeInterface.h +++ b/Framework/include/QualityControl/UserCodeInterface.h @@ -23,6 +23,7 @@ #include "QualityControl/ConditionAccess.h" #include "QualityControl/CustomParameters.h" +#include "QualityControl/DatabaseInterface.h" namespace o2::quality_control::core { @@ -48,12 +49,14 @@ class UserCodeInterface : public ConditionAccess const std::string& getName() const; void setName(const std::string& name); + void setDatabase(std::unordered_map dbConfig); protected: CustomParameters mCustomParameters; std::string mName; + std::shared_ptr mDatabase; - ClassDef(UserCodeInterface, 3) + ClassDef(UserCodeInterface, 4) }; } // namespace o2::quality_control::core diff --git a/Framework/src/Aggregator.cxx b/Framework/src/Aggregator.cxx index c5269c00cb..5c0c25ce7d 100644 --- a/Framework/src/Aggregator.cxx +++ b/Framework/src/Aggregator.cxx @@ -52,6 +52,7 @@ void Aggregator::init() mAggregatorInterface->setName(mAggregatorConfig.name); mAggregatorInterface->setCustomParameters(mAggregatorConfig.customParameters); mAggregatorInterface->setCcdbUrl(mAggregatorConfig.conditionUrl); + mAggregatorInterface->setDatabase(mAggregatorConfig.database); mAggregatorInterface->configure(); } catch (...) { std::string diagnostic = boost::current_exception_diagnostic_information(); @@ -217,18 +218,20 @@ AggregatorConfig Aggregator::extractConfig(const core::CommonSpec& commonSpec, c } return { - aggregatorSpec.aggregatorName, aggregatorSpec.moduleName, aggregatorSpec.className, aggregatorSpec.detectorName, + commonSpec.consulUrl, aggregatorSpec.customParameters, + commonSpec.conditionDBUrl, + commonSpec.database, + aggregatorSpec.aggregatorName, updatePolicy, std::move(objectNames), checkAllObjects, std::move(inputs), createOutputSpec(aggregatorSpec.detectorName, aggregatorSpec.aggregatorName), - sources, - commonSpec.conditionDBUrl + sources }; } diff --git a/Framework/src/Check.cxx b/Framework/src/Check.cxx index 9b232deae1..042d5a1609 100644 --- a/Framework/src/Check.cxx +++ b/Framework/src/Check.cxx @@ -28,6 +28,7 @@ #include "QualityControl/QcInfoLogger.h" #include "QualityControl/Quality.h" #include "QualityControl/HashDataDescription.h" +#include "QualityControl/runnerUtils.h" #include @@ -70,6 +71,7 @@ void Check::init() try { mCheckInterface = root_class_factory::create(mCheckConfig.moduleName, mCheckConfig.className); mCheckInterface->setName(mCheckConfig.name); + mCheckInterface->setDatabase(mCheckConfig.database); mCheckInterface->setCustomParameters(mCheckConfig.customParameters); mCheckInterface->setCcdbUrl(mCheckConfig.conditionUrl); } catch (...) { @@ -250,18 +252,20 @@ CheckConfig Check::extractConfig(const CommonSpec& commonSpec, const CheckSpec& } return { - checkSpec.checkName, checkSpec.moduleName, checkSpec.className, checkSpec.detectorName, + commonSpec.consulUrl, checkSpec.customParameters, + commonSpec.conditionDBUrl, + commonSpec.database, + checkSpec.checkName, updatePolicy, std::move(objectNames), checkAllObjects, allowBeautify, std::move(inputs), createOutputSpec(checkSpec.detectorName, checkSpec.checkName), - commonSpec.conditionDBUrl }; } diff --git a/Framework/src/CheckRunner.cxx b/Framework/src/CheckRunner.cxx index ac96a12a65..b677294e5e 100644 --- a/Framework/src/CheckRunner.cxx +++ b/Framework/src/CheckRunner.cxx @@ -190,7 +190,6 @@ void CheckRunner::init(framework::InitContext& iCtx) updatePolicyManager.reset(); for (auto& [checkName, check] : mChecks) { check.init(); - check.setDatabase(mDatabase); updatePolicyManager.addPolicy(check.getName(), check.getUpdatePolicyType(), check.getObjectsNames(), check.getAllObjectsOption(), false); } } catch (...) { diff --git a/Framework/src/PostProcessingConfig.cxx b/Framework/src/PostProcessingConfig.cxx index af431790db..49a521fda4 100644 --- a/Framework/src/PostProcessingConfig.cxx +++ b/Framework/src/PostProcessingConfig.cxx @@ -24,12 +24,6 @@ namespace o2::quality_control::postprocessing PostProcessingConfig::PostProcessingConfig(const std::string& id, const boost::property_tree::ptree& config) // : id(id), taskName(config.get("qc.postprocessing." + id + ".taskName", id)), - moduleName(config.get("qc.postprocessing." + id + ".moduleName")), - className(config.get("qc.postprocessing." + id + ".className")), - detectorName(config.get("qc.postprocessing." + id + ".detectorName", "MISC")), - qcdbUrl(config.get("qc.config.database.implementation") == "CCDB" ? config.get("qc.config.database.host") : ""), - ccdbUrl(config.get("qc.config.conditionDB.url", "")), - consulUrl(config.get("qc.config.consul.url", "")), activity(config.get("qc.config.Activity.number", 0), config.get("qc.config.Activity.type", "NONE"), config.get("qc.config.Activity.periodName", ""), @@ -40,6 +34,17 @@ PostProcessingConfig::PostProcessingConfig(const std::string& id, const boost::p matchAnyRunNumber(config.get("qc.config.postprocessing.matchAnyRunNumber", false)), critical(true) { + moduleName = config.get("qc.postprocessing." + id + ".moduleName"); + className = config.get("qc.postprocessing." + id + ".className"); + detectorName = config.get("qc.postprocessing." + id + ".detectorName", "MISC"); + consulUrl = config.get("qc.config.consul.url", ""); + conditionUrl = config.get("qc.config.conditionDB.url", ""); + std::unordered_map dbConfig { + {"implementation", config.get("qc.config.database.implementation")}, + {"host", config.get("qc.config.database.host")} + }; + database = dbConfig; + for (const auto& initTrigger : config.get_child("qc.postprocessing." + id + ".initTrigger")) { initTriggers.push_back(initTrigger.second.get_value()); } diff --git a/Framework/src/PostProcessingFactory.cxx b/Framework/src/PostProcessingFactory.cxx index 600e6cc10a..dbe1f376fd 100644 --- a/Framework/src/PostProcessingFactory.cxx +++ b/Framework/src/PostProcessingFactory.cxx @@ -29,6 +29,7 @@ PostProcessingInterface* PostProcessingFactory::create(const PostProcessingConfi { auto* result = root_class_factory::create(config.moduleName, config.className); result->setCustomParameters(config.customParameters); + result->setDatabase(config.database); return result; } diff --git a/Framework/src/PostProcessingRunner.cxx b/Framework/src/PostProcessingRunner.cxx index b2a7636c82..4a192c0802 100644 --- a/Framework/src/PostProcessingRunner.cxx +++ b/Framework/src/PostProcessingRunner.cxx @@ -114,7 +114,7 @@ void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig, mTask->setID(mTaskConfig.id); mTask->setName(mTaskConfig.taskName); mTask->setCustomParameters(mTaskConfig.customParameters); - mTask->setCcdbUrl(mTaskConfig.ccdbUrl); + mTask->setCcdbUrl(mTaskConfig.conditionUrl); mTask->configure(mRunnerConfig.configTree); } else { throw std::runtime_error("Failed to create the task '" + mTaskConfig.taskName + "' (det " + mTaskConfig.detectorName + ")"); diff --git a/Framework/src/TaskRunner.cxx b/Framework/src/TaskRunner.cxx index 9f1a418104..813a43482a 100644 --- a/Framework/src/TaskRunner.cxx +++ b/Framework/src/TaskRunner.cxx @@ -49,6 +49,7 @@ #include "QualityControl/ActivityHelpers.h" #include "QualityControl/WorkflowType.h" #include "QualityControl/HashDataDescription.h" +#include "QualityControl/runnerUtils.h" #include #include @@ -122,6 +123,7 @@ void TaskRunner::init(InitContext& iCtx) mTask.reset(TaskFactory::create(mTaskConfig, mObjectsManager)); mTask->setMonitoring(mCollector); mTask->setGlobalTrackingDataRequest(mTaskConfig.globalTrackingDataRequest); + mTask->setDatabase(mTaskConfig.database); // load config params if (!ConfigParamGlo::keyValues.empty()) { diff --git a/Framework/src/TaskRunnerFactory.cxx b/Framework/src/TaskRunnerFactory.cxx index b562e0ab39..654dcb1bae 100644 --- a/Framework/src/TaskRunnerFactory.cxx +++ b/Framework/src/TaskRunnerFactory.cxx @@ -168,22 +168,23 @@ TaskRunnerConfig TaskRunnerFactory::extractConfig(const CommonSpec& globalConfig o2::globaltracking::RecoContainer rd; return { - deviceName, - taskSpec.taskName, taskSpec.moduleName, taskSpec.className, + InfrastructureSpecReader::validateDetectorName(taskSpec.detectorName), + globalConfig.consulUrl, + taskSpec.customParameters, + globalConfig.conditionDBUrl, + globalConfig.database, + deviceName, + taskSpec.taskName, multipleCycleDurations, taskSpec.maxNumberCycles, taskSpec.critical, - globalConfig.consulUrl, - globalConfig.conditionDBUrl, globalConfig.monitoringUrl, globalConfig.bookkeepingUrl, inputs, monitorObjectsSpec, options, - taskSpec.customParameters, - InfrastructureSpecReader::validateDetectorName(taskSpec.detectorName), parallelTaskID, taskSpec.saveObjectsToFile, resetAfterCycles.value_or(taskSpec.resetAfterCycles), @@ -192,7 +193,7 @@ TaskRunnerConfig TaskRunnerFactory::extractConfig(const CommonSpec& globalConfig grpGeomRequest, globalTrackingDataRequest, taskSpec.movingWindows, - taskSpec.disableLastCycle + taskSpec.disableLastCycle, }; } diff --git a/Framework/src/TriggerHelpers.cxx b/Framework/src/TriggerHelpers.cxx index 22cd309c80..d1c4f14df2 100644 --- a/Framework/src/TriggerHelpers.cxx +++ b/Framework/src/TriggerHelpers.cxx @@ -104,15 +104,15 @@ TriggerFcn triggerFactory(const std::string& trigger, const PostProcessingConfig return triggers::EndOfFill(activity); } else if (triggerLowerCase.find("newobject") != std::string::npos) { const auto [db, objectPath] = parseDbTriggers(trigger, "newobject"); - const std::string& dbUrl = db == "qcdb" ? config.qcdbUrl : config.ccdbUrl; + const std::string& dbUrl = db == "qcdb" ? config.database.at("host") : config.conditionUrl; return triggers::NewObject(dbUrl, db, objectPath, activity, trigger); } else if (triggerLowerCase.find("foreachobject") != std::string::npos) { const auto [db, objectPath] = parseDbTriggers(trigger, "foreachobject"); - const std::string& dbUrl = db == "qcdb" ? config.qcdbUrl : config.ccdbUrl; + const std::string& dbUrl = db == "qcdb" ? config.database.at("host") : config.conditionUrl; return triggers::ForEachObject(dbUrl, db, objectPath, activity, trigger); } else if (triggerLowerCase.find("foreachlatest") != std::string::npos) { const auto [db, objectPath] = parseDbTriggers(trigger, "foreachlatest"); - const std::string& dbUrl = db == "qcdb" ? config.qcdbUrl : config.ccdbUrl; + const std::string& dbUrl = db == "qcdb" ? config.database.at("host") : config.conditionUrl; return triggers::ForEachLatest(dbUrl, db, objectPath, activity, trigger); } else if (auto seconds = string2Seconds(triggerLowerCase); seconds.has_value()) { if (seconds.value() < 0) { diff --git a/Framework/src/UserCodeInterface.cxx b/Framework/src/UserCodeInterface.cxx index 977a13f53c..565716ab2a 100644 --- a/Framework/src/UserCodeInterface.cxx +++ b/Framework/src/UserCodeInterface.cxx @@ -15,8 +15,12 @@ /// #include "QualityControl/UserCodeInterface.h" +#include +#include "QualityControl/QcInfoLogger.h" +#include "QualityControl/DatabaseFactory.h" using namespace o2::ccdb; +using namespace std; namespace o2::quality_control::core { @@ -27,8 +31,24 @@ void UserCodeInterface::setCustomParameters(const CustomParameters& parameters) configure(); } -const std::string& UserCodeInterface::getName() const { return mName; } +const std::string& UserCodeInterface::getName() const { + return mName; +} + +void UserCodeInterface::setName(const std::string& name) { + mName = name; +} -void UserCodeInterface::setName(const std::string& name) { mName = name; } +void UserCodeInterface::setDatabase(std::unordered_map dbConfig) +{ + if(dbConfig.count("implementation") == 0 || dbConfig.count("host") == 0) { + ILOG(Error, Devel) << "dbConfig is incomplete, we don't build the user code database instance" << ENDM; + throw std::invalid_argument("Cannot set database in UserCodeInterface"); + } + + mDatabase = repository::DatabaseFactory::create(dbConfig.at("implementation")); + mDatabase->connect(dbConfig); + ILOG(Debug, Devel) << "Database that is going to be used > Implementation : " << dbConfig.at("implementation") << " / Host : " << dbConfig.at("host") << ENDM; +} } // namespace o2::quality_control::core \ No newline at end of file diff --git a/Framework/test/testCheck.cxx b/Framework/test/testCheck.cxx index 806fca9f7d..3fe74db26d 100644 --- a/Framework/test/testCheck.cxx +++ b/Framework/test/testCheck.cxx @@ -144,14 +144,18 @@ TEST_CASE("test_check_postprocessing") TEST_CASE("test_check_activity") { - Check check({ "test", - "QcSkeleton", + Check check({ "QcSkeleton", "o2::quality_control_modules::skeleton::SkeletonCheck", "TST", + "", {}, + "", + {}, + "test", UpdatePolicyType::OnAny, {}, - true }); + true } + ); std::map> moMap{ { "abcTask/test1", dummyMO("test1") }, diff --git a/Framework/test/testCheckWorkflow.cxx b/Framework/test/testCheckWorkflow.cxx index 03ea526ca1..8a769cbc27 100644 --- a/Framework/test/testCheckWorkflow.cxx +++ b/Framework/test/testCheckWorkflow.cxx @@ -62,7 +62,7 @@ using namespace o2::configuration; * It is expected to terminate as soon as all task publish for the first time. */ -class Receiver : public framework::Task +class Receiver : public o2::framework::Task { public: Receiver(std::string configurationSource) @@ -78,7 +78,7 @@ class Receiver : public framework::Task ~Receiver() override{}; /// \brief Receiver process callback - void run(framework::ProcessingContext& pctx) override + void run(o2::framework::ProcessingContext& pctx) override { std::vector namesToErase; diff --git a/Framework/test/testTaskInterface.cxx b/Framework/test/testTaskInterface.cxx index 77cbc1de19..7b93b3ad21 100644 --- a/Framework/test/testTaskInterface.cxx +++ b/Framework/test/testTaskInterface.cxx @@ -167,10 +167,15 @@ TEST_CASE("test_invoke_all_TaskRunnerConfig_methods") TEST_CASE("test_task_factory") { TaskRunnerConfig config{ - "SkeletonTaskRunner", - "skeletonTask", "QcSkeleton", "o2::quality_control_modules::skeleton::SkeletonTask", + "TST", + "", + {}, + "", + {}, + "SkeletonTaskRunner", + "skeletonTask", { { 10, 1 } }, -1, true, diff --git a/Framework/test/testTriggerHelpers.cxx b/Framework/test/testTriggerHelpers.cxx index a8552eca0c..7d21eba536 100644 --- a/Framework/test/testTriggerHelpers.cxx +++ b/Framework/test/testTriggerHelpers.cxx @@ -63,8 +63,8 @@ TEST_CASE("test_factory") // generating new object trigger PostProcessingConfig configWithDBs; - configWithDBs.qcdbUrl = CCDB_ENDPOINT; - configWithDBs.ccdbUrl = CCDB_ENDPOINT; + configWithDBs.database["host"] = CCDB_ENDPOINT; + configWithDBs.conditionUrl = CCDB_ENDPOINT; CHECK_NOTHROW(trigger_helpers::triggerFactory("newobject:qcdb:qc/asdf/vcxz", configWithDBs)); CHECK_NOTHROW(trigger_helpers::triggerFactory("newobject:ccdb:qc/asdf/vcxz", configWithDBs)); CHECK_NOTHROW(trigger_helpers::triggerFactory("newobject:QCDB:qc/asdf/vcxz", configWithDBs)); From 8581b15f6f8bd77db2aeb9a77ff20d263caf7162 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Tue, 22 Oct 2024 15:18:58 +0200 Subject: [PATCH 2/7] Format --- Framework/src/PostProcessingConfig.cxx | 6 +++--- Framework/src/UserCodeInterface.cxx | 8 +++++--- Framework/test/testCheck.cxx | 5 ++--- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/Framework/src/PostProcessingConfig.cxx b/Framework/src/PostProcessingConfig.cxx index 49a521fda4..53d6f893db 100644 --- a/Framework/src/PostProcessingConfig.cxx +++ b/Framework/src/PostProcessingConfig.cxx @@ -39,9 +39,9 @@ PostProcessingConfig::PostProcessingConfig(const std::string& id, const boost::p detectorName = config.get("qc.postprocessing." + id + ".detectorName", "MISC"); consulUrl = config.get("qc.config.consul.url", ""); conditionUrl = config.get("qc.config.conditionDB.url", ""); - std::unordered_map dbConfig { - {"implementation", config.get("qc.config.database.implementation")}, - {"host", config.get("qc.config.database.host")} + std::unordered_map dbConfig{ + { "implementation", config.get("qc.config.database.implementation") }, + { "host", config.get("qc.config.database.host") } }; database = dbConfig; diff --git a/Framework/src/UserCodeInterface.cxx b/Framework/src/UserCodeInterface.cxx index 565716ab2a..96ee5bcdb3 100644 --- a/Framework/src/UserCodeInterface.cxx +++ b/Framework/src/UserCodeInterface.cxx @@ -31,17 +31,19 @@ void UserCodeInterface::setCustomParameters(const CustomParameters& parameters) configure(); } -const std::string& UserCodeInterface::getName() const { +const std::string& UserCodeInterface::getName() const +{ return mName; } -void UserCodeInterface::setName(const std::string& name) { +void UserCodeInterface::setName(const std::string& name) +{ mName = name; } void UserCodeInterface::setDatabase(std::unordered_map dbConfig) { - if(dbConfig.count("implementation") == 0 || dbConfig.count("host") == 0) { + if (dbConfig.count("implementation") == 0 || dbConfig.count("host") == 0) { ILOG(Error, Devel) << "dbConfig is incomplete, we don't build the user code database instance" << ENDM; throw std::invalid_argument("Cannot set database in UserCodeInterface"); } diff --git a/Framework/test/testCheck.cxx b/Framework/test/testCheck.cxx index 3fe74db26d..a4d336affe 100644 --- a/Framework/test/testCheck.cxx +++ b/Framework/test/testCheck.cxx @@ -144,7 +144,7 @@ TEST_CASE("test_check_postprocessing") TEST_CASE("test_check_activity") { - Check check({ "QcSkeleton", + Check check({ "QcSkeleton", "o2::quality_control_modules::skeleton::SkeletonCheck", "TST", "", @@ -154,8 +154,7 @@ TEST_CASE("test_check_activity") "test", UpdatePolicyType::OnAny, {}, - true } - ); + true }); std::map> moMap{ { "abcTask/test1", dummyMO("test1") }, From 24da4c228d9c41a4e06a884bc478ab9a2c84f0be Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Wed, 23 Oct 2024 09:01:51 +0200 Subject: [PATCH 3/7] [QC-1206] Make CTP Scalers available to user code --- Framework/CMakeLists.txt | 1 + .../QualityControl/UserCodeInterface.h | 22 +++++- Framework/src/UserCodeInterface.cxx | 69 +++++++++++++++++-- Modules/Skeleton/src/SkeletonCheck.cxx | 4 ++ doc/Advanced.md | 3 + 5 files changed, 93 insertions(+), 6 deletions(-) diff --git a/Framework/CMakeLists.txt b/Framework/CMakeLists.txt index 104115d9b9..a34c66dedc 100644 --- a/Framework/CMakeLists.txt +++ b/Framework/CMakeLists.txt @@ -155,6 +155,7 @@ target_link_libraries(O2QualityControl O2::DataFormatsQualityControl O2::DetectorsBase O2::GlobalTracking + O2::DataFormatsCTP O2QualityControlKafkaProtos ${RDKAFKA_LIB} PRIVATE Boost::system diff --git a/Framework/include/QualityControl/UserCodeInterface.h b/Framework/include/QualityControl/UserCodeInterface.h index 8cd26dd310..cc58294b8d 100644 --- a/Framework/include/QualityControl/UserCodeInterface.h +++ b/Framework/include/QualityControl/UserCodeInterface.h @@ -25,6 +25,10 @@ #include "QualityControl/CustomParameters.h" #include "QualityControl/DatabaseInterface.h" +namespace o2::ctp { +class CTPRateFetcher; +} + namespace o2::quality_control::core { @@ -51,14 +55,28 @@ class UserCodeInterface : public ConditionAccess void setName(const std::string& name); void setDatabase(std::unordered_map dbConfig); + private: + /// \brief Just the callback for the thread for the scalers retrieval. + void regularCallback(int intervalMinutes); + /// \brief Retrieve fresh scalers from the QCDB (with cache) + void updateScalers(); + std::shared_ptr mCtpFetcher; + std::chrono::steady_clock::time_point mScalersLastUpdate; + bool mScalersEnabled = false; + protected: + /// \brief Call it to enable the retrieval of CTP scalers and use `getScalers` later + void enableCtpScalers(size_t runNumber, std::string ccdbUrl); + /// \brief Get the scalers's value for the given source + double getScalersValue(std::string sourceName, size_t runNumber); + CustomParameters mCustomParameters; std::string mName; std::shared_ptr mDatabase; - ClassDef(UserCodeInterface, 4) + ClassDef(UserCodeInterface, 5) }; } // namespace o2::quality_control::core -#endif // QUALITYCONTROL_USERCODEINTERFACE_H +#endif // QUALITYCONTROL_USERCODEINTERFACE_H \ No newline at end of file diff --git a/Framework/src/UserCodeInterface.cxx b/Framework/src/UserCodeInterface.cxx index 96ee5bcdb3..ae83243d41 100644 --- a/Framework/src/UserCodeInterface.cxx +++ b/Framework/src/UserCodeInterface.cxx @@ -15,6 +15,7 @@ /// #include "QualityControl/UserCodeInterface.h" +#include #include #include "QualityControl/QcInfoLogger.h" #include "QualityControl/DatabaseFactory.h" @@ -31,16 +32,76 @@ void UserCodeInterface::setCustomParameters(const CustomParameters& parameters) configure(); } -const std::string& UserCodeInterface::getName() const -{ +const std::string& UserCodeInterface::getName() const { return mName; } -void UserCodeInterface::setName(const std::string& name) -{ +void UserCodeInterface::setName(const std::string& name) { mName = name; } +void UserCodeInterface::enableCtpScalers(size_t runNumber, std::string ccdbUrl) +{ + // TODO bail if we are in async + ILOG(Debug, Devel) << "Enabling CTP scalers" << ENDM; + mCtpFetcher = make_shared(); + mScalersEnabled = true; + auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); + ccdbManager.setURL(ccdbUrl); + mCtpFetcher->setupRun(runNumber, &ccdbManager, /*1726300234140*/ getCurrentTimestamp(), false); + + mScalersLastUpdate = std::chrono::steady_clock::time_point::min(); + cout << "mScalersLastUpdate : " << std::chrono::duration_cast(mScalersLastUpdate.time_since_epoch()).count() << endl; + updateScalers(); // initial value + ILOG(Debug, Devel) << "Enabled CTP scalers" << ENDM; +} + +void UserCodeInterface::updateScalers() +{ + if(!mScalersEnabled) { + ILOG(Error, Ops) << "CTP scalers not enabled, impossible to get them." << ENDM; + return; // TODO should we throw ? probably yes + } + ILOG(Debug, Devel) << "Updating scalers." << ENDM; + + if(! mDatabase) { + ILOG(Error, Devel) << "Database not set ! Cannot update scalers." << ENDM; + mScalersEnabled = false; + + return; + // todo handle the case when database is not set + } + + auto now = std::chrono::steady_clock::now(); + auto minutesSinceLast = std::chrono::duration_cast(now - mScalersLastUpdate); + + // TODO get the interval from config + if (minutesSinceLast.count() >= 0 /*first time it is neg*/ && minutesSinceLast.count() < 5) { + ILOG(Debug, Devel) << "getScalers was called less than 5 minutes ago, use the cached value" << ENDM; + return; + } + + std::map meta; + void* rawResult = mDatabase->retrieveAny(typeid(o2::ctp::CTPRunScalers), "qc/CTP/Scalers", meta); + o2::ctp::CTPRunScalers* ctpScalers = static_cast(rawResult); + mCtpFetcher->updateScalers(*ctpScalers); + mScalersLastUpdate = now; + ILOG(Debug, Devel) << "Scalers updated." << ENDM; +} + +double UserCodeInterface::getScalersValue(std::string sourceName, size_t runNumber) +{ + if(!mScalersEnabled) { + ILOG(Error, Ops) << "CTP scalers not enabled, impossible to get the value." << ENDM; + return 0; + } + updateScalers(); // from QCDB + auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); + auto result = mCtpFetcher->fetchNoPuCorr(&ccdbManager, getCurrentTimestamp(), runNumber, sourceName); + ILOG(Debug, Devel) << "Returning scaler value : " << result << ENDM; + return result; +} + void UserCodeInterface::setDatabase(std::unordered_map dbConfig) { if (dbConfig.count("implementation") == 0 || dbConfig.count("host") == 0) { diff --git a/Modules/Skeleton/src/SkeletonCheck.cxx b/Modules/Skeleton/src/SkeletonCheck.cxx index 8f1f1cbf99..eef9552566 100644 --- a/Modules/Skeleton/src/SkeletonCheck.cxx +++ b/Modules/Skeleton/src/SkeletonCheck.cxx @@ -49,6 +49,9 @@ Quality SkeletonCheck::check(std::mapmId); + ILOG(Info, Devel) << "\"T0VTX\" : " << t0vtx << ENDM; + // This is an example of accessing the histogram 'example' created by SkeletonTask for (auto& [moName, mo] : *moMap) { if (mo->getName() == "example") { @@ -124,6 +127,7 @@ void SkeletonCheck::startOfActivity(const Activity& activity) // THUS FUNCTION BODY IS AN EXAMPLE. PLEASE REMOVE EVERYTHING YOU DO NOT NEED. ILOG(Debug, Devel) << "SkeletonCheck::start : " << activity.mId << ENDM; mActivity = make_shared(activity); + enableCtpScalers(activity.mId, "alice-ccdb.cern.ch"); // TODO get it from the config } void SkeletonCheck::endOfActivity(const Activity& activity) diff --git a/doc/Advanced.md b/doc/Advanced.md index 3f4690c4dc..9849992397 100644 --- a/doc/Advanced.md +++ b/doc/Advanced.md @@ -1727,7 +1727,10 @@ In the example above, the quality goes to bad when there are 3 cycles in a row w In consul go to `o2/runtime/aliecs/defaults` and modify the file corresponding to the detector: [det]_qc_shm_segment_size +## CTP Scalers +Get a certificate for development : https://alice-doc.github.io/alice-analysis-tutorial/start/cert.html#test-your-certificate +JAliEn-ROOT --- From 3cf10ca3e501f5c70f1b63d8cc6662ffffd99187 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Wed, 23 Oct 2024 09:03:10 +0200 Subject: [PATCH 4/7] format --- Framework/include/QualityControl/UserCodeInterface.h | 5 +++-- Framework/src/UserCodeInterface.cxx | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Framework/include/QualityControl/UserCodeInterface.h b/Framework/include/QualityControl/UserCodeInterface.h index cc58294b8d..f3b7fe99ff 100644 --- a/Framework/include/QualityControl/UserCodeInterface.h +++ b/Framework/include/QualityControl/UserCodeInterface.h @@ -25,7 +25,8 @@ #include "QualityControl/CustomParameters.h" #include "QualityControl/DatabaseInterface.h" -namespace o2::ctp { +namespace o2::ctp +{ class CTPRateFetcher; } @@ -61,7 +62,7 @@ class UserCodeInterface : public ConditionAccess /// \brief Retrieve fresh scalers from the QCDB (with cache) void updateScalers(); std::shared_ptr mCtpFetcher; - std::chrono::steady_clock::time_point mScalersLastUpdate; + std::chrono::steady_clock::time_point mScalersLastUpdate; bool mScalersEnabled = false; protected: diff --git a/Framework/src/UserCodeInterface.cxx b/Framework/src/UserCodeInterface.cxx index ae83243d41..954d166f52 100644 --- a/Framework/src/UserCodeInterface.cxx +++ b/Framework/src/UserCodeInterface.cxx @@ -58,13 +58,13 @@ void UserCodeInterface::enableCtpScalers(size_t runNumber, std::string ccdbUrl) void UserCodeInterface::updateScalers() { - if(!mScalersEnabled) { + if (!mScalersEnabled) { ILOG(Error, Ops) << "CTP scalers not enabled, impossible to get them." << ENDM; return; // TODO should we throw ? probably yes } ILOG(Debug, Devel) << "Updating scalers." << ENDM; - if(! mDatabase) { + if (!mDatabase) { ILOG(Error, Devel) << "Database not set ! Cannot update scalers." << ENDM; mScalersEnabled = false; @@ -91,7 +91,7 @@ void UserCodeInterface::updateScalers() double UserCodeInterface::getScalersValue(std::string sourceName, size_t runNumber) { - if(!mScalersEnabled) { + if (!mScalersEnabled) { ILOG(Error, Ops) << "CTP scalers not enabled, impossible to get the value." << ENDM; return 0; } From f994efcad818556dafb939350ac241143b6e8ab5 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Wed, 23 Oct 2024 16:58:36 +0200 Subject: [PATCH 5/7] disable CTP scalers in async --- Framework/src/UserCodeInterface.cxx | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Framework/src/UserCodeInterface.cxx b/Framework/src/UserCodeInterface.cxx index 954d166f52..8d59cbadcc 100644 --- a/Framework/src/UserCodeInterface.cxx +++ b/Framework/src/UserCodeInterface.cxx @@ -43,6 +43,13 @@ void UserCodeInterface::setName(const std::string& name) { void UserCodeInterface::enableCtpScalers(size_t runNumber, std::string ccdbUrl) { // TODO bail if we are in async + + auto deploymentMode = framework::DefaultsHelpers::deploymentMode(); + if(deploymentMode == framework::DeploymentMode::Grid) { + ILOG(Info, Ops) << "Async mode detected, CTP scalers cannot be enabled." << ENDM; + return; + } + ILOG(Debug, Devel) << "Enabling CTP scalers" << ENDM; mCtpFetcher = make_shared(); mScalersEnabled = true; From a3fb58f8a779f91151b8cc2de195587378e1aa8e Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Thu, 31 Oct 2024 15:17:49 +0100 Subject: [PATCH 6/7] properly set the run number --- .../QualityControl/UserCodeInterface.h | 3 +- Framework/src/UserCodeInterface.cxx | 40 +++++++++++++------ 2 files changed, 29 insertions(+), 14 deletions(-) diff --git a/Framework/include/QualityControl/UserCodeInterface.h b/Framework/include/QualityControl/UserCodeInterface.h index f3b7fe99ff..f575d5e9ce 100644 --- a/Framework/include/QualityControl/UserCodeInterface.h +++ b/Framework/include/QualityControl/UserCodeInterface.h @@ -60,7 +60,8 @@ class UserCodeInterface : public ConditionAccess /// \brief Just the callback for the thread for the scalers retrieval. void regularCallback(int intervalMinutes); /// \brief Retrieve fresh scalers from the QCDB (with cache) - void updateScalers(); + /// \return true if success, false if failure + bool updateScalers(size_t runNumber); std::shared_ptr mCtpFetcher; std::chrono::steady_clock::time_point mScalersLastUpdate; bool mScalersEnabled = false; diff --git a/Framework/src/UserCodeInterface.cxx b/Framework/src/UserCodeInterface.cxx index 8d59cbadcc..bb5ab40351 100644 --- a/Framework/src/UserCodeInterface.cxx +++ b/Framework/src/UserCodeInterface.cxx @@ -58,25 +58,25 @@ void UserCodeInterface::enableCtpScalers(size_t runNumber, std::string ccdbUrl) mCtpFetcher->setupRun(runNumber, &ccdbManager, /*1726300234140*/ getCurrentTimestamp(), false); mScalersLastUpdate = std::chrono::steady_clock::time_point::min(); - cout << "mScalersLastUpdate : " << std::chrono::duration_cast(mScalersLastUpdate.time_since_epoch()).count() << endl; - updateScalers(); // initial value - ILOG(Debug, Devel) << "Enabled CTP scalers" << ENDM; + if(updateScalers(runNumber)) { // initial value + ILOG(Debug, Devel) << "Enabled CTP scalers" << ENDM; + } else { + ILOG(Debug, Devel) << "CTP scalers not enabled, failure to get them." << ENDM; + } } -void UserCodeInterface::updateScalers() +bool UserCodeInterface::updateScalers(size_t runNumber) { if (!mScalersEnabled) { - ILOG(Error, Ops) << "CTP scalers not enabled, impossible to get them." << ENDM; - return; // TODO should we throw ? probably yes + ILOG(Error, Ops) << "CTP scalers not enabled, impossible to update them." << ENDM; + return false; } ILOG(Debug, Devel) << "Updating scalers." << ENDM; if (!mDatabase) { ILOG(Error, Devel) << "Database not set ! Cannot update scalers." << ENDM; mScalersEnabled = false; - - return; - // todo handle the case when database is not set + return false; } auto now = std::chrono::steady_clock::now(); @@ -85,15 +85,26 @@ void UserCodeInterface::updateScalers() // TODO get the interval from config if (minutesSinceLast.count() >= 0 /*first time it is neg*/ && minutesSinceLast.count() < 5) { ILOG(Debug, Devel) << "getScalers was called less than 5 minutes ago, use the cached value" << ENDM; - return; + return true; } std::map meta; - void* rawResult = mDatabase->retrieveAny(typeid(o2::ctp::CTPRunScalers), "qc/CTP/Scalers", meta); + meta["runNumber"] = std::to_string(runNumber); + std::map headers; + auto validity = mDatabase->getLatestObjectValidity("qc/CTP/Scalers", meta); + void* rawResult = mDatabase->retrieveAny(typeid(o2::ctp::CTPRunScalers), "qc/CTP/Scalers", meta, validity.getMax() - 1, &headers); + if (!rawResult) { + ILOG(Error, Devel) << "Could not retrieve the CTP Scalers" << ENDM; + return false; + } else { + ILOG(Debug, Devel) << "object retrieved" << ENDM; + } + o2::ctp::CTPRunScalers* ctpScalers = static_cast(rawResult); mCtpFetcher->updateScalers(*ctpScalers); mScalersLastUpdate = now; ILOG(Debug, Devel) << "Scalers updated." << ENDM; + return true; } double UserCodeInterface::getScalersValue(std::string sourceName, size_t runNumber) @@ -102,9 +113,12 @@ double UserCodeInterface::getScalersValue(std::string sourceName, size_t runNumb ILOG(Error, Ops) << "CTP scalers not enabled, impossible to get the value." << ENDM; return 0; } - updateScalers(); // from QCDB + if(!updateScalers(runNumber)) { // from QCDB + ILOG(Debug, Devel) << "Could not update the scalers, returning 0" << ENDM; + return 0; + } auto& ccdbManager = o2::ccdb::BasicCCDBManager::instance(); - auto result = mCtpFetcher->fetchNoPuCorr(&ccdbManager, getCurrentTimestamp(), runNumber, sourceName); + auto result = mCtpFetcher->fetchNoPuCorr(&ccdbManager, getCurrentTimestamp()*1000, runNumber, sourceName); ILOG(Debug, Devel) << "Returning scaler value : " << result << ENDM; return result; } From 7f846a03e5aab19221a69fdcf4164ecdf5dccb3d Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Fri, 1 Nov 2024 14:52:38 +0100 Subject: [PATCH 7/7] doc --- doc/Advanced.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/Advanced.md b/doc/Advanced.md index 9849992397..ec81604b7a 100644 --- a/doc/Advanced.md +++ b/doc/Advanced.md @@ -1730,7 +1730,11 @@ In consul go to `o2/runtime/aliecs/defaults` and modify the file corresponding t ## CTP Scalers Get a certificate for development : https://alice-doc.github.io/alice-analysis-tutorial/start/cert.html#test-your-certificate -JAliEn-ROOT + +Build JAlien-ROOT + +alienv enter QualityControl/latest JAliEn-ROOT/latest + ---