From aa8ab7d23887a2afe5edc03954a1203b32201a25 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Thu, 11 Jul 2024 11:13:17 +0200 Subject: [PATCH 1/4] [QC-1026] Separate source / target QCDB for postprocessing --- .../QualityControl/PostProcessingRunner.h | 4 ++- .../PostProcessingRunnerConfig.h | 3 +- .../QualityControl/PostProcessingTaskSpec.h | 2 ++ Framework/src/InfrastructureSpecReader.cxx | 10 +++++++ Framework/src/PostProcessingRunner.cxx | 29 ++++++++++++++----- 5 files changed, 38 insertions(+), 10 deletions(-) diff --git a/Framework/include/QualityControl/PostProcessingRunner.h b/Framework/include/QualityControl/PostProcessingRunner.h index fbf8daad75..ce67fa157d 100644 --- a/Framework/include/QualityControl/PostProcessingRunner.h +++ b/Framework/include/QualityControl/PostProcessingRunner.h @@ -119,7 +119,9 @@ class PostProcessingRunner core::Activity mActivity; PostProcessingConfig mTaskConfig; PostProcessingRunnerConfig mRunnerConfig; - std::shared_ptr mDatabase; + std::shared_ptr mSourceDatabase; + std::shared_ptr mDestinationDatabase; + std::unique_ptr configureDatabase(std::unordered_map& dbConfig, const std::string& name); }; MOCPublicationCallback publishToDPL(o2::framework::DataAllocator&, std::string outputBinding); diff --git a/Framework/include/QualityControl/PostProcessingRunnerConfig.h b/Framework/include/QualityControl/PostProcessingRunnerConfig.h index 134f239e84..3c2285bec7 100644 --- a/Framework/include/QualityControl/PostProcessingRunnerConfig.h +++ b/Framework/include/QualityControl/PostProcessingRunnerConfig.h @@ -29,7 +29,8 @@ struct PostProcessingRunnerConfig { std::string id; std::string taskName; std::string detectorName; - std::unordered_map database; + std::unordered_map sourceDatabase; + std::unordered_map destinationDatabase; std::string consulUrl{}; std::string bookkeepingUrl{}; core::LogDiscardParameters infologgerDiscardParameters; diff --git a/Framework/include/QualityControl/PostProcessingTaskSpec.h b/Framework/include/QualityControl/PostProcessingTaskSpec.h index 4847416eab..4691a234e9 100644 --- a/Framework/include/QualityControl/PostProcessingTaskSpec.h +++ b/Framework/include/QualityControl/PostProcessingTaskSpec.h @@ -45,6 +45,8 @@ struct PostProcessingTaskSpec { std::string detectorName = "Invalid"; boost::property_tree::ptree tree = {}; core::CustomParameters customParameters; + std::unordered_map sourceDatabase; + std::unordered_map destinationDatabase; }; } // namespace o2::quality_control::core diff --git a/Framework/src/InfrastructureSpecReader.cxx b/Framework/src/InfrastructureSpecReader.cxx index f1079e3a7d..61f52c0cf4 100644 --- a/Framework/src/InfrastructureSpecReader.cxx +++ b/Framework/src/InfrastructureSpecReader.cxx @@ -381,6 +381,16 @@ PostProcessingTaskSpec ppts.active = ppTaskTree.get("active", ppts.active); ppts.critical = ppTaskTree.get("critical", ppts.critical); ppts.detectorName = ppTaskTree.get("detectorName", ppts.detectorName); + if (ppTaskTree.count("sourceRepo") > 0) { + for (const auto& [key, value] : ppTaskTree.get_child("sourceRepo")) { + ppts.sourceDatabase.emplace(key, value.get_value()); + } + } + if (ppTaskTree.count("destinationRepo") > 0) { + for (const auto& [key, value] : ppTaskTree.get_child("destinationRepo")) { + ppts.destinationDatabase.emplace(key, value.get_value()); + } + } ppts.tree = wholeTree; return ppts; diff --git a/Framework/src/PostProcessingRunner.cxx b/Framework/src/PostProcessingRunner.cxx index d18dd63110..3959fb8227 100644 --- a/Framework/src/PostProcessingRunner.cxx +++ b/Framework/src/PostProcessingRunner.cxx @@ -66,6 +66,16 @@ void PostProcessingRunner::init(const boost::property_tree::ptree& config, core: init(PostProcessingRunner::extractConfig(specs.common, *ppTaskSpec), PostProcessingConfig{ mID, config }); } +std::unique_ptr PostProcessingRunner::configureDatabase(std::unordered_map& dbConfig, const std::string& name) +{ + auto database = DatabaseFactory::create(dbConfig.at("implementation")); + database->connect(dbConfig); + ILOG(Info, Support) << name << " database that is going to be used > Implementation : " << dbConfig.at("implementation") << " / " + << " Host : " << dbConfig.at("host") << ENDM; + return database; +} + + void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig, const PostProcessingConfig& taskConfig) { QcInfoLogger::init(("post/" + taskConfig.taskName).substr(0, QcInfoLogger::maxFacilityLength), runnerConfig.infologgerDiscardParameters); @@ -82,16 +92,14 @@ void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig, } // configuration of the database - mDatabase = DatabaseFactory::create(mRunnerConfig.database.at("implementation")); - mDatabase->connect(mRunnerConfig.database); - ILOG(Info, Support) << "Database that is going to be used > Implementation : " << mRunnerConfig.database.at("implementation") << " / " - << " Host : " << mRunnerConfig.database.at("host") << ENDM; + mSourceDatabase = configureDatabase(mRunnerConfig.sourceDatabase, "Source"); + mDestinationDatabase = configureDatabase(mRunnerConfig.destinationDatabase, "Destination"); mObjectManager = std::make_shared(mTaskConfig.taskName, mTaskConfig.className, mTaskConfig.detectorName, mRunnerConfig.consulUrl); mObjectManager->setActivity(mActivity); - mServices.registerService(mDatabase.get()); + mServices.registerService(mSourceDatabase.get()); if (mPublicationCallback == nullptr) { - mPublicationCallback = publishToRepository(*mDatabase); + mPublicationCallback = publishToRepository(*mDestinationDatabase); } Bookkeeping::getInstance().init(runnerConfig.bookkeepingUrl); @@ -232,7 +240,8 @@ void PostProcessingRunner::reset() mTaskState = TaskState::INVALID; mTask.reset(); - mDatabase.reset(); + mSourceDatabase.reset(); + mDestinationDatabase.reset(); mServices = framework::ServiceRegistry(); mObjectManager.reset(); @@ -327,11 +336,15 @@ const std::string& PostProcessingRunner::getID() const PostProcessingRunnerConfig PostProcessingRunner::extractConfig(const CommonSpec& commonSpec, const PostProcessingTaskSpec& ppTaskSpec) { + auto sourceDatabase = ppTaskSpec.sourceDatabase.empty() ? commonSpec.database : ppTaskSpec.sourceDatabase; + auto destinationDatabase = ppTaskSpec.destinationDatabase.empty() ? commonSpec.database : ppTaskSpec.destinationDatabase; + return { ppTaskSpec.id, ppTaskSpec.taskName, ppTaskSpec.detectorName, - commonSpec.database, + sourceDatabase, + destinationDatabase, commonSpec.consulUrl, commonSpec.bookkeepingUrl, commonSpec.infologgerDiscardParameters, From 970b2a4477d26940089c1b42f8d8ea0f846bc2e5 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Thu, 11 Jul 2024 11:21:01 +0200 Subject: [PATCH 2/4] doc --- doc/PostProcessing.md | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/doc/PostProcessing.md b/doc/PostProcessing.md index 40231fee95..4664ffd86d 100644 --- a/doc/PostProcessing.md +++ b/doc/PostProcessing.md @@ -142,7 +142,7 @@ Checks can be applied to the results of Post-processing Tasks just as for normal ... ``` -## Definition and access of user-specific configuration +#### Definition and access of user-specific configuration A postprocessing task can access custom parameters declared in the configuration file at `qc.postprocessing..extendedTaskParameters`. They are stored inside an object of type `CustomParameters` named `mCustomParameters`, which is a protected member of `TaskInterface`. @@ -165,6 +165,34 @@ Each of the three methods can be invoked by one or more triggers. Below are list * `"once"` - Once - triggers only first time it is checked * `"always"` - Always - triggers each time it is checked +#### Using different databases + +It might happen that one wants to get data and store data in different databases. Typically if you want to test with +production data but store the object in test. + +This can be achieved by setting extra parameters in the task. You have to add it to all your tasks as this is not a global +parameter. + +`sourceRepo` and `destinationRepo` are both optional. You can specify none, either or both. + +``` + "postprocessing": { + "MyPostProcessingTaskID": { + ... + "sourceRepo": { + "implementation": "CCDB", + "host": "another-test.cern.ch:8080" + }, + "destinationRepo": { + "implementation": "CCDB", + "host": "another-test.cern.ch:8080" + }, + ... + } + ... + } +``` + ### Running it The post-processing tasks can be run in three ways. First uses the usual `o2-qc` executable which relies on DPL and From 1931170dcd4570f9b766baa1fcb3979dc77d9078 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Fri, 12 Jul 2024 08:00:43 +0200 Subject: [PATCH 3/4] format --- Framework/src/PostProcessingRunner.cxx | 1 - 1 file changed, 1 deletion(-) diff --git a/Framework/src/PostProcessingRunner.cxx b/Framework/src/PostProcessingRunner.cxx index 3959fb8227..c8f22f1ad2 100644 --- a/Framework/src/PostProcessingRunner.cxx +++ b/Framework/src/PostProcessingRunner.cxx @@ -75,7 +75,6 @@ std::unique_ptr PostProcessingRunner::configureDatabase(std:: return database; } - void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig, const PostProcessingConfig& taskConfig) { QcInfoLogger::init(("post/" + taskConfig.taskName).substr(0, QcInfoLogger::maxFacilityLength), runnerConfig.infologgerDiscardParameters); From 8101d2874a406671b81a69e64f69719be5267584 Mon Sep 17 00:00:00 2001 From: Barthelemy Date: Tue, 20 Aug 2024 10:11:30 +0200 Subject: [PATCH 4/4] only keep sourceDatabase, destination is always the global database. I kept "destinationDatabase" in the `PostProcessingRunnerConfig` because there is a "sourceDatabase" and having "mDatabase" seemed confusing. --- .../include/QualityControl/PostProcessingTaskSpec.h | 1 - Framework/src/InfrastructureSpecReader.cxx | 5 ----- Framework/src/PostProcessingRunner.cxx | 3 +-- doc/PostProcessing.md | 10 +++------- 4 files changed, 4 insertions(+), 15 deletions(-) diff --git a/Framework/include/QualityControl/PostProcessingTaskSpec.h b/Framework/include/QualityControl/PostProcessingTaskSpec.h index 4691a234e9..1d88578f0e 100644 --- a/Framework/include/QualityControl/PostProcessingTaskSpec.h +++ b/Framework/include/QualityControl/PostProcessingTaskSpec.h @@ -46,7 +46,6 @@ struct PostProcessingTaskSpec { boost::property_tree::ptree tree = {}; core::CustomParameters customParameters; std::unordered_map sourceDatabase; - std::unordered_map destinationDatabase; }; } // namespace o2::quality_control::core diff --git a/Framework/src/InfrastructureSpecReader.cxx b/Framework/src/InfrastructureSpecReader.cxx index 61f52c0cf4..d22d140229 100644 --- a/Framework/src/InfrastructureSpecReader.cxx +++ b/Framework/src/InfrastructureSpecReader.cxx @@ -386,11 +386,6 @@ PostProcessingTaskSpec ppts.sourceDatabase.emplace(key, value.get_value()); } } - if (ppTaskTree.count("destinationRepo") > 0) { - for (const auto& [key, value] : ppTaskTree.get_child("destinationRepo")) { - ppts.destinationDatabase.emplace(key, value.get_value()); - } - } ppts.tree = wholeTree; return ppts; diff --git a/Framework/src/PostProcessingRunner.cxx b/Framework/src/PostProcessingRunner.cxx index c8f22f1ad2..ac8b1a1c46 100644 --- a/Framework/src/PostProcessingRunner.cxx +++ b/Framework/src/PostProcessingRunner.cxx @@ -336,14 +336,13 @@ const std::string& PostProcessingRunner::getID() const PostProcessingRunnerConfig PostProcessingRunner::extractConfig(const CommonSpec& commonSpec, const PostProcessingTaskSpec& ppTaskSpec) { auto sourceDatabase = ppTaskSpec.sourceDatabase.empty() ? commonSpec.database : ppTaskSpec.sourceDatabase; - auto destinationDatabase = ppTaskSpec.destinationDatabase.empty() ? commonSpec.database : ppTaskSpec.destinationDatabase; return { ppTaskSpec.id, ppTaskSpec.taskName, ppTaskSpec.detectorName, sourceDatabase, - destinationDatabase, + commonSpec.database, commonSpec.consulUrl, commonSpec.bookkeepingUrl, commonSpec.infologgerDiscardParameters, diff --git a/doc/PostProcessing.md b/doc/PostProcessing.md index 4664ffd86d..83dbd88f13 100644 --- a/doc/PostProcessing.md +++ b/doc/PostProcessing.md @@ -170,10 +170,10 @@ Each of the three methods can be invoked by one or more triggers. Below are list It might happen that one wants to get data and store data in different databases. Typically if you want to test with production data but store the object in test. -This can be achieved by setting extra parameters in the task. You have to add it to all your tasks as this is not a global -parameter. +This can be achieved by setting the extra parameter `sourceRepo` in the task. You have to add it to all your tasks as this is not a global +parameter. It is optional. -`sourceRepo` and `destinationRepo` are both optional. You can specify none, either or both. +The destination repository is always the global one defined in the global configuration under `qc.config.database`. ``` "postprocessing": { @@ -183,10 +183,6 @@ parameter. "implementation": "CCDB", "host": "another-test.cern.ch:8080" }, - "destinationRepo": { - "implementation": "CCDB", - "host": "another-test.cern.ch:8080" - }, ... } ...