diff --git a/Framework/include/QualityControl/PostProcessingRunner.h b/Framework/include/QualityControl/PostProcessingRunner.h index fbf8daad75..ce67fa157d 100644 --- a/Framework/include/QualityControl/PostProcessingRunner.h +++ b/Framework/include/QualityControl/PostProcessingRunner.h @@ -119,7 +119,9 @@ class PostProcessingRunner core::Activity mActivity; PostProcessingConfig mTaskConfig; PostProcessingRunnerConfig mRunnerConfig; - std::shared_ptr mDatabase; + std::shared_ptr mSourceDatabase; + std::shared_ptr mDestinationDatabase; + std::unique_ptr configureDatabase(std::unordered_map& dbConfig, const std::string& name); }; MOCPublicationCallback publishToDPL(o2::framework::DataAllocator&, std::string outputBinding); diff --git a/Framework/include/QualityControl/PostProcessingRunnerConfig.h b/Framework/include/QualityControl/PostProcessingRunnerConfig.h index 134f239e84..3c2285bec7 100644 --- a/Framework/include/QualityControl/PostProcessingRunnerConfig.h +++ b/Framework/include/QualityControl/PostProcessingRunnerConfig.h @@ -29,7 +29,8 @@ struct PostProcessingRunnerConfig { std::string id; std::string taskName; std::string detectorName; - std::unordered_map database; + std::unordered_map sourceDatabase; + std::unordered_map destinationDatabase; std::string consulUrl{}; std::string bookkeepingUrl{}; core::LogDiscardParameters infologgerDiscardParameters; diff --git a/Framework/include/QualityControl/PostProcessingTaskSpec.h b/Framework/include/QualityControl/PostProcessingTaskSpec.h index 4847416eab..1d88578f0e 100644 --- a/Framework/include/QualityControl/PostProcessingTaskSpec.h +++ b/Framework/include/QualityControl/PostProcessingTaskSpec.h @@ -45,6 +45,7 @@ struct PostProcessingTaskSpec { std::string detectorName = "Invalid"; boost::property_tree::ptree tree = {}; core::CustomParameters customParameters; + std::unordered_map sourceDatabase; }; } // namespace o2::quality_control::core diff --git a/Framework/src/InfrastructureSpecReader.cxx b/Framework/src/InfrastructureSpecReader.cxx index f1079e3a7d..d22d140229 100644 --- a/Framework/src/InfrastructureSpecReader.cxx +++ b/Framework/src/InfrastructureSpecReader.cxx @@ -381,6 +381,11 @@ PostProcessingTaskSpec ppts.active = ppTaskTree.get("active", ppts.active); ppts.critical = ppTaskTree.get("critical", ppts.critical); ppts.detectorName = ppTaskTree.get("detectorName", ppts.detectorName); + if (ppTaskTree.count("sourceRepo") > 0) { + for (const auto& [key, value] : ppTaskTree.get_child("sourceRepo")) { + ppts.sourceDatabase.emplace(key, value.get_value()); + } + } ppts.tree = wholeTree; return ppts; diff --git a/Framework/src/PostProcessingRunner.cxx b/Framework/src/PostProcessingRunner.cxx index d18dd63110..ac8b1a1c46 100644 --- a/Framework/src/PostProcessingRunner.cxx +++ b/Framework/src/PostProcessingRunner.cxx @@ -66,6 +66,15 @@ void PostProcessingRunner::init(const boost::property_tree::ptree& config, core: init(PostProcessingRunner::extractConfig(specs.common, *ppTaskSpec), PostProcessingConfig{ mID, config }); } +std::unique_ptr PostProcessingRunner::configureDatabase(std::unordered_map& dbConfig, const std::string& name) +{ + auto database = DatabaseFactory::create(dbConfig.at("implementation")); + database->connect(dbConfig); + ILOG(Info, Support) << name << " database that is going to be used > Implementation : " << dbConfig.at("implementation") << " / " + << " Host : " << dbConfig.at("host") << ENDM; + return database; +} + void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig, const PostProcessingConfig& taskConfig) { QcInfoLogger::init(("post/" + taskConfig.taskName).substr(0, QcInfoLogger::maxFacilityLength), runnerConfig.infologgerDiscardParameters); @@ -82,16 +91,14 @@ void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig, } // configuration of the database - mDatabase = DatabaseFactory::create(mRunnerConfig.database.at("implementation")); - mDatabase->connect(mRunnerConfig.database); - ILOG(Info, Support) << "Database that is going to be used > Implementation : " << mRunnerConfig.database.at("implementation") << " / " - << " Host : " << mRunnerConfig.database.at("host") << ENDM; + mSourceDatabase = configureDatabase(mRunnerConfig.sourceDatabase, "Source"); + mDestinationDatabase = configureDatabase(mRunnerConfig.destinationDatabase, "Destination"); mObjectManager = std::make_shared(mTaskConfig.taskName, mTaskConfig.className, mTaskConfig.detectorName, mRunnerConfig.consulUrl); mObjectManager->setActivity(mActivity); - mServices.registerService(mDatabase.get()); + mServices.registerService(mSourceDatabase.get()); if (mPublicationCallback == nullptr) { - mPublicationCallback = publishToRepository(*mDatabase); + mPublicationCallback = publishToRepository(*mDestinationDatabase); } Bookkeeping::getInstance().init(runnerConfig.bookkeepingUrl); @@ -232,7 +239,8 @@ void PostProcessingRunner::reset() mTaskState = TaskState::INVALID; mTask.reset(); - mDatabase.reset(); + mSourceDatabase.reset(); + mDestinationDatabase.reset(); mServices = framework::ServiceRegistry(); mObjectManager.reset(); @@ -327,10 +335,13 @@ const std::string& PostProcessingRunner::getID() const PostProcessingRunnerConfig PostProcessingRunner::extractConfig(const CommonSpec& commonSpec, const PostProcessingTaskSpec& ppTaskSpec) { + auto sourceDatabase = ppTaskSpec.sourceDatabase.empty() ? commonSpec.database : ppTaskSpec.sourceDatabase; + return { ppTaskSpec.id, ppTaskSpec.taskName, ppTaskSpec.detectorName, + sourceDatabase, commonSpec.database, commonSpec.consulUrl, commonSpec.bookkeepingUrl, diff --git a/doc/PostProcessing.md b/doc/PostProcessing.md index 40231fee95..83dbd88f13 100644 --- a/doc/PostProcessing.md +++ b/doc/PostProcessing.md @@ -142,7 +142,7 @@ Checks can be applied to the results of Post-processing Tasks just as for normal ... ``` -## Definition and access of user-specific configuration +#### Definition and access of user-specific configuration A postprocessing task can access custom parameters declared in the configuration file at `qc.postprocessing..extendedTaskParameters`. They are stored inside an object of type `CustomParameters` named `mCustomParameters`, which is a protected member of `TaskInterface`. @@ -165,6 +165,30 @@ Each of the three methods can be invoked by one or more triggers. Below are list * `"once"` - Once - triggers only first time it is checked * `"always"` - Always - triggers each time it is checked +#### Using different databases + +It might happen that one wants to get data and store data in different databases. Typically if you want to test with +production data but store the object in test. + +This can be achieved by setting the extra parameter `sourceRepo` in the task. You have to add it to all your tasks as this is not a global +parameter. It is optional. + +The destination repository is always the global one defined in the global configuration under `qc.config.database`. + +``` + "postprocessing": { + "MyPostProcessingTaskID": { + ... + "sourceRepo": { + "implementation": "CCDB", + "host": "another-test.cern.ch:8080" + }, + ... + } + ... + } +``` + ### Running it The post-processing tasks can be run in three ways. First uses the usual `o2-qc` executable which relies on DPL and