[QC-1026] Separate source / target QCDB for postprocessing (#2371)

* [QC-1026] Separate source / target QCDB for postprocessing

* doc

* format

* only keep sourceDatabase, destination is always the global database.

I kept "destinationDatabase" in the `PostProcessingRunnerConfig` because there is a "sourceDatabase" and having "mDatabase" seemed confusing.

Framework/include/QualityControl/PostProcessingRunner.h

@@ -119,7 +119,9 @@ class PostProcessingRunner
   core::Activity mActivity;
   PostProcessingConfig mTaskConfig;
   PostProcessingRunnerConfig mRunnerConfig;
-  std::shared_ptr<o2::quality_control::repository::DatabaseInterface> mDatabase;
+  std::shared_ptr<o2::quality_control::repository::DatabaseInterface> mSourceDatabase;
+  std::shared_ptr<o2::quality_control::repository::DatabaseInterface> mDestinationDatabase;
+  std::unique_ptr<repository::DatabaseInterface> configureDatabase(std::unordered_map<std::string, std::string>& dbConfig, const std::string& name);
 };
 
 MOCPublicationCallback publishToDPL(o2::framework::DataAllocator&, std::string outputBinding);

Framework/include/QualityControl/PostProcessingRunnerConfig.h

@@ -29,7 +29,8 @@ struct PostProcessingRunnerConfig {
   std::string id;
   std::string taskName;
   std::string detectorName;
-  std::unordered_map<std::string, std::string> database;
+  std::unordered_map<std::string, std::string> sourceDatabase;
+  std::unordered_map<std::string, std::string> destinationDatabase;
   std::string consulUrl{};
   std::string bookkeepingUrl{};
   core::LogDiscardParameters infologgerDiscardParameters;

Framework/include/QualityControl/PostProcessingTaskSpec.h

@@ -45,6 +45,7 @@ struct PostProcessingTaskSpec {
   std::string detectorName = "Invalid";
   boost::property_tree::ptree tree = {};
   core::CustomParameters customParameters;
+  std::unordered_map<std::string, std::string> sourceDatabase;
 };
 
 } // namespace o2::quality_control::core

Framework/src/InfrastructureSpecReader.cxx

@@ -381,6 +381,11 @@ PostProcessingTaskSpec
   ppts.active = ppTaskTree.get<bool>("active", ppts.active);
   ppts.critical = ppTaskTree.get<bool>("critical", ppts.critical);
   ppts.detectorName = ppTaskTree.get<std::string>("detectorName", ppts.detectorName);
+  if (ppTaskTree.count("sourceRepo") > 0) {
+    for (const auto& [key, value] : ppTaskTree.get_child("sourceRepo")) {
+      ppts.sourceDatabase.emplace(key, value.get_value<std::string>());
+    }
+  }
   ppts.tree = wholeTree;
 
   return ppts;

Framework/src/PostProcessingRunner.cxx

@@ -66,6 +66,15 @@ void PostProcessingRunner::init(const boost::property_tree::ptree& config, core:
   init(PostProcessingRunner::extractConfig(specs.common, *ppTaskSpec), PostProcessingConfig{ mID, config });
 }
 
+std::unique_ptr<DatabaseInterface> PostProcessingRunner::configureDatabase(std::unordered_map<std::string, std::string>& dbConfig, const std::string& name)
+{
+  auto database = DatabaseFactory::create(dbConfig.at("implementation"));
+  database->connect(dbConfig);
+  ILOG(Info, Support) << name << " database that is going to be used > Implementation : " << dbConfig.at("implementation") << " / "
+                      << " Host : " << dbConfig.at("host") << ENDM;
+  return database;
+}
+
 void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig, const PostProcessingConfig& taskConfig)
 {
   QcInfoLogger::init(("post/" + taskConfig.taskName).substr(0, QcInfoLogger::maxFacilityLength), runnerConfig.infologgerDiscardParameters);
@@ -82,16 +91,14 @@ void PostProcessingRunner::init(const PostProcessingRunnerConfig& runnerConfig,
   }
 
   // configuration of the database
-  mDatabase = DatabaseFactory::create(mRunnerConfig.database.at("implementation"));
-  mDatabase->connect(mRunnerConfig.database);
-  ILOG(Info, Support) << "Database that is going to be used > Implementation : " << mRunnerConfig.database.at("implementation") << " / "
-                      << " Host : " << mRunnerConfig.database.at("host") << ENDM;
+  mSourceDatabase = configureDatabase(mRunnerConfig.sourceDatabase, "Source");
+  mDestinationDatabase = configureDatabase(mRunnerConfig.destinationDatabase, "Destination");
 
   mObjectManager = std::make_shared<ObjectsManager>(mTaskConfig.taskName, mTaskConfig.className, mTaskConfig.detectorName, mRunnerConfig.consulUrl);
   mObjectManager->setActivity(mActivity);
-  mServices.registerService<DatabaseInterface>(mDatabase.get());
+  mServices.registerService<DatabaseInterface>(mSourceDatabase.get());
   if (mPublicationCallback == nullptr) {
-    mPublicationCallback = publishToRepository(*mDatabase);
+    mPublicationCallback = publishToRepository(*mDestinationDatabase);
   }
   Bookkeeping::getInstance().init(runnerConfig.bookkeepingUrl);
 
@@ -232,7 +239,8 @@ void PostProcessingRunner::reset()
   mTaskState = TaskState::INVALID;
 
   mTask.reset();
-  mDatabase.reset();
+  mSourceDatabase.reset();
+  mDestinationDatabase.reset();
   mServices = framework::ServiceRegistry();
   mObjectManager.reset();
 
@@ -327,10 +335,13 @@ const std::string& PostProcessingRunner::getID() const
 
 PostProcessingRunnerConfig PostProcessingRunner::extractConfig(const CommonSpec& commonSpec, const PostProcessingTaskSpec& ppTaskSpec)
 {
+  auto sourceDatabase = ppTaskSpec.sourceDatabase.empty() ? commonSpec.database : ppTaskSpec.sourceDatabase;
+
   return {
     ppTaskSpec.id,
     ppTaskSpec.taskName,
     ppTaskSpec.detectorName,
+    sourceDatabase,
     commonSpec.database,
     commonSpec.consulUrl,
     commonSpec.bookkeepingUrl,

doc/PostProcessing.md

@@ -142,7 +142,7 @@ Checks can be applied to the results of Post-processing Tasks just as for normal
 ...
 ```
 
-## Definition and access of user-specific configuration
+#### Definition and access of user-specific configuration
 
 A postprocessing task can access custom parameters declared in the configuration file at `qc.postprocessing.<task_id>.extendedTaskParameters`. They are stored inside an object of type `CustomParameters` named `mCustomParameters`, which is a protected member of `TaskInterface`.
 
@@ -165,6 +165,30 @@ Each of the three methods can be invoked by one or more triggers. Below are list
  * `"once"` - Once - triggers only first time it is checked
  * `"always"` - Always - triggers each time it is checked
 
+####  Using different databases
+
+It might happen that one wants to get data and store data in different databases. Typically if you want to test with
+production data but store the object in test. 
+
+This can be achieved by setting the extra parameter `sourceRepo` in the task. You have to add it to all your tasks as this is not a global
+parameter. It is optional.
+
+The destination repository is always the global one defined in the global configuration under `qc.config.database`.
+
+```
+    "postprocessing": {
+      "MyPostProcessingTaskID": {
+        ...
+        "sourceRepo": {
+          "implementation": "CCDB",
+          "host": "another-test.cern.ch:8080"
+        },
+        ...
+      }
+      ...
+    }
+```
+
 ### Running it
 
 The post-processing tasks can be run in three ways. First uses the usual `o2-qc` executable which relies on DPL and