Skip to content

Commit

Permalink
QC-1214 Updates and improvements to QO -> Flag conversion
Browse files Browse the repository at this point in the history
The converter has been adapted to the rules below:
- **Good QOs with no Flags are not onverted to any Flags**
- **Bad and Medium QOs with no Flags are converted to Flag 14 (Unknown)**
- **Null QOs with no Flags are converted to Flag 1**
- **All QOs with Flags are converted to Flags, while Quality is ignored**. A warning should be printed if a Check associates a GOOD flag to BAD quality, or a BAD flag to GOOD quality.
- **Timespans not covered by a given QO are filled with Flag 1 (Unknown Quality)**
- **Overlapping or adjacent Flags with the same ID, comment and source (QO) are merged, even if they were associated to different Qualities.**
- **Flag 1 (UnknownQuality) gets overwritten by any other Flag**.
- **Good and Bad flags do not affect each other, they may coexist**.
- **Flags for different QOs do not affect each other, Flag 1 is added separately for each.**
- **The order of QO arrival should not matter**. Today, `QualitiesToFlagCollectionConverter` assumes it should receive quality flags in chronological order. However, this may not always be the case in sync QC, where validity start might slightly move to the past due unaligned validities of objects coming from EPNs, being merged in the merger nodes.

Additionally the tests were extended and updated.
  • Loading branch information
knopers8 committed Sep 16, 2024
1 parent 158680c commit 4e78a5c
Show file tree
Hide file tree
Showing 11 changed files with 1,195 additions and 331 deletions.
8 changes: 5 additions & 3 deletions Framework/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,8 @@ add_library(O2QualityControl
src/TimekeeperFactory.cxx
src/RootFileStorage.cxx
src/ReductorHelpers.cxx
src/KafkaPoller.cxx)
src/KafkaPoller.cxx
src/FlagHelpers.cxx)

target_include_directories(
O2QualityControl
Expand Down Expand Up @@ -276,7 +277,9 @@ add_executable(o2-qc-test-core
test/testMonitorObjectCollection.cxx
test/testTrendingTask.cxx
test/testKafkaTests.cxx
)
test/testFlagHelpers.cxx
test/testQualitiesToFlagCollectionConverter.cxx
)
set_property(TARGET o2-qc-test-core
PROPERTY RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/tests)
target_link_libraries(o2-qc-test-core PRIVATE O2QualityControl O2::Catch2)
Expand Down Expand Up @@ -305,7 +308,6 @@ set(TEST_SRCS
test/testCheckWorkflow.cxx
test/testWorkflow.cxx
test/testRepoPathUtils.cxx
test/testQualitiesToFlagCollectionConverter.cxx
test/testUserCodeInterface.cxx
test/testStringUtils.cxx
test/testRunnerUtils.cxx
Expand Down
10 changes: 6 additions & 4 deletions Framework/include/QualityControl/BookkeepingQualitySink.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@
#ifndef QUALITYCONTROL_BOOKKEEPINGQUALITYSINK_H
#define QUALITYCONTROL_BOOKKEEPINGQUALITYSINK_H

#include <DataFormatsQualityControl/QualityControlFlagCollection.h>
#include <Framework/CompletionPolicy.h>
#include <Framework/DataProcessorSpec.h>
#include <Framework/Task.h>
#include "Provenance.h"
#include "QualityControl/QualitiesToFlagCollectionConverter.h"
#include "QualityControl/Provenance.h"

namespace o2::quality_control::core
{
Expand All @@ -31,7 +31,9 @@ class BookkeepingQualitySink : public framework::Task
{
public:
// we are using map here instead of the set, because items in the map are changeable, however items of the set are not.
using FlagsMap = std::unordered_map<std::string /*detector*/, std::unique_ptr<QualityControlFlagCollection>>;
using FlagsMap = std::unordered_map<std::string /*detector*/,
std::unordered_map<std::string /* QO name */,
std::unique_ptr<QualitiesToFlagCollectionConverter>>>;
using SendCallback = std::function<void(const std::string& grpcUri, const FlagsMap&, Provenance)>;

// sendCallback is mainly used for testing without the necessity to do grpc calls
Expand All @@ -50,7 +52,7 @@ class BookkeepingQualitySink : public framework::Task
std::string mGrpcUri;
Provenance mProvenance;
SendCallback mSendCallback;
FlagsMap mQualityObjectsMap;
FlagsMap mFlagsMap;

void sendAndClear();
};
Expand Down
47 changes: 47 additions & 0 deletions Framework/include/QualityControl/FlagHelpers.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
// All rights not expressly granted are reserved.
//
// This software is distributed under the terms of the GNU General Public
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

///
/// \file FlagHelpers.h
/// \author Piotr Konopka
///

#ifndef QUALITYCONTROL_FLAGHELPERS_H
#define QUALITYCONTROL_FLAGHELPERS_H

#include "DataFormatsQualityControl/QualityControlFlag.h"
#include "QualityControl/ValidityInterval.h"
#include <vector>
#include <optional>

namespace o2::quality_control::core::flag_helpers
{

/// \brief returns true if the provided intervals are valid and are overlapping or adjacent
bool intervalsConnect(const ValidityInterval& one, const ValidityInterval& other);

/// \brief returns true if the provided intervals are valid and are overlapping (there is at least one 1ms common)
bool intervalsOverlap(const ValidityInterval& one, const ValidityInterval& other);

/// \brief Removes the provided interval from the flag.
///
/// Removes the provided interval from the flag. A result can be:
/// - an empty vector if the interval fully covers the flag's interval
/// - a vector with one flag if the interval covers the flag's interval on one side
/// - a vector with two flags if the interval is fully contained by the flag's interval
std::vector<QualityControlFlag> excludeInterval(const QualityControlFlag& flag, ValidityInterval interval);

/// Trims the provided flag to the intersection with the provided interval.
/// If the intersection does not exist, it returns nullopt
std::optional<QualityControlFlag> intersection(const QualityControlFlag& flag, ValidityInterval interval);

} // namespace o2::quality_control::core::flag_helpers
#endif // QUALITYCONTROL_FLAGHELPERS_H
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,22 @@

#include <DataFormatsQualityControl/QualityControlFlag.h>
#include <DataFormatsQualityControl/QualityControlFlagCollection.h>
#include <QualityControl/ValidityInterval.h>

#include <memory>
#include <vector>
#include <functional>

namespace o2::quality_control::core
{

class QualityObject;

/// \brief Converts a set of chronologically provided Qualities from the same path into a QualityControlFlagCollection.
/// \brief Converts series of Quality Objects from the same path into a QualityControlFlagCollection.
class QualitiesToFlagCollectionConverter
{
public:
QualitiesToFlagCollectionConverter(std::unique_ptr<QualityControlFlagCollection> qcfc, std::string qoPath);
QualitiesToFlagCollectionConverter(std::unique_ptr<QualityControlFlagCollection> emptyQcfc, std::string qoPath);

~QualitiesToFlagCollectionConverter() = default;

Expand All @@ -44,14 +47,26 @@ class QualitiesToFlagCollectionConverter
size_t getQOsIncluded() const;
size_t getWorseThanGoodQOs() const;

/// Sets the provided validity interval, trims affected flags and fills extensions with UnknownQuality
void updateValidityInterval(const ValidityInterval validityInterval);

private:
std::string mQOPath; // this is only to indicate what is the missing Quality in QC Flag
/// \brief inserts the provided flag to the buffer, takes care of merging and trimming
void insert(QualityControlFlag&& flag);

std::unique_ptr<QualityControlFlagCollection> mConverted;
/// \brief trims all buffered flags which match the predicate using the provided interval
void trimBufferWithInterval(ValidityInterval interval,
const std::function<bool(const QualityControlFlag&)>& predicate = [](const auto&) { return true; });

uint64_t mCurrentStartTime = 0;
uint64_t mCurrentEndTime;
std::vector<QualityControlFlag> mCurrentFlags;
/// \brief trims the provided flag with all buffered flags which match the predicate
///
/// The result is a vector, because a flag interval split in the middle becomes two flags.
std::vector<QualityControlFlag> trimFlagAgainstBuffer(const QualityControlFlag& newFlag,
const std::function<bool(const QualityControlFlag&)>& predicate = [](const auto&){ return true; });

std::string mQOPath; // this is only to indicate what is the missing Quality in QC Flag
std::unique_ptr<QualityControlFlagCollection> mConverted;
std::set<QualityControlFlag> mFlagBuffer;
size_t mQOsIncluded = 0;
size_t mWorseThanGoodQOs = 0;
};
Expand Down
85 changes: 55 additions & 30 deletions Framework/src/BookkeepingQualitySink.cxx
Original file line number Diff line number Diff line change
Expand Up @@ -19,13 +19,15 @@
#include <Framework/InputRecordWalker.h>
#include <Framework/CompletionPolicyHelpers.h>
#include <Framework/DeviceSpec.h>
#include <DataFormatsQualityControl/QualityControlFlagCollection.h>
#include "QualityControl/Bookkeeping.h"
#include "QualityControl/QualitiesToFlagCollectionConverter.h"
#include "QualityControl/QualityObject.h"
#include "QualityControl/QcInfoLogger.h"
#include <BookkeepingApi/QcFlagServiceClient.h>
#include <BookkeepingApi/BkpClientFactory.h>
#include <stdexcept>
#include <utility>

namespace o2::quality_control::core
{
Expand All @@ -44,36 +46,59 @@ void BookkeepingQualitySink::send(const std::string& grpcUri, const BookkeepingQ
auto bkpClient = o2::bkp::api::BkpClientFactory::create(grpcUri);
auto& qcClient = bkpClient->qcFlag();

for (const auto& [detector, flagCollection] : flags) {
std::optional<int> runNumber;
std::optional<std::string> passName;
std::optional<std::string> periodName;

for (auto& [detector, qoMap] : flags) {
ILOG(Info, Support) << "Sending " << flags.size() << " flags for detector: " << detector << ENDM;

if (flagCollection->size() == 0) {
continue;
std::vector<QcFlag> bkpQcFlags{};
for (auto& [qoName, converter] : qoMap) {
if (converter == nullptr) {
continue;
}
auto flagCollection = converter->getResult();
if (flagCollection == nullptr) {
continue;
}
if (!runNumber.has_value()) {
runNumber = flagCollection->getRunNumber();
}
if (!passName.has_value()) {
passName = flagCollection->getPassName();
}
if (!periodName.has_value()) {
periodName = flagCollection->getPeriodName();
}

for (const auto& flag : *flagCollection) {
// BKP uses start/end of run for missing time values, so we are using this functionality in order to avoid
// determining these values by ourselves (see TaskRunner::start() for details). mtichak checked with mboulais that
// it is okay to do so.
bkpQcFlags.emplace_back(QcFlag{
.flagTypeId = flag.getFlag().getID(),
.from = flag.getStart() == gFullValidityInterval.getMin() ? std::nullopt : std::optional<uint64_t>{ flag.getStart() },
.to = flag.getEnd() == gFullValidityInterval.getMax() ? std::nullopt : std::optional<uint64_t>{ flag.getEnd() },
.origin = flag.getSource(),
.comment = flag.getComment() });
}
}

std::vector<QcFlag> bkpQcFlags{};
for (const auto& flag : *flagCollection) {
// BKP uses start/end of run for missing time values, so we are using this functionality in order to avoid
// determining these values by ourselves (see TaskRunner::start() for details). mtichak checked with mboulais that
// it is okay to do so.
bkpQcFlags.emplace_back(QcFlag{
.flagTypeId = flag.getFlag().getID(),
.from = flag.getStart() == gFullValidityInterval.getMin() ? std::nullopt : std::optional<uint64_t>{ flag.getStart() },
.to = flag.getEnd() == gFullValidityInterval.getMax() ? std::nullopt : std::optional<uint64_t>{ flag.getEnd() },
.origin = flag.getSource(),
.comment = flag.getComment() });
if (bkpQcFlags.empty()) {
continue;
}

try {
switch (provenance) {
case Provenance::SyncQC:
qcClient->createForSynchronous(flagCollection->getRunNumber(), detector, bkpQcFlags);
qcClient->createForSynchronous(runNumber.value(), detector, bkpQcFlags);
break;
case Provenance::AsyncQC:
qcClient->createForDataPass(flagCollection->getRunNumber(), flagCollection->getPassName(), detector, bkpQcFlags);
qcClient->createForDataPass(runNumber.value(), passName.value(), detector, bkpQcFlags);
break;
case Provenance::MCQC:
qcClient->createForSimulationPass(flagCollection->getRunNumber(), flagCollection->getPeriodName(), detector, bkpQcFlags);
qcClient->createForSimulationPass(runNumber.value(), periodName.value(), detector, bkpQcFlags);
break;
}
} catch (const std::runtime_error& err) {
Expand All @@ -84,14 +109,7 @@ void BookkeepingQualitySink::send(const std::string& grpcUri, const BookkeepingQ
}

BookkeepingQualitySink::BookkeepingQualitySink(const std::string& grpcUri, Provenance provenance, SendCallback sendCallback)
: mGrpcUri{ grpcUri }, mProvenance{ provenance }, mSendCallback{ sendCallback } {}

auto merge(std::unique_ptr<QualityControlFlagCollection>&& collection, const std::unique_ptr<QualityObject>& qualityObject) -> std::unique_ptr<QualityControlFlagCollection>
{
QualitiesToFlagCollectionConverter converter(std::move(collection), qualityObject->getPath());
converter(*qualityObject);
return converter.getResult();
}
: mGrpcUri{ grpcUri }, mProvenance{ provenance }, mSendCallback{ std::move(sendCallback) } {}

auto collectionForQualityObject(const QualityObject& qualityObject) -> std::unique_ptr<QualityControlFlagCollection>
{
Expand All @@ -108,13 +126,18 @@ auto collectionForQualityObject(const QualityObject& qualityObject) -> std::uniq
void BookkeepingQualitySink::run(framework::ProcessingContext& context)
{
for (auto const& ref : framework::InputRecordWalker(context.inputs())) {
std::unique_ptr<QualityObject> qualityObject;
try {
auto qualityObject = framework::DataRefUtils::as<QualityObject>(ref);
auto [emplacedIt, _] = mQualityObjectsMap.emplace(qualityObject->getDetectorName(), collectionForQualityObject(*qualityObject));
emplacedIt->second = merge(std::move(emplacedIt->second), qualityObject);
qualityObject = framework::DataRefUtils::as<QualityObject>(ref);
} catch (...) {
ILOG(Warning, Support) << "Unexpected message received, QualityObject expected" << ENDM;
continue;
}
auto& converter = mFlagsMap[qualityObject->getDetectorName()][qualityObject->getName()];
if (converter == nullptr) {
converter = std::make_unique<QualitiesToFlagCollectionConverter>(collectionForQualityObject(*qualityObject), qualityObject->getPath());
}
(*converter)(*qualityObject);
}
}

Expand All @@ -130,8 +153,10 @@ void BookkeepingQualitySink::stop()

void BookkeepingQualitySink::sendAndClear()
{
mSendCallback(mGrpcUri, mQualityObjectsMap, mProvenance);
mQualityObjectsMap.clear();
if (!mFlagsMap.empty()) {
mSendCallback(mGrpcUri, mFlagsMap, mProvenance);
}
mFlagsMap.clear();
}

} // namespace o2::quality_control::core
73 changes: 73 additions & 0 deletions Framework/src/FlagHelpers.cxx
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
// All rights not expressly granted are reserved.
//
// This software is distributed under the terms of the GNU General Public
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
//
// In applying this license CERN does not waive the privileges and immunities
// granted to it by virtue of its status as an Intergovernmental Organization
// or submit itself to any jurisdiction.

///
/// \file FlagHelpers.cxx
/// \author Piotr Konopka
///

#include "QualityControl/FlagHelpers.h"

namespace o2::quality_control::core::flag_helpers
{

bool intervalsConnect(const ValidityInterval& one, const ValidityInterval& other)
{
// Object validity in CCDB is a right-open range, which means it includes the beginning and excludes the ending.
// In other words, for the validity [1, 10), 9 is the last integer to be included.
// Thus, ranges [1, 10) and [10, 20) are considered adjacent, while [1, 10) and [11, 20) are already separate
// and should not be merged.
return one.isValid() && other.isValid() && one.getMax() >= other.getMin() && one.getMin() <= other.getMax();
}

bool intervalsOverlap(const ValidityInterval& one, const ValidityInterval& other)
{
return one.isValid() && other.isValid() && one.getMax() > other.getMin() && one.getMin() < other.getMax();
}

std::vector<QualityControlFlag> excludeInterval(const QualityControlFlag& flag, ValidityInterval interval)
{
std::vector<QualityControlFlag> result;

if (flag.getInterval().isInvalid()) {
return result;
}

if (auto overlap = flag.getInterval().getOverlap(interval); overlap.isInvalid() || overlap.isZeroLength()) {
result.push_back(flag);
return result;
}

if (interval.getMin() > flag.getStart()) {
result.emplace_back(flag.getStart(), interval.getMin(), flag.getFlag(), flag.getComment(), flag.getSource());
}
if (interval.getMax() < flag.getEnd()) {
result.emplace_back(interval.getMax(), flag.getEnd(), flag.getFlag(), flag.getComment(), flag.getSource());
}
return result;
}

std::optional<QualityControlFlag> intersection(const QualityControlFlag& flag, ValidityInterval interval)
{
if (flag.getInterval().isInvalid()) {
return std::nullopt;
}
if (interval.isInvalid()) {
return flag;
}
auto intersection = flag.getInterval().getOverlap(interval);
if (intersection.isInvalid()) {
return std::nullopt;
}
return QualityControlFlag{ intersection.getMin(), intersection.getMax(), flag.getFlag(), flag.getComment(), flag.getSource() };
}

} // namespace o2::quality_control::core::flag_helpers
Loading

0 comments on commit 4e78a5c

Please sign in to comment.