-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* added function (+ tests) to extract mdat modules * added mdat so we can write start and end times * moved code into cpp file * added doc and checks to stop time tests * clang formatting * improved error message * more clang format * black formatting * cppcheck warnings fixed * fix whoopsie * clang formatting * fixed test code * intergration test with isoformat * use f strings to make it tidier * add milliseconds to expected output * need to compare against byte string * black formatting * start and end time checks need to be utc * urggh timestamp formatting * urggh timestamp formatting... * oops should be end_time in json too * update docs * add debug message * write stop time even on historical jobs * Update mdat_Writer.cpp missing newline * json example was malformed * Simplified by introducing a private class * removed duplication
- Loading branch information
1 parent
a411421
commit b216d06
Showing
15 changed files
with
343 additions
and
34 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,42 @@ | ||
| # mdat writer module | ||
|
|
||
| ## Stream configuration fields | ||
|
|
||
| This module is different to other writer modules in that it doesn't use Kafka. Instead metadata data values are set via | ||
| code. | ||
| It isn't a general metadata writer, there are only a discrete set of named values it will work with. Other values are | ||
| ignored. | ||
|
|
||
| Currently, it only supports start and stop times. | ||
|
|
||
| ## Example | ||
| Example `nexus_structure` to write start and stop times: | ||
|
|
||
| ```json | ||
| { | ||
| "nexus_structure": { | ||
| "children": [ | ||
| { | ||
| "type": "group", | ||
| "name": "entry", | ||
| "children": [ | ||
| { | ||
| "module": "mdat", | ||
| "config": { | ||
| "name": "start_time" | ||
| } | ||
| }, | ||
| { | ||
| "module": "mdat", | ||
| "config": { | ||
| "name": "end_time" | ||
| } | ||
| } | ||
| ] | ||
| } | ||
| ] | ||
| } | ||
| } | ||
| ``` | ||
|
|
||
|
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| set(mdat_SRC | ||
| mdat_Writer.cpp | ||
| ) | ||
|
|
||
| set(mdat_INC | ||
| mdat_Writer.h | ||
| ) | ||
|
|
||
| create_writer_module(mdat) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| // SPDX-License-Identifier: BSD-2-Clause | ||
| // | ||
| // This code has been produced by the European Spallation Source | ||
| // and its partner institutes under the BSD 2 Clause License. | ||
| // | ||
| // See LICENSE.md at the top level for license information. | ||
| // | ||
| // Screaming Udder! https://esss.se | ||
|
|
||
| #include "mdat_Writer.h" | ||
| #include "FileWriterTask.h" | ||
| #include "HDFOperations.h" | ||
| #include "ModuleHDFInfo.h" | ||
| #include "MultiVector.h" | ||
| #include "TimeUtility.h" | ||
| #include "json.h" | ||
|
|
||
| namespace WriterModule::mdat { | ||
| void mdat_Writer::defineMetadata(std::vector<ModuleHDFInfo> const &Modules) { | ||
| Writables = extractDetails(Modules); | ||
| } | ||
|
|
||
| void mdat_Writer::setStartTime(time_point startTime) { | ||
| setWritableValueIfDefined(StartTime, startTime); | ||
| } | ||
|
|
||
| void mdat_Writer::setStopTime(time_point stopTime) { | ||
| setWritableValueIfDefined(EndTime, stopTime); | ||
| } | ||
|
|
||
| void mdat_Writer::setWritableValueIfDefined(std::string const &Name, | ||
| time_point const &Time) { | ||
| if (auto Result = Writables.find(Name); Result != Writables.end()) { | ||
| Result->second.Value = toUTCDateTime(Time); | ||
| } | ||
| } | ||
|
|
||
| void mdat_Writer::writeMetadata(FileWriter::FileWriterTask const *Task) const { | ||
| for (auto const &[Name, Value] : Writables) { | ||
| if (std::find(AllowedNames.cbegin(), AllowedNames.cend(), Name) == | ||
| AllowedNames.end()) { | ||
| continue; | ||
| } | ||
| if (Value.isWritable()) { | ||
| writeStringValue(Task, Name, Value.Path, Value.Value); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| void mdat_Writer::writeStringValue(FileWriter::FileWriterTask const *Task, | ||
| std::string const &Name, | ||
| std::string const &Path, | ||
| std::string const &Value) { | ||
| try { | ||
| auto StringVec = MultiVector<std::string>{{1}}; | ||
| StringVec.at({0}) = Value; | ||
| auto Group = hdf5::node::get_group(Task->hdfGroup(), Path); | ||
| HDFOperations::writeStringDataset(Group, Name, StringVec); | ||
| } catch (std::exception &Error) { | ||
| LOG_ERROR("Failed to write mdat string value: {}", Error.what()); | ||
| } | ||
| } | ||
|
|
||
| std::unordered_map<std::string, mdat_Writer::Writable> | ||
| mdat_Writer::extractDetails(std::vector<ModuleHDFInfo> const &Modules) const { | ||
| std::unordered_map<std::string, Writable> Details; | ||
|
|
||
| for (auto const &Module : Modules) { | ||
| if (Module.WriterModule != "mdat") { | ||
| continue; | ||
| } | ||
|
|
||
| auto const name = extractName(Module.ConfigStream); | ||
| if (name && std::find(AllowedNames.begin(), AllowedNames.end(), name) != | ||
| AllowedNames.end()) { | ||
| Writable NewWritable; | ||
| NewWritable.Path = Module.HDFParentName; | ||
| Details[name.value()] = NewWritable; | ||
| } | ||
| } | ||
| return Details; | ||
| } | ||
|
|
||
| std::optional<std::string> | ||
| mdat_Writer::extractName(std::string const &configJson) { | ||
| nlohmann::json json = nlohmann::json::parse(configJson); | ||
| for (auto it = json.begin(); it != json.end(); ++it) { | ||
| if (it.key() == "name" && !it.value().empty()) { | ||
| return it.value(); | ||
| } | ||
| } | ||
| return {}; | ||
| } | ||
| } // namespace WriterModule::mdat |
Oops, something went wrong.