Commit fc911f3c authored by Wouter Deconinck's avatar Wouter Deconinck
Browse files

More changes to Podio layer

parent dfba5903
......@@ -25,7 +25,7 @@ endif()
find_package(EICD REQUIRED)
find_package(EDM4HEP REQUIRED)
find_package(podio 0.11...0.14 REQUIRED)
find_package(podio 0.13...0.14 REQUIRED)
add_definitions("-Dpodio_VERSION_MAJOR=${podio_VERSION_MAJOR}")
add_definitions("-Dpodio_VERSION_MINOR=${podio_VERSION_MINOR}")
add_definitions("-Dpodio_VERSION_PATCH=${podio_VERSION_PATCH}")
......
#include "PodioOutput.h"
#include "podio/podioVersion.h"
#include "GaudiKernel/ISvcLocator.h"
#include "JugBase/PodioDataSvc.h"
#include "TFile.h"
#include "rootutils.h"
......@@ -11,13 +9,12 @@ PodioOutput::PodioOutput(const std::string& name, ISvcLocator* svcLoc)
: GaudiAlgorithm(name, svcLoc), m_firstEvent(true) {}
StatusCode PodioOutput::initialize() {
if (GaudiAlgorithm::initialize().isFailure())
return StatusCode::FAILURE;
if (GaudiAlgorithm::initialize().isFailure()) return StatusCode::FAILURE;
// check whether we have the PodioEvtSvc active
m_podioDataSvc = dynamic_cast<PodioDataSvc*>(evtSvc().get());
if (!m_podioDataSvc) {
error() << "Failed to get the DataSvc" << endmsg;
if (nullptr == m_podioDataSvc) {
error() << "Could not get DataSvc!" << endmsg;
return StatusCode::FAILURE;
}
......@@ -27,94 +24,104 @@ StatusCode PodioOutput::initialize() {
m_datatree = m_podioDataSvc->eventDataTree();
m_datatree->SetDirectory(m_file.get());
m_metadatatree = new TTree("metadata", "Metadata tree");
m_switch = KeepDropSwitch(m_outputCommands);
m_runMDtree =new TTree("run_metadata", "Run metadata tree");
m_evtMDtree = new TTree("evt_metadata", "Event metadata tree");
m_colMDtree = new TTree("col_metadata", "Collection metadata tree");
m_evtMDtree->Branch("evtMD", "GenericParameters", m_podioDataSvc->getProvider().eventMetaDataPtr() ) ;
m_switch = KeepDropSwitch(m_outputCommands);
return StatusCode::SUCCESS;
}
void PodioOutput::resetBranches(const std::vector<std::pair<std::string, podio::CollectionBase*>>& collections) {
for (auto& [collName, collBuffers] : collections) {
#if podio_VERSION_MAJOR == 0 && (podio_VERSION_MINOR < 13 || (podio_VERSION_MINOR == 13 && podio_VERSION_PATCH < 2))
auto data = collBuffers->getBufferAddress();
auto references = collBuffers->referenceCollections();
auto vecmembers = collBuffers->vectorMembers();
#else
auto buffers = collBuffers->getBuffers();
auto data = buffers.data;
auto references = buffers.references;
auto vecmembers = buffers.vectorMembers;
#endif
for (auto& collNamePair : collections) {
auto collName = collNamePair.first;
if (m_switch.isOn(collName)) {
// Reconnect branches and collections
m_datatree->SetBranchAddress(collName.c_str(), data);
auto colls = references;
if (colls) {
int j = 0;
for (auto& c : (*colls)) {
const auto brName = podio::root_utils::refBranch(collName, j);
m_datatree->SetBranchAddress(brName.c_str(), &c);
++j;
const auto collBuffers = collNamePair.second->getBuffers();
m_datatree->SetBranchAddress(collName.c_str(), collBuffers.data);
auto colls = collBuffers.references;
if (colls != nullptr) {
for (size_t j = 0; j < colls->size(); ++j) {
// We need to make sure that we pass the actual pointer to
// SetBranchAddress instead of the pointer to the unique_ptr
auto collPtr = (*colls)[j].get();
m_datatree->SetBranchAddress((collName + "#" + std::to_string(j)).c_str(), &collPtr);
}
}
auto colls_v = vecmembers;
if (colls_v) {
auto colls_v = collBuffers.vectorMembers;
if (colls_v != nullptr) {
int j = 0;
for (auto& [dataType, add] : (*colls_v)) {
const auto brName = podio::root_utils::vecBranch(collName, j);
m_datatree->SetBranchAddress(brName.c_str(), add);
for (auto& c : (*colls_v)) {
void* add = c.second ;
m_datatree->SetBranchAddress((collName + "_" + std::to_string(j)).c_str(), add);
++j;
}
}
}
collBuffers->prepareForWrite();
collNamePair.second->prepareForWrite();
}
}
void PodioOutput::createBranches(const std::vector<std::pair<std::string, podio::CollectionBase*>>& collections) {
for (auto& [collName, collBuffers] : collections) {
#if podio_VERSION_MAJOR == 0 && (podio_VERSION_MINOR < 13 || (podio_VERSION_MINOR == 13 && podio_VERSION_PATCH < 2))
auto data = collBuffers->getBufferAddress();
auto references = collBuffers->referenceCollections();
auto vecmembers = collBuffers->vectorMembers();
#else
auto buffers = collBuffers->getBuffers();
auto data = buffers.data;
auto references = buffers.references;
auto vecmembers = buffers.vectorMembers;
#endif
const std::string className = collBuffers->getValueTypeName();
const std::string collClassName = "vector<" + className + "Data>";
// collectionID, collection type, subset collection
std::vector<std::tuple<int, std::string, bool>>* collectionInfo = new std::vector<std::tuple<int, std::string, bool>>();
collectionInfo->reserve( collections.size());
for (auto& collNamePair : collections) {
auto collName = collNamePair.first;
std::string className( collNamePair.second->getValueTypeName() ) ;
std::string collClassName = "vector<" + className + "Data>";
int isOn = 0;
if (m_switch.isOn(collName)) {
isOn = 1;
m_datatree->Branch(collName.c_str(), collClassName.c_str(), data);
const auto collBuffers = collNamePair.second->getBuffers();
m_datatree->Branch(collName.c_str(), collClassName.c_str(), collBuffers.data);
// Create branches for collections holding relations
if (auto refColls = references) {
int j = 0;
if (auto refColls = collBuffers.references) {
int i = 0;
for (auto& c : (*refColls)) {
const auto brName = podio::root_utils::refBranch(collName, j);
m_datatree->Branch(brName.c_str(), c);
++j;
const auto brName = podio::root_utils::refBranch(collName, i);
m_datatree->Branch(brName.c_str(), c.get());
++i;
}
}
// vector members
if (auto vminfo = vecmembers) {
int j = 0;
for (auto& [dataType, add] : (*vminfo)) {
const std::string typeName = "vector<" + dataType + ">";
const auto brName = podio::root_utils::vecBranch(collName, j);
m_datatree->Branch(brName.c_str(), typeName.c_str(), add);
++j;
// ---- vector members
auto vminfo = collBuffers.vectorMembers;
if (vminfo != nullptr){
int i = 0;
for(auto& c : (*vminfo)){
std::string typeName = "vector<"+c.first+">" ;
void* add = c.second ;
m_datatree->Branch((collName+"_"+std::to_string(i)).c_str(),
typeName.c_str(),
add);
++i;
}
}
}
}
const auto collID = m_podioDataSvc->getCollectionIDs()->collectionID(collName);
// No check necessary, only registered collections possible
auto coll = collNamePair.second;
const auto collType = coll->getValueTypeName() + "Collection";
collectionInfo->emplace_back(collID, std::move(collType), coll->isSubsetCollection());
//}
debug() << isOn << " Registering collection " << collClassName << " " << collName.c_str() << " containing type "
<< className << endmsg;
collBuffers->prepareForWrite();
collNamePair.second->prepareForWrite();
}
m_metadatatree->Branch("CollectionTypeInfo", collectionInfo);
}
StatusCode PodioOutput::execute() {
......@@ -122,74 +129,73 @@ StatusCode PodioOutput::execute() {
// register for writing
if (m_firstEvent) {
createBranches(m_podioDataSvc->getCollections());
createBranches(m_podioDataSvc->getReadCollections());
} else {
resetBranches(m_podioDataSvc->getCollections());
resetBranches(m_podioDataSvc->getReadCollections());
}
m_firstEvent = false;
if (msgLevel(MSG::DEBUG)) {
debug() << "Filling DataTree .." << endmsg;
}
debug() << "Filling DataTree .." << endmsg;
m_datatree->Fill();
m_evtMDtree->Fill();
return StatusCode::SUCCESS;
}
/** PodioOutput::finalize
* has to happen after all algorithms that touch the data store finish.
* Here the job options are retrieved and stored to disk as a branch
* in the metadata tree.
*
*/
* has to happen after all algorithms that touch the data store finish.
* Here the job options are retrieved and stored to disk as a branch
* in the metadata tree.
*
*/
StatusCode PodioOutput::finalize() {
info() << "Finalizing output algorithm" << endmsg;
if (GaudiAlgorithm::finalize().isFailure()) {
return StatusCode::FAILURE;
}
if (GaudiAlgorithm::finalize().isFailure()) return StatusCode::FAILURE;
//// prepare job options metadata ///////////////////////
// retrieve the configuration of the job
// and write it to file as vector of strings
debug() << "Preparing job options metadata" << endmsg;
std::vector<std::string> config_data;
const auto& jobOptionsSvc = Gaudi::svcLocator()->getOptsSvc();
const auto& jobOptionsSvc = Gaudi::svcLocator()->getOptsSvc();
const auto& configured_properties = jobOptionsSvc.items();
for (const auto& per_property : configured_properties) {
std::stringstream config_stream;
// sample output:
// HepMCToEDMConverter.genparticles = "GenParticles";
// Note that quotes are added to all property values,
// which leads to problems with ints, lists, dicts and bools.
// which leads to problems with ints, lists, dicts and bools.
// For theses types, the quotes must be removed in postprocessing.
config_stream << std::get<0>(per_property) << " = \"" << std::get<1>(per_property) << "\";" << std::endl;
config_data.push_back(config_stream.str());
}
// Some default components are not captured by the job option service
// and have to be traversed like this. Note that Gaudi!577 will improve this.
debug() << "Appending default component metadata" << endmsg;
for (const auto* name : {"ApplicationMgr", "MessageSvc", "NTupleSvc"}) {
std::stringstream config_stream;
auto svc = service<IProperty>(name);
if (!svc.isValid()) {
continue;
}
auto svc = service<IProperty>( name );
if (!svc.isValid()) continue;
for (const auto* property : svc->getProperties()) {
config_stream << name << "." << property->name() << " = \"" << property->toString() << "\";" << std::endl;
}
config_data.push_back(config_stream.str());
}
//// finalize trees and file //////////////////////////////
debug() << "Finalizing trees and output file" << endmsg;
m_file->cd();
m_metadatatree->Branch("gaudiConfigOptions", &config_data);
m_metadatatree->Branch("CollectionIDs", m_podioDataSvc->getCollectionIDs());
m_metadatatree->Fill();
m_colMDtree->Branch("colMD", "std::map<int,podio::GenericParameters>", m_podioDataSvc->getProvider().getColMetaDataMap() ) ;
m_colMDtree->Fill();
m_runMDtree->Branch("runMD", "std::map<int,podio::GenericParameters>", m_podioDataSvc->getProvider().getRunMetaDataMap() ) ;
m_runMDtree->Fill();
m_datatree->Write();
m_file->Write();
m_file->Close();
info() << "Data written to: " << m_filename.value() << endmsg;
info() << "Data written to: " << m_filename.value();
if (!m_filenameRemote.value().empty()) {
TFile::Cp(m_filename.value().c_str(), m_filenameRemote.value().c_str(), false);
info() << " and copied to: " << m_filenameRemote.value() << endmsg;
info() << " and copied to: " << m_filenameRemote.value() << endmsg;
}
return StatusCode::SUCCESS;
}
......@@ -38,7 +38,7 @@ private:
Gaudi::Property<std::vector<std::string>> m_outputCommands{
this, "outputCommands", {"keep *"}, "A set of commands to declare which collections to keep or drop."};
Gaudi::Property<std::string> m_filenameRemote{
this, "filenameRemote", "", "An optional file path to copy the outputfile to."};
this, "remoteFilename", "", "An optional file path to copy the outputfile to."};
/// Switch for keeping or dropping outputs
KeepDropSwitch m_switch;
/// Needed for collection ID table
......@@ -49,6 +49,10 @@ private:
TTree* m_datatree;
/// The tree to be filled with meta data
TTree* m_metadatatree;
TTree* m_runMDtree;
TTree* m_evtMDtree;
TTree* m_colMDtree;
/// The stored collections
std::vector<podio::CollectionBase*> m_storedCollections;
};
......
#ifndef PODIO_ROOT_UTILS_H
#define PODIO_ROOT_UTILS_H
#include "podio/podioVersion.h"
#include "podio/CollectionBase.h"
#include "podio/CollectionBranches.h"
......@@ -29,30 +28,21 @@ inline std::string vecBranch(const std::string& name, size_t index) {
inline void setCollectionAddresses(podio::CollectionBase* collection, const CollectionBranches& branches) {
#if podio_VERSION_MAJOR == 0 && (podio_VERSION_MINOR < 13 || (podio_VERSION_MINOR == 13 && podio_VERSION_PATCH < 2))
auto data = collection->getBufferAddress();
auto references = collection->referenceCollections();
auto vecmembers = collection->vectorMembers();
#else
auto buffers = collection->getBuffers();
auto data = buffers.data;
auto references = buffers.references;
auto vecmembers = buffers.vectorMembers;
#endif
if (data) {
branches.data->SetAddress(data);
const auto collBuffers = collection->getBuffers();
if (auto buffer = collBuffers.data) {
branches.data->SetAddress(buffer);
}
if (references) {
for (size_t i = 0; i < references->size(); ++i) {
branches.refs[i]->SetAddress(&(*references)[i]);
if (auto refCollections = collBuffers.references) {
for (size_t i = 0; i < refCollections->size(); ++i) {
branches.refs[i]->SetAddress(&(*refCollections)[i]);
}
}
if (vecmembers) {
for (size_t i = 0; i < vecmembers->size(); ++i) {
branches.vecs[i]->SetAddress((*vecmembers)[i].second);
if (auto vecMembers = collBuffers.vectorMembers) {
for (size_t i = 0; i < vecMembers->size(); ++i) {
branches.vecs[i]->SetAddress((*vecMembers)[i].second);
}
}
}
......@@ -60,4 +50,3 @@ inline void setCollectionAddresses(podio::CollectionBase* collection, const Coll
}
#endif
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment