Run ML training on 100 events for every pipeline (manual for 100k)

benchmarks/imaging_ecal/analysis/emcal_barrel_pion_rejection_analysis.cxx

@@ -15,8 +15,6 @@
 #include "eicd/CalorimeterHitData.h"
 #include "eicd/ClusterCollection.h"
 #include "eicd/ClusterData.h"
-#include "eicd/ClusterLayerData.h"
-
 #include "eicd/RawCalorimeterHitCollection.h"
 #include "eicd/RawCalorimeterHitData.h"
 
@@ -229,7 +227,7 @@ void emcal_barrel_pion_rejection_analysis(
 
   // EcalBarrelImagingClustersLayers Functions////////////////////////////////////////////////////////////////////////////////
   // Number of hits
-  auto nhitsClusLayerImg = [] (const std::vector<eicd::ClusterLayerData>& evt) {
+  auto nhitsClusLayerImg = [] (const std::vector<eicd::ClusterData>& evt) {
     int nhitsTot = 0;
     for (const auto &i : evt){
       nhitsTot += i.nhits;
@@ -238,10 +236,10 @@ void emcal_barrel_pion_rejection_analysis(
   };
 
   // Number of clusters
-  auto nClusLayerImg = [] (const std::vector<eicd::ClusterLayerData>& evt) {return (int) evt.size(); };
+  auto nClusLayerImg = [] (const std::vector<eicd::ClusterData>& evt) {return (int) evt.size(); };
 
   // Energy deposition in cluster [GeV]
-  auto EClusLayerImg = [](const std::vector<eicd::ClusterLayerData>& evt) {
+  auto EClusLayerImg = [](const std::vector<eicd::ClusterData>& evt) {
     double total_edep = 0.0;
     for (const auto& i: evt){
       total_edep += i.energy;
@@ -250,7 +248,7 @@ void emcal_barrel_pion_rejection_analysis(
   };
 
   // Max Energy deposition in cluster [GeV]
-  auto EClusLayerMaxImg = [](const std::vector<eicd::ClusterLayerData>& evt) {
+  auto EClusLayerMaxImg = [](const std::vector<eicd::ClusterData>& evt) {
     double max = 0.0;
     for (const auto& i: evt){
       if (i.energy > max){max = i.energy;}
@@ -259,7 +257,7 @@ void emcal_barrel_pion_rejection_analysis(
   };
 
   // Min Energy deposition in cluster [GeV]
-  auto EClusLayerMax2Img = [](const std::vector<eicd::ClusterLayerData>& evt) {
+  auto EClusLayerMax2Img = [](const std::vector<eicd::ClusterData>& evt) {
     double max1 = 0.0;
     double max2 = 0.0;
     for (const auto& i: evt){

benchmarks/imaging_shower_ML/config.yml

-ml_shower:tagging_epi :
+ml_shower:tagging_epimuphka_100:
+  extends: .rec_benchmark
+  stage: benchmarks1
+  script:
+    - pwd
+    - ls -l
+    - python3 benchmarks/imaging_shower_ML/sim_rec_tag.py -t imcal_epimuphka_100 -n 100 --particles "electron,pion-,muon,photon,kaon-"
+      --pmin 0.5 --pmax 10
+
+ml_shower:tagging_epimuphka:
   extends: .rec_benchmark
   when: manual
   stage: benchmarks1
   script:
-    - python3 benchmarks/imaging_shower_ML/sim_rec_tag.py -t imcal_epi -n 100000 
-      --particles "electron,pion-,pion-" --pmin 0.5 --pmax 10
+    - ls -hal
+    - python3 benchmarks/imaging_shower_ML/sim_rec_tag.py -t imcal_epimuphka -n 10000 --particles "electron,pion-,muon,photon,kaon-"
+      --pmin 0.5 --pmax 10
 
-ml_shower:tagging_e :
+ml_shower:tagging_e:
   extends: .rec_benchmark
   when: manual
   stage: benchmarks1
   script:
-    - python3 benchmarks/imaging_shower_ML/sim_rec_tag.py -t imcal_e -n 100000 
+    - python3 benchmarks/imaging_shower_ML/sim_rec_tag.py -t imcal_e -n 100 
       --particles "electron" --pmin 0.5 --pmax 10
 
-ml_shower:tagging_pi :
+ml_shower:tagging_pi:
   extends: .rec_benchmark
   when: manual
   stage: benchmarks1
   script:
-    - python3 benchmarks/imaging_shower_ML/sim_rec_tag.py -t imcal_pi -n 100000 
+    - python3 benchmarks/imaging_shower_ML/sim_rec_tag.py -t imcal_pi -n 100 
       --particles "pion-" --pmin 0.5 --pmax 10
 
+ml_shower:training_100:
+  extends: .rec_benchmark
+  stage: process
+  needs: ["ml_shower:tagging_epimuphka_100"]#, "ml_shower:tagging_e", "ml_shower:tagging_pi"]
+  script:
+    - pip install tensorflow particle
+    - python3 benchmarks/imaging_shower_ML/scripts/ml_training.py -t imcal_epimuphka_100 --pmin 0.5 --pmax 10
+
 ml_shower:training:
   extends: .rec_benchmark
-  when: manual
   stage: process
-  needs: ["ml_shower:tagging_epi", "ml_shower:tagging_e", "ml_shower:tagging_pi"]
+  when: manual
+  needs: ["ml_shower:tagging_epimuphka"]#, "ml_shower:tagging_e", "ml_shower:tagging_pi"]
   script:
-    - ls -lrth
-    # TODO
+    - pip install tensorflow particle
+    - python3 benchmarks/imaging_shower_ML/scripts/ml_training.py -t imcal_epimuphka --pmin 0.5 --pmax 10
 
 ml_shower:test:
   extends: .rec_benchmark

benchmarks/imaging_shower_ML/options/imaging_ml_data.py

@@ -60,13 +60,13 @@ becal_img_reco = CalHitReco('becal_img_reco',
         **becal_img_daq)
 
 becal_img_merger = MLDataMerger('becal_img_merger',
-        inputHitCollection=becal_img_reco.outputHitCollection,
-        outputHitCollection='EcalBarrelImagingHitsSeg',
+        inputHits=becal_img_reco.outputHitCollection,
+        outputHits='EcalBarrelImagingHitsSeg',
         etaSize=0.001,
         phiSize=0.001)
 
 becal_img_sorter = MLDataSorter('becal_img_sorter',
-        inputHitCollection=becal_img_merger.outputHitCollection,
+        inputHitCollection=becal_img_merger.outputHits,
         outputHitCollection='EcalBarrelImagingHitsML',
         numberOfLayers=kwargs['img_nlayers'],
         numberOfHits=kwargs['nhits'])
@@ -117,12 +117,14 @@ becal_scfi_sorter = MLDataSorter('becal_scfi_sorter',
 
 # combine layers
 becal_combiner = MLDataCombiner('becal_combiner',
-        inputHitCollection1=becal_img_sorter.outputHitCollection,
-        inputHitCollection2=becal_scfi_sorter.outputHitCollection,
-        outputHitCollection='EcalBarrelHitsCombinedML',
+        inputHits1=becal_img_sorter.outputHitCollection,
+        inputHits2=becal_scfi_sorter.outputHitCollection,
+        outputHits='EcalBarrelHitsCombinedML',
         layerIncrement=100,
         rule=kwargs['combine'])
 
+        
+
 podout.outputCommands = [
 #        'keep *',
         'drop *',

benchmarks/imaging_shower_ML/scripts/prepare_tf_dataset.py

@@ -145,7 +145,8 @@ if __name__ == '__main__':
 
     event_ids = df['event'].unique()
     data = df.set_index('event')[featcols].clip(0, 1).values.reshape([len(event_ids)] + dshape)
-    tags = dfm.loc[event_ids, ['pdgID', 'p', 'pT', 'eta', 'phi', 'mass']]
+
+    tags = dfm.loc[event_ids, ['PDG', 'p', 'pT', 'eta', 'phi', 'mass']]
     # also save Edep per layers
     # merge the sandwich layers
     df['layer'] = df['layer'] % 100

benchmarks/imaging_shower_ML/sim_rec_tag.py

@@ -30,13 +30,13 @@ parser.add_argument('--physics-list', type=str, default='FTFP_BERT', help='Path
 args = parser.parse_args()
 kwargs = vars(args)
 
-for mdir in ['gen_data', 'sim_data', 'rec_data', 'tag_data']:
+for mdir in ['sim_output/gen_data', 'sim_output/sim_data', 'sim_output/rec_data', 'sim_output/tag_data']:
     os.makedirs(mdir, exist_ok=True)
 
-gen_file = os.path.join('gen_data', '{nametag}_{pmin}_{pmax}.hepmc'.format(**kwargs))
-sim_file = os.path.join('sim_data', '{nametag}_{pmin}_{pmax}.edm4hep.root'.format(**kwargs))
-rec_file = os.path.join('rec_data', '{nametag}_{pmin}_{pmax}.root'.format(**kwargs))
-tag_dir = os.path.join('tag_data', '{nametag}_{pmin}_{pmax}'.format(**kwargs))
+gen_file = os.path.join('sim_output/gen_data', '{nametag}_{pmin}_{pmax}.hepmc'.format(**kwargs))
+sim_file = os.path.join('sim_output/sim_data', '{nametag}_{pmin}_{pmax}.edm4hep.root'.format(**kwargs))
+rec_file = os.path.join('sim_output/rec_data', '{nametag}_{pmin}_{pmax}.root'.format(**kwargs))
+tag_dir = os.path.join('sim_output/tag_data', '{nametag}_{pmin}_{pmax}'.format(**kwargs))
 
 procs = [p.strip() for p in args.process.split(',')]
 sdir = os.path.dirname(os.path.realpath(__file__))