Merge pull request #29 from eic/pr/tracking_performances_ci

benchmarks/tracking_performances: implement CI

.gitlab-ci.yml

@@ -131,6 +131,7 @@ include:
   - local: 'benchmarks/backgrounds/config.yml'
   - local: 'benchmarks/ecal_gaps/config.yml'
   - local: 'benchmarks/tracking_detectors/config.yml'
+  - local: 'benchmarks/tracking_performances/config.yml'
   - local: 'benchmarks/barrel_ecal/config.yml'
   - local: 'benchmarks/barrel_hcal/config.yml'
   - local: 'benchmarks/zdc/config.yml'

Snakefile

@@ -2,6 +2,7 @@ include: "benchmarks/backgrounds/Snakefile"
 include: "benchmarks/barrel_ecal/Snakefile"
 include: "benchmarks/ecal_gaps/Snakefile"
 include: "benchmarks/material_scan/Snakefile"
+include: "benchmarks/tracking_performances/Snakefile"
 
 
 rule fetch_epic:

benchmarks/ecal_gaps/Snakefile

@@ -6,9 +6,9 @@ rule ecal_gaps_sim:
         steering_file="EPIC/EVGEN/SINGLE/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.steer",
         warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
     output:
-        "sim_output/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root",
+        "sim_output/ecal_gaps/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root",
     log:
-        "sim_output/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root.log",
+        "sim_output/ecal_gaps/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root.log",
     wildcard_constraints:
         PARTICLE="e-",
         ENERGY="(500MeV|5GeV|20GeV)",
@@ -33,11 +33,11 @@ ddsim \
 
 rule ecal_gaps_recon:
     input:
-        "sim_output/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root",
+        "sim_output/ecal_gaps/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root",
     output:
-        "sim_output/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.eicrecon.tree.edm4eic.root",
+        "sim_output/ecal_gaps/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.eicrecon.tree.edm4eic.root",
     log:
-        "sim_output/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.eicrecon.tree.edm4eic.root.log",
+        "sim_output/ecal_gaps/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.eicrecon.tree.edm4eic.root.log",
     wildcard_constraints:
         INDEX="\d{4}",
     shell: """
@@ -54,7 +54,7 @@ rule ecal_gaps:
         script="benchmarks/ecal_gaps/ecal_gaps.py",
         # TODO pass as a file list?
         _=expand(
-            "sim_output/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX:04d}.eicrecon.tree.edm4eic.root",
+            "sim_output/ecal_gaps/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX:04d}.eicrecon.tree.edm4eic.root",
             DETECTOR_CONFIG=DETECTOR_CONFIG,
             PARTICLE=["e-"],
             ENERGY=["500MeV", "5GeV", "20GeV"],

benchmarks/ecal_gaps/ecal_gaps.org

@@ -72,9 +72,9 @@ axis_eta_coarse = bh.axis.Regular(100, -4, 4)
 def get_events(particle="e-", energy="20GeV", num_files=1):
     events = uproot.dask(
         {}
-        | {f"sim_output/{DETECTOR_CONFIG}/{particle}/{energy}/3to50deg/{particle}_{energy}_3to50deg.{INDEX:04d}.eicrecon.tree.edm4eic.root": "events" for INDEX in range(num_files)}
-        | {f"sim_output/{DETECTOR_CONFIG}/{particle}/{energy}/45to135deg/{particle}_{energy}_45to135deg.{INDEX:04d}.eicrecon.tree.edm4eic.root": "events" for INDEX in range(num_files)}
-        | {f"sim_output/{DETECTOR_CONFIG}/{particle}/{energy}/130to177deg/{particle}_{energy}_130to177deg.{INDEX:04d}.eicrecon.tree.edm4eic.root": "events" for INDEX in range(num_files)}
+        | {f"sim_output/ecal_gaps/{DETECTOR_CONFIG}/{particle}/{energy}/3to50deg/{particle}_{energy}_3to50deg.{INDEX:04d}.eicrecon.tree.edm4eic.root": "events" for INDEX in range(num_files)}
+        | {f"sim_output/ecal_gaps/{DETECTOR_CONFIG}/{particle}/{energy}/45to135deg/{particle}_{energy}_45to135deg.{INDEX:04d}.eicrecon.tree.edm4eic.root": "events" for INDEX in range(num_files)}
+        | {f"sim_output/ecal_gaps/{DETECTOR_CONFIG}/{particle}/{energy}/130to177deg/{particle}_{energy}_130to177deg.{INDEX:04d}.eicrecon.tree.edm4eic.root": "events" for INDEX in range(num_files)}
         ,
         filter_name=filter_name, open_files=False, steps_per_file=1,
     )

benchmarks/tracking_performances/Script_widebin.sh

@@ -35,12 +35,13 @@ done
 for ((iparticle=0; iparticle<${#particle_array[@]}; iparticle++)); do
 # truth seeding
 for ((i=0; i<${#mom_array[@]}; i++)); do
-root -b -l -q Tracking_Performances.C'("'${filename}'","'${particle_array[iparticle]}'",'${mom_array[i]}',0.15,"")'
+Form("./%s_%1.1f",,mom)
+root -b -l -q Tracking_Performances.C'("./'${filename}$(printf "%.1f" ${mom_array[i]})'.edm4eic.root","'${particle_array[iparticle]}'",'${mom_array[i]}',0.15,"")'
 done
 
 # real seeding
 for ((i=0; i<${#mom_array[@]}; i++)); do
-root -b -l -q Tracking_Performances.C'("'${filename}'","'${particle_array[iparticle]}'",'${mom_array[i]}',0.15,"Seeded")'
+root -b -l -q Tracking_Performances.C'("./'${filename}$(printf "%.1f" ${mom_array[i]})'.edm4eic.root","'${particle_array[iparticle]}'",'${mom_array[i]}',0.15,"Seeded")'
 done
 done
 cd truthseed/pi-/dca

benchmarks/tracking_performances/Snakefile

+rule tracking_performance_sim:
+    input:
+        steering_file=provider.remote(remote_path("EPIC/EVGEN/SINGLE/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.steer")),
+        warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
+    output:
+        "sim_output/tracking_performance/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root",
+    log:
+        "sim_output/tracking_performance/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root.log",
+    wildcard_constraints:
+        PARTICLE="pi-",
+        ENERGY="[0-9]+[kMG]eV",
+        PHASE_SPACE="(3to50|45to135|130to177)deg",
+        INDEX="\d{4}",
+    params:
+        N_EVENTS=10000
+    shell:
+        """
+ddsim \
+  --runType batch \
+  --enableGun \
+  --steeringFile "{input.steering_file}" \
+  --random.seed 1{wildcards.INDEX} \
+  --filter.tracker edep0 \
+  -v WARNING \
+  --numberOfEvents {params.N_EVENTS} \
+  --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
+  --outputFile {output}
+"""
+
+
+rule tracking_performance_recon:
+    input:
+        "sim_output/tracking_performance/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.edm4hep.root",
+    output:
+        "sim_output/tracking_performance/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.eicrecon.tree.edm4eic.root",
+    log:
+        "sim_output/tracking_performance/{DETECTOR_CONFIG}/{PARTICLE}/{ENERGY}/{PHASE_SPACE}/{PARTICLE}_{ENERGY}_{PHASE_SPACE}.{INDEX}.eicrecon.tree.edm4eic.root.log",
+    wildcard_constraints:
+        INDEX="\d{4}",
+    shell: """
+env DETECTOR_CONFIG={wildcards.DETECTOR_CONFIG} \
+  eicrecon {input} -Ppodio:output_file={output} \
+  -Ppodio:output_include_collections=MCParticles,CentralCKFTrajectories,CentralCKFTrackParameters,CentralCKFSeededTrackParameters,CentralTrackVertices
+"""
+
+
+rule tracking_performance_at_momentum:
+    input:
+        script="benchmarks/tracking_performances/Tracking_Performances.C",
+        # TODO pass as a file list?
+        sim=lambda wildcards: expand(
+            "sim_output/tracking_performance/{DETECTOR_CONFIG}/{{PARTICLE}}/{ENERGY}/{PHASE_SPACE}/{{PARTICLE}}_{ENERGY}_{PHASE_SPACE}.{INDEX:04d}.eicrecon.tree.edm4eic.root",
+            DETECTOR_CONFIG="epic_craterlake_tracking_only",
+            ENERGY=f"{float(wildcards.MOMENTUM):.0f}GeV" if float(wildcards.MOMENTUM) >= 1 else f"{float(wildcards.MOMENTUM) * 1000:.0f}MeV",
+            PHASE_SPACE=["3to50deg", "45to135deg", "130to177deg"],
+            INDEX=range(1),
+        ),
+    output:
+        "{SEEDING}/pi-/mom/Performances_mom_{MOMENTUM}_mom_resol_{SEEDING_IGNORE}_{PARTICLE}.root",
+        "{SEEDING}/pi-/dca/Performances_dca_{MOMENTUM}_dca_resol_{SEEDING_IGNORE}_{PARTICLE}.root",
+        combined_root=temp("sim_{SEEDING}_{MOMENTUM}_{SEEDING_IGNORE}_{PARTICLE}.root"),
+    shell:
+        """
+if [[ "{wildcards.SEEDING}" == "truthseed" ]]; then
+        SEEDING=""
+elif [[ "{wildcards.SEEDING}" == "realseed" ]]; then
+        SEEDING="Seeded"
+fi
+hadd {output.combined_root} {input.sim}
+root -l -b -q {input.script}'("{output.combined_root}", "{wildcards.PARTICLE}", {wildcards.MOMENTUM}, 0.15, "'$SEEDING'")'
+"""
+
+
+rule tracking_performance_summary_at_eta:
+    input:
+        expand(
+            [
+                "truthseed/pi-/mom/Performances_mom_{MOMENTUM:.1f}_mom_resol_truth_pi-.root",
+                "truthseed/pi-/dca/Performances_dca_{MOMENTUM:.1f}_dca_resol_truth_pi-.root",
+                "realseed/pi-/mom/Performances_mom_{MOMENTUM:.1f}_mom_resol_realseed_pi-.root",
+                "realseed/pi-/dca/Performances_dca_{MOMENTUM:.1f}_dca_resol_realseed_pi-.root",
+            ],
+            MOMENTUM=[0.5, 1.0, 2.0, 5.0, 10.0, 15.0],
+        ),
+        script="benchmarks/tracking_performances/doCompare_truth_real_widebins_mom.C",
+    output:
+        expand(
+            "Debug_Plots/{SEEDING}/pi-/mom/{SEEDING}_mom_resol_mom{MOMENTUM:.1f}_{{ETA_MIN}}_eta_{{ETA_MAX}}.png",
+            SEEDING=["real", "truth"],
+            MOMENTUM=[0.5, 1.0, 2.0, 5.0, 10.0, 15.0],
+        ),
+        "Final_Results/pi-/mom/mom_resol_{ETA_MIN}_eta_{ETA_MAX}.png",
+        "Final_Results/pi-/mom/mom_resol_{ETA_MIN}_eta_{ETA_MAX}.root",
+    shell:
+        """
+root -l -b -q {input.script}'("pi-", {wildcards.ETA_MIN}, {wildcards.ETA_MAX}, 1.)'
+"""
+
+
+TRACKING_PERFORMANCE_ETA_BINS = [-3.5, -2.5, -1.0, 1.0, 2.5, 3.5]
+
+rule tracking_performance:
+    input:
+        expand(
+            [
+                "Final_Results/pi-/mom/mom_resol_{ETA_BIN}.png",
+                "Final_Results/pi-/mom/mom_resol_{ETA_BIN}.root",
+            ],
+            ETA_BIN=[f"{eta_min:.1f}_eta_{eta_max:.1f}" for eta_min, eta_max in zip(TRACKING_PERFORMANCE_ETA_BINS[:-1], TRACKING_PERFORMANCE_ETA_BINS[1:])],
+        ),
+    output:
+        directory("results/tracking_performances")
+    shell:
+        """
+mkdir {output}
+cp {input} {output}
+"""

benchmarks/tracking_performances/Tracking_Performances.C

@@ -46,7 +46,7 @@ void Tracking_Performances(TString filename="tracking_output",TString particle="
    histp[i]->SetName(Form("hist_mom_%1.1f_%1.1f_pmax_%1.1f",mom,eta[i],eta[i+1]));
    }
    
-   TFile* file = TFile::Open(Form("./%s_%1.1f.edm4eic.root",filename.Data(),mom));
+   TFile* file = TFile::Open(filename.Data());
    if (!file) {printf("file not found !!!"); return;}
    TTreeReader myReader("events", file); // name of tree and file
    if (debug) cout<<"Filename: "<<file->GetName()<<"\t NEvents: "<<myReader.GetEntries()<<endl;

benchmarks/tracking_performances/config.yml

+sim:tracking_performance:
+  extends: .det_benchmark
+  stage: simulate
+  parallel:
+    matrix:
+      - PARTICLE: ["pi-"]
+        MOMENTUM: ["500MeV", "1GeV", "2GeV", "5GeV", "10GeV", "15GeV"]
+  script:
+    - |
+      snakemake --cores 1 \
+        sim_output/tracking_performance/epic_craterlake_tracking_only/${PARTICLE}/${MOMENTUM}/3to50deg/${PARTICLE}_${MOMENTUM}_3to50deg.0000.eicrecon.tree.edm4eic.root \
+        sim_output/tracking_performance/epic_craterlake_tracking_only/${PARTICLE}/${MOMENTUM}/45to135deg/${PARTICLE}_${MOMENTUM}_45to135deg.0000.eicrecon.tree.edm4eic.root \
+        sim_output/tracking_performance/epic_craterlake_tracking_only/${PARTICLE}/${MOMENTUM}/130to177deg/${PARTICLE}_${MOMENTUM}_130to177deg.0000.eicrecon.tree.edm4eic.root
+
+bench:tracking_performance:
+  extends: .det_benchmark
+  stage: benchmarks
+  needs:
+    - ["sim:tracking_performance"]
+  script:
+    - snakemake --cores 1 tracking_performance
+
+collect_results:tracking_performance:
+  extends: .det_benchmark
+  stage: collect
+  needs:
+    - "bench:tracking_performance"
+  script:
+    - ls -lrht