Restructuring benchmarks like physics_benchmarks

 - moved benchmarks to benchmarks directory

rich/rich_config.yml → benchmarks/rich/config.yml

 rich_job_x:
   image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
-  tags:
-    - silicon
-  needs: ["configure"]
+  needs: ["detector"]
   timeout: 24 hours
   artifacts:
     expire_in: 20 weeks
@@ -10,5 +8,7 @@ rich_job_x:
       - results/
   stage: run
   script:
-    - bash rich/forward_hadrons.sh
+    - bash benchmarks/rich/forward_hadrons.sh
+  allow_failure: true
+
 

rich/forward_hadrons.sh → benchmarks/rich/forward_hadrons.sh

 #!/bin/bash
 
-if [[ ! -n  "${JUGGLER_DETECTOR}" ]] ; then
-  export JUGGLER_DETECTOR="topside"
-fi
+./util/print_env.sh
+
+## To run the reconstruction, we need the following global variables:
+## - JUGGLER_INSTALL_PREFIX:   Install prefix for Juggler (simu/recon)
+## - JUGGLER_DETECTOR:         the detector package we want to use for this benchmark
+## - JUGGLER_DETECTOR_VERSION: the detector package we want to use for this benchmark
+## - DETECTOR_PATH:            full path to the detector definitions
+##
+## You can ready options/env.sh for more in-depth explanations of the variables
+## and how they can be controlled.
+source options/env.sh
+export JUGGLER_DETECTOR_PATH=${DETECTOR_PATH}
 
 if [[ ! -n  "${JUGGLER_N_EVENTS}" ]] ; then 
   export JUGGLER_N_EVENTS=100
@@ -18,37 +27,52 @@ echo "JUGGLER_N_EVENTS = ${JUGGLER_N_EVENTS}"
 echo "JUGGLER_DETECTOR = ${JUGGLER_DETECTOR}"
 
 
-# Build the detector constructors.
-git clone https://eicweb.phy.anl.gov/EIC/detectors/${JUGGLER_DETECTOR}.git
-mkdir ${JUGGLER_DETECTOR}/build
-pushd ${JUGGLER_DETECTOR}/build
-cmake ../. -DCMAKE_INSTALL_PREFIX=/usr/local && make -j30 install
-popd
-
 # generate the input events
 # note datasets is now only used to develop datasets.
 #git clone https://eicweb.phy.anl.gov/EIC/datasets.git datasets
-python rich/scripts/rich_data_gen.py \
+python benchmarks/rich/scripts/rich_data_gen.py \
        ${JUGGLER_FILE_NAME_TAG}.hepmc \
        -n ${JUGGLER_N_EVENTS} \
        --pmin 5.0 \
        --pmax 100.0 \
        --angmin 3.0 \
        --angmax 8.0
+if [[ "$?" -ne "0" ]] ; then
+  echo "ERROR running script"
+  exit 1
+fi
 
-pushd ${JUGGLER_DETECTOR}
-ls -l
-# run geant4 simulations
-python options/ForwardRICH/simu.py \
-       --compact=${JUGGLER_DETECTOR}.xml \
-       -i ../${JUGGLER_FILE_NAME_TAG}.hepmc \
-       -o ${JUGGLER_SIM_FILE}
-       -n ${JUGGLER_N_EVENTS}
+# This script appears to be missing
+## run geant4 simulations
+#python benchmakrs/options/ForwardRICH/simu.py \
+#       --compact=${DETECTOR_PATH}/${JUGGLER_DETECTOR}.xml \
+#       -i ${JUGGLER_FILE_NAME_TAG}.hepmc \
+#       -o ${JUGGLER_SIM_FILE}
+#       -n ${JUGGLER_N_EVENTS}
+#if [[ "$?" -ne "0" ]] ; then
+#  echo "ERROR running script"
+#  exit 1
+#fi
+
+npsim --runType batch \
+      --part.minimalKineticEnergy 1000*GeV  \
+      -v WARNING \
+      --numberOfEvents ${JUGGLER_N_EVENTS} \
+      --compactFile ${DETECTOR_PATH}/${JUGGLER_DETECTOR}.xml \
+      --inputFiles  ${JUGGLER_FILE_NAME_TAG}.hepmc \
+      --outputFile  ${JUGGLER_SIM_FILE}
+if [[ "$?" -ne "0" ]] ; then
+  echo "ERROR running npdet"
+  exit 1
+fi
 
 # @TODO changeable simulation file name and detector xml file name
-xenv -x /usr/local/Juggler.xenv gaudirun.py ../rich/options/rich_reco.py
-ls -l
-popd
+xenv -x ${JUGGLER_INSTALL_PREFIX}/Juggler.xenv \
+  gaudirun.py benchmarks/rich/options/rich_reco.py
+if [[ "$?" -ne "0" ]] ; then
+  echo "ERROR running juggler"
+  exit 1
+fi
 
 # @TODO add analysis scripts
 #root_filesize=$(stat --format=%s "${JUGGLER_DETECTOR}/${JUGGLER_REC_FILE}")

rich/options/rich_reco.py → benchmarks/rich/options/rich_reco.py

@@ -7,6 +7,8 @@ from Configurables import ApplicationMgr, EICDataSvc, PodioOutput, GeoSvc
 detector_name = "topside"
 if "JUGGLER_DETECTOR" in os.environ :
   detector_name = str(os.environ["JUGGLER_DETECTOR"])
+if "JUGGLER_DETECTOR_PATH" in os.environ :
+  detector_name = str(os.environ["JUGGLER_DETECTOR_PATH"])+"/"+detector_name
 
 # todo add checks
 input_sim_file  = str(os.environ["JUGGLER_SIM_FILE"])

tracking/central_electrons.sh → benchmarks/tracking/central_electrons.sh

 #!/bin/bash
-
-if [[ ! -n  "${JUGGLER_DETECTOR}" ]] ; then 
-  export JUGGLER_DETECTOR="topside"
-fi
+./util/print_env.sh
+
+## To run the reconstruction, we need the following global variables:
+## - JUGGLER_INSTALL_PREFIX:   Install prefix for Juggler (simu/recon)
+## - JUGGLER_DETECTOR:         the detector package we want to use for this benchmark
+## - JUGGLER_DETECTOR_VERSION: the detector package we want to use for this benchmark
+## - DETECTOR_PATH:            full path to the detector definitions
+##
+## You can ready options/env.sh for more in-depth explanations of the variables
+## and how they can be controlled.
+source options/env.sh
+export JUGGLER_DETECTOR_PATH=${DETECTOR_PATH}
 
 if [[ ! -n  "${JUGGLER_N_EVENTS}" ]] ; then 
   export JUGGLER_N_EVENTS=100
 fi
 
-if [[ ! -n  "${JUGGLER_INSTALL_PREFIX}" ]] ; then 
-  export JUGGLER_INSTALL_PREFIX="/usr/local"
-fi
 
 export JUGGLER_FILE_NAME_TAG="central_electrons"
 export JUGGLER_GEN_FILE="${JUGGLER_FILE_NAME_TAG}.hepmc"
@@ -22,34 +27,21 @@ echo "JUGGLER_N_EVENTS = ${JUGGLER_N_EVENTS}"
 echo "JUGGLER_DETECTOR = ${JUGGLER_DETECTOR}"
 
 
-### Build the detector constructors.
-git clone -b acts_v4 https://eicweb.phy.anl.gov/EIC/detectors/${JUGGLER_DETECTOR}.git
-git clone https://eicweb.phy.anl.gov/EIC/detectors/accelerator.git
-pushd ${JUGGLER_DETECTOR}
-ln -s ../accelerator/eic
-popd
-mkdir ${JUGGLER_DETECTOR}/build
-pushd ${JUGGLER_DETECTOR}/build
-cmake ../. -DCMAKE_INSTALL_PREFIX=/usr/local && make -j30 install
-popd
-
 ## generate the input events
-root -b -q "tracking/scripts/gen_central_electrons.cxx(${JUGGLER_N_EVENTS}, \"${JUGGLER_FILE_NAME_TAG}.hepmc\")"
+root -b -q "benchmarks/tracking/scripts/gen_central_electrons.cxx(${JUGGLER_N_EVENTS}, \"${JUGGLER_FILE_NAME_TAG}.hepmc\")"
 if [[ "$?" -ne "0" ]] ; then
   echo "ERROR running script"
   exit 1
 fi
 
-#
-pushd ${JUGGLER_DETECTOR}
 
 ## run geant4 simulations
 npsim --runType batch \
       --part.minimalKineticEnergy 1000*GeV  \
       -v WARNING \
       --numberOfEvents ${JUGGLER_N_EVENTS} \
-      --compactFile ${JUGGLER_DETECTOR}.xml \
-      --inputFiles ../${JUGGLER_FILE_NAME_TAG}.hepmc \
+      --compactFile ${DETECTOR_PATH}/${JUGGLER_DETECTOR}.xml \
+      --inputFiles  ${JUGGLER_FILE_NAME_TAG}.hepmc \
       --outputFile  ${JUGGLER_SIM_FILE}
 if [[ "$?" -ne "0" ]] ; then
   echo "ERROR running script"
@@ -57,29 +49,26 @@ if [[ "$?" -ne "0" ]] ; then
 fi
 
 # Need to figure out how to pass file name to juggler from the commandline
-xenv -x ${JUGGLER_INSTALL_PREFIX}/Juggler.xenv gaudirun.py ../tracking/options/tracker_reconstruction.py
-
+xenv -x ${JUGGLER_INSTALL_PREFIX}/Juggler.xenv gaudirun.py benchmarks/tracking/options/tracker_reconstruction.py
 if [[ "$?" -ne "0" ]] ; then
   echo "ERROR running juggler"
   exit 1
 fi
-ls -l
-popd
 
-pwd
+
 mkdir -p results/tracking
 
-root -b -q "tracking/scripts/rec_central_electrons.cxx(\"${JUGGLER_DETECTOR}/${JUGGLER_REC_FILE}\")"
+root -b -q "benchmarks/tracking/scripts/rec_central_electrons.cxx(\"${JUGGLER_REC_FILE}\")"
 if [[ "$?" -ne "0" ]] ; then
   echo "ERROR running root script"
   exit 1
 fi
 
-root_filesize=$(stat --format=%s "${JUGGLER_DETECTOR}/${JUGGLER_REC_FILE}")
+root_filesize=$(stat --format=%s "${JUGGLER_REC_FILE}")
 if [[ "${JUGGLER_N_EVENTS}" -lt "500" ]] ; then 
   # file must be less than 10 MB to upload
   if [[ "${root_filesize}" -lt "10000000" ]] ; then 
-    cp ${JUGGLER_DETECTOR}/${JUGGLER_REC_FILE} results/.
+    cp ${JUGGLER_REC_FILE} results/.
   fi
 fi
 

tracking/tracking_config.yml → benchmarks/tracking/config.yml

 tracking_central_electrons:
   image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
-  tags:
-    - silicon
-  needs: ["configure"]
+  needs: ["detector"]
   timeout: 24 hours
   artifacts:
     expire_in: 20 weeks
@@ -10,5 +8,5 @@ tracking_central_electrons:
       - results/
   stage: run
   script:
-    - bash tracking/central_electrons.sh
+    - bash benchmarks/tracking/central_electrons.sh
 

tracking/options/tracker_reconstruction.py → benchmarks/tracking/options/tracker_reconstruction.py

@@ -7,6 +7,8 @@ from GaudiKernel import SystemOfUnits as units
 detector_name = "topside"
 if "JUGGLER_DETECTOR" in os.environ :
   detector_name = str(os.environ["JUGGLER_DETECTOR"])
+if "JUGGLER_DETECTOR_PATH" in os.environ :
+  detector_name = str(os.environ["JUGGLER_DETECTOR_PATH"])+"/"+detector_name
 
 # todo add checks
 input_sim_file  = str(os.environ["JUGGLER_SIM_FILE"])

clustering/clustering_config.yml

-clustering_barrel_electrons:
-  image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:$JUGGLER_TAG
-  tags:
-    - silicon
-  needs: ["configure"]
-  timeout: 24 hours
-  artifacts:
-    expire_in: 20 weeks
-    paths:
-      - results/
-  stage: run
-  script:
-    - bash clustering/barrel_clusters.sh
-

include/benchmark.h

+#ifndef BENCHMARK_H
+#define BENCHMARK_H
+
+#include "exception.h"
+#include <fmt/core.h>
+#include <fstream>
+#include <iomanip>
+#include <iostream>
+#include <nlohmann/json.hpp>
+#include <string>
+
+// Bookkeeping of test data to store data of one or more tests in a json file to
+// facilitate future accounting.
+//
+// Usage Example 1 (single test):
+// ==============================
+// 1. define our test
+//      eic::util::Test test1{
+//        {{"name", "example_test"},
+//        {"title", "Example Test"},
+//        {"description", "This is an example of a test definition"},
+//        {"quantity", "efficiency"},
+//        {"target", "1"}}};
+// 2. set pass/fail/error status and return value (in this case .99)
+//      test1.pass(0.99)
+// 3. write our test data to a json file
+//      eic::util::write_test(test1, "test1.json");
+//
+// Usage Example 2 (multiple tests):
+// =================================
+// 1. define our tests
+//      eic::util::Test test1{
+//        {{"name", "example_test"},
+//        {"title", "Example Test"},
+//        {"description", "This is an example of a test definition"},
+//        {"quantity", "efficiency"},
+//        {"target", "1"}}};
+//      eic::util::Test test2{
+//        {{"name", "another_test"},
+//        {"title", "Another example Test"},
+//        {"description", "This is a second example of a test definition"},
+//        {"quantity", "resolution"},
+//        {"target", "3."}}};
+// 2. set pass/fail/error status and return value (in this case .99)
+//      test1.fail(10)
+// 3. write our test data to a json file
+//      eic::util::write_test({test1, test2}, "test.json");
+
+// Namespace for utility scripts, FIXME this should be part of an independent
+// library
+namespace eic::util {
+
+  struct TestDefinitionError : Exception {
+    TestDefinitionError(std::string_view msg) : Exception(msg, "test_definition_error") {}
+  };
+
+  // Wrapper for our test data json, with three methods to set the status
+  // after test completion (to pass, fail or error). The default value
+  // is error.
+  // The following fields should be defined in the definitions json
+  // for the test to make sense:
+  //  - name: unique identifier for this test
+  //  - title: Slightly more verbose identifier for this test
+  //  - description: Concise description of what is tested
+  //  - quantity: What quantity is tested? Unites of value/target
+  //  - target: Target value of <quantity> that we want to reach
+  //  - value: Actual value of <quantity>
+  //  - weight: Weight for this test (this is defaulted to 1.0 if not specified)
+  //  - result: pass/fail/error
+  struct Test {
+    // note: round braces for the json constructor, as else it will pick the wrong
+    //       initializer-list constructur (it will put everything in an array)
+    Test(const std::map<std::string, std::string>& definition) : json(definition)
+    {
+      // std::cout << json.dump() << std::endl;
+      // initialize with error (as we don't have a value yet)
+      error();
+      // Check that all required fields are present
+      for (const auto& field :
+           {"name", "title", "description", "quantity", "target", "value", "result"}) {
+        if (json.find(field) == json.end()) {
+          throw TestDefinitionError{
+              fmt::format("Error in test definition: field '{}' missing", field)};
+        }
+      }
+      // Default "weight" to 1 if not set
+      if (json.find("weight") == json.end()) {
+        json["weight"] = 1.0;
+      }
+    }
+    // Set this test to pass/fail/error
+    void pass(double value) { update_result("pass", value); }
+    void fail(double value) { update_result("fail", value); }
+    void error(double value = 0) { update_result("error", value); }
+
+    nlohmann::json json;
+
+  private:
+    void update_result(std::string_view status, double value)
+    {
+      json["result"] = status;
+      json["value"]  = value;
+    }
+  };
+
+  void write_test(const std::vector<Test>& data, const std::string& fname)
+  {
+    nlohmann::json test;
+    for (auto& entry : data) {
+      test["tests"].push_back(entry.json);
+    }
+    std::cout << fmt::format("Writing test data to {}\n", fname);
+    std::ofstream output_file(fname);
+    output_file << std::setw(4) << test << "\n";
+  }
+  void write_test(const Test& data, const std::string& fname)
+  {
+    std::vector<Test> vtd{data};
+    write_test(vtd, fname);
+  }
+
+} // namespace eic::util
+
+#endif

include/exception.h

+#ifndef UTIL_EXCEPTION_H
+#define UTIL_EXCEPTION_H
+
+#include <exception>
+#include <string>
+
+namespace eic::util {
+  class Exception : public std::exception {
+  public:
+    Exception(std::string_view msg, std::string_view type = "exception") : msg_{msg}, type_{type} {}
+
+    virtual const char* what() const throw() { return msg_.c_str(); }
+    virtual const char* type() const throw() { return type_.c_str(); }
+    virtual ~Exception() throw() {}
+
+  private:
+    std::string msg_;
+    std::string type_;
+  };
+} // namespace eic::util
+
+#endif

include/mt.h

+#ifndef MT_H
+#define MT_H
+
+// Defines the number of threads to run within the ROOT analysis scripts.
+// TODO: make this a file configured by the CI scripts so we can specify
+//       the number of threads (and the number of processes) at a global
+//       level
+
+constexpr const int kNumThreads = 8;
+
+#endif