Merge branch 'analyze-one-line-per-file' into 'main'

Reorg timing analysis

See merge request !29

.gitlab-ci.yml

 image: eicweb.phy.anl.gov:4567/eic/juggler/juggler:latest
 
 variables:
-  JOBS: 6
+  JOBS: 1
 
 default:
   timeout: 3 hours
@@ -20,7 +20,7 @@ stages:
   stage: gzip
   script:
     - mc config host add S3rw https://dtn01.sdcc.bnl.gov:9000 ${S3RW_ACCESS_KEY} ${S3RW_SECRET_KEY}
-    - cat $DATA | grep -v "^\#" | parallel -j 1 --colsep "," scripts/gzip_hepmc.sh {1} {2} {3}
+    - cat $DATA | grep -v "^\#" | parallel -j $JOBS --colsep "," scripts/gzip_hepmc.sh {1} {2} {3}
 
 .nevents:
   stage: nevents
@@ -34,7 +34,11 @@ stages:
   script:
     - mc config host add S3 https://dtn01.sdcc.bnl.gov:9000 ${S3_ACCESS_KEY} ${S3_SECRET_KEY}
     - mkdir -p $(dirname results/datasets/timings/$DATA)
-    - cat results/datasets/nevents/$DATA | grep -v "^\#" | parallel -j $JOBS --colsep "," scripts/determine_timing.sh {1} {2} {3} | sort | tee results/datasets/timings/$DATA
+    - cat results/datasets/nevents/$DATA | grep -v "^\#" | head -n 1 | parallel -j $JOBS --colsep "," scripts/determine_timing.sh {1} {2} {3} | sort | tee results/datasets/timings/$DATA && echo $?
+    - |
+      IFS="," read file nevents dt0 dt1 < results/datasets/timings/$DATA
+      export dt0 dt1
+      cat results/datasets/nevents/$DATA | grep -v "^\#" | cat       | parallel -j $JOBS --colsep "," scripts/determine_timing.sh {1} {2} {3} | sort | tee results/datasets/timings/$DATA
 
 .collect:
   stage: collect

DIS/CC/DIS_CC_hepmc2.csv

-EVGEN/DIS/CC_hepmc2/18x275/minQ2=100/pythia8CCDIS_18x275_minQ2=100_beamEffects_xAngle=-0.025_hiDiv_1.hepmc,1000000,70

DIS/CC/config.yml

-DIS:CC:gzip:
-  extends: .gzip
-  parallel:
-    matrix:
-      - DATA: "DIS/CC/DIS_CC_hepmc2.csv"
-
-DIS:CC:nevents:
-  extends: .nevents
-  parallel:
-    matrix:
-      - DATA: "DIS/CC/DIS_CC_hepmc2.csv"
-
-DIS:CC:timings:
-  extends: .timings
-  needs: 
-    - DIS:CC:nevents
-  parallel:
-    matrix:
-      - DATA: "DIS/CC/DIS_CC_hepmc2.csv"
-
-DIS:CC:collect:
-  extends: .collect
-  needs: 
-    - DIS:CC:timings

DIS/NC/10x100/DIS_NC_10x100.csv

+DIS/NC/10x100/minQ2=1/DIS_NC_10x100_minQ2=1.csv
+DIS/NC/10x100/minQ2=10/DIS_NC_10x100_minQ2=10.csv
+DIS/NC/10x100/minQ2=100/DIS_NC_10x100_minQ2=100.csv
+DIS/NC/10x100/minQ2=1000/DIS_NC_10x100_minQ2=1000.csv

DIS/NC/10x100/config.yml

+DIS:NC:10x100:gzip:
+  extends: .gzip
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/NC/10x100/minQ2=1000/DIS_NC_10x100_minQ2=1000.csv"
+        - "DIS/NC/10x100/minQ2=100/DIS_NC_10x100_minQ2=100.csv"
+        - "DIS/NC/10x100/minQ2=10/DIS_NC_10x100_minQ2=10.csv"
+        - "DIS/NC/10x100/minQ2=1/DIS_NC_10x100_minQ2=1.csv"
+
+DIS:NC:10x100:nevents:
+  extends: .nevents
+  needs:
+    - DIS:NC:10x100:gzip
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/NC/10x100/minQ2=1000/DIS_NC_10x100_minQ2=1000.csv"
+        - "DIS/NC/10x100/minQ2=100/DIS_NC_10x100_minQ2=100.csv"
+        - "DIS/NC/10x100/minQ2=10/DIS_NC_10x100_minQ2=10.csv"
+        - "DIS/NC/10x100/minQ2=1/DIS_NC_10x100_minQ2=1.csv"
+
+DIS:NC:10x100:timings:
+  extends: .timings
+  needs: 
+    - DIS:NC:10x100:nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/NC/10x100/minQ2=1000/DIS_NC_10x100_minQ2=1000.csv"
+        - "DIS/NC/10x100/minQ2=100/DIS_NC_10x100_minQ2=100.csv"
+        - "DIS/NC/10x100/minQ2=10/DIS_NC_10x100_minQ2=10.csv"
+        - "DIS/NC/10x100/minQ2=1/DIS_NC_10x100_minQ2=1.csv"
+
+DIS:NC:10x100:collect:
+  extends: .collect
+  needs: 
+    - DIS:NC:10x100:timings

DIS/NC/10x275/DIS_NC_10x275.csv

+DIS/NC/10x275/minQ2=1/DIS_NC_10x275_minQ2=1.csv
+DIS/NC/10x275/minQ2=10/DIS_NC_10x275_minQ2=10.csv
+DIS/NC/10x275/minQ2=100/DIS_NC_10x275_minQ2=100.csv
+DIS/NC/10x275/minQ2=1000/DIS_NC_10x275_minQ2=1000.csv

DIS/NC/10x275/config.yml

+DIS:NC:10x275:gzip:
+  extends: .gzip
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/NC/10x275/minQ2=1000/DIS_NC_10x275_minQ2=1000.csv"
+        - "DIS/NC/10x275/minQ2=100/DIS_NC_10x275_minQ2=100.csv"
+        - "DIS/NC/10x275/minQ2=10/DIS_NC_10x275_minQ2=10.csv"
+        - "DIS/NC/10x275/minQ2=1/DIS_NC_10x275_minQ2=1.csv"
+
+DIS:NC:10x275:nevents:
+  extends: .nevents
+  needs:
+    - DIS:NC:10x275:gzip
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/NC/10x275/minQ2=1000/DIS_NC_10x275_minQ2=1000.csv"
+        - "DIS/NC/10x275/minQ2=100/DIS_NC_10x275_minQ2=100.csv"
+        - "DIS/NC/10x275/minQ2=10/DIS_NC_10x275_minQ2=10.csv"
+        - "DIS/NC/10x275/minQ2=1/DIS_NC_10x275_minQ2=1.csv"
+
+DIS:NC:10x275:timings:
+  extends: .timings
+  needs: 
+    - DIS:NC:10x275:nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/NC/10x275/minQ2=1000/DIS_NC_10x275_minQ2=1000.csv"
+        - "DIS/NC/10x275/minQ2=100/DIS_NC_10x275_minQ2=100.csv"
+        - "DIS/NC/10x275/minQ2=10/DIS_NC_10x275_minQ2=10.csv"
+        - "DIS/NC/10x275/minQ2=1/DIS_NC_10x275_minQ2=1.csv"
+
+DIS:NC:10x275:collect:
+  extends: .collect
+  needs: 
+    - DIS:NC:10x275:timings

DIS/NC/18x275/DIS_NC_18x275.csv

+DIS/NC/18x275/minQ2=1/DIS_NC_18x275_minQ2=1.csv
+DIS/NC/18x275/minQ2=10/DIS_NC_18x275_minQ2=10.csv
+DIS/NC/18x275/minQ2=100/DIS_NC_18x275_minQ2=100.csv
+DIS/NC/18x275/minQ2=1000/DIS_NC_18x275_minQ2=1000.csv

DIS/NC/18x275/config.yml

+DIS:NC:18x275:gzip:
+  extends: .gzip
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/NC/18x275/minQ2=1000/DIS_NC_18x275_minQ2=1000.csv"
+        - "DIS/NC/18x275/minQ2=100/DIS_NC_18x275_minQ2=100.csv"
+        - "DIS/NC/18x275/minQ2=10/DIS_NC_18x275_minQ2=10.csv"
+        - "DIS/NC/18x275/minQ2=1/DIS_NC_18x275_minQ2=1.csv"
+
+DIS:NC:18x275:nevents:
+  extends: .nevents
+  needs:
+    - DIS:NC:18x275:gzip
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/NC/18x275/minQ2=1000/DIS_NC_18x275_minQ2=1000.csv"
+        - "DIS/NC/18x275/minQ2=100/DIS_NC_18x275_minQ2=100.csv"
+        - "DIS/NC/18x275/minQ2=10/DIS_NC_18x275_minQ2=10.csv"
+        - "DIS/NC/18x275/minQ2=1/DIS_NC_18x275_minQ2=1.csv"
+
+DIS:NC:18x275:timings:
+  extends: .timings
+  needs: 
+    - DIS:NC:18x275:nevents
+  parallel:
+    matrix:
+      - DATA:
+        - "DIS/NC/18x275/minQ2=1000/DIS_NC_18x275_minQ2=1000.csv"
+        - "DIS/NC/18x275/minQ2=100/DIS_NC_18x275_minQ2=100.csv"
+        - "DIS/NC/18x275/minQ2=10/DIS_NC_18x275_minQ2=10.csv"
+        - "DIS/NC/18x275/minQ2=1/DIS_NC_18x275_minQ2=1.csv"
+
+DIS:NC:18x275:collect:
+  extends: .collect
+  needs: 
+    - DIS:NC:18x275:timings