demp,diffractive_vm: drop use of S3RemoteProvider, stream from XRootD

Snakefile

@@ -5,6 +5,7 @@ if ROOT_BUILD_DIR is not None:
 else:
     ROOT_BUILD_DIR_PREFIX = ""
 
+
 rule compile_analysis:
     input:
         "{path}/{filename}.cxx",
@@ -17,6 +18,15 @@ rule compile_analysis:
 root -l -b -q -e '.L {input}+'
 """
 
+
+rule fetch_epic:
+    output:
+        filepath="EPIC/{PATH}"
+    shell: """
+xrdcp root://dtn-eic.jlab.org//work/eic2/{output.filepath} {output.filepath}
+"""
+
+
 rule warmup_run:
     output:
         "warmup/{DETECTOR_CONFIG}.edm4hep.root",

benchmarks/demp/Snakefile

 import os
-import shutil
-
-from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider
-
-
-S3 = S3RemoteProvider(
-    endpoint_url="https://eics3.sdcc.bnl.gov:9000",
-    access_key_id=os.environ["S3_ACCESS_KEY"],
-    secret_access_key=os.environ["S3_SECRET_KEY"],
-)
 
 
 rule demp_compile:
@@ -16,18 +6,9 @@ rule demp_compile:
         ROOT_BUILD_DIR_PREFIX + "benchmarks/demp/analysis/demp_analysis_cxx.so",
         ROOT_BUILD_DIR_PREFIX + "benchmarks/demp/analysis/demp_plots_cxx.so",
 
-rule demp_get:
-    input:
-        S3.remote("eictest/EPIC/EVGEN/EXCLUSIVE/DEMP/{EBEAM}on{PBEAM}/eic_DEMPGen_{EBEAM}on{PBEAM}_ip6_pi+_1B_{INDEX}.hepmc"),
-    output:
-        "input/demp/eic_DEMPGen_{EBEAM}on{PBEAM}_ip6_pi+_1B_{INDEX}.hepmc",
-    run:
-        shutil.move(input[0], output[0])
-
 
 rule demp_sim:
     input:
-        hepmc="input/demp/eic_DEMPGen_{EBEAM}on{PBEAM}_ip6_pi+_1B_{INDEX}.hepmc",
         warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
     output:
         "sim/{DETECTOR_CONFIG}/demp_{EBEAM}on{PBEAM}_{INDEX}.edm4hep.root",
@@ -46,7 +27,7 @@ ddsim \
   -v WARNING \
   --numberOfEvents {params.N_EVENTS} \
   --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
-  --inputFiles {input.hepmc} \
+  --inputFiles root://dtn-eic.jlab.org//work/eic2/EPIC/EVGEN/EXCLUSIVE/DEMP/{wildcards.EBEAM}on{wildcards.PBEAM}/eic_DEMPGen_{wildcards.EBEAM}on{wildcards.PBEAM}_ip6_pi+_1B_{wildcards.INDEX}.hepmc3.tree.root \
   --outputFile {output}
 """
 

benchmarks/demp/config.yml

@@ -4,18 +4,9 @@ demp:compile:
   script:
     - snakemake --cores 1 demp_compile
 
-demp:generate:
-  stage: generate
-  extends: .phy_benchmark
-  needs: ["common:detector", "demp:compile"]
-  timeout: 1 hours
-  script:
-    - snakemake --cores 1 input/demp/eic_DEMPGen_5on41_ip6_pi+_1B_{1,2,3,4,5}.hepmc
-
 demp:simulate:
   stage: simulate
   extends: .phy_benchmark
-  needs: ["demp:generate"]
   timeout: 2 hours
   script:
     - snakemake --cores 5 demp_run_locally

benchmarks/diffractive_vm/Snakefile

 import os
 
-from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider
-
-
-S3 = S3RemoteProvider(
-    endpoint_url="https://eics3.sdcc.bnl.gov:9000",
-    access_key_id=os.environ["S3_ACCESS_KEY"],
-    secret_access_key=os.environ["S3_SECRET_KEY"],
-)
-
 
 rule diffractive_vm_compile:
     input:
@@ -16,31 +7,19 @@ rule diffractive_vm_compile:
         ROOT_BUILD_DIR_PREFIX + "benchmarks/diffractive_vm/analysis/plot_cxx.so",
 
 
-rule diffractive_vm_get:
-    input:
-        lambda wildcards: S3.remote(f"eictest/EPIC/EVGEN/EXCLUSIVE/DIFFRACTIVE_{wildcards.PARTICLE.upper()}_ABCONV/Sartre/Coherent/sartre_bnonsat_Au_{wildcards.PARTICLE}_ab_eAu_1_{wildcards.INDEX}.hepmc.gz"),
-    output:
-        "input/diffractive_vm/sartre_{PARTICLE}_{INDEX}.hepmc",
-    shell:
-        """
-gunzip -c {input} > {output}
-"""
-
-
 rule diffractive_vm_campaign_reco_get:
     input:
-        lambda wildcards: S3.remote(f"eictest/EPIC/RECO/{wildcards.RELEASE_TAG}/{wildcards.DETECTOR_CONFIG}/EXCLUSIVE/DIFFRACTIVE_{wildcards.PARTICLE.upper()}_ABCONV/Sartre/Coherent/sartre_bnonsat_Au_{wildcards.PARTICLE}_ab_eAu_1.0{wildcards.INDEX}.eicrecon.tree.edm4eic.root")
+        lambda wildcards: f"EPIC/RECO/{wildcards.RELEASE_TAG}/{wildcards.DETECTOR_CONFIG}/EXCLUSIVE/DIFFRACTIVE_{wildcards.PARTICLE.upper()}_ABCONV/Sartre/Coherent/sartre_bnonsat_Au_{wildcards.PARTICLE}_ab_eAu_1.0{wildcards.INDEX}.eicrecon.tree.edm4eic.root",
     output:
-        "reco/{DETECTOR_CONFIG}/campaign_{RELEASE_TAG}_sartre_{PARTICLE}_{INDEX}.edm4eic.root",
+        temp("reco/{DETECTOR_CONFIG}/campaign_{RELEASE_TAG}_sartre_{PARTICLE}_{INDEX}.edm4eic.root"),
     shell:
         """
-ln {input} {output}
+mv {input} {output}
 """
 
 
 rule diffractive_vm_sim:
     input:
-        hepmc="input/diffractive_vm/sartre_{PARTICLE}_{INDEX}.hepmc",
         warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
     output:
         "sim/{DETECTOR_CONFIG}/sartre_{PARTICLE}_{INDEX}.edm4hep.root",
@@ -53,9 +32,10 @@ ddsim \
   --part.minimalKineticEnergy 100*GeV  \
   --filter.tracker edep0 \
   -v WARNING \
+  --skipNEvents $( command expr "{wildcards.INDEX}" "*" "{params.N_EVENTS}" ) \
   --numberOfEvents {params.N_EVENTS} \
   --compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
-  --inputFiles {input.hepmc} \
+  --inputFiles root://dtn-eic.jlab.org//work/eic2/EPIC/EVGEN/EXCLUSIVE/DIFFRACTIVE_$( echo -n "{wildcards.PARTICLE}" | tr "[:lower:]" "[:upper:]" )_ABCONV/Sartre/Coherent/sartre_bnonsat_Au_{wildcards.PARTICLE}_ab_eAu_1.hepmc3.tree.root \
   --outputFile {output}
 """
 

benchmarks/diffractive_vm/config.yml

@@ -4,22 +4,9 @@ diffractive_vm:compile:
   script:
     - snakemake --cores 1 diffractive_vm_compile
 
-diffractive_vm:generate:
-  stage: generate
-  extends: .phy_benchmark
-  needs: ["common:detector", "diffractive_vm:compile"]
-  parallel:
-    matrix:
-      - VM: phi
-      - VM: jpsi
-  timeout: 1 hours
-  script:
-    - snakemake --cores 1 input/diffractive_vm/sartre_${VM}_{000,001,002,003,004,005,006,007,008,009}.hepmc
-
 diffractive_vm:simulate:
   stage: simulate
   extends: .phy_benchmark
-  needs: ["diffractive_vm:generate"]
   parallel:
     matrix:
       - VM: phi