From 6ad447657ea218271afc100250f9be1b31f5712c Mon Sep 17 00:00:00 2001
From: Dmitry Kalinkin <dmitry.kalinkin@gmail.com>
Date: Sun, 7 Jul 2024 04:14:06 -0400
Subject: [PATCH] demp,diffractive_vm: drop use of S3RemoteProvider, stream
from XRootD
---
Snakefile | 10 ++++++++++
benchmarks/demp/Snakefile | 21 +------------------
benchmarks/demp/config.yml | 9 ---------
benchmarks/diffractive_vm/Snakefile | 30 +++++-----------------------
benchmarks/diffractive_vm/config.yml | 13 ------------
5 files changed, 16 insertions(+), 67 deletions(-)
diff --git a/Snakefile b/Snakefile
index 5546d862..73b03c2b 100644
--- a/Snakefile
+++ b/Snakefile
@@ -5,6 +5,7 @@ if ROOT_BUILD_DIR is not None:
else:
ROOT_BUILD_DIR_PREFIX = ""
+
rule compile_analysis:
input:
"{path}/{filename}.cxx",
@@ -17,6 +18,15 @@ rule compile_analysis:
root -l -b -q -e '.L {input}+'
"""
+
+rule fetch_epic:
+ output:
+ filepath="EPIC/{PATH}"
+ shell: """
+xrdcp root://dtn-eic.jlab.org//work/eic2/{output.filepath} {output.filepath}
+"""
+
+
rule warmup_run:
output:
"warmup/{DETECTOR_CONFIG}.edm4hep.root",
diff --git a/benchmarks/demp/Snakefile b/benchmarks/demp/Snakefile
index 3ec788ae..920a0e91 100644
--- a/benchmarks/demp/Snakefile
+++ b/benchmarks/demp/Snakefile
@@ -1,14 +1,4 @@
import os
-import shutil
-
-from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider
-
-
-S3 = S3RemoteProvider(
- endpoint_url="https://eics3.sdcc.bnl.gov:9000",
- access_key_id=os.environ["S3_ACCESS_KEY"],
- secret_access_key=os.environ["S3_SECRET_KEY"],
-)
rule demp_compile:
@@ -16,18 +6,9 @@ rule demp_compile:
ROOT_BUILD_DIR_PREFIX + "benchmarks/demp/analysis/demp_analysis_cxx.so",
ROOT_BUILD_DIR_PREFIX + "benchmarks/demp/analysis/demp_plots_cxx.so",
-rule demp_get:
- input:
- S3.remote("eictest/EPIC/EVGEN/EXCLUSIVE/DEMP/{EBEAM}on{PBEAM}/eic_DEMPGen_{EBEAM}on{PBEAM}_ip6_pi+_1B_{INDEX}.hepmc"),
- output:
- "input/demp/eic_DEMPGen_{EBEAM}on{PBEAM}_ip6_pi+_1B_{INDEX}.hepmc",
- run:
- shutil.move(input[0], output[0])
-
rule demp_sim:
input:
- hepmc="input/demp/eic_DEMPGen_{EBEAM}on{PBEAM}_ip6_pi+_1B_{INDEX}.hepmc",
warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
output:
"sim/{DETECTOR_CONFIG}/demp_{EBEAM}on{PBEAM}_{INDEX}.edm4hep.root",
@@ -46,7 +27,7 @@ ddsim \
-v WARNING \
--numberOfEvents {params.N_EVENTS} \
--compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
- --inputFiles {input.hepmc} \
+ --inputFiles root://dtn-eic.jlab.org//work/eic2/EPIC/EVGEN/EXCLUSIVE/DEMP/{wildcards.EBEAM}on{wildcards.PBEAM}/eic_DEMPGen_{wildcards.EBEAM}on{wildcards.PBEAM}_ip6_pi+_1B_{wildcards.INDEX}.hepmc3.tree.root \
--outputFile {output}
"""
diff --git a/benchmarks/demp/config.yml b/benchmarks/demp/config.yml
index eef56547..1afda42d 100644
--- a/benchmarks/demp/config.yml
+++ b/benchmarks/demp/config.yml
@@ -4,18 +4,9 @@ demp:compile:
script:
- snakemake --cores 1 demp_compile
-demp:generate:
- stage: generate
- extends: .phy_benchmark
- needs: ["common:detector", "demp:compile"]
- timeout: 1 hours
- script:
- - snakemake --cores 1 input/demp/eic_DEMPGen_5on41_ip6_pi+_1B_{1,2,3,4,5}.hepmc
-
demp:simulate:
stage: simulate
extends: .phy_benchmark
- needs: ["demp:generate"]
timeout: 2 hours
script:
- snakemake --cores 5 demp_run_locally
diff --git a/benchmarks/diffractive_vm/Snakefile b/benchmarks/diffractive_vm/Snakefile
index 76dc9144..0908563a 100644
--- a/benchmarks/diffractive_vm/Snakefile
+++ b/benchmarks/diffractive_vm/Snakefile
@@ -1,14 +1,5 @@
import os
-from snakemake.remote.S3 import RemoteProvider as S3RemoteProvider
-
-
-S3 = S3RemoteProvider(
- endpoint_url="https://eics3.sdcc.bnl.gov:9000",
- access_key_id=os.environ["S3_ACCESS_KEY"],
- secret_access_key=os.environ["S3_SECRET_KEY"],
-)
-
rule diffractive_vm_compile:
input:
@@ -16,31 +7,19 @@ rule diffractive_vm_compile:
ROOT_BUILD_DIR_PREFIX + "benchmarks/diffractive_vm/analysis/plot_cxx.so",
-rule diffractive_vm_get:
- input:
- lambda wildcards: S3.remote(f"eictest/EPIC/EVGEN/EXCLUSIVE/DIFFRACTIVE_{wildcards.PARTICLE.upper()}_ABCONV/Sartre/Coherent/sartre_bnonsat_Au_{wildcards.PARTICLE}_ab_eAu_1_{wildcards.INDEX}.hepmc.gz"),
- output:
- "input/diffractive_vm/sartre_{PARTICLE}_{INDEX}.hepmc",
- shell:
- """
-gunzip -c {input} > {output}
-"""
-
-
rule diffractive_vm_campaign_reco_get:
input:
- lambda wildcards: S3.remote(f"eictest/EPIC/RECO/{wildcards.RELEASE_TAG}/{wildcards.DETECTOR_CONFIG}/EXCLUSIVE/DIFFRACTIVE_{wildcards.PARTICLE.upper()}_ABCONV/Sartre/Coherent/sartre_bnonsat_Au_{wildcards.PARTICLE}_ab_eAu_1.0{wildcards.INDEX}.eicrecon.tree.edm4eic.root")
+ lambda wildcards: f"EPIC/RECO/{wildcards.RELEASE_TAG}/{wildcards.DETECTOR_CONFIG}/EXCLUSIVE/DIFFRACTIVE_{wildcards.PARTICLE.upper()}_ABCONV/Sartre/Coherent/sartre_bnonsat_Au_{wildcards.PARTICLE}_ab_eAu_1.0{wildcards.INDEX}.eicrecon.tree.edm4eic.root",
output:
- "reco/{DETECTOR_CONFIG}/campaign_{RELEASE_TAG}_sartre_{PARTICLE}_{INDEX}.edm4eic.root",
+ temp("reco/{DETECTOR_CONFIG}/campaign_{RELEASE_TAG}_sartre_{PARTICLE}_{INDEX}.edm4eic.root"),
shell:
"""
-ln {input} {output}
+mv {input} {output}
"""
rule diffractive_vm_sim:
input:
- hepmc="input/diffractive_vm/sartre_{PARTICLE}_{INDEX}.hepmc",
warmup="warmup/{DETECTOR_CONFIG}.edm4hep.root",
output:
"sim/{DETECTOR_CONFIG}/sartre_{PARTICLE}_{INDEX}.edm4hep.root",
@@ -53,9 +32,10 @@ ddsim \
--part.minimalKineticEnergy 100*GeV \
--filter.tracker edep0 \
-v WARNING \
+ --skipNEvents $( command expr "{wildcards.INDEX}" "*" "{params.N_EVENTS}" ) \
--numberOfEvents {params.N_EVENTS} \
--compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
- --inputFiles {input.hepmc} \
+ --inputFiles root://dtn-eic.jlab.org//work/eic2/EPIC/EVGEN/EXCLUSIVE/DIFFRACTIVE_$( echo -n "{wildcards.PARTICLE}" | tr "[:lower:]" "[:upper:]" )_ABCONV/Sartre/Coherent/sartre_bnonsat_Au_{wildcards.PARTICLE}_ab_eAu_1.hepmc3.tree.root \
--outputFile {output}
"""
diff --git a/benchmarks/diffractive_vm/config.yml b/benchmarks/diffractive_vm/config.yml
index e5551d86..6353a610 100644
--- a/benchmarks/diffractive_vm/config.yml
+++ b/benchmarks/diffractive_vm/config.yml
@@ -4,22 +4,9 @@ diffractive_vm:compile:
script:
- snakemake --cores 1 diffractive_vm_compile
-diffractive_vm:generate:
- stage: generate
- extends: .phy_benchmark
- needs: ["common:detector", "diffractive_vm:compile"]
- parallel:
- matrix:
- - VM: phi
- - VM: jpsi
- timeout: 1 hours
- script:
- - snakemake --cores 1 input/diffractive_vm/sartre_${VM}_{000,001,002,003,004,005,006,007,008,009}.hepmc
-
diffractive_vm:simulate:
stage: simulate
extends: .phy_benchmark
- needs: ["diffractive_vm:generate"]
parallel:
matrix:
- VM: phi
--
GitLab