Skip to content
Snippets Groups Projects
Unverified Commit 21fca951 authored by Dmitry Kalinkin's avatar Dmitry Kalinkin Committed by GitHub
Browse files

snakemake.yml: allow using S3 (XRootD is down sometimes) (#45)

parent 5b6564ca
No related branches found
No related tags found
No related merge requests found
Pipeline #100363 canceled with stages
......@@ -113,6 +113,7 @@ get_data:
needs:
- ["get_data","common:detector"]
before_script:
- mc config host add S3 https://eics3.sdcc.bnl.gov:9000 ${S3_ACCESS_KEY} ${S3_SECRET_KEY}
- source .local/bin/env.sh
- ls -lrtha
- ln -s "${LOCAL_DATA_PATH}/sim_output" sim_output
......
configfile: "snakemake.yml"
include: "benchmarks/backgrounds/Snakefile"
include: "benchmarks/barrel_ecal/Snakefile"
include: "benchmarks/ecal_gaps/Snakefile"
......@@ -5,11 +7,29 @@ include: "benchmarks/material_scan/Snakefile"
include: "benchmarks/tracking_performances/Snakefile"
include: "benchmarks/zdc_lyso/Snakefile"
use_s3 = config["remote_provider"].lower() == "s3"
use_xrootd = config["remote_provider"].lower() == "xrootd"
def get_remote_path(path):
if use_s3:
return f"s3https://eics3.sdcc.bnl.gov:9000/eictest/{path}"
elif use_xrootd:
return f"root://dtn-eic.jlab.org//work/eic2/{path}"
else:
raise runtime_exception('Unexpected value for config["remote_provider"]: {config["remote_provider"]}')
rule fetch_epic:
output:
filepath="EPIC/{PATH}"
shell: """
xrdcp root://dtn-eic.jlab.org//work/eic2/{output.filepath} {output.filepath}
""" if use_xrootd else """
mc cp S3/eictest/{output.filepath} {output.filepath}
""" if use_s3 else f"""
echo 'Unexpected value for config["remote_provider"]: {config["remote_provider"]}'
exit 1
"""
......
......@@ -10,7 +10,8 @@ rule backgrounds_sim:
log:
"sim_output/{DETECTOR_CONFIG}/backgrounds/{PATH}.edm4hep.root.log",
params:
N_EVENTS=100
N_EVENTS=100,
hepmc=lambda wildcards: get_remote_path(f"{wildcards.PATH}.hepmc3.tree.root"),
shell:
"""
set -m # monitor mode to prevent lingering processes
......@@ -21,7 +22,7 @@ exec ddsim \
-v WARNING \
--numberOfEvents {params.N_EVENTS} \
--compactFile $DETECTOR_PATH/{wildcards.DETECTOR_CONFIG}.xml \
--inputFiles root://dtn-eic.jlab.org//work/eic2/{wildcards.PATH}.hepmc3.tree.root \
--inputFiles {params.hepmc} \
--outputFile {output}
"""
......
remote_provider: XRootD
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment