Resolve "view14 is storing 2 GB of artifacts"

.gitlab-ci.yml

@@ -33,7 +33,6 @@ stages:
   - build
   - test
   - docs
-  - test 
   - collect
   - finalize 
   - deploy
@@ -77,22 +76,18 @@ build:simple:
     - mkdir -p build && cd build && cmake ../. -DCMAKE_INSTALL_PREFIX=/usr/local && make -j20 && make install  && cd ..
     - echo "Build successful."
 
+
 .views:
   stage: docs
   before_script:
     - source .local/bin/env.sh
-      #    - sed -i 's?<support inside?<\!--support inside?' compact/ecal_barrel_hybrid.xml 
-      #    - sed -i 's?</support>?</support-->?' compact/ecal_barrel_hybrid.xml
-      #    - sed -i 's?<fiber material?<\!--fiber material?' compact/ecal_barrel_interlayers.xml 
-      #    - sed -i 's?</fiber>?</fiber-->?' compact/ecal_barrel_interlayers.xml
-      #    - echo $DETECTOR_PATH
-      #    - cp compact/ecal_barrel_interlayers.xml ${DETECTOR_PATH}/compact/ecal_barrel_interlayers.xml
     - env
     - pip3 install psutil
   needs:
     - ["common:detector"]
 
 include:
+  - local: 'views/generate_prim_files.yml'
   - local: 'views/view1.yml'
   - local: 'views/view2.yml'
   - local: 'views/view3.yml'

bin/generate_prim_file

+#!/usr/local/bin/python
+
+# same as make_dawn_views but stops at generating the prim file.
+# W. Armstrong (ANL), original bash script
+# C. Peng (ANL), translate to python and add flexible run time for simulation
+
+import os
+import signal
+import subprocess
+import argparse
+import atexit
+import time
+from datetime import datetime
+import fcntl
+import psutil
+
+
+def readline_nonblocking(output):
+    fd = output.fileno()
+    fl = fcntl.fcntl(fd, fcntl.F_GETFL)
+    fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
+    try:
+        return output.readline()
+    except:
+        return ''
+
+
+# arguments
+parser = argparse.ArgumentParser()
+
+parser.add_argument('-c', '--compact-file', type=str, dest='compact',
+        default=os.path.join(os.environ.get('DETECTOR_PATH', '.'), 'athena.xml'),
+        help='Top level compact file for detectors')
+
+parser.add_argument('-s', '--skip', type=int,
+        default=0,
+        help='Number of events number to skip in the input')
+
+parser.add_argument('-i', '--input', type=str,
+        default='scripts/input_data/few_events.hepmc',
+        help='Input hepmc file')
+
+parser.add_argument('-o', '--output-dir', type=str, dest='out_dir',
+        default='sim_output',
+        help='output directory')
+
+parser.add_argument('-D', '--detector-only', action='store_true', dest='detector_only',
+        help='only generate the prim files for the detector geometry')
+
+parser.add_argument('-t', '--tag', type=str,dest='file_tag',
+        default='view',
+        help='Output file tag')
+
+parser.add_argument('--timeout', type=int,
+        default=60,
+        help='Timeout in seconds')
+        
+parser.add_argument('passthrough', nargs='*')
+
+args = parser.parse_args()
+
+macro = 'macro/dawn_picture.mac' if args.detector_only else 'macro/dawn_picture2.mac'
+
+# raise error if cannot create a temporary working dir
+# os.makedirs('dawn_view_tmp', exist_ok=False)
+os.makedirs(args.out_dir, exist_ok=True)
+
+# use absolute path so the chdir does not affect them
+args.input = os.path.abspath(args.input)
+args.out_dir = os.path.abspath(args.out_dir)
+args.compact = os.path.abspath(args.compact)
+macro = os.path.abspath(macro)
+
+prim_file = 'g4_0000.prim'
+dawn_env = os.environ.copy()
+dawn_env['DAWN_BATCH'] = 'a'
+# sdir = os.path.dirname(os.path.realpath(__file__))
+
+# Using a python warpper such as npsim introduces some problem in managing the subprocess.
+# The process1 managed by subprocess will generate another process with proc2_pid = proc1_pid + 1, which will not
+# be terminated by terminating or killing the process1.
+# In addition, running Geant4 with vis mode will never exit automatically (it waits for input).
+# Thus the created process 2 will occupy the system resources.
+sim_cmd = ['npsim', '--runType', 'vis',
+        '--compact', args.compact,
+        '--inputFiles', args.input,
+        '--outputFile', 'derp.root',
+        '--numberOfEvents', '1',
+        '--skipNEvents', str(args.skip),
+        '--macroFile', macro]
+
+start = datetime.now()
+elapse = datetime.now() - start
+last_update = datetime.now()
+finished = False
+
+# run simulation
+print(' '.join(sim_cmd))
+p = subprocess.Popen(args=sim_cmd, env=dawn_env,
+                     stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
+__child_pid = p.pid
+while elapse.seconds < args.timeout:
+    line = readline_nonblocking(p.stdout)
+    elapse = datetime.now() - start
+    time_left = args.timeout - elapse.seconds
+    time_str = '[{:02d}:{:02d}]'.format(elapse.seconds // 60, elapse.seconds % 60)
+
+    if time_left < 10:
+        print('{} === TIMEOUT ===: Terminating in {:d} seconds'.format(time_str, time_left))
+
+    if line:
+        decoded_line = line.decode('utf-8').strip()
+        print('{} {}'.format(time_str, decoded_line))
+        # what we are looking for
+        if decoded_line == 'File  {}  is generated.'.format(prim_file):
+            print('{} === FINISHED ===: Got the prim file, terminating.'.format(time_str))
+            finished = True
+            break
+        if decoded_line == 'Idle>':
+            p.stdin.write(b'exit')
+            break
+        # do not sleep
+        continue
+
+    # ended early before file
+    if p.poll() is not None:
+        print(p.poll())
+        break
+
+    time.sleep(1)
+
+p.kill()
+# use to kill the subprocess generated from the python wrapper
+# this is unsafe so maybe more checks required
+for proc in psutil.process_iter():
+    pinfo = proc.as_dict(attrs=['pid', 'name', 'create_time'])
+    if pinfo['pid'] == p.pid + 1 and pinfo['name'] == 'python':
+        print('kill {}, generated from {}'.format(pinfo, p.pid))
+        os.kill(pinfo['pid'], signal.SIGTERM)
+
+line = b'stderr outputs:\n'
+while line:
+    print(line.decode('utf-8'), end='')
+    line = readline_nonblocking(p.stderr)
+
+if finished:
+    print('Simulation finished')
+else:
+    print('Simulation failed')
+    exit(1)
+
+# move the prim files (which can be quite large)
+# to the local pipeline storage path
+os.system('mv g4_0000.prim {}/{}.prim'.format(args.out_dir,args.file_tag))
+
+

bin/make_dawn_views

@@ -36,8 +36,8 @@ parser.add_argument('-s', '--skip', type=int,
         default=0,
         help='Number of events number to skip in the input')
 
-parser.add_argument('-i', '--input', type=str,
-        default='scripts/input_data/few_events.hepmc',
+parser.add_argument('-i', '--input', type=str, dest='input',
+        default='sim_output',
         help='Input hepmc file')
 
 parser.add_argument('-o', '--output-dir', type=str, dest='out_dir',
@@ -51,7 +51,7 @@ parser.add_argument('-d', '--dawn-dir', type=str, dest='dawn_dir',
         default='scripts/view1',
         help='Directory to dawn script dir (with .DAWN files and a generate_eps script)')
 
-parser.add_argument('-t', '--tag', type=str,
+parser.add_argument('-t', '--tag', type=str,dest='file_tag',
         default='view',
         help='Output file tag')
 
@@ -80,90 +80,20 @@ dawn_env = os.environ.copy()
 dawn_env['DAWN_BATCH'] = 'a'
 # sdir = os.path.dirname(os.path.realpath(__file__))
 
-# Using a python warpper such as npsim introduces some problem in managing the subprocess.
-# The process1 managed by subprocess will generate another process with proc2_pid = proc1_pid + 1, which will not
-# be terminated by terminating or killing the process1.
-# In addition, running Geant4 with vis mode will never exit automatically (it waits for input).
-# Thus the created process 2 will occupy the system resources.
-sim_cmd = ['npsim', '--runType', 'vis',
-        '--compact', args.compact,
-        '--inputFiles', args.input,
-        '--outputFile', 'derp.root',
-        '--numberOfEvents', '1',
-        '--skipNEvents', str(args.skip),
-        '--macroFile', macro]
-
-start = datetime.now()
-elapse = datetime.now() - start
-last_update = datetime.now()
-finished = False
-
-# run simulation
-print(' '.join(sim_cmd))
-p = subprocess.Popen(args=sim_cmd, env=dawn_env,
-                     stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
-__child_pid = p.pid
-while elapse.seconds < args.timeout:
-    line = readline_nonblocking(p.stdout)
-    elapse = datetime.now() - start
-    time_left = args.timeout - elapse.seconds
-    time_str = '[{:02d}:{:02d}]'.format(elapse.seconds // 60, elapse.seconds % 60)
-
-    if time_left < 10:
-        print('{} === TIMEOUT ===: Terminating in {:d} seconds'.format(time_str, time_left))
-
-    if line:
-        decoded_line = line.decode('utf-8').strip()
-        print('{} {}'.format(time_str, decoded_line))
-        # what we are looking for
-        if decoded_line == 'File  {}  is generated.'.format(prim_file):
-            print('{} === FINISHED ===: Got the prim file, terminating.'.format(time_str))
-            finished = True
-            break
-        if decoded_line == 'Idle>':
-            p.stdin.write(b'exit')
-            break
-        # do not sleep
-        continue
-
-    # ended early before file
-    if p.poll() is not None:
-        print(p.poll())
-        break
-
-    time.sleep(1)
-
-p.kill()
-# use to kill the subprocess generated from the python wrapper
-# this is unsafe so maybe more checks required
-for proc in psutil.process_iter():
-    pinfo = proc.as_dict(attrs=['pid', 'name', 'create_time'])
-    if pinfo['pid'] == p.pid + 1 and pinfo['name'] == 'python':
-        print('kill {}, generated from {}'.format(pinfo, p.pid))
-        os.kill(pinfo['pid'], signal.SIGTERM)
-
-line = b'stderr outputs:\n'
-while line:
-    print(line.decode('utf-8'), end='')
-    line = readline_nonblocking(p.stderr)
-
-if finished:
-    print('Simulation finished')
-else:
-    print('Simulation failed')
-    exit(1)
-
 
 # generate DAWN images
-prim_file = os.path.abspath(prim_file)
+out_dir = os.path.abspath(args.out_dir)
+input_file = os.path.abspath(args.input)
+#prim_file = '{}/{}.prim'.format(input_dir,args.file_tag)
+#prim_file = os.path.abspath(prim_file)
 owd = os.getcwd()
 os.chdir(args.dawn_dir)
 subprocess.run(['pwd'])
-subprocess.run(['./generate_eps', '-t', args.tag, '-i', prim_file] + args.passthrough)
+subprocess.run(['./generate_eps', '-t', args.file_tag, '-i', input_file] + args.passthrough)
 subprocess.run(['ls', '-lrth'])
 
 # upload the results
-os.system('cp *.pdf {}'.format(args.out_dir))
-os.system('cp *.png {}'.format(args.out_dir))
+os.system('cp *.pdf {}'.format(out_dir))
+os.system('cp *.png {}'.format(out_dir))
 os.chdir(owd)
 

scripts/view11/.DAWN_1.history

+1.34392e+07
+90
+180
+0
+0
+0
+25000
+2.0
+1
+0.001
+0
+1
+1
+1
+0.5
+0.5
+0.5
+19
+71
+0.001
+0.001
+0.001
+3
+71
+0.001
+0
+0
+1
+evince
+0
+0

scripts/view12/.DAWN_1.history

+1.34392e+07
+0
+180
+0
+0
+0
+0
+8
+1
+0.001
+0
+1
+1
+1
+0.5
+0.5
+0.5
+19
+71
+0.01
+0.01
+0.01
+3
+70
+0.01
+1
+1
+1
+evince
+0
+0

scripts/view12/generate_eps

@@ -66,17 +66,6 @@ set -- "${POSITIONAL[@]}" # restore positional parameters
 
 echo "view12 produces a series of slightly rotated XY slices a different z locations. Along beamline"
 
-# slice at z = 2m
-# note the offset has to change with sign of the direction to cut in the opposite direction.
-dawncut 0  0  1  2005 ${INPUT_FILE} ${FILE_TAG}b_temp0.prim 
-dawncut 0  0 -1 -2000 ${FILE_TAG}b_temp0.prim  ${FILE_TAG}b.prim
-dawn -d ${FILE_TAG}b.prim 
-ps2pdf ${FILE_TAG}b.eps ${FILE_TAG}b_full.pdf
-gs -o ${FILE_TAG}b.pdf -sDEVICE=pdfwrite \
-  -c "[/CropBox [50 175 550 675] /PAGES pdfmark" \
-  -f ${FILE_TAG}b_full.pdf
-pdftoppm ${FILE_TAG}b.pdf ${FILE_TAG}b -png -singlefile -cropbox
-
 original_file_tag="${FILE_TAG}"
 
 make_slice(){

scripts/view13/generate_eps

@@ -9,7 +9,7 @@ function print_the_help {
   exit 
 }
 
-FILE_TAG="view2"
+FILE_TAG="view13"
 INPUT_FILE="g4_0000.prim"
 
 

scripts/view14/.DAWN_1.history

+0.0
+72.5
+189.5
+0
+0
+0
+1500
+55
+3
+0.001
+0
+1
+1
+1
+0.5
+0.5
+0.5
+25.5
+71
+0.001
+0.001
+0.001
+1
+70
+0.001
+1
+0
+1
+evince
+0
+0

scripts/view14/generate_eps

@@ -69,22 +69,6 @@ set -- "${POSITIONAL[@]}" # restore positional parameters
 
 # units are mm
 
-echo "view14 produces a series of slightly rotated XY slices a different z locations. Along beamline"
-
-# slice at z = 2m
-# note the offset has to change with sign of the direction to cut in the opposite direction.
-dawncut 0  0  1  10005 ${INPUT_FILE} ${FILE_TAG}b_temp0.prim 
-dawncut 0  0 -1 -1000 ${FILE_TAG}b_temp0.prim  ${FILE_TAG}b.prim
-dawn -d ${FILE_TAG}b.prim 
-ps2pdf ${FILE_TAG}b.eps ${FILE_TAG}b_full.pdf
-gs -o ${FILE_TAG}b.pdf -sDEVICE=pdfwrite \
-  -c "[/CropBox [50 175 550 675] /PAGES pdfmark" \
-  -f ${FILE_TAG}b_full.pdf
-pdftoppm ${FILE_TAG}b.pdf ${FILE_TAG}b -png -singlefile -cropbox
-
-echo "done ..."
-
-
 original_file_tag="${FILE_TAG}"
 
 make_slice(){

scripts/view15/.DAWN_1.history

+0.0
+72.5
+189.5
+0
+0
+0
+1500
+50
+3
+0.001
+0
+1
+1
+1
+0.5
+0.5
+0.5
+25.5
+71
+0.001
+0.001
+0.001
+1
+70
+0.001
+1
+0
+1
+evince
+0
+0

scripts/view2/.DAWN_1.history

@@ -6,7 +6,7 @@
 0
 0
 8
-3
+1
 0.001
 0
 1

scripts/view5/.DAWN_1.history

+1.34392e+07
+0
+0
+1
+0
+0
+491.1
+1.2
+5
+0.001
+0
+1
+1
+1
+0.5
+0.5
+0.5
+19
+71
+0.01
+0.01
+0.01
+3
+70
+0.01
+1
+1
+1
+evince
+0
+0

scripts/view6/.DAWN_1.history

+1.34392e+07
+0
+180
+0
+0
+0
+0
+8.6
+1
+0.001
+0
+1
+1
+1
+0.5
+0.5
+0.5
+19
+71
+0.01
+0.01
+0.01
+3
+70
+0.01
+1
+1
+1
+evince
+0
+0

views/generate_prim_files.yml

+view_prim:detector_only:
+  extends: .views
+  stage: test
+  script:
+    - ./bin/generate_prim_file -o ${LOCAL_DATA_PATH} -D -t detector_view
+    - ls -lrth && ls -lrth ${LOCAL_DATA_PATH}
+
+view_prim:ev001:
+  extends: .views
+  stage: test
+  rules:
+    - if: '$DETECTOR_EVENT_VIEWS == "ON"'
+  script:
+    - ./bin/generate_prim_file -o ${LOCAL_DATA_PATH} -t view_ev001 -s 1
+
+view_prim:ev002:
+  extends: .views
+  stage: test
+  rules:
+    - if: '$DETECTOR_EVENT_VIEWS == "ON"'
+  script:
+    - ./bin/generate_prim_file -o ${LOCAL_DATA_PATH} -t view_ev002 -s 2
+
+view_prim:ev003:
+  extends: .views
+  stage: test
+  rules:
+    - if: '$DETECTOR_EVENT_VIEWS == "ON"'
+  script:
+    - ./bin/generate_prim_file -o ${LOCAL_DATA_PATH} -t view_ev003 -s 3
+
+view_prim:ev004:
+  extends: .views
+  stage: test
+  rules:
+    - if: '$DETECTOR_EVENT_VIEWS == "ON"'
+  script:
+    - ./bin/generate_prim_file -o ${LOCAL_DATA_PATH} -t view_ev004 -s 4

views/view1.yml

 dawn_view_01:detector:
   extends: .views
+  needs:
+    - job: view_prim:detector_only
+      optional: false
   script:
-    - ./bin/make_dawn_views -t view01 -d scripts/view1 -D
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/detector_view.prim -t view01 -d scripts/view1 -D
+
 dawn_view_01:ev001:
   extends: .views
   rules:
     - if: '$DETECTOR_EVENT_VIEWS == "ON"'
+  needs:
+    - job: view_prim:ev001
+      optional: true
   script:
-    - ./bin/make_dawn_views -t view01_ev001 -d scripts/view1 -s 1
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/view_ev001.prim -t view01_ev001 -d scripts/view1 -s 1
+
 dawn_view_01:ev002:
   extends: .views
   rules:
     - if: '$DETECTOR_EVENT_VIEWS == "ON"'
+  needs:
+    - job: view_prim:ev002
+      optional: true
   script:
-    - ./bin/make_dawn_views -t view01_ev002 -d scripts/view1 -s 2
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/view_ev002.prim -t view01_ev002 -d scripts/view1 -s 2
 
 view_01:
   stage: collect
-  rules:
-    - if: '$CI_SERVER_HOST == "eicweb.phy.anl.gov"' 
   needs:
     - job: dawn_view_01:detector
       optional: false

views/view11.yml

 dawn_view_11:detector:
   extends: .views
+  needs:
+    - job: view_prim:detector_only
+      optional: false
   script:
-    - ./bin/make_dawn_views -t view11 -d scripts/view11 -D
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/detector_view.prim -t view11 -d scripts/view11 -D
+
 dawn_view_11:ev000:
   rules:
     - if: '$DETECTOR_EVENT_VIEWS == "ON"'
   extends: .views
+  needs:
+    - job: view_prim:ev001
+      optional: true
   script:
-    - ./bin/make_dawn_views -t view11 -d scripts/view11
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/view_ev001.prim  -t view11 -d scripts/view11
+
 dawn_view_11:ev001:
   rules:
     - if: '$DETECTOR_EVENT_VIEWS == "ON"'
   extends: .views
+  needs:
+    - job: view_prim:ev001
+      optional: true
   script:
-    - ./bin/make_dawn_views -t view11 -d scripts/view11 -s 1
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/view_ev002.prim  -t view11 -d scripts/view11 -s 1
+
 dawn_view_11:ev002:
   rules:
     - if: '$DETECTOR_EVENT_VIEWS == "ON"'
   extends: .views
+  needs:
+    - job: view_prim:ev002
+      optional: true
   script:
-    - ./bin/make_dawn_views -t view11 -d scripts/view11 -s 2
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/view_ev003.prim  -t view11 -d scripts/view11 -s 2
 dawn_view_11:ev003:
   rules:
     - if: '$DETECTOR_EVENT_VIEWS == "ON"'
   extends: .views
+  needs:
+    - job: view_prim:ev003
+      optional: true
   script:
-    - ./bin/make_dawn_views -t view11 -d scripts/view11 -s 3
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH} -t view11 -d scripts/view11 -s 3
+
 dawn_view_11:ev004:
   rules:
     - if: '$DETECTOR_EVENT_VIEWS == "ON"'
   extends: .views
+  needs:
+    - job: view_prim:ev004
+      optional: true
   script:
-    - ./bin/make_dawn_views -t view11 -d scripts/view11 -s 4
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH} -t view11 -d scripts/view11 -s 4
 
 view_11:
   stage: collect

views/view12.yml

 dawn_view_12:detector:
   extends: .views
+  needs:
+    - job: view_prim:detector_only
+      optional: false
   script:
-    - ./bin/make_dawn_views -t view12 -d scripts/view12 -D -- ${SLICE}
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/detector_view.prim -t view12 -d scripts/view12 -D -- ${SLICE}
     - ls -lrth *
     - ls -lrth images/*
   parallel:

views/view13.yml

 dawn_view_13:detector:
   extends: .views
+  needs:
+    - job: view_prim:detector_only
+      optional: false
   script:
-    - ./bin/make_dawn_views -t view13 -d scripts/view13 -D
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/detector_view.prim -t view13 -d scripts/view13 -D
 
 view_13:
   stage: collect

views/view14.yml

 dawn_view_14:detector:
   extends: .views
+  needs:
+    - job: view_prim:detector_only
+      optional: false
   script:
-    - ./bin/make_dawn_views -t view14 -d scripts/view14 -D -- ${SLICE}
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/detector_view.prim -t view14 -d scripts/view14 -D -- ${SLICE}
     - ls -lrth *
     - ls -lrth images/*
   parallel:

views/view15.yml

 dawn_view_15:detector:
   extends: .views
+  needs:
+    - job: view_prim:detector_only
+      optional: false
   script:
-    - ./bin/make_dawn_views -t view15 -d scripts/view15 -D -- ${SLICE}
+    - ./bin/make_dawn_views -i ${LOCAL_DATA_PATH}/detector_view.prim -t view15 -d scripts/view15 -D -- ${SLICE}
     - ls -lrth *
     - ls -lrth images/*
   parallel: