Draft: run du

.gitlab-ci.yml

@@ -6,6 +6,7 @@ variables:
   DETECTOR_REPOSITORYURL: 'https://github.com/eic/epic.git'
   GITHUB_SHA: ''
   GITHUB_REPOSITORY: ''
+  RM: ''
 
 workflow:
   name: '$PIPELINE_NAME'
@@ -66,124 +67,51 @@ stages:
       fi
 
 
-benchmarks:detector:pending:
-  stage: status-pending
-  extends: .status
-  variables:
-    STATE: "pending"
-    DESCRIPTION: "Started..."
-  when: always
+
 
 common:setup:
   stage: config
   before_script:
   script:
     - |
-      if [[ "${COMMON_BENCH_VERSION}" == "" ]] ; then
-        export COMMON_BENCH_VERSION="master" 
+      uptime
+      df -h
+      mount
+      if [ -n "${RM}" ] ; then rm -rf $RM ; fi
+      cat >filter_broken.py <<EOF
+      import sys
+      import ROOT
+      ROOT.gEnv.SetValue("TFile.Recover", 0)
+      for filename in sys.stdin:
+          filename = filename.rstrip()
+          bad = False;
+          try:
+              f = ROOT.TFile(filename)
+              bad = bad or f.IsZombie()
+          except OSError:
+              bad = True
+          if bad:
+              print(filename)
+      EOF
+      :>list
+      if [ -n "${REPAIR}" ]; then
+        echo "removing broken ROOT files from cache"
+        for file in $(find /scratch/snakemake-cache -type f); do
+          if file -b "$file" | grep -E '^ROOT file' >/dev/null; then
+            echo "$file" >> list
+          fi
+        done
+        echo "located $(cat list | wc -l) ROOT files"
+        cat list | python filter_broken.py | tee broken
+        cat broken | xargs --no-run-if-empty rm -v
       fi
-      echo "COMMON_BENCH_VERSION = ${COMMON_BENCH_VERSION}" 
-      echo "COMMON_BENCH_VERSION=${COMMON_BENCH_VERSION}" >> .env
-      git clone -b "${COMMON_BENCH_VERSION}" https://eicweb.phy.anl.gov/EIC/benchmarks/common_bench.git setup 
-      echo "BENCHMARKS_TAG: ${BENCHMARKS_TAG}"
-      echo "BENCHMARKS_CONTAINER: ${BENCHMARKS_CONTAINER}"
-      echo "BENCHMARKS_REGISTRY: ${BENCHMARKS_REGISTRY}"
-    - source setup/bin/env.sh && ./setup/bin/install_common.sh
-
-
-common:detector:
-  stage: initialize
-  needs: ["common:setup"]
-  script:
-    - source .local/bin/env.sh && build_detector.sh
-    - mkdir_local_data_link sim_output
-    - mkdir -p results
-    - mkdir -p config
-    - print_env.sh
-
-get_data:
-  stage: data_init
-  needs: ["common:detector"]
-  script:
-    - source .local/bin/env.sh
-    - ln -s "${LOCAL_DATA_PATH}/sim_output" sim_output
-    - ln -s "${LOCAL_DATA_PATH}/datasets/data" data
-    - cd ${LOCAL_DATA_PATH} && git clone --depth=1 https://eicweb.phy.anl.gov/EIC/datasets.git datasets
-
-.det_benchmark:
-  needs:
-    - ["get_data","common:detector"]
-  before_script:
-    - mc config host add S3 https://eics3.sdcc.bnl.gov:9000 ${S3_ACCESS_KEY} ${S3_SECRET_KEY}
-    - source .local/bin/env.sh
-    - ls -lrtha 
-    - ln -s "${LOCAL_DATA_PATH}/sim_output" sim_output
-    - ln -s "${LOCAL_DATA_PATH}/datasets/data" data
-    # snakemake support
-    - mkdir "${DETECTOR_CONFIG}"
-    - ln -s "${LOCAL_DATA_PATH}/sim_output" "${DETECTOR_CONFIG}/sim_output"
-    - ln -s "../results" "${DETECTOR_CONFIG}/results"
-    # cache downloaded artifacts
-    - if [ -d /scratch ]; then mkdir -p /scratch/EPIC; ln -sf /scratch/EPIC ./EPIC; fi
-    - du -hs /scratch/EPIC
-    - ls -lrtha
-  retry:
-    max: 2
-    when:
-      - runner_system_failure
-
-include: 
-  - local: 'benchmarks/backgrounds/config.yml'
-  - local: 'benchmarks/backwards_ecal/config.yml'
-  - local: 'benchmarks/ecal_gaps/config.yml'
-  - local: 'benchmarks/tracking_detectors/config.yml'
-  - local: 'benchmarks/tracking_performances/config.yml'
-  - local: 'benchmarks/barrel_ecal/config.yml'
-  - local: 'benchmarks/barrel_hcal/config.yml'
-  - local: 'benchmarks/zdc/config.yml'
-  - local: 'benchmarks/zdc_lyso/config.yml'
-  - local: 'benchmarks/material_maps/config.yml'
-  - local: 'benchmarks/material_scan/config.yml'
-  - local: 'benchmarks/pid/config.yml'
-  - local: 'benchmarks/timing/config.yml'
-  - local: 'benchmarks/b0_tracker/config.yml'
-
-deploy_results:
-  stage: deploy
-  needs:
-    - ["collect_results:zdc","collect_results:barrel_ecal","collect_results:barrel_hcal","collect_results:material_scan"]
-  script:
-    - echo "deploy results!"
-    - find results -print | sort | tee summary.txt
-
-benchmarks:detector:success:
-  stage: status-report
-  dependencies: []
-  extends: .status
-  variables:
-    STATE: "success"
-    DESCRIPTION: "Succeeded!"
-  when: on_success
-
-benchmarks:detector:failure:
-  stage: status-report
-  dependencies: []
-  extends: .status
-  variables:
-    STATE: "failure"
-    DESCRIPTION: "Failed!"
-  when: on_failure
-
-pages:
-  stage: deploy
-  rules:
-    - if: '$CI_COMMIT_BRANCH == "master" && $CI_SERVER_HOST != "eicweb.phy.anl.gov"'
-  cache:
-    paths:
-      - node_modules/
-  image: node:latest
-  script:
-    - mkdir public && cp doc/main.html public/index.html
-  artifacts:
-    paths:
-      - public
+      echo "removing broken symlinks"
+      find `realpath /scratch` -xtype l -print -delete
+      echo "listing most offending directories"
+      find `realpath /scratch` -type d -links 2 -print0 | (du -sL --files0-from=- || true) | sort -n > /tmp/df
+      #echo "listing cached entries by size"
+      #find `realpath /scratch` -type l -print0 | (du -sL --files0-from=- || true) | sort -n > /tmp/df
+      #find `realpath /scratch` -type f -print0 | du -s --files0-from=- || true | sort -n > /tmp/df
+      head /tmp/df
+      tail -n 1000 /tmp/df
+      df -h
\ No newline at end of file