diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2b2d037fa8790b654d9a751cd50b65d12dc915ee..1079442e08fb272d3d9a7aaff7b7eb75aabe57ca 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -37,7 +37,7 @@ generate_config:
     - sodium
   script:
     - mkdir -p config && ./bin/gen_ci_config -p test_ -i dummy > config/dummy_config.yml
-    - mkdir -p config && ./bin/gen_ci_config -p dis_ -i dis > config/dis_config.yml
+    - mkdir -p config && ./bin/gen_ci_config --no-failure -p dis_ -i dis > config/dis_config.yml
 
 child-pipeline:
   stage: run
diff --git a/bin/gen_ci_config b/bin/gen_ci_config
index da8803e03decd673c8d05ddebff0b8c298e12cb0..781607b0ce797ac9986e96d05f763d3e80e2afaf 100755
--- a/bin/gen_ci_config
+++ b/bin/gen_ci_config
@@ -2,9 +2,10 @@
 set -o nounset
 set -o errexit
 
-CI_TAG=sodium
 BENCHMARK_SCRIPT_DIR=./dummy
+CI_TAG=sodium
 CI_JOB_PREFIX=test_
+CI_FAILURE="true"
 
 function print_the_help {
   echo "USAGE: $0 [-t <runner_tag>] "
@@ -12,6 +13,7 @@ function print_the_help {
   echo "            -i,--input         Input scripts directory "
   echo "            -t,--tag           Gitlab Runner tag"
   echo "            -p,--prefix        job name prefix"
+  echo "            --no-failure       do not allow failures"
   exit 
 }
 
@@ -39,6 +41,10 @@ do
       shift # past argument
       shift # past value
       ;;
+    --no-failure)
+      CI_FAILURE="false"
+      shift # past argument
+      ;;
     *)    # unknown option
       #POSITIONAL+=("$1") # save it in an array for later
       echo "unknown option $1"
@@ -71,7 +77,7 @@ ${CI_JOB_PREFIX}${ifile}_${filename_noext}:
   stage: benchmarks
   script:
     - bash ${script_name}
-  allow_failure: true
+  allow_failure: ${CI_FAILURE}
 
 EOF
 done
@@ -88,7 +94,7 @@ ${CI_JOB_PREFIX}${ifile}_${filename_noext}:
   stage: benchmarks
   script:
     - root -b -q ${script_name}
-  allow_failure: true
+  allow_failure: ${CI_FAILURE}
 
 EOF
 done
diff --git a/dis/dummy_fail_test.sh b/dis/dummy_fail_test.sh
new file mode 100644
index 0000000000000000000000000000000000000000..f8c506b3ea3d71d5acb606b58de5e31bbac6b62d
--- /dev/null
+++ b/dis/dummy_fail_test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "Dummy Test..."
+echo "..."
+echo "Fails!"
+
+exit 1
diff --git a/dis/dummy_test.sh b/dis/dummy_test.sh
new file mode 100644
index 0000000000000000000000000000000000000000..5af1dc34f2f597a6327a3a5ef7d8ca734e413c1b
--- /dev/null
+++ b/dis/dummy_test.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "Dummy Test..."
+echo "..."
+echo "Passes!"
+
+#exit 1
diff --git a/dis/dummy_test2.sh b/dis/dummy_test2.sh
new file mode 100644
index 0000000000000000000000000000000000000000..dfede272a5390cb72e5417698f6390b0e7c55349
--- /dev/null
+++ b/dis/dummy_test2.sh
@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "Dummy Test number 2..."
+echo "..."
+echo "Passes!"
+
+#exit 1