Index: src/tests/.expect/concurrent/sched-int-multi2.txt
===================================================================
--- src/tests/.expect/concurrent/sched-int-multi2.txt	(revision 0428aad6959f9d162a6d7a1872fa2d093f6d1348)
+++ src/tests/.expect/concurrent/sched-int-multi2.txt	(revision 0428aad6959f9d162a6d7a1872fa2d093f6d1348)
@@ -0,0 +1,12 @@
+Waiting 1
+Waiting 2
+Waiting 3
+Waiting 4
+Signaling ABC
+Signaling AB
+Signaling BC
+Signaling AC
+Waking 4
+Waking 3
+Waking 2
+Waking 1
Index: src/tests/pybin/__init__.py
===================================================================
--- src/tests/pybin/__init__.py	(revision 0428aad6959f9d162a6d7a1872fa2d093f6d1348)
+++ src/tests/pybin/__init__.py	(revision 0428aad6959f9d162a6d7a1872fa2d093f6d1348)
@@ -0,0 +1,1 @@
+#This file is empty but needs to exist for python import to work
Index: src/tests/pybin/tools.py
===================================================================
--- src/tests/pybin/tools.py	(revision 0428aad6959f9d162a6d7a1872fa2d093f6d1348)
+++ src/tests/pybin/tools.py	(revision 0428aad6959f9d162a6d7a1872fa2d093f6d1348)
@@ -0,0 +1,96 @@
+import __main__
+import argparse
+import os
+import re
+import stat
+
+from subprocess import Popen, PIPE, STDOUT
+
+# helper functions to run terminal commands
+def sh(cmd, dry_run = False, print2stdout = True):
+	if dry_run : 	# if this is a dry_run, only print the commands that would be ran
+		print("cmd: %s" % cmd)
+		return 0, None
+	else :			# otherwise create a pipe and run the desired command
+		proc = Popen(cmd, stdout=None if print2stdout else PIPE, stderr=STDOUT, shell=True)
+		out, err = proc.communicate()
+		return proc.returncode, out
+
+# Remove 1 or more files silently
+def rm( files, dry_run = False ):
+	try:
+		for file in files:
+			sh("rm -f %s > /dev/null 2>&1" % file, dry_run)
+	except TypeError:
+		sh("rm -f %s > /dev/null 2>&1" % files, dry_run)
+
+def chdir( dest = __main__.__file__ ):
+	abspath = os.path.abspath(dest)
+	dname = os.path.dirname(abspath)
+	os.chdir(dname)
+
+# helper function to replace patterns in a file
+def file_replace(fname, pat, s_after):
+    # first, see if the pattern is even in the file.
+    with open(fname) as f:
+        if not any(re.search(pat, line) for line in f):
+            return # pattern does not occur in file so we are done.
+
+    # pattern is in the file, so perform replace operation.
+    with open(fname) as f:
+        out_fname = fname + ".tmp"
+        out = open(out_fname, "w")
+        for line in f:
+            out.write(re.sub(pat, s_after, line))
+        out.close()
+        os.rename(out_fname, fname)
+
+# helper function to check if a files contains only a spacific string
+def fileContainsOnly(file, text) :
+	with open(file) as f:
+		ff = f.read().strip()
+		result = ff == text.strip()
+
+		return result;
+
+# check whether or not a file is executable
+def fileIsExecutable(file) :
+	try :
+		fileinfo = os.stat(file)
+		return bool(fileinfo.st_mode & stat.S_IXUSR)
+	except Exception as inst:
+		print(type(inst))    # the exception instance
+		print(inst.args)     # arguments stored in .args
+		print(inst)
+		return False
+
+# check if arguments is yes or no
+def yes_no(string):
+	if string == "yes" :
+		return True
+	if string == "no" :
+		return False
+	raise argparse.ArgumentTypeError(msg)
+	return False
+
+# diff two files
+def diff( lhs, rhs, dry_run ):
+	# diff the output of the files
+	diff_cmd = ("diff --ignore-all-space "
+				"--ignore-blank-lines "
+				"--old-group-format='\t\tmissing lines :\n"
+				"%%<' \\\n"
+				"--new-group-format='\t\tnew lines :\n"
+				"%%>' \\\n"
+				"--unchanged-group-format='%%=' \\"
+				"--changed-group-format='\t\texpected :\n"
+				"%%<\n"
+				"\t\tgot :\n"
+				"%%>' \\\n"
+				"--new-line-format='\t\t%%dn\t%%L' \\\n"
+				"--old-line-format='\t\t%%dn\t%%L' \\\n"
+				"--unchanged-line-format='' \\\n"
+				"%s %s")
+
+	# fetch return code and error from the diff command
+	return sh(diff_cmd % (lhs, rhs), dry_run, False)	
Index: src/tests/sched-int-multi2.c
===================================================================
--- src/tests/sched-int-multi2.c	(revision 0428aad6959f9d162a6d7a1872fa2d093f6d1348)
+++ src/tests/sched-int-multi2.c	(revision 0428aad6959f9d162a6d7a1872fa2d093f6d1348)
@@ -0,0 +1,119 @@
+#include <fstream>
+#include <kernel>
+#include <monitor>
+#include <thread>
+
+monitor global_t {};
+
+global_t globalA;
+global_t globalB;
+global_t globalC;
+
+condition condAB, condAC, condBC, condABC;
+
+thread Signaler {};
+thread WaiterAB {};
+thread WaiterAC {};
+thread WaiterBC {};
+thread WaiterABC{};
+
+int state;
+
+/*
+multi phase
+*/
+
+//----------------------------------------------------------------------------------------------------
+// Tools
+void signal( condition * cond, global_t * mutex a, global_t * mutex b ) {
+	signal( cond );
+}
+
+void signal( condition * cond, global_t * mutex a, global_t * mutex b, global_t * mutex c ) {
+	signal( cond );
+}
+
+void wait( condition * cond, global_t * mutex a, global_t * mutex b ) {
+	state++;
+	sout | "Waiting" | state | endl;
+	wait( cond );
+	sout | "Waking" | state | endl;
+	state--;
+}
+
+void wait( condition * cond, global_t * mutex a, global_t * mutex b, global_t * mutex c ) {
+	state++;
+	sout | "Waiting" | state | endl;
+	wait( cond );
+	sout | "Waking" | state | endl;
+	state--;
+}
+
+//----------------------------------------------------------------------------------------------------
+// Signaler
+// signals respectively AB, AC, BC, ABC
+void signalerABC( global_t * mutex a, global_t * mutex b, global_t * mutex c ) {
+	sout | "Signaling ABC" | endl;
+	signal( &condABC, a, b, c );
+	sout | "Signaling AB" | endl;
+	signal( &condAB , a, b );
+	sout | "Signaling BC" | endl;
+	signal( &condBC , b, c );
+	sout | "Signaling AC" | endl;
+	signal( &condAC , a, c );
+}
+
+void signalerAB( global_t * mutex a, global_t * mutex b, global_t * c ) {
+	signalerABC(a, b, c);
+}
+
+void signalerA( global_t * mutex a, global_t * b, global_t * c ) {
+	signalerAB (a, b, c);
+}
+
+void main( Signaler* this ) {
+	while( state != 4 ) { yield(); }
+	signalerA( &globalA, &globalB, &globalC );
+}
+
+//----------------------------------------------------------------------------------------------------
+// Waiter ABC
+void main( WaiterABC* this ) {
+	while( state != 0 ) { yield(); }
+	wait( &condABC, &globalA, &globalB, &globalC );
+}
+
+//----------------------------------------------------------------------------------------------------
+// Waiter AB
+void main( WaiterAB* this ) {
+	while( state != 1 ) { yield(); }
+	wait( &condAB , &globalA, &globalB );
+}
+
+//----------------------------------------------------------------------------------------------------
+// Waiter AC
+void main( WaiterAC* this ) {
+	while( state != 2 ) { yield(); }
+	wait( &condAC , &globalA, &globalC );
+}
+
+//----------------------------------------------------------------------------------------------------
+// Waiter BC
+void main( WaiterBC* this ) {
+	while( state != 3 ) { yield(); }
+	wait( &condBC , &globalB, &globalC );
+}
+
+//----------------------------------------------------------------------------------------------------
+// Main
+int main(int argc, char* argv[]) {
+	state = 0;
+	processor p;
+	{
+		WaiterABC a;
+		WaiterAB  b;
+		WaiterBC  c;
+		WaiterAC  d;
+		Signaler  e;
+	}
+}
Index: src/tests/test.py
===================================================================
--- src/tests/test.py	(revision f3fda4e7176490b94b751856b2d719c027070bbe)
+++ src/tests/test.py	(revision 0428aad6959f9d162a6d7a1872fa2d093f6d1348)
@@ -6,5 +6,5 @@
 from os import listdir, environ
 from os.path import isfile, join, splitext
-from subprocess import Popen, PIPE, STDOUT
+from pybin.tools import *
 
 import argparse
@@ -13,5 +13,4 @@
 import re
 import signal
-import stat
 import sys
 
@@ -28,5 +27,5 @@
 def getMachineType():
 	sh('echo "void ?{}(int*a,int b){}int main(){return 0;}" > .dummy.c')
-	ret, out = sh("make .dummy -s", print2stdout=False)
+	ret, out = sh("make .dummy -s", print2stdout=True)
 	
 	if ret != 0:
@@ -34,11 +33,10 @@
 		print(out)
 		print("Stopping")
-		sh("rm -f .dummy.c > /dev/null 2>&1")
-		sh("rm -f .dummy > /dev/null 2>&1")
+		rm( (".dummy.c",".dummy") )
 		sys.exit(1)
 
 	_, out = sh("file .dummy", print2stdout=False)
-	sh("rm -f .dummy.c > /dev/null 2>&1")
-	sh("rm -f .dummy > /dev/null 2>&1")
+	rm( (".dummy.c",".dummy") )
+
 	return re.search("ELF\s([0-9]+)-bit", out).group(1)
 
@@ -69,240 +67,6 @@
 	return generic_list + typed_list + concurrent_list;
 
-# helper functions to run terminal commands
-def sh(cmd, dry_run = False, print2stdout = True):
-	if dry_run : 	# if this is a dry_run, only print the commands that would be ran
-		print("cmd: %s" % cmd)
-		return 0, None
-	else :			# otherwise create a pipe and run the desired command
-		proc = Popen(cmd, stdout=None if print2stdout else PIPE, stderr=STDOUT, shell=True)
-		out, err = proc.communicate()
-		return proc.returncode, out
-
-# helper function to replace patterns in a file
-def file_replace(fname, pat, s_after):
-    # first, see if the pattern is even in the file.
-    with open(fname) as f:
-        if not any(re.search(pat, line) for line in f):
-            return # pattern does not occur in file so we are done.
-
-    # pattern is in the file, so perform replace operation.
-    with open(fname) as f:
-        out_fname = fname + ".tmp"
-        out = open(out_fname, "w")
-        for line in f:
-            out.write(re.sub(pat, s_after, line))
-        out.close()
-        os.rename(out_fname, fname)
-
-# tests output may differ depending on the depth of the makefile
-def fix_MakeLevel(file) :
-	if environ.get('MAKELEVEL') :
-		file_replace(file, "make\[%i\]" % int(environ.get('MAKELEVEL')), 'make' )
-
-# helper function to check if a files contains only a spacific string
-def fileContainsOnly(file, text) :
-	with open(file) as f:
-		ff = f.read().strip()
-		result = ff == text.strip()
-
-		return result;
-
-# check whether or not a file is executable
-def fileIsExecutable(file) :
-	try :
-		fileinfo = os.stat(file)
-		return bool(fileinfo.st_mode & stat.S_IXUSR)
-	except Exception as inst:
-		print(type(inst))    # the exception instance
-		print(inst.args)     # arguments stored in .args
-		print(inst)
-		return False
-
-################################################################################
-#               running test functions
-################################################################################
-def run_single_test(test, generate, dry_run, debug):
-
-	try : 
-		# find the output file based on the test name and options flag
-		out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path)
-		err_file = ".err/%s.log" % test.name
-
-		# remove any outputs from the previous tests to prevent side effects
-		sh("rm -f %s" % out_file, dry_run)
-		sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run)
-
-		options = "-debug" if debug else "-nodebug";
-
-		# build, skipping to next test on error
-		make_ret, _ = sh("""%s test=yes EXTRA_FLAGS="-quiet %s" %s 2> %s 1> /dev/null""" % (make_cmd, options, test.name, out_file), dry_run)
-
-		# if the make command succeds continue otherwise skip to diff
-		if make_ret == 0 :
-			# fetch optional input
-			stdinput = "< .in/%s.txt" % test.name if isfile(".in/%s.txt" % test.path) else ""
-
-			if fileIsExecutable(test.name) :
-				# run test
-				sh("./%s %s > %s 2>&1" % (test.name, stdinput, out_file), dry_run)
-			else :
-				# simply cat the result into the output
-				sh("cat %s > %s" % (test.name, out_file), dry_run)
-
-		else :
-			# command failed save the log to less temporary file
-			sh("mv %s %s" % (err_file, out_file), dry_run)
-
-		retcode = 0
-		error = None
-
-		# # fix output to prevent make depth to cause issues
-		# fix_MakeLevel(out_file)
-
-		if generate :
-			# if we are ounly generating the output we still need to check that the test actually exists
-			if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.name) :
-				retcode = 1;
-				error = "\t\tNo make target for test %s!" % test.name
-				sh("rm %s" % out_file, False)
-
-		else :
-			# diff the output of the files
-			diff_cmd = ("diff --ignore-all-space "
-						"--ignore-blank-lines "
-						"--old-group-format='\t\tmissing lines :\n"
-						"%%<' \\\n"
-						"--new-group-format='\t\tnew lines :\n"
-						"%%>' \\\n"
-						"--unchanged-group-format='%%=' \\"
-						"--changed-group-format='\t\texpected :\n"
-						"%%<\n"
-						"\t\tgot :\n"
-						"%%>' \\\n"
-						"--new-line-format='\t\t%%dn\t%%L' \\\n"
-						"--old-line-format='\t\t%%dn\t%%L' \\\n"
-						"--unchanged-line-format='' \\\n"
-						".expect/%s.txt .out/%s.log")
-
-			# fetch return code and error from the diff command
-			retcode, error = sh(diff_cmd % (test.path, test.name), dry_run, False)	
-	finally :
-		# clean the executable
-		sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run)
-
-	return retcode, error
-
-def run_test_instance(t, generate, dry_run, debug) :
-
-	signal.signal(signal.SIGINT, signal.SIG_DFL)
-	# print formated name
-	name_txt = "%20s  " % t.name
-
-	#run the test instance and collect the result
-	test_failed, error = run_single_test(t, generate, dry_run, debug)
-
-	# update output based on current action
-	if generate :
-		failed_txt = "ERROR"
-		success_txt = "Done"
-	else :
-		failed_txt = "FAILED"
-		success_txt = "PASSED"
-
-	#print result with error if needed
-	text = name_txt + (failed_txt if test_failed else success_txt)
-	out = sys.stdout
-	if error :
-		text = text + "\n" + error
-		out = sys.stderr
-
-	print(text, file = out);
-	sys.stdout.flush()
-	sys.stderr.flush()
-	signal.signal(signal.SIGINT, signal.SIG_IGN)
-
-	return test_failed
-
-
-# run the given list of tests with the given parameters
-def run_tests(tests, generate, dry_run, jobs, debug) :
-	# clean the sandbox from previous commands
-	sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
-
-	#make sure the required folder are present
-	sh('mkdir -p .out .expect .err', dry_run)
-
-	if generate :
-		print( "Regenerate tests for: " )
-
-	# for each test to run
-	original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
-	pool = Pool(jobs)
-	signal.signal(signal.SIGINT, original_sigint_handler)
-	try :
-		results = pool.map_async(partial(run_test_instance, generate=generate, dry_run=dry_run, debug=debug), tests ).get(9999)
-	except KeyboardInterrupt:
-		pool.terminate()
-		print("Tests interrupted by user")
-		sys.exit(1)
-
-	#clean the workspace
-	sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
-
-	for failed in results:
-		if failed :
-			return 1
-
-	return 0
-
-def yes_no(string):
-	if string == "yes" :
-		return True
-	if string == "no" :
-		return False
-	raise argparse.ArgumentTypeError(msg)
-	return False
-
-
-################################################################################
-#               main loop
-################################################################################
-abspath = os.path.abspath(__file__)
-dname = os.path.dirname(abspath)
-os.chdir(dname)
-
-# create a parser with the arguments for the tests script
-parser = argparse.ArgumentParser(description='Script which runs cforall tests')
-parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='no')
-parser.add_argument('--concurrent', help='Run concurrent tests', type=yes_no, default='yes')
-parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
-parser.add_argument('--list', help='List all test available', action='store_true')
-parser.add_argument('--all', help='Run all test available', action='store_true')
-parser.add_argument('--regenerate-expected', help='Regenerate the .expect by running the specified tets, can be used with --all option', action='store_true')
-parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='8')
-parser.add_argument('--list-comp', help='List all valide arguments', action='store_true')
-parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run')
-
-
-# parse the command line arguments
-options = parser.parse_args()
-do_list = options.list or options.list_comp
-
-# script must have at least some tests to run
-if (len(options.tests) > 0  and     options.all and not do_list) \
-or (len(options.tests) == 0 and not options.all and not do_list) :
-	print('ERROR: must have option \'--all\' or non-empty test list', file=sys.stderr)
-	parser.print_help()
-	sys.exit(1)
-
-# fetch the liest of all valid tests
-allTests = listTests( options.concurrent )
-
-# if user wants all tests than no other treatement of the test list is required
-if options.all or do_list :
-	tests = allTests
-
-else :
-	#otherwise we need to validate that the test list that was entered is valid
+# from the found tests, filter all the valid tests/desired tests
+def validTests( options ):
 	tests = []
 
@@ -311,5 +75,5 @@
 	if options.regenerate_expected :
 		for testname in options.tests :
-			if testname.endswith(".c") or testname.endswith(".cc") or testname.endswith(".cpp") :
+			if testname.endswith( (".c", ".cc", ".cpp") ):
 				print('ERROR: "%s", tests are not allowed to end with a C/C++/CFA extension, ignoring it' % testname, file=sys.stderr)
 			else :
@@ -332,16 +96,36 @@
 		sys.exit(1)
 
-# sort the test alphabetically for convenience
-tests.sort(key=lambda t: t.name)
-
-# users may want to simply list the tests
-if options.list_comp :
-	print("-h --help --debug --concurrent --dry-run --list --all --regenerate-expected -j --jobs ", end='')
-	print(" ".join(map(lambda t: "%s" % (t.name), tests)))
-
-elif options.list :
-	print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests)))
-
-else :
+	return tests
+
+# parses the option
+def getOptions():
+	# create a parser with the arguments for the tests script
+	parser = argparse.ArgumentParser(description='Script which runs cforall tests')
+	parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='no')
+	parser.add_argument('--concurrent', help='Run concurrent tests', type=yes_no, default='yes')
+	parser.add_argument('--dry-run', help='Don\'t run the tests, only output the commands', action='store_true')
+	parser.add_argument('--list', help='List all test available', action='store_true')
+	parser.add_argument('--all', help='Run all test available', action='store_true')
+	parser.add_argument('--regenerate-expected', help='Regenerate the .expect by running the specified tets, can be used with --all option', action='store_true')
+	parser.add_argument('-j', '--jobs', help='Number of tests to run simultaneously', type=int, default='8')
+	parser.add_argument('--list-comp', help='List all valide arguments', action='store_true')
+	parser.add_argument('tests', metavar='test', type=str, nargs='*', help='a list of tests to run')
+
+	options =  parser.parse_args()
+
+	# script must have at least some tests to run or be listing
+	listing    = options.list or options.list_comp
+	all_tests  = options.all
+	some_tests = len(options.tests) > 0
+
+	# check that exactly one of the booleans is set to true
+	if not sum( (listing, all_tests, some_tests) ) == 1 :
+		print('ERROR: must have option \'--all\', \'--list\' or non-empty test list', file=sys.stderr)
+		parser.print_help()
+		sys.exit(1)
+
+	return options
+
+def jobCount( options ):
 	# check if the user already passed in a number of jobs for multi-threading
 	make_flags = environ.get('MAKEFLAGS')
@@ -359,9 +143,163 @@
 		sys.exit(1)
 
-	options.jobs = min( options.jobs, len(tests) )
-
-	print('Running (%s) on %i cores' % ("debug" if options.debug else "no debug", options.jobs))
-	make_cmd = "make" if make_flags else ("make -j%i" % options.jobs)
-
-	# otherwise run all tests and make sure to return the correct error code
-	sys.exit( run_tests(tests, options.regenerate_expected, options.dry_run, options.jobs, options.debug) )
+	return min( options.jobs, len(tests) ), True if make_flags else False
+
+################################################################################
+#               running test functions
+################################################################################
+# logic to run a single test and return the result (No handling of printing or other test framework logic)
+def run_single_test(test, generate, dry_run, debug):
+
+	# find the output file based on the test name and options flag
+	out_file = (".out/%s.log" % test.name) if not generate else (".expect/%s.txt" % test.path)
+	err_file = ".err/%s.log" % test.name
+
+	# remove any outputs from the previous tests to prevent side effects
+	rm( (out_file, test.name), dry_run )
+
+	options = "-debug" if debug else "-nodebug"
+
+	# build, skipping to next test on error
+	make_ret, _ = sh("""%s test=yes EXTRA_FLAGS="-quiet %s" %s 2> %s 1> /dev/null""" % (make_cmd, options, test.name, out_file), dry_run)
+
+	# if the make command succeds continue otherwise skip to diff
+	if make_ret == 0 :
+		# fetch optional input
+		stdinput = "< .in/%s.txt" % test.name if isfile(".in/%s.txt" % test.path) else ""
+
+		if fileIsExecutable(test.name) :
+			# run test
+			sh("./%s %s > %s 2>&1" % (test.name, stdinput, out_file), dry_run)
+		else :
+			# simply cat the result into the output
+			sh("cat %s > %s" % (test.name, out_file), dry_run)
+
+	else :
+		# command failed save the log to less temporary file
+		sh("mv %s %s" % (err_file, out_file), dry_run)
+
+	retcode = 0
+	error = None
+
+	if generate :
+		# if we are ounly generating the output we still need to check that the test actually exists
+		if not dry_run and fileContainsOnly(out_file, "make: *** No rule to make target `%s'.  Stop." % test.name) :
+			retcode = 1;
+			error = "\t\tNo make target for test %s!" % test.name
+			sh("rm %s" % out_file, False)
+
+	else :
+		# fetch return code and error from the diff command
+		retcode, error = diff(".expect/%s.txt" % test.path, ".out/%s.log" % test.name, dry_run)
+	
+	# clean the executable
+	sh("rm -f %s > /dev/null 2>&1" % test.name, dry_run)
+
+	return retcode, error
+
+# run a single test and handle the errors, outputs, printing, exception handling, etc.
+def run_test_worker(t, generate, dry_run, debug) :
+
+	signal.signal(signal.SIGINT, signal.SIG_DFL)
+	# print formated name
+	name_txt = "%20s  " % t.name
+
+	#run the test instance and collect the result
+	test_failed, error = run_single_test(t, generate, dry_run, debug)
+
+	# update output based on current action
+	if generate :
+		failed_txt = "ERROR"
+		success_txt = "Done"
+	else :
+		failed_txt = "FAILED"
+		success_txt = "PASSED"
+
+	#print result with error if needed
+	text = name_txt + (failed_txt if test_failed else success_txt)
+	out = sys.stdout
+	if error :
+		text = text + "\n" + error
+		out = sys.stderr
+
+	print(text, file = out);
+	sys.stdout.flush()
+	sys.stderr.flush()
+	signal.signal(signal.SIGINT, signal.SIG_IGN)
+
+	return test_failed
+
+# run the given list of tests with the given parameters
+def run_tests(tests, generate, dry_run, jobs, debug) :
+	# clean the sandbox from previous commands
+	sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
+
+	# make sure the required folder are present
+	sh('mkdir -p .out .expect .err', dry_run)
+
+	if generate :
+		print( "Regenerate tests for: " )
+
+	# create the executor for our jobs and handle the signal properly
+	original_sigint_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)
+	pool = Pool(jobs)
+	signal.signal(signal.SIGINT, original_sigint_handler)
+
+	# for each test to run
+	try :
+		results = pool.map_async(partial(run_test_worker, generate=generate, dry_run=dry_run, debug=debug), tests ).get(3600)
+	except KeyboardInterrupt:
+		pool.terminate()
+		print("Tests interrupted by user")
+		sys.exit(1)
+
+	# clean the workspace
+	sh("%s clean > /dev/null 2>&1" % make_cmd, dry_run)
+
+	for failed in results:
+		if failed :
+			return 1
+
+	return 0
+
+
+################################################################################
+#               main loop
+################################################################################
+if __name__ == "__main__":
+	#always run from same folder
+	chdir() 
+	
+	# parse the command line arguments
+	options = getOptions()
+
+	# fetch the liest of all valid tests
+	allTests = listTests( options.concurrent )
+
+	# if user wants all tests than no other treatement of the test list is required
+	if options.all or options.list or options.list_comp :
+		tests = allTests
+
+	else :
+		#otherwise we need to validate that the test list that was entered is valid
+		tests = validTests( options )
+
+	# sort the test alphabetically for convenience
+	tests.sort(key=lambda t: t.name)
+
+	# users may want to simply list the tests
+	if options.list_comp :
+		print("-h --help --debug --concurrent --dry-run --list --all --regenerate-expected -j --jobs ", end='')
+		print(" ".join(map(lambda t: "%s" % (t.name), tests)))
+
+	elif options.list :
+		print("\n".join(map(lambda t: "%s (%s)" % (t.name, t.path), tests)))
+
+	else :
+		options.jobs, forceJobs = jobCount( options )
+
+		print('Running (%s) on %i cores' % ("debug" if options.debug else "no debug", options.jobs))
+		make_cmd = "make" if forceJobs else ("make -j%i" % options.jobs)
+
+		# otherwise run all tests and make sure to return the correct error code
+		sys.exit( run_tests(tests, options.regenerate_expected, options.dry_run, options.jobs, options.debug) )