Index: tests/test.py
===================================================================
--- tests/test.py	(revision 29806675e245a4300bb8eeae405bb961d95519f6)
+++ tests/test.py	(revision 136f86b8d410dcce05db552ea01f4f4caa406aab)
@@ -6,4 +6,5 @@
 
 import argparse
+import itertools
 import re
 import sys
@@ -29,6 +30,5 @@
 			test.path = match.group(1)
 			test.arch = match.group(3)[1:] if match.group(3) else None
-			if settings.arch.match(test.arch):
-				expected.append(test)
+			expected.append(test)
 
 	path_walk( match_test )
@@ -52,4 +52,7 @@
 			x.target().startswith( tuple(excludes) )
 		]
+
+	# sort the test alphabetically for convenience
+	test_list.sort(key=lambda t: ('~' if t.arch else '') + t.target() + (t.arch if t.arch else ''))
 
 	return test_list
@@ -76,5 +79,5 @@
 
 			if test :
-				tests.append( test[0] )
+				tests.extend( test )
 			else :
 				print('ERROR: No expected file for test %s, ignoring it' % testname, file=sys.stderr)
@@ -86,7 +89,8 @@
 	# create a parser with the arguments for the tests script
 	parser = argparse.ArgumentParser(description='Script which runs cforall tests')
-	parser.add_argument('--debug', help='Run all tests in debug or release', type=yes_no, default='yes')
-	parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=yes_no, default='no')
-	parser.add_argument('--arch', help='Test for specific architecture', type=str, default='')
+	parser.add_argument('--debug', help='Run all tests in debug or release', type=comma_separated(yes_no), default='yes')
+	parser.add_argument('--install', help='Run all tests based on installed binaries or tree binaries', type=comma_separated(yes_no), default='no')
+	parser.add_argument('--arch', help='Test for specific architecture', type=comma_separated(str), default='')
+	parser.add_argument('--continue', help='When multiple specifications are passed (debug/install/arch), sets whether or not to continue if the last specification failed', type=yes_no, default='yes', dest='continue_')
 	parser.add_argument('--timeout', help='Maximum duration in seconds after a single test is considered to have timed out', type=int, default=60)
 	parser.add_argument('--global-timeout', help='Maximum cumulative duration in seconds after the ALL tests are considered to have timed out', type=int, default=7200)
@@ -279,5 +283,5 @@
 	make('clean', output_file=subprocess.DEVNULL, error=subprocess.DEVNULL)
 
-	return 1 if failed else 0
+	return failed
 
 
@@ -298,7 +302,4 @@
 		tests = list_tests( None, None )
 
-		# sort the test alphabetically for convenience
-		tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
-
 		# print the possible options
 		print("-h --help --debug --dry-run --list --arch --all --regenerate-expected --archive-errors --install --timeout --global-timeout --timeout-with-gdb -j --jobs ", end='')
@@ -309,9 +310,5 @@
 		tests = list_tests( options.include, options.exclude )
 
-		# sort the test alphabetically for convenience
-		tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
-
 		# print the available tests
-		print("Listing for %s:%s"% (settings.arch.string, settings.debug.string))
 		fancy_print("\n".join(map(lambda t: t.toString(), tests)))
 
@@ -333,22 +330,39 @@
 			sys.exit(1)
 
-		# sort the test alphabetically for convenience
-		tests.sort(key=lambda t: (t.arch if t.arch else '') + t.target())
-
-		# check the build configuration works
+		# prep invariants
 		settings.prep_output(tests)
-		settings.validate()
-
-		options.jobs, forceJobs = job_count( options, tests )
-		settings.update_make_cmd(forceJobs, options.jobs)
-
-		print('%s %i tests on %i cores (%s:%s)' % (
-			'Regenerating' if settings.generating else 'Running',
-			len(tests),
-			options.jobs,
-			settings.arch.string,
-			settings.debug.string
-		))
-
-		# otherwise run all tests and make sure to return the correct error code
-		sys.exit( run_tests(tests, options.jobs) )
+		failed = 0
+
+		# for each build configurations, run the test
+		for arch, debug, install in itertools.product(settings.all_arch, settings.all_debug, settings.all_install):
+			settings.arch    = arch
+			settings.debug   = debug
+			settings.install = install
+
+			# filter out the tests for a different architecture
+			# tests are the same across debug/install
+			local_tests = settings.arch.filter( tests )
+			options.jobs, forceJobs = job_count( options, local_tests )
+			settings.update_make_cmd(forceJobs, options.jobs)
+
+			# check the build configuration works
+			settings.validate()
+
+			# print configuration
+			print('%s %i tests on %i cores (%s:%s)' % (
+				'Regenerating' if settings.generating else 'Running',
+				len(local_tests),
+				options.jobs,
+				settings.arch.string,
+				settings.debug.string
+			))
+
+			# otherwise run all tests and make sure to return the correct error code
+			failed = run_tests(local_tests, options.jobs)
+			if failed:
+				result = 1
+				if not settings.continue_:
+					break
+
+
+		sys.exit( failed )
