Added mutation test summary and made it exit with relevant exit code

2 files changed, 27 insertions, 1 deletions

diff --git a/python/servo/mutation/init.py b/python/servo/mutation/init.py
index 6087dc2500d..aa7b86c5f00 100644
--- a/python/servo/mutation/init.py
+++ b/python/servo/mutation/init.py
@@ -12,6 +12,12 @@ from os.path import isfile, isdir, join
 import json
 import sys
 import test
+test_summary = {
+    test.Status.PASSED: 0,
+    test.Status.FAILED: 0,
+    test.Status.SKIPPED: 0,
+    test.Status.UNEXPECTED: 0
+}
 
 
 def get_folders_list(path):
@@ -30,11 +36,18 @@ def mutation_test_for(mutation_path):
         test_mapping = json.loads(json_data)
         # Run mutation test for all source files in mapping file.
         for src_file in test_mapping.keys():
-            test.mutation_test(join(mutation_path, src_file.encode('utf-8')), test_mapping[src_file])
+            status = test.mutation_test(join(mutation_path, src_file.encode('utf-8')), test_mapping[src_file])
+            test_summary[status] += 1
         # Run mutation test in all folder in the path.
         for folder in get_folders_list(mutation_path):
             mutation_test_for(folder)
     else:
         print("This folder {0} has no test mapping file.".format(mutation_path))
 
+
 mutation_test_for(sys.argv[1])
+print "\nTest Summary:"
+for test_status in test_summary:
+    print "{0} : {1}".format(test_status.name, test_summary[test_status])
+if test_summary[test.Status.FAILED] or test_summary[test.Status.UNEXPECTED]:
+    sys.exit(1)
diff --git a/python/servo/mutation/test.py b/python/servo/mutation/test.py
index b004dc3fcd9..46bd3950778 100644
--- a/python/servo/mutation/test.py
+++ b/python/servo/mutation/test.py
@@ -13,9 +13,17 @@ import subprocess
 import sys
 import os
 import random
+from enum import Enum
 DEVNULL = open(os.devnull, 'wb')
 
 
+class Status(Enum):
+    PASSED = 0
+    FAILED = 1
+    SKIPPED = 2
+    UNEXPECTED = 3
+
+
 def mutate_random_line(file_name):
     line_numbers = []
     for line in fileinput.input(file_name):
@@ -33,8 +41,10 @@ def mutate_random_line(file_name):
 
 
 def mutation_test(file_name, tests):
+    status = Status.UNEXPECTED
     local_changes_present = subprocess.call('git diff --quiet {0}'.format(file_name), shell=True)
     if local_changes_present == 1:
+        status = Status.SKIPPED
         print "{0} has local changes, please commit/remove changes before running the test".format(file_name)
     else:
         mutated_line = mutate_random_line(file_name)
@@ -53,8 +63,10 @@ def mutation_test(file_name, tests):
                     print "mutated file {0} diff".format(file_name)
                     sys.stdout.flush()
                     subprocess.call('git --no-pager diff {0}'.format(file_name), shell=True)
+                    status = Status.FAILED
                 else:
                     print("Success: Mutation killed by {0}".format(test.encode('utf-8')))
+                    status = Status.PASSED
                     break
             print "reverting mutant {0}:{1}".format(file_name, mutated_line)
             sys.stdout.flush()
@@ -62,3 +74,4 @@ def mutation_test(file_name, tests):
         else:
             print "Cannot mutate {0}".format(file_name)
         print "-" * 80 + "\n"
+    return status