Mutation Testing

Introduced strategy design pattern

Added Strategies:
if true and if false
Replace String literals
Remove if blocks which do not have else.
Modify Comparision Statement - changing <= to < and changing >= to >
Duplicating statements
Added plus to minus and minus to plus mutaiton strategy

Classifying build failures for mutant as unexpected error - Skipping test run for mutant with compilation failure
Added logger messages instead of print
Randomized the mutation test order
Try new strategy on failure to mutate on a file
Updated Readme - Adding mutation strategy and mutation test execution flow

1 files changed, 37 insertions, 47 deletions

diff --git a/python/servo/mutation/test.py b/python/servo/mutation/test.py
index 2248f81ccd3..d209328ac22 100644
--- a/python/servo/mutation/test.py
+++ b/python/servo/mutation/test.py
@@ -7,15 +7,17 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-import fileinput
-import re
 import subprocess
-import sys
 import os
 import random
+import logging
+
+from mutator import Mutator, get_strategies
 from enum import Enum
 DEVNULL = open(os.devnull, 'wb')
 
+logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG)
+
 
 class Status(Enum):
     KILLED = 0
@@ -24,55 +26,43 @@ class Status(Enum):
     UNEXPECTED = 3
 
 
-def mutate_random_line(file_name, strategy):
-    line_numbers = []
-    for line in fileinput.input(file_name):
-        if re.search(strategy['regex'], line):
-            line_numbers.append(fileinput.lineno())
-    if len(line_numbers) == 0:
-        return -1
-    else:
-        mutation_line_number = line_numbers[random.randint(0, len(line_numbers) - 1)]
-        for line in fileinput.input(file_name, inplace=True):
-            if fileinput.lineno() == mutation_line_number:
-                line = re.sub(strategy['regex'], strategy['replaceString'], line)
-            print line.rstrip()
-        return mutation_line_number
-
-
 def mutation_test(file_name, tests):
     status = Status.UNEXPECTED
     local_changes_present = subprocess.call('git diff --quiet {0}'.format(file_name), shell=True)
     if local_changes_present == 1:
         status = Status.SKIPPED
-        print "{0} has local changes, please commit/remove changes before running the test".format(file_name)
+        logging.warning("{0} has local changes, please commit/remove changes before running the test".format(file_name))
     else:
-        strategy = {'regex': r'\s&&\s', 'replaceString': ' || '}
-        mutated_line = mutate_random_line(file_name, strategy)
-        if mutated_line != -1:
-            print "Mutating {0} at line {1}".format(file_name, mutated_line)
-            print "compiling mutant {0}:{1}".format(file_name, mutated_line)
-            sys.stdout.flush()
-            subprocess.call('python mach build --release', shell=True, stdout=DEVNULL)
-            for test in tests:
-                test_command = "python mach test-wpt {0} --release".format(test.encode('utf-8'))
-                print "running `{0}` test for mutant {1}:{2}".format(test, file_name, mutated_line)
-                sys.stdout.flush()
-                test_status = subprocess.call(test_command, shell=True, stdout=DEVNULL)
-                if test_status != 0:
-                    print("Failed: while running `{0}`".format(test_command))
-                    print "mutated file {0} diff".format(file_name)
-                    sys.stdout.flush()
-                    subprocess.call('git --no-pager diff {0}'.format(file_name), shell=True)
-                    status = Status.SURVIVED
+        strategies = list(get_strategies())
+        while len(strategies):
+            strategy = random.choice(strategies)
+            strategies.remove(strategy)
+            mutator = Mutator(strategy())
+            mutated_line = mutator.mutate(file_name)
+            if mutated_line != -1:
+                logging.info("Mutated {0} at line {1}".format(file_name, mutated_line))
+                logging.info("compiling mutant {0}:{1}".format(file_name, mutated_line))
+                if subprocess.call('python mach build --release', shell=True, stdout=DEVNULL):
+                    logging.error("Compilation Failed: Unexpected error")
+                    status = Status.UNEXPECTED
                 else:
-                    print("Success: Mutation killed by {0}".format(test.encode('utf-8')))
-                    status = Status.KILLED
-                    break
-            print "reverting mutant {0}:{1}".format(file_name, mutated_line)
-            sys.stdout.flush()
-            subprocess.call('git checkout {0}'.format(file_name), shell=True)
-        else:
-            print "Cannot mutate {0}".format(file_name)
-        print "-" * 80 + "\n"
+                    for test in tests:
+                        test_command = "python mach test-wpt {0} --release".format(test.encode('utf-8'))
+                        logging.info("running `{0}` test for mutant {1}:{2}".format(test, file_name, mutated_line))
+                        test_status = subprocess.call(test_command, shell=True, stdout=DEVNULL)
+                        if test_status != 0:
+                            logging.error("Failed: while running `{0}`".format(test_command))
+                            logging.error("mutated file {0} diff".format(file_name))
+                            subprocess.call('git --no-pager diff {0}'.format(file_name), shell=True)
+                            status = Status.SURVIVED
+                        else:
+                            logging.info("Success: Mutation killed by {0}".format(test.encode('utf-8')))
+                            status = Status.KILLED
+                            break
+                logging.info("reverting mutant {0}:{1}\n".format(file_name, mutated_line))
+                subprocess.call('git checkout {0}'.format(file_name), shell=True)
+                break
+            elif not len(strategies):
+                # All strategies are tried
+                logging.info("\nCannot mutate {0}\n".format(file_name))
     return status