Auto merge of #19417 - dsandeephegde:master, r=asajeffrey,jdm

Mutation Test: with more mutation strategies

<!-- Please describe your changes on the following line: -->

1. Added following mutation strategies:
 - If True (make if always true)
 - If False(make if always false)
 - Modify Comparision (<= to <, >= to >)
 - Plus To Minus
 - Minus To Plus
 - Changing Atomic Strings (make string constant empty)
 - Duplicate Line
 - Delete If Block
2. Randomized the test order.
3. Introduced logging instead of print.
4. Added retry mechanism when mutation cannot be performed on a file by a strategy.

---
<!-- Thank you for contributing to Servo! Please replace each `[ ]` by `[X]` when the step is complete, and replace `__` with appropriate data: -->
- [x] `./mach build -d` does not report any errors
- [X] `./mach test-tidy` does not report any errors
- [X] These changes #18529 (github issue number if applicable).

<!-- Either: -->
- [X] These changes do not require tests because it is a python script to run mutation test and does not change any behavior.

<!-- Also, please make sure that "Allow edits from maintainers" checkbox is checked, so that we can help you if you get stuck somewhere along the way.-->

<!-- Pull requests that do not address these steps are welcome, but they will require additional verification as part of the review process. -->

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/19417)
<!-- Reviewable:end -->

1 files changed, 37 insertions, 47 deletions

diff --git a/python/servo/mutation/test.py b/python/servo/mutation/test.py
index 2248f81ccd3..d209328ac22 100644
--- a/python/servo/mutation/test.py
+++ b/python/servo/mutation/test.py
@@ -7,15 +7,17 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-import fileinput
-import re
 import subprocess
-import sys
 import os
 import random
+import logging
+
+from mutator import Mutator, get_strategies
 from enum import Enum
 DEVNULL = open(os.devnull, 'wb')
 
+logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG)
+
 
 class Status(Enum):
     KILLED = 0
@@ -24,55 +26,43 @@ class Status(Enum):
     UNEXPECTED = 3
 
 
-def mutate_random_line(file_name, strategy):
-    line_numbers = []
-    for line in fileinput.input(file_name):
-        if re.search(strategy['regex'], line):
-            line_numbers.append(fileinput.lineno())
-    if len(line_numbers) == 0:
-        return -1
-    else:
-        mutation_line_number = line_numbers[random.randint(0, len(line_numbers) - 1)]
-        for line in fileinput.input(file_name, inplace=True):
-            if fileinput.lineno() == mutation_line_number:
-                line = re.sub(strategy['regex'], strategy['replaceString'], line)
-            print line.rstrip()
-        return mutation_line_number
-
-
 def mutation_test(file_name, tests):
     status = Status.UNEXPECTED
     local_changes_present = subprocess.call('git diff --quiet {0}'.format(file_name), shell=True)
     if local_changes_present == 1:
         status = Status.SKIPPED
-        print "{0} has local changes, please commit/remove changes before running the test".format(file_name)
+        logging.warning("{0} has local changes, please commit/remove changes before running the test".format(file_name))
     else:
-        strategy = {'regex': r'\s&&\s', 'replaceString': ' || '}
-        mutated_line = mutate_random_line(file_name, strategy)
-        if mutated_line != -1:
-            print "Mutating {0} at line {1}".format(file_name, mutated_line)
-            print "compiling mutant {0}:{1}".format(file_name, mutated_line)
-            sys.stdout.flush()
-            subprocess.call('python mach build --release', shell=True, stdout=DEVNULL)
-            for test in tests:
-                test_command = "python mach test-wpt {0} --release".format(test.encode('utf-8'))
-                print "running `{0}` test for mutant {1}:{2}".format(test, file_name, mutated_line)
-                sys.stdout.flush()
-                test_status = subprocess.call(test_command, shell=True, stdout=DEVNULL)
-                if test_status != 0:
-                    print("Failed: while running `{0}`".format(test_command))
-                    print "mutated file {0} diff".format(file_name)
-                    sys.stdout.flush()
-                    subprocess.call('git --no-pager diff {0}'.format(file_name), shell=True)
-                    status = Status.SURVIVED
+        strategies = list(get_strategies())
+        while len(strategies):
+            strategy = random.choice(strategies)
+            strategies.remove(strategy)
+            mutator = Mutator(strategy())
+            mutated_line = mutator.mutate(file_name)
+            if mutated_line != -1:
+                logging.info("Mutated {0} at line {1}".format(file_name, mutated_line))
+                logging.info("compiling mutant {0}:{1}".format(file_name, mutated_line))
+                if subprocess.call('python mach build --release', shell=True, stdout=DEVNULL):
+                    logging.error("Compilation Failed: Unexpected error")
+                    status = Status.UNEXPECTED
                 else:
-                    print("Success: Mutation killed by {0}".format(test.encode('utf-8')))
-                    status = Status.KILLED
-                    break
-            print "reverting mutant {0}:{1}".format(file_name, mutated_line)
-            sys.stdout.flush()
-            subprocess.call('git checkout {0}'.format(file_name), shell=True)
-        else:
-            print "Cannot mutate {0}".format(file_name)
-        print "-" * 80 + "\n"
+                    for test in tests:
+                        test_command = "python mach test-wpt {0} --release".format(test.encode('utf-8'))
+                        logging.info("running `{0}` test for mutant {1}:{2}".format(test, file_name, mutated_line))
+                        test_status = subprocess.call(test_command, shell=True, stdout=DEVNULL)
+                        if test_status != 0:
+                            logging.error("Failed: while running `{0}`".format(test_command))
+                            logging.error("mutated file {0} diff".format(file_name))
+                            subprocess.call('git --no-pager diff {0}'.format(file_name), shell=True)
+                            status = Status.SURVIVED
+                        else:
+                            logging.info("Success: Mutation killed by {0}".format(test.encode('utf-8')))
+                            status = Status.KILLED
+                            break
+                logging.info("reverting mutant {0}:{1}\n".format(file_name, mutated_line))
+                subprocess.call('git checkout {0}'.format(file_name), shell=True)
+                break
+            elif not len(strategies):
+                # All strategies are tried
+                logging.info("\nCannot mutate {0}\n".format(file_name))
     return status