Add ./mach filter-intermittents for catching intermittents on CI

1 files changed, 48 insertions, 0 deletions

diff --git a/python/servo/testing_commands.py b/python/servo/testing_commands.py
index f24750df245..dffc38c9646 100644
--- a/python/servo/testing_commands.py
+++ b/python/servo/testing_commands.py
@@ -17,6 +17,9 @@ import os.path as path
 import copy
 from collections import OrderedDict
 from time import time
+import json
+import urllib2
+import base64
 
 from mach.registrar import Registrar
 from mach.decorators import (
@@ -470,6 +473,51 @@ class MachCommands(CommandBase):
         execfile(run_file, run_globals)
         return run_globals["update_tests"](**kwargs)
 
+    @Command('filter-intermittents',
+             description='Given a WPT error summary file, filter out intermittents and other cruft.',
+             category='testing')
+    @CommandArgument('summary',
+                     help="Error summary log to take un")
+    @CommandArgument('--output', default=None,
+                     help='Print filtered log to file')
+    @CommandArgument('--auth', default=None,
+                     help='File containing basic authorization credentials for Github API (format `username:password`)')
+    def filter_intermittents(self, summary, output, auth):
+        encoded_auth = None
+        if auth:
+            with open(auth, "r") as file:
+                encoded_auth = base64.encodestring(file.read().strip()).replace('\n', '')
+        failures = []
+        with open(summary, "r") as file:
+            for line in file:
+                line_json = json.loads(line)
+                if 'status' in line_json:
+                    failures += [line_json]
+        actual_failures = []
+        for failure in failures:
+            qstr = "repo:servo/servo+label:I-intermittent+type:issue+state:open+%s" % failure['test']
+            # we want `/` to get quoted, but not `+` (github's API doesn't like that), so we set `safe` to `+`
+            query = urllib2.quote(qstr, safe='+')
+            request = urllib2.Request("https://api.github.com/search/issues?q=%s" % query)
+            if encoded_auth:
+                request.add_header("Authorization", "Basic %s" % encoded_auth)
+            search = urllib2.urlopen(request)
+            data = json.load(search)
+            if data['total_count'] == 0:
+                actual_failures += [failure]
+
+        if len(actual_failures) == 0:
+            return 0
+
+        output = open(output, "w") if output else sys.stdout
+        for failure in actual_failures:
+            json.dump(failure, output)
+            print("\n", end='', file=output)
+
+        if output is not sys.stdout:
+            output.close()
+        return 1
+
     @Command('test-jquery',
              description='Run the jQuery test suite',
              category='testing')