Auto merge of #12781 - UK992:tidycheck-rebased, r=Wafflespeanut

Improve tidy's license validation logic

Rebased and fixed https://github.com/servo/servo/pull/10721, which is inactive for months.
Fixes https://github.com/servo/servo/issues/10716

r? @larsbergstrom or @edunham

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/servo/12781)
<!-- Reviewable:end -->

2 files changed, 61 insertions, 103 deletions

diff --git a/python/tidy/servo_tidy/licenseck.py b/python/tidy/servo_tidy/licenseck.py
index 3a17f1fdf04..a4953e6eabb 100644
--- a/python/tidy/servo_tidy/licenseck.py
+++ b/python/tidy/servo_tidy/licenseck.py
@@ -7,97 +7,24 @@
 # option. This file may not be copied, modified, or distributed
 # except according to those terms.
 
-
-# These licenses are valid for use in Servo
-licenses = [
-
-"""\
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
-""",
-
-"""\
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-""",
-
-"""\
-#!/usr/bin/env bash
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-""",
-
-"""\
-#!/usr/bin/env python
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-""",
-
-"""\
-#!/usr/bin/env python3
-
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-""",
-
-"""\
-// This Source Code Form is subject to the terms of the Mozilla Public
-// License, v. 2.0. If a copy of the MPL was not distributed with this
-// file, You can obtain one at http://mozilla.org/MPL/2.0/.
-""",
-
-"""\
-// Copyright 2013 The Servo Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-""",
-
-"""\
-# Copyright 2013 The Servo Project Developers. See the COPYRIGHT
-# file at the top-level directory of this distribution.
-#
-# Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-# http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-# <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-# option. This file may not be copied, modified, or distributed
-# except according to those terms.
-""",
-
-"""\
-// Copyright 2015 The Rust Project Developers. See the COPYRIGHT
-// file at the top-level directory of this distribution and at
-// http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-""",
-
-"""\
-// Copyright 2012-2014 The Rust Project Developers.
-// See http://rust-lang.org/COPYRIGHT.
-//
-// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
-// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
-// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
-// option. This file may not be copied, modified, or distributed
-// except according to those terms.
-""",
-]  # noqa: Indicate to flake8 that we do not want to check indentation here
+MPL = """\
+This Source Code Form is subject to the terms of the Mozilla Public \
+License, v. 2.0. If a copy of the MPL was not distributed with this \
+file, You can obtain one at http://mozilla.org/MPL/2.0/.\
+"""
+
+APACHE = """\
+Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or \
+http://www.apache.org/licenses/LICENSE-2.0> or the MIT license \
+<LICENSE-MIT or http://opensource.org/licenses/MIT>, at your \
+option. This file may not be copied, modified, or distributed \
+except according to those terms.\
+"""
+
+COPYRIGHT = [
+    "See the COPYRIGHT file at the top-level directory of this distribution",
+    "See http://rust-lang.org/COPYRIGHT",
+]
 
 # The valid licenses, in the form we'd expect to see them in a Cargo.toml file.
 licenses_toml = [
diff --git a/python/tidy/servo_tidy/tidy.py b/python/tidy/servo_tidy/tidy.py
index b21d65259fb..d6d6344db54 100644
--- a/python/tidy/servo_tidy/tidy.py
+++ b/python/tidy/servo_tidy/tidy.py
@@ -17,12 +17,10 @@ import site
 import StringIO
 import subprocess
 import sys
-from licenseck import licenses, licenses_toml, licenses_dep_toml
+from licenseck import MPL, APACHE, COPYRIGHT, licenses_toml, licenses_dep_toml
+import colorama
 
-# License and header checks
-EMACS_HEADER = "/* -*- Mode:"
-VIM_HEADER = "/* vim:"
-MAX_LICENSE_LINESPAN = max(len(license.splitlines()) for license in licenses)
+COMMENTS = ["// ", "# ", " *", "/* "]
 
 # File patterns to include in the non-WPT tidy check.
 FILE_PATTERNS_TO_CHECK = ["*.rs", "*.rc", "*.cpp", "*.c",
@@ -60,7 +58,6 @@ IGNORED_FILES = [
 IGNORED_DIRS = [
     # Upstream
     os.path.join(".", "support", "android", "apk"),
-    os.path.join(".", "support", "rust-task_info"),
     os.path.join(".", "tests", "wpt", "css-tests"),
     os.path.join(".", "tests", "wpt", "harness"),
     os.path.join(".", "tests", "wpt", "update"),
@@ -152,13 +149,45 @@ def filter_files(start_dir, only_changed_files, progress):
         yield file_name
 
 
+def uncomment(line):
+    for c in COMMENTS:
+        if line.startswith(c):
+            if line.endswith("*/"):
+                return line[len(c):(len(line) - 3)].strip()
+            return line[len(c):].strip()
+
+
+def licensed_mpl(header):
+    return MPL in header
+
+
+def licensed_apache(header):
+    if APACHE in header:
+        return any(c in header for c in COPYRIGHT)
+
+
 def check_license(file_name, lines):
     if any(file_name.endswith(ext) for ext in (".toml", ".lock", ".json", ".html")):
         raise StopIteration
-    while lines and (lines[0].startswith(EMACS_HEADER) or lines[0].startswith(VIM_HEADER)):
-        lines = lines[1:]
-    contents = "".join(lines[:MAX_LICENSE_LINESPAN])
-    valid_license = any(contents.startswith(license) for license in licenses)
+
+    if lines[0].startswith("#!") and lines[1].strip():
+        yield (1, "missing blank line after shebang")
+
+    blank_lines = 0
+    max_blank_lines = 2 if lines[0].startswith("#!") else 1
+    license_block = []
+
+    for l in lines:
+        l = l.rstrip('\n')
+        if not l.strip():
+            blank_lines += 1
+        if blank_lines >= max_blank_lines:
+            break
+        line = uncomment(l)
+        if line is not None:
+            license_block.append(line)
+    contents = " ".join(license_block)
+    valid_license = licensed_mpl(contents) or licensed_apache(contents)
     acknowledged_bad_license = "xfail-license" in contents
     if not (valid_license or acknowledged_bad_license):
         yield (1, "incorrect license")
@@ -310,8 +339,8 @@ def check_toml(file_name, lines):
     for idx, line in enumerate(lines):
         if line.find("*") != -1:
             yield (idx + 1, "found asterisk instead of minimum version number")
-        for license in licenses_toml:
-            ok_licensed |= (license in line)
+        for license_line in licenses_toml:
+            ok_licensed |= (license_line in line)
     if not ok_licensed:
         yield (0, ".toml file should contain a valid license.")
 
@@ -762,8 +791,10 @@ def scan(only_changed_files=False, progress=True):
     errors = itertools.chain(errors, dep_license_errors, wpt_lint_errors)
     error = None
     for error in errors:
+        colorama.init()
         print "\r\033[94m{}\033[0m:\033[93m{}\033[0m: \033[91m{}\033[0m".format(*error)
     print
     if error is None:
+        colorama.init()
         print "\033[92mtidy reported no errors.\033[0m"
     return int(error is not None)